Class | Gem::RemoteFetcher |
In: |
lib/rubygems/remote_fetcher.rb
|
Parent: | Object |
RemoteFetcher handles the details of fetching gems and gem information from a remote source.
Cached RemoteFetcher instance.
# File lib/rubygems/remote_fetcher.rb, line 40 40: def self.fetcher 41: @fetcher ||= self.new Gem.configuration[:http_proxy] 42: end
Initialize a remote fetcher using the source URI and possible proxy information.
proxy
variable setting
HTTP_PROXY_PASS)
# File lib/rubygems/remote_fetcher.rb, line 55 55: def initialize(proxy = nil) 56: require 'net/http' 57: require 'stringio' 58: require 'time' 59: require 'uri' 60: 61: Socket.do_not_reverse_lookup = true 62: 63: @connections = {} 64: @requests = Hash.new 0 65: @proxy_uri = 66: case proxy 67: when :no_proxy then nil 68: when nil then get_proxy_from_env 69: when URI::HTTP then proxy 70: else URI.parse(proxy) 71: end 72: end
Creates or an HTTP connection based on uri, or retrieves an existing connection, using a proxy if needed.
# File lib/rubygems/remote_fetcher.rb, line 244 244: def connection_for(uri) 245: net_http_args = [uri.host, uri.port] 246: 247: if @proxy_uri then 248: net_http_args += [ 249: @proxy_uri.host, 250: @proxy_uri.port, 251: @proxy_uri.user, 252: @proxy_uri.password 253: ] 254: end 255: 256: connection_id = [Thread.current.object_id, *net_http_args].join ':' 257: @connections[connection_id] ||= Net::HTTP.new(*net_http_args) 258: connection = @connections[connection_id] 259: 260: if uri.scheme == 'https' and not connection.started? then 261: require 'net/https' 262: connection.use_ssl = true 263: connection.verify_mode = OpenSSL::SSL::VERIFY_NONE 264: end 265: 266: connection.start unless connection.started? 267: 268: connection 269: rescue Errno::EHOSTDOWN => e 270: raise FetchError.new(e.message, uri) 271: end
Moves the gem spec from source_uri to the cache dir unless it is already there. If the source_uri is local the gem cache dir copy is always replaced.
# File lib/rubygems/remote_fetcher.rb, line 79 79: def download(spec, source_uri, install_dir = Gem.dir) 80: Gem.ensure_gem_subdirectories(install_dir) rescue nil 81: 82: if File.writable?(install_dir) 83: cache_dir = File.join install_dir, 'cache' 84: else 85: cache_dir = File.join(Gem.user_dir, 'cache') 86: end 87: 88: gem_file_name = spec.file_name 89: local_gem_path = File.join cache_dir, gem_file_name 90: 91: FileUtils.mkdir_p cache_dir rescue nil unless File.exist? cache_dir 92: 93: # Always escape URI's to deal with potential spaces and such 94: unless URI::Generic === source_uri 95: source_uri = URI.parse(URI.const_defined?(:DEFAULT_PARSER) ? 96: URI::DEFAULT_PARSER.escape(source_uri) : 97: URI.escape(source_uri)) 98: end 99: 100: scheme = source_uri.scheme 101: 102: # URI.parse gets confused by MS Windows paths with forward slashes. 103: scheme = nil if scheme =~ /^[a-z]$/i 104: 105: case scheme 106: when 'http', 'https' then 107: unless File.exist? local_gem_path then 108: begin 109: say "Downloading gem #{gem_file_name}" if 110: Gem.configuration.really_verbose 111: 112: remote_gem_path = source_uri + "gems/#{gem_file_name}" 113: 114: gem = self.fetch_path remote_gem_path 115: rescue Gem::RemoteFetcher::FetchError 116: raise if spec.original_platform == spec.platform 117: 118: alternate_name = "#{spec.original_name}.gem" 119: 120: say "Failed, downloading gem #{alternate_name}" if 121: Gem.configuration.really_verbose 122: 123: remote_gem_path = source_uri + "gems/#{alternate_name}" 124: 125: gem = self.fetch_path remote_gem_path 126: end 127: 128: File.open local_gem_path, 'wb' do |fp| 129: fp.write gem 130: end 131: end 132: when 'file' then 133: begin 134: path = source_uri.path 135: path = File.dirname(path) if File.extname(path) == '.gem' 136: 137: remote_gem_path = File.join(path, 'gems', gem_file_name) 138: 139: FileUtils.cp(remote_gem_path, local_gem_path) 140: rescue Errno::EACCES 141: local_gem_path = source_uri.to_s 142: end 143: 144: say "Using local gem #{local_gem_path}" if 145: Gem.configuration.really_verbose 146: when nil then # TODO test for local overriding cache 147: source_path = if Gem.win_platform? && source_uri.scheme && 148: !source_uri.path.include?(':') then 149: "#{source_uri.scheme}:#{source_uri.path}" 150: else 151: source_uri.path 152: end 153: 154: source_path = unescape source_path 155: 156: begin 157: FileUtils.cp source_path, local_gem_path unless 158: File.expand_path(source_path) == File.expand_path(local_gem_path) 159: rescue Errno::EACCES 160: local_gem_path = source_uri.to_s 161: end 162: 163: say "Using local gem #{local_gem_path}" if 164: Gem.configuration.really_verbose 165: else 166: raise Gem::InstallError, "unsupported URI scheme #{source_uri.scheme}" 167: end 168: 169: local_gem_path 170: end
# File lib/rubygems/remote_fetcher.rb, line 196 196: def escape(str) 197: return unless str 198: @uri_parser ||= uri_escaper 199: @uri_parser.escape str 200: end
Downloads uri and returns it as a String.
# File lib/rubygems/remote_fetcher.rb, line 175 175: def fetch_path(uri, mtime = nil, head = false) 176: data = open_uri_or_path uri, mtime, head 177: data = Gem.gunzip data if data and not head and uri.to_s =~ /gz$/ 178: data 179: rescue FetchError 180: raise 181: rescue Timeout::Error 182: raise FetchError.new('timed out', uri) 183: rescue IOError, SocketError, SystemCallError => e 184: raise FetchError.new("#{e.class}: #{e}", uri) 185: end
Returns the size of uri in bytes.
# File lib/rubygems/remote_fetcher.rb, line 190 190: def fetch_size(uri) # TODO: phase this out 191: response = fetch_path(uri, nil, true) 192: 193: response['content-length'].to_i 194: end
Returns an HTTP proxy URI if one is set in the environment variables.
# File lib/rubygems/remote_fetcher.rb, line 217 217: def get_proxy_from_env 218: env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY'] 219: 220: return nil if env_proxy.nil? or env_proxy.empty? 221: 222: uri = URI.parse(normalize_uri(env_proxy)) 223: 224: if uri and uri.user.nil? and uri.password.nil? then 225: # Probably we have http_proxy_* variables? 226: uri.user = escape(ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER']) 227: uri.password = escape(ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS']) 228: end 229: 230: uri 231: end
Normalize the URI by adding "http://" if it is missing.
# File lib/rubygems/remote_fetcher.rb, line 236 236: def normalize_uri(uri) 237: (uri =~ /^(https?|ftp|file):/) ? uri : "http://#{uri}" 238: end
Read the data from the (source based) URI, but if it is a file:// URI, read from the filesystem instead.
# File lib/rubygems/remote_fetcher.rb, line 277 277: def open_uri_or_path(uri, last_modified = nil, head = false, depth = 0) 278: raise "block is dead" if block_given? 279: 280: uri = URI.parse uri unless URI::Generic === uri 281: 282: # This check is redundant unless Gem::RemoteFetcher is likely 283: # to be used directly, since the scheme is checked elsewhere. 284: # - Daniel Berger 285: unless ['http', 'https', 'file'].include?(uri.scheme) 286: raise ArgumentError, 'uri scheme is invalid' 287: end 288: 289: if uri.scheme == 'file' 290: path = uri.path 291: 292: # Deal with leading slash on Windows paths 293: if path[0].chr == '/' && path[1].chr =~ /[a-zA-Z]/ && path[2].chr == ':' 294: path = path[1..-1] 295: end 296: 297: return Gem.read_binary(path) 298: end 299: 300: fetch_type = head ? Net::HTTP::Head : Net::HTTP::Get 301: response = request uri, fetch_type, last_modified 302: 303: case response 304: when Net::HTTPOK, Net::HTTPNotModified then 305: head ? response : response.body 306: when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther, 307: Net::HTTPTemporaryRedirect then 308: raise FetchError.new('too many redirects', uri) if depth > 10 309: 310: open_uri_or_path(response['Location'], last_modified, head, depth + 1) 311: else 312: raise FetchError.new("bad response #{response.message} #{response.code}", uri) 313: end 314: end
Performs a Net::HTTP request of type request_class on uri returning a Net::HTTP response object. request maintains a table of persistent connections to reduce connect overhead.
# File lib/rubygems/remote_fetcher.rb, line 321 321: def request(uri, request_class, last_modified = nil) 322: request = request_class.new uri.request_uri 323: 324: unless uri.nil? || uri.user.nil? || uri.user.empty? then 325: request.basic_auth uri.user, uri.password 326: end 327: 328: ua = "RubyGems/#{Gem::VERSION} #{Gem::Platform.local}" 329: ua << " Ruby/#{RUBY_VERSION} (#{RUBY_RELEASE_DATE}" 330: ua << " patchlevel #{RUBY_PATCHLEVEL}" if defined? RUBY_PATCHLEVEL 331: ua << ")" 332: 333: request.add_field 'User-Agent', ua 334: request.add_field 'Connection', 'keep-alive' 335: request.add_field 'Keep-Alive', '30' 336: 337: if last_modified then 338: last_modified = last_modified.utc 339: request.add_field 'If-Modified-Since', last_modified.rfc2822 340: end 341: 342: yield request if block_given? 343: 344: connection = connection_for uri 345: 346: retried = false 347: bad_response = false 348: 349: begin 350: @requests[connection.object_id] += 1 351: 352: say "#{request.method} #{uri}" if 353: Gem.configuration.really_verbose 354: 355: file_name = File.basename(uri.path) 356: # perform download progress reporter only for gems 357: if request.response_body_permitted? && file_name =~ /\.gem$/ 358: reporter = ui.download_reporter 359: response = connection.request(request) do |incomplete_response| 360: if Net::HTTPOK === incomplete_response 361: reporter.fetch(file_name, incomplete_response.content_length) 362: downloaded = 0 363: data = '' 364: 365: incomplete_response.read_body do |segment| 366: data << segment 367: downloaded += segment.length 368: reporter.update(downloaded) 369: end 370: reporter.done 371: if incomplete_response.respond_to? :body= 372: incomplete_response.body = data 373: else 374: incomplete_response.instance_variable_set(:@body, data) 375: end 376: end 377: end 378: else 379: response = connection.request request 380: end 381: 382: say "#{response.code} #{response.message}" if 383: Gem.configuration.really_verbose 384: 385: rescue Net::HTTPBadResponse 386: say "bad response" if Gem.configuration.really_verbose 387: 388: reset connection 389: 390: raise FetchError.new('too many bad responses', uri) if bad_response 391: 392: bad_response = true 393: retry 394: # HACK work around EOFError bug in Net::HTTP 395: # NOTE Errno::ECONNABORTED raised a lot on Windows, and make impossible 396: # to install gems. 397: rescue EOFError, Timeout::Error, 398: Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE 399: 400: requests = @requests[connection.object_id] 401: say "connection reset after #{requests} requests, retrying" if 402: Gem.configuration.really_verbose 403: 404: raise FetchError.new('too many connection resets', uri) if retried 405: 406: reset connection 407: 408: retried = true 409: retry 410: end 411: 412: response 413: end
Resets HTTP connection connection.
# File lib/rubygems/remote_fetcher.rb, line 418 418: def reset(connection) 419: @requests.delete connection.object_id 420: 421: connection.finish 422: connection.start 423: end
# File lib/rubygems/remote_fetcher.rb, line 202 202: def unescape(str) 203: return unless str 204: @uri_parser ||= uri_escaper 205: @uri_parser.unescape str 206: end