Skip to content

Commit 86cbb81

Browse files
authored
Merge pull request #9786 from ruby/claude/eager-poitras-5f95f1
Fetch Bundler metadata through Gem::Request instead of net-http-persistent
2 parents 9143e91 + 4cf080c commit 86cbb81

29 files changed

Lines changed: 253 additions & 2247 deletions

Manifest.txt

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ lib/bundler/feature_flag.rb
6969
lib/bundler/fetcher.rb
7070
lib/bundler/fetcher/base.rb
7171
lib/bundler/fetcher/compact_index.rb
72+
lib/bundler/fetcher/connection_pools.rb
7273
lib/bundler/fetcher/dependency.rb
7374
lib/bundler/fetcher/downloader.rb
7475
lib/bundler/fetcher/gem_remote_fetcher.rb
@@ -257,19 +258,8 @@ lib/bundler/ui/silent.rb
257258
lib/bundler/uri_credentials_filter.rb
258259
lib/bundler/uri_normalizer.rb
259260
lib/bundler/vendor/.document
260-
lib/bundler/vendor/connection_pool/LICENSE
261-
lib/bundler/vendor/connection_pool/lib/connection_pool.rb
262-
lib/bundler/vendor/connection_pool/lib/connection_pool/fork.rb
263-
lib/bundler/vendor/connection_pool/lib/connection_pool/timed_stack.rb
264-
lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb
265-
lib/bundler/vendor/connection_pool/lib/connection_pool/wrapper.rb
266261
lib/bundler/vendor/fileutils/COPYING
267262
lib/bundler/vendor/fileutils/lib/fileutils.rb
268-
lib/bundler/vendor/net-http-persistent/README.rdoc
269-
lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb
270-
lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb
271-
lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb
272-
lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb
273263
lib/bundler/vendor/thor/LICENSE.md
274264
lib/bundler/vendor/thor/lib/thor.rb
275265
lib/bundler/vendor/thor/lib/thor/actions.rb
@@ -308,7 +298,6 @@ lib/bundler/vendor/thor/lib/thor/util.rb
308298
lib/bundler/vendor/thor/lib/thor/version.rb
309299
lib/bundler/vendored_fileutils.rb
310300
lib/bundler/vendored_net_http.rb
311-
lib/bundler/vendored_persistent.rb
312301
lib/bundler/vendored_pub_grub.rb
313302
lib/bundler/vendored_securerandom.rb
314303
lib/bundler/vendored_thor.rb

lib/bundler/fetcher.rb

Lines changed: 5 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# frozen_string_literal: true
22

3-
require_relative "vendored_persistent"
3+
require_relative "vendored_net_http"
44
require_relative "vendored_timeout"
55
require_relative "vendored_securerandom"
66
require "zlib"
@@ -10,6 +10,7 @@ module Bundler
1010
class Fetcher
1111
autoload :Base, File.expand_path("fetcher/base", __dir__)
1212
autoload :CompactIndex, File.expand_path("fetcher/compact_index", __dir__)
13+
autoload :ConnectionPools, File.expand_path("fetcher/connection_pools", __dir__)
1314
autoload :Downloader, File.expand_path("fetcher/downloader", __dir__)
1415
autoload :Dependency, File.expand_path("fetcher/dependency", __dir__)
1516
autoload :Index, File.expand_path("fetcher/index", __dir__)
@@ -137,7 +138,7 @@ def initialize(remote)
137138
@remote = remote
138139

139140
Socket.do_not_reverse_lookup = true
140-
connection # create persistent connection
141+
connection # set up the connection pools eagerly so SSL support is checked upfront
141142
end
142143

143144
def uri
@@ -237,7 +238,7 @@ def user_agent
237238
end
238239

239240
def http_proxy
240-
return unless uri = connection.proxy_uri
241+
return unless uri = connection.proxy_for(remote_uri)
241242
uri.to_s
242243
end
243244

@@ -307,28 +308,7 @@ def connection
307308
end
308309
end
309310

310-
con = Gem::Net::HTTP::Persistent.new name: "bundler", proxy: :ENV
311-
if gem_proxy = Gem.configuration[:http_proxy]
312-
con.proxy = Gem::URI.parse(gem_proxy) if gem_proxy != :no_proxy
313-
end
314-
315-
if remote_uri.scheme == "https"
316-
con.verify_mode = (Bundler.settings[:ssl_verify_mode] ||
317-
OpenSSL::SSL::VERIFY_PEER)
318-
con.cert_store = bundler_cert_store
319-
end
320-
321-
ssl_client_cert = Bundler.settings[:ssl_client_cert] ||
322-
(Gem.configuration.ssl_client_cert if
323-
Gem.configuration.respond_to?(:ssl_client_cert))
324-
if ssl_client_cert
325-
pem = File.read(ssl_client_cert)
326-
con.cert = OpenSSL::X509::Certificate.new(pem)
327-
con.key = OpenSSL::PKey.read(pem)
328-
end
329-
330-
con.read_timeout = Fetcher.api_timeout
331-
con.open_timeout = Fetcher.api_timeout
311+
con = ConnectionPools.new(size: Bundler.settings.processor_count, timeout: Fetcher.api_timeout)
332312
con.override_headers["User-Agent"] = user_agent
333313
con.override_headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri
334314
con
@@ -341,25 +321,6 @@ def gemspec_cached_path(spec_file_name)
341321
paths.find {|path| File.file? path }
342322
end
343323

344-
def bundler_cert_store
345-
store = OpenSSL::X509::Store.new
346-
ssl_ca_cert = Bundler.settings[:ssl_ca_cert] ||
347-
(Gem.configuration.ssl_ca_cert if
348-
Gem.configuration.respond_to?(:ssl_ca_cert))
349-
if ssl_ca_cert
350-
if File.directory? ssl_ca_cert
351-
store.add_path ssl_ca_cert
352-
else
353-
store.add_file ssl_ca_cert
354-
end
355-
else
356-
store.set_default_paths
357-
require "rubygems/request"
358-
Gem::Request.get_cert_files.each {|c| store.add_file c }
359-
end
360-
store
361-
end
362-
363324
def remote_uri
364325
@remote.uri
365326
end
Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# frozen_string_literal: true
2+
3+
require "rubygems/request"
4+
require "rubygems/remote_fetcher"
5+
6+
module Bundler
7+
class Fetcher
8+
# Hands out pools of configured Gem::Net::HTTP connections, compatible
9+
# with the checkout/checkin interface that Gem::Request expects.
10+
# Connection policy (proxy, SSL, timeouts) follows Bundler settings
11+
# first, falling back to RubyGems configuration.
12+
class ConnectionPools
13+
# A fixed-size pool of connections to a single host, sharing the
14+
# interface of Gem::Request::HTTPPool.
15+
class Pool
16+
attr_reader :cert_files, :proxy_uri
17+
18+
def initialize(connections, uri, proxy_uri, size)
19+
@connections = connections
20+
@uri = uri
21+
@proxy_uri = proxy_uri
22+
@cert_files = connections.cert_files
23+
@queue = Thread::SizedQueue.new(size)
24+
size.times { @queue.push(nil) }
25+
end
26+
27+
def checkout
28+
@queue.pop || @connections.build_connection(@uri, @proxy_uri)
29+
end
30+
31+
def checkin(connection)
32+
@queue.push(connection)
33+
end
34+
end
35+
36+
attr_reader :override_headers, :cert_files
37+
38+
def initialize(size:, timeout:)
39+
@size = size
40+
@timeout = timeout
41+
@override_headers = {}
42+
@cert_files = Gem::Request.get_cert_files
43+
@pools = {}
44+
@pool_mutex = Thread::Mutex.new
45+
end
46+
47+
# Performs a GET request through Gem::Request, which handles resetting
48+
# and retrying stale connections, and returns the raw response.
49+
def request(uri, headers = nil)
50+
Gem::Request.new(uri, Gem::Net::HTTP::Get, nil, pool_for(uri)).fetch do |request|
51+
@override_headers.each {|key, value| request[key] = value }
52+
headers&.each {|key, value| request[key] = value }
53+
end
54+
end
55+
56+
def pool_for(uri)
57+
key = [uri.scheme, uri.hostname, uri.port]
58+
@pool_mutex.synchronize do
59+
@pools[key] ||= Pool.new(self, uri, proxy_for(uri), @size)
60+
end
61+
end
62+
63+
# The proxy that will be used for +uri+, or nil. RubyGems configuration
64+
# takes precedence over the environment, like Gem::RemoteFetcher.
65+
# Unlike Gem::Request, an https URI falls back to `http_proxy` when no
66+
# https-specific proxy is set, which is what the previous
67+
# net-http-persistent based transport did.
68+
def proxy_for(uri)
69+
proxy = if config_proxy = Gem.configuration[:http_proxy]
70+
Gem::Request.proxy_uri(config_proxy)
71+
else
72+
env_proxy_for(uri)
73+
end
74+
return unless proxy
75+
return unless Gem::URI::Generic.use_proxy?(uri.hostname, nil, uri.port, no_proxy_env)
76+
proxy
77+
end
78+
79+
def build_connection(uri, proxy_uri) # :nodoc:
80+
args = [uri.hostname, uri.port]
81+
args += if proxy_uri
82+
[proxy_uri.hostname, proxy_uri.port,
83+
Gem::UriFormatter.new(proxy_uri.user).unescape,
84+
Gem::UriFormatter.new(proxy_uri.password).unescape]
85+
else
86+
[nil, nil]
87+
end
88+
89+
connection = Gem::Request::ConnectionPools.client.new(*args)
90+
configure_ssl(connection) if uri.scheme == "https"
91+
connection.open_timeout = @timeout
92+
connection.read_timeout = @timeout
93+
connection.start
94+
connection
95+
end
96+
97+
private
98+
99+
def env_proxy_for(uri)
100+
proxy = Gem::Request.get_proxy_from_env(uri.scheme)
101+
proxy = Gem::Request.get_proxy_from_env("http") if proxy == :no_proxy && uri.scheme == "https"
102+
proxy == :no_proxy ? nil : proxy
103+
end
104+
105+
def no_proxy_env
106+
ENV["no_proxy"] || ENV["NO_PROXY"] || ""
107+
end
108+
109+
def configure_ssl(connection)
110+
connection.use_ssl = true
111+
connection.verify_mode = Bundler.settings[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
112+
connection.cert_store = bundler_cert_store
113+
114+
ssl_client_cert = Bundler.settings[:ssl_client_cert] ||
115+
(Gem.configuration.ssl_client_cert if
116+
Gem.configuration.respond_to?(:ssl_client_cert))
117+
return unless ssl_client_cert
118+
119+
pem = File.read(ssl_client_cert)
120+
connection.cert = OpenSSL::X509::Certificate.new(pem)
121+
connection.key = OpenSSL::PKey.read(pem)
122+
end
123+
124+
def bundler_cert_store
125+
store = OpenSSL::X509::Store.new
126+
ssl_ca_cert = Bundler.settings[:ssl_ca_cert] ||
127+
(Gem.configuration.ssl_ca_cert if
128+
Gem.configuration.respond_to?(:ssl_ca_cert))
129+
if ssl_ca_cert
130+
if File.directory? ssl_ca_cert
131+
store.add_path ssl_ca_cert
132+
else
133+
store.add_file ssl_ca_cert
134+
end
135+
else
136+
store.set_default_paths
137+
cert_files.each {|c| store.add_file c }
138+
end
139+
store
140+
end
141+
end
142+
end
143+
end

lib/bundler/fetcher/downloader.rb

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,19 @@ class Downloader
66
HTTP_NON_RETRYABLE_ERRORS = [
77
SocketError,
88
Errno::EADDRNOTAVAIL,
9+
Errno::ECONNREFUSED,
10+
Errno::EHOSTDOWN,
11+
Errno::EHOSTUNREACH,
912
Errno::ENETDOWN,
1013
Errno::ENETUNREACH,
11-
Gem::Net::HTTP::Persistent::Error,
12-
Errno::EHOSTUNREACH,
1314
].freeze
1415

16+
# The vendored net-http raises Gem::Timeout::Error, but when Gem::Net is
17+
# the real Net (hosts without a vendored net-http), timeouts are plain
18+
# Timeout::Error subclasses instead.
1519
HTTP_RETRYABLE_ERRORS = [
1620
Gem::Timeout::Error,
21+
*(::Timeout::Error if defined?(::Timeout::Error)),
1722
EOFError,
1823
Errno::EINVAL,
1924
Errno::ECONNRESET,
@@ -25,11 +30,11 @@ class Downloader
2530
Zlib::BufError,
2631
].freeze
2732

28-
attr_reader :connection
33+
attr_reader :connections
2934
attr_reader :redirect_limit
3035

31-
def initialize(connection, redirect_limit)
32-
@connection = connection
36+
def initialize(connections, redirect_limit)
37+
@connections = connections
3338
@redirect_limit = redirect_limit
3439
end
3540

@@ -79,23 +84,25 @@ def request(uri, headers)
7984
filtered_uri = URICredentialsFilter.credential_filtered_uri(uri)
8085

8186
Bundler.ui.debug "HTTP GET #{filtered_uri}"
82-
req = Gem::Net::HTTP::Get.new uri.request_uri, headers
83-
if uri.user
84-
user = CGI.unescape(uri.user)
85-
password = uri.password ? CGI.unescape(uri.password) : nil
86-
req.basic_auth(user, password)
87+
connections.request(uri, headers)
88+
rescue Gem::RemoteFetcher::FetchError => e
89+
Bundler.ui.trace e
90+
91+
case e.message
92+
when /certificate verify failed/
93+
raise CertificateFailureError.new(uri)
94+
when /host is down|host down/i
95+
raise network_down_error(uri, filtered_uri)
96+
else
97+
raise HTTPError, "Network error while fetching #{filtered_uri}" \
98+
" (#{e})"
8799
end
88-
connection.request(uri, req)
89100
rescue OpenSSL::SSL::SSLError
90101
raise CertificateFailureError.new(uri)
91102
rescue *HTTP_NON_RETRYABLE_ERRORS => e
92103
Bundler.ui.trace e
93104

94-
host = uri.host
95-
host_port = "#{host}:#{uri.port}"
96-
host = host_port if filtered_uri.to_s.include?(host_port)
97-
raise NetworkDownError, "Could not reach host #{host}. Check your network " \
98-
"connection and try again."
105+
raise network_down_error(uri, filtered_uri)
99106
rescue *HTTP_RETRYABLE_ERRORS => e
100107
Bundler.ui.trace e
101108

@@ -105,6 +112,14 @@ def request(uri, headers)
105112

106113
private
107114

115+
def network_down_error(uri, filtered_uri)
116+
host = uri.host
117+
host_port = "#{host}:#{uri.port}"
118+
host = host_port if filtered_uri.to_s.include?(host_port)
119+
NetworkDownError.new("Could not reach host #{host}. Check your network " \
120+
"connection and try again.")
121+
end
122+
108123
def validate_uri_scheme!(uri)
109124
return if /\Ahttps?\z/.match?(uri.scheme)
110125
raise InvalidOption,

lib/bundler/vendor/connection_pool/LICENSE

Lines changed: 0 additions & 20 deletions
This file was deleted.

0 commit comments

Comments
 (0)