diff --git a/lib/bundler/cli/pristine.rb b/lib/bundler/cli/pristine.rb index b8545fe4c9c3eb..f463f0bce824f9 100644 --- a/lib/bundler/cli/pristine.rb +++ b/lib/bundler/cli/pristine.rb @@ -53,7 +53,7 @@ def run true end.map(&:name) - jobs = installer.send(:installation_parallelization) + jobs = Bundler.settings.installation_parallelization pristine_count = definition.specs.count - installed_specs.count # allow a pristining a single gem to skip the parallel worker jobs = [jobs, pristine_count].min diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index efc749e9b32626..d9abc85d228f7d 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -1122,7 +1122,9 @@ def source_requirements end def preload_git_source_worker - @preload_git_source_worker ||= Bundler::Worker.new(5, "Git source preloading", ->(source, _) { source.specs }) + workers = Bundler.settings.installation_parallelization + + @preload_git_source_worker ||= Bundler::Worker.new(workers, "Git source preloading", ->(source, _) { source.specs }) end def preload_git_sources diff --git a/lib/bundler/fetcher/gem_remote_fetcher.rb b/lib/bundler/fetcher/gem_remote_fetcher.rb index 3c3c1826a1b1e2..3159e056880a40 100644 --- a/lib/bundler/fetcher/gem_remote_fetcher.rb +++ b/lib/bundler/fetcher/gem_remote_fetcher.rb @@ -8,7 +8,7 @@ class GemRemoteFetcher < Gem::RemoteFetcher def initialize(*) super - @pool_size = 5 + @pool_size = Bundler.settings.installation_parallelization end def request(*args) diff --git a/lib/bundler/installer.rb b/lib/bundler/installer.rb index c5fd75431f41e6..3455f72c2143b5 100644 --- a/lib/bundler/installer.rb +++ b/lib/bundler/installer.rb @@ -189,21 +189,13 @@ def install(options) standalone = options[:standalone] force = options[:force] local = options[:local] || options[:"prefer-local"] - jobs = installation_parallelization + jobs = Bundler.settings.installation_parallelization spec_installations = ParallelInstaller.call(self, @definition.specs, jobs, standalone, force, local: local) spec_installations.each do |installation| post_install_messages[installation.name] = installation.post_install_message if installation.has_post_install_message? end end - def installation_parallelization - if jobs = Bundler.settings[:jobs] - return jobs - end - - Bundler.settings.processor_count - end - def load_plugins Gem.load_plugins diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index 000fe664da6c3f..ed66ba9a48175c 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -146,7 +146,7 @@ When set, no post install messages will be printed\. To silence a single gem, us Generate a \fBgems\.rb\fR instead of a \fBGemfile\fR when running \fBbundle init\fR\. .TP \fBjobs\fR (\fBBUNDLE_JOBS\fR) -The number of gems Bundler can install in parallel\. Defaults to the number of available processors\. +The number of gems Bundler can download and install in parallel\. Defaults to the number of available processors\. .TP \fBlockfile\fR (\fBBUNDLE_LOCKFILE\fR) The path to the lockfile that bundler should use\. By default, Bundler adds \fB\.lock\fR to the end of the \fBgemfile\fR entry\. Can be set to \fBfalse\fR in the Gemfile to disable lockfile creation entirely (see gemfile(5))\. diff --git a/lib/bundler/man/bundle-config.1.ronn b/lib/bundler/man/bundle-config.1.ronn index a8670a36709d49..b70293cfeddadc 100644 --- a/lib/bundler/man/bundle-config.1.ronn +++ b/lib/bundler/man/bundle-config.1.ronn @@ -192,8 +192,8 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). * `init_gems_rb` (`BUNDLE_INIT_GEMS_RB`): Generate a `gems.rb` instead of a `Gemfile` when running `bundle init`. * `jobs` (`BUNDLE_JOBS`): - The number of gems Bundler can install in parallel. Defaults to the number of - available processors. + The number of gems Bundler can download and install in parallel. + Defaults to the number of available processors. * `lockfile` (`BUNDLE_LOCKFILE`): The path to the lockfile that bundler should use. By default, Bundler adds `.lock` to the end of the `gemfile` entry. Can be set to `false` in the diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index 6c12ca946602d7..d120faca61fa1c 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -303,6 +303,10 @@ def app_cache_path @app_cache_path ||= self[:cache_path] || "vendor/cache" end + def installation_parallelization + self[:jobs] || processor_count + end + def validate! all.each do |raw_key| [@local_config, @env_config, @global_config].each do |settings| diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb index 8f625d29c52448..72f7dc771038ac 100644 --- a/lib/bundler/source/git/git_proxy.rb +++ b/lib/bundler/source/git/git_proxy.rb @@ -57,6 +57,29 @@ class GitProxy attr_accessor :path, :uri, :branch, :tag, :ref, :explicit_ref attr_writer :revision + def self.version + @version ||= full_version[/((\.?\d+)+).*/, 1] + end + + def self.full_version + @full_version ||= begin + raise GitNotInstalledError.new unless Bundler.git_present? + + require "open3" + out, err, status = Open3.capture3("git", "--version") + + raise GitCommandError.new("--version", SharedHelpers.pwd, err) unless status.success? + Bundler.ui.warn err unless err.empty? + + out.sub(/git version\s*/, "").strip + end + end + + def self.reset + @version = nil + @full_version = nil + end + def initialize(path, uri, options = {}, revision = nil, git = nil) @path = path @uri = uri @@ -92,11 +115,11 @@ def contains?(commit) end def version - @version ||= full_version.match(/((\.?\d+)+).*/)[1] + self.class.version end def full_version - @full_version ||= git_local("--version").sub(/git version\s*/, "").strip + self.class.full_version end def checkout diff --git a/prism/extension.c b/prism/extension.c index 147434c9759680..c248b0c12356d2 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -208,10 +208,10 @@ build_options_i(VALUE key, VALUE value, VALUE argument) { if (!pm_options_version_set(options, ruby_version, 3)) { // Prism doesn't know this specific version. Is it lower? if (ruby_version[0] < '3' || (ruby_version[0] == '3' && ruby_version[2] < '3')) { - options->version == PM_OPTIONS_VERSION_CRUBY_3_3; + options->version = PM_OPTIONS_VERSION_CRUBY_3_3; } else { // Must be higher. - options->version == PM_OPTIONS_VERSION_LATEST; + options->version = PM_OPTIONS_VERSION_LATEST; } } } else if (!pm_options_version_set(options, version, RSTRING_LEN(value))) { diff --git a/prism/parser.h b/prism/parser.h index 8187d8685af323..66df791244bcb4 100644 --- a/prism/parser.h +++ b/prism/parser.h @@ -100,13 +100,6 @@ typedef struct { pm_heredoc_indent_t indent; } pm_heredoc_lex_mode_t; -/** - * When lexing Ruby source, the lexer has a small amount of state to tell which - * kind of token it is currently lexing. For example, when we find the start of - * a string, the first token that we return is a TOKEN_STRING_BEGIN token. After - * that the lexer is now in the PM_LEX_STRING mode, and will return tokens that - * are found as part of a string. - */ /** * The size of the breakpoints and strpbrk cache charset buffers. All * breakpoint arrays and the strpbrk cache charset must share this size so @@ -114,6 +107,13 @@ typedef struct { */ #define PM_STRPBRK_CACHE_SIZE 16 +/** + * When lexing Ruby source, the lexer has a small amount of state to tell which + * kind of token it is currently lexing. For example, when we find the start of + * a string, the first token that we return is a TOKEN_STRING_BEGIN token. After + * that the lexer is now in the PM_LEX_STRING mode, and will return tokens that + * are found as part of a string. + */ typedef struct pm_lex_mode { /** The type of this lex mode. */ enum { diff --git a/spec/bundler/bundler/env_spec.rb b/spec/bundler/bundler/env_spec.rb index e0ab0a45e33004..259b4ee9dc2f71 100644 --- a/spec/bundler/bundler/env_spec.rb +++ b/spec/bundler/bundler/env_spec.rb @@ -217,8 +217,9 @@ def with_clear_paths(env_var, env_value) context "when the git version is OS specific" do it "includes OS specific information with the version number" do - expect(git_proxy_stub).to receive(:git_local).with("--version"). - and_return("git version 1.2.3 (Apple Git-BS)") + status = double("success?" => true) + expect(Open3).to receive(:capture3).with("git", "--version"). + and_return(["git version 1.2.3 (Apple Git-BS)", "", status]) expect(Bundler::Source::Git::GitProxy).to receive(:new).and_return(git_proxy_stub) expect(described_class.report).to include("Git 1.2.3 (Apple Git-BS)") diff --git a/spec/bundler/bundler/gem_helper_spec.rb b/spec/bundler/bundler/gem_helper_spec.rb index 83c2dd237adcec..acf783908d6fb5 100644 --- a/spec/bundler/bundler/gem_helper_spec.rb +++ b/spec/bundler/bundler/gem_helper_spec.rb @@ -222,7 +222,7 @@ def sha512_hexdigest(path) mock_confirm_message "#{app_name} (#{app_version}) installed." subject.install_gem(nil, :local) expect(app_gem_path).to exist - gem_command :list + installed_gems_list expect(out).to include("#{app_name} (#{app_version})") end end diff --git a/spec/bundler/bundler/source/git/git_proxy_spec.rb b/spec/bundler/bundler/source/git/git_proxy_spec.rb index 24b1c2e709998b..1f10ca4b0776fb 100644 --- a/spec/bundler/bundler/source/git/git_proxy_spec.rb +++ b/spec/bundler/bundler/source/git/git_proxy_spec.rb @@ -101,7 +101,7 @@ describe "#version" do context "with a normal version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). + expect(described_class).to receive(:full_version). and_return("git version 1.2.3") end @@ -116,7 +116,7 @@ context "with a OSX version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). + expect(described_class).to receive(:full_version). and_return("git version 1.2.3 (Apple Git-BS)") end @@ -131,7 +131,7 @@ context "with a msysgit version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). + expect(described_class).to receive(:full_version). and_return("git version 1.2.3.msysgit.0") end @@ -148,8 +148,9 @@ describe "#full_version" do context "with a normal version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). - and_return("git version 1.2.3") + status = double("success?" => true) + expect(Open3).to receive(:capture3).with("git", "--version"). + and_return(["git version 1.2.3", "", status]) end it "returns the git version number" do @@ -159,8 +160,9 @@ context "with a OSX version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). - and_return("git version 1.2.3 (Apple Git-BS)") + status = double("success?" => true) + expect(Open3).to receive(:capture3).with("git", "--version"). + and_return(["git version 1.2.3 (Apple Git-BS)", "", status]) end it "does not strip out OSX specific additions in the version string" do @@ -170,8 +172,9 @@ context "with a msysgit version number" do before do - expect(git_proxy).to receive(:git_local).with("--version"). - and_return("git version 1.2.3.msysgit.0") + status = double("success?" => true) + expect(Open3).to receive(:capture3).with("git", "--version"). + and_return(["git version 1.2.3.msysgit.0", "", status]) end it "does not strip out msysgit specific additions in the version string" do diff --git a/spec/bundler/commands/check_spec.rb b/spec/bundler/commands/check_spec.rb index 72da24fb0b2d0b..4dce7813a624f8 100644 --- a/spec/bundler/commands/check_spec.rb +++ b/spec/bundler/commands/check_spec.rb @@ -164,7 +164,7 @@ bundle "config set --local path vendor/bundle" bundle :cache - gem_command "uninstall myrack", env: { "GEM_HOME" => vendored_gems.to_s } + uninstall_gem("myrack", env: { "GEM_HOME" => vendored_gems.to_s }) bundle "check", raise_on_error: false expect(err).to include("* myrack (1.0.0)") diff --git a/spec/bundler/commands/clean_spec.rb b/spec/bundler/commands/clean_spec.rb index 81209388ae9783..582bfd5fd1c3b1 100644 --- a/spec/bundler/commands/clean_spec.rb +++ b/spec/bundler/commands/clean_spec.rb @@ -379,7 +379,7 @@ def should_not_have_gems(*gems) gem "myrack" G - gem_command :list + installed_gems_list expect(out).to include("myrack (1.0.0)").and include("thin (1.0)") end @@ -498,7 +498,7 @@ def should_not_have_gems(*gems) end bundle :update, all: true - gem_command :list + installed_gems_list expect(out).to include("foo (1.0.1, 1.0)") end @@ -522,7 +522,7 @@ def should_not_have_gems(*gems) bundle "clean --force" expect(out).to include("Removing foo (1.0)") - gem_command :list + installed_gems_list expect(out).not_to include("foo (1.0)") expect(out).to include("myrack (1.0.0)") end @@ -556,7 +556,7 @@ def should_not_have_gems(*gems) expect(err).to include(system_gem_path.to_s) expect(err).to include("grant write permissions") - gem_command :list + installed_gems_list expect(out).to include("foo (1.0)") expect(out).to include("myrack (1.0.0)") end diff --git a/spec/bundler/install/gems/compact_index_spec.rb b/spec/bundler/install/gems/compact_index_spec.rb index bb4d4011f5b94f..d082b9be72ce59 100644 --- a/spec/bundler/install/gems/compact_index_spec.rb +++ b/spec/bundler/install/gems/compact_index_spec.rb @@ -997,7 +997,7 @@ def start gem "activemerchant" end G - gem_command "uninstall activemerchant" + uninstall_gem("activemerchant") bundle "update rails", artifice: "compact_index" count = lockfile.match?("CHECKSUMS") ? 2 : 1 # Once in the specs, and once in CHECKSUMS expect(lockfile.scan(/activemerchant \(/).size).to eq(count) diff --git a/spec/bundler/support/builders.rb b/spec/bundler/support/builders.rb index a58b575b63b38a..43ab7e053dfb6b 100644 --- a/spec/bundler/support/builders.rb +++ b/spec/bundler/support/builders.rb @@ -664,7 +664,7 @@ def _build(opts) Bundler.rubygems.build(@spec, opts[:skip_validation]) end elsif opts[:skip_validation] - @context.gem_command "build --force #{@spec.name}", dir: lib_path + Dir.chdir(lib_path) { Gem::Package.build(@spec, true) } else Dir.chdir(lib_path) { Gem::Package.build(@spec) } end diff --git a/spec/bundler/support/helpers.rb b/spec/bundler/support/helpers.rb index 52e6ff5d9a3140..6a6cfc8b0084ae 100644 --- a/spec/bundler/support/helpers.rb +++ b/spec/bundler/support/helpers.rb @@ -25,6 +25,7 @@ def reset! FileUtils.mkdir_p(home) FileUtils.mkdir_p(tmpdir) Bundler.reset! + Bundler::Source::Git::GitProxy.reset Gem.clear_paths end @@ -182,19 +183,6 @@ def gembin(cmd, options = {}) sys_exec(cmd.to_s, options) end - def gem_command(command, options = {}) - env = options[:env] || {} - env["RUBYOPT"] = opt_add(opt_add("-r#{hax}", env["RUBYOPT"]), ENV["RUBYOPT"]) - options[:env] = env - - # Sometimes `gem install` commands hang at dns resolution, which has a - # default timeout of 60 seconds. When that happens, the timeout for a - # command is expired too. So give `gem install` commands a bit more time. - options[:timeout] = 120 - - sys_exec("#{Path.gem_bin} #{command}", options) - end - def sys_exec(cmd, options = {}, &block) env = options[:env] || {} env["RUBYOPT"] = opt_add(opt_add("-r#{spec_dir}/support/switch_rubygems.rb", env["RUBYOPT"]), ENV["RUBYOPT"]) @@ -326,9 +314,20 @@ def self.install_dev_bundler def install_gem(path, install_dir, default = false) raise ArgumentError, "`#{path}` does not exist!" unless File.exist?(path) - args = "--no-document --ignore-dependencies --verbose --local --install-dir #{install_dir}" - - gem_command "install #{args} '#{path}'" + require "rubygems/installer" + + with_simulated_platform do + installer = Gem::Installer.at( + path.to_s, + install_dir: install_dir.to_s, + document: [], + ignore_dependencies: true, + wrappers: true, + env_shebang: true, + force: true + ) + installer.install + end if default gem = Pathname.new(path).basename.to_s.match(/(.*)\.gem/)[1] @@ -343,6 +342,57 @@ def install_gem(path, install_dir, default = false) end end + def uninstall_gem(name, options = {}) + require "rubygems/uninstaller" + + gem_home = options.dig(:env, "GEM_HOME") || system_gem_path.to_s + + with_env_vars("GEM_HOME" => gem_home) do + Gem.clear_paths + + uninstaller = Gem::Uninstaller.new( + name, + ignore: true, + executables: true, + all: true + ) + uninstaller.uninstall + ensure + Gem.clear_paths + end + end + + def installed_gems_list(options = {}) + gem_home = options.dig(:env, "GEM_HOME") || system_gem_path.to_s + + # Temporarily set GEM_HOME for the command + old_gem_home = ENV["GEM_HOME"] + ENV["GEM_HOME"] = gem_home + Gem.clear_paths + + begin + require "rubygems/commands/list_command" + + # Capture output from the list command + output_io = StringIO.new + cmd = Gem::Commands::ListCommand.new + cmd.ui = Gem::StreamUI.new(StringIO.new, output_io, StringIO.new, false) + cmd.invoke + output = output_io.string.strip + ensure + ENV["GEM_HOME"] = old_gem_home + Gem.clear_paths + end + + # Create a fake command execution so `out` helper works + command_execution = Spec::CommandExecution.new("gem list", timeout: 60) + command_execution.original_stdout << output + command_execution.exitstatus = 0 + command_executions << command_execution + + output + end + def with_built_bundler(version = nil, opts = {}, &block) require_relative "builders" @@ -374,6 +424,36 @@ def without_env_side_effects ENV.replace(backup) end + # Simulate the platform set by BUNDLER_SPEC_PLATFORM for in-process + # operations, mirroring what hax.rb does for subprocesses. + def with_simulated_platform + spec_platform = ENV["BUNDLER_SPEC_PLATFORM"] + unless spec_platform + return yield + end + + old_arch = RbConfig::CONFIG["arch"] + old_host_os = RbConfig::CONFIG["host_os"] + + if /mingw|mswin/.match?(spec_platform) + Gem.class_variable_set(:@@win_platform, nil) # rubocop:disable Style/ClassVars + RbConfig::CONFIG["host_os"] = spec_platform.gsub(/^[^-]+-/, "").tr("-", "_") + end + + RbConfig::CONFIG["arch"] = spec_platform + Gem::Platform.instance_variable_set(:@local, nil) + Gem.instance_variable_set(:@platforms, []) + + yield + ensure + if spec_platform + RbConfig::CONFIG["arch"] = old_arch + RbConfig::CONFIG["host_os"] = old_host_os + Gem::Platform.instance_variable_set(:@local, nil) + Gem.instance_variable_set(:@platforms, []) + end + end + def with_path_added(path) with_path_as([path.to_s, ENV["PATH"]].join(File::PATH_SEPARATOR)) do yield diff --git a/test/prism/magic_comment_test.rb b/test/prism/magic_comment_test.rb index ccfe5a5d0a5bc2..7985bae5689ac0 100644 --- a/test/prism/magic_comment_test.rb +++ b/test/prism/magic_comment_test.rb @@ -69,6 +69,10 @@ def test_emacs_multiple assert_magic_encoding(Encoding::US_ASCII, "# -*- foo: bar; encoding: ascii -*-") end + def test_emacs_missing_delimiter + assert_magic_encoding(Encoding::US_ASCII, '# -*- \1; encoding: ascii -*-') + end + def test_coding_whitespace assert_magic_encoding(Encoding::ASCII_8BIT, "# coding \t \r \v : \t \v \r ascii-8bit") end