[Content Addressable] Bundler Local Cache and Lockfile

lib/bundler/lazy_specification.rb

@@ -66,7 +66,7 @@ def source_changed?
     end
 
     def full_name
-      @full_name ||= if @content_address
+      @full_name ||= if Gem::ContentAddress.match?(@content_address) && platform != Gem::Platform::RUBY
         "#{@name}-#{@version}-#{@content_address}"
       elsif platform == Gem::Platform::RUBY
         "#{@name}-#{@version}"
@@ -80,7 +80,7 @@ def lock_name
     end
 
     def name_tuple
-      Gem::NameTuple.new(@name, @version, @platform)
+      Gem::NameTuple.new(@name, @version, @platform, content_address: @content_address)
     end
 
     def ==(other)
@@ -117,7 +117,11 @@ def satisfies?(dependency)
 
     def to_lock
       out = String.new
-      out << "    #{lock_name}\n"
+      out << "    #{lock_name}"
+      # Append the platform additionally for content-addressable gems that contain a SHA
+      # where the platform would otherwise be
+      out << " #{platform}" if Gem::ContentAddress.match?(content_address) && platform != Gem::Platform::RUBY
+      out << "\n"
 
       dependencies.sort_by(&:to_s).uniq.each do |dep|
         next if dep.type == :development

lib/bundler/lockfile_parser.rb

@@ -264,11 +264,13 @@ def parse_checksum(line)
       checksums = $6
       name = $2
       version = $3
-      platform = $4
+      content_address = $4 if Gem::ContentAddress.match?($4)
+      platform = $4 unless content_address
 
       version = Gem::Version.new(version)
       platform = platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY
-      full_name = Gem::NameTuple.new(name, version, platform).full_name
+      name_tuple = Gem::NameTuple.new(name, version, platform, content_address: content_address)
+      full_name = name_tuple.full_name
       spec = @specs[full_name]
 
       if name == "bundler"
@@ -295,11 +297,17 @@ def parse_spec(line)
 
       if spaces.size == 4
         # only load platform for non-dependency (spec) line
-        platform = $4
+        if Gem::ContentAddress.match?($4) && $6 && $6 != Gem::Platform::RUBY.to_s
+          content_address = $4
+          platform = $6
+        else
+          platform = $4
+          content_address = $6 if Gem::ContentAddress.match?($6)
+        end
 
         version = Gem::Version.new(version)
         platform = platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY
-        @current_spec = LazySpecification.new(name, version, platform, @current_source, strict: @strict)
+        @current_spec = LazySpecification.new(name, version, platform, @current_source, content_address: content_address, strict: @strict)
         @current_source.add_dependency_names(name)
 
         @specs[@current_spec.full_name] = @current_spec

lib/bundler/remote_specification.rb

@@ -10,18 +10,19 @@ class RemoteSpecification
     include MatchPlatform
     include Comparable
 
-    attr_reader :name, :version, :platform
+    attr_reader :name, :version, :platform, :content_address
     attr_writer :dependencies
     attr_accessor :source, :remote, :locked_platform, :created_at
 
-    def initialize(name, version, platform, spec_fetcher)
+    def initialize(name, version, platform, spec_fetcher, content_address: nil)
       @name         = name
       @version      = Gem::Version.create version
       @original_platform = platform || Gem::Platform::RUBY
       @platform     = Gem::Platform.new(platform)
       @spec_fetcher = spec_fetcher
       @dependencies = nil
       @locked_platform = nil
+      @content_address = content_address
     end
 
     def insecurely_materialized?
@@ -35,7 +36,9 @@ def fetch_platform
     end
 
     def full_name
-      @full_name ||= if @platform == Gem::Platform::RUBY
+      @full_name ||= if Gem::ContentAddress.match?(@content_address) && @platform != Gem::Platform::RUBY
+        "#{@name}-#{@version}-#{@content_address}"
+      elsif @platform == Gem::Platform::RUBY
         "#{@name}-#{@version}"
       else
         "#{@name}-#{@version}-#{@platform}"

lib/bundler/rubygems_ext.rb

@@ -23,6 +23,10 @@
   Gem::NameTuple.attr_reader :content_address
 end
 
+unless Gem::Installer.private_method_defined?(:assign_content_address)
+  Gem::Installer.send(:define_method, :assign_content_address) {}
+end
+
 module Gem
   # Can be removed once RubyGems 4.0.0 support is dropped
   unless defined?(Gem::ContentAddress)
@@ -454,6 +458,15 @@ def initialize(name, version, platform = Gem::Platform::RUBY, content_address =
       end
     end
 
+    unless instance_method(:initialize).parameters.any? {|kind, name| kind == :key && name == :content_address }
+      alias_method :initialize_without_content_address, :initialize
+
+      def initialize(name, version, platform = Gem::Platform::RUBY, content_address: nil)
+        initialize_without_content_address(name, version, platform)
+        @content_address = content_address
+      end
+    end
+
     def lock_name
       return "#{name} (#{version}-#{content_address})" if Gem::ContentAddress.match?(content_address)
 

lib/bundler/rubygems_gem_installer.rb

@@ -14,6 +14,8 @@ def check_executable_overwrite(filename)
     end
 
     def install
+      assign_content_address
+
       pre_install_checks
 
       run_pre_install_hooks

lib/bundler/rubygems_integration.rb

@@ -144,7 +144,12 @@ def ext_lock
 
     def spec_from_gem(path)
       require "rubygems/package"
-      Gem::Package.new(path).spec
+      package = Gem::Package.new(path)
+      spec = package.spec
+      if package.respond_to?(:content_address)
+        spec.content_address = package.content_address
+      end
+      spec
     end
 
     def build_gem(gem_dir, spec)

lib/bundler/stub_specification.rb

@@ -4,7 +4,8 @@ module Bundler
   class StubSpecification < RemoteSpecification
     def self.from_stub(stub)
       return stub if stub.is_a?(Bundler::StubSpecification)
-      spec = new(stub.name, stub.version, stub.platform, nil)
+      content_address = stub.content_address
+      spec = new(stub.name, stub.version, stub.platform, nil, content_address: content_address)
       spec.stub = stub
       spec
     end

spec/bundler/lockfile_parser_spec.rb

@@ -145,6 +145,65 @@
 
     include_examples "parsing"
 
+    context "when a spec has a content address" do
+      let(:lockfile_contents) do
+        <<~L
+          GEM
+            remote: https://rubygems.org/
+            specs:
+              mygem (1.0-abcdef1234) x86_64-linux
+
+          PLATFORMS
+            x86_64-linux
+
+          DEPENDENCIES
+            mygem
+
+          CHECKSUMS
+            mygem (1.0-abcdef1234) sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8
+
+          BUNDLED WITH
+             1.12.0.rc.2
+        L
+      end
+
+      it "parses the platform and content address" do
+        spec = subject.specs.find {|s| s.name == "mygem" }
+
+        expect(spec.platform).to eq(Gem::Platform.new("x86_64-linux"))
+        expect(spec.content_address).to eq("abcdef1234")
+
+        checksums = subject.sources.first.checksum_store.to_lock(spec)
+        expect(checksums).to eq("#{spec.lock_name} sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8")
+      end
+    end
+
+    context "when a Ruby-platform suffix resembles a content address but no platform is present" do
+      let(:lockfile_contents) do
+        <<~L
+          GEM
+            remote: https://rubygems.org/
+            specs:
+              mygem (1.0-abcdef1234)
+
+          PLATFORMS
+            ruby
+
+          DEPENDENCIES
+            mygem
+
+          BUNDLED WITH
+             1.12.0.rc.2
+        L
+      end
+
+      it "does not parse the suffix as a content address" do
+        spec = subject.specs.find {|s| s.name == "mygem" }
+
+        expect(spec.content_address).to be_nil
+      end
+    end
+
     context "when an extra section is at the end" do
       let(:lockfile_contents) { super() + "\n\nFOO BAR\n  baz\n   baa\n    qux\n" }
       include_examples "parsing"

spec/install/gemfile/content_addressable_spec.rb

@@ -33,8 +33,43 @@
 
       cached_files = Dir.glob(default_bundle_path("cache", "mygem-1.0-*.gem").to_s)
       expect(cached_files.size).to eq(1), "expected exactly one cached gem file, found: #{cached_files}"
-      expect(cached_files.first).to match(/mygem-1\.0-[0-9a-f]{8}\.gem$/)
+      expect(cached_files.first).to match(/mygem-1\.0-[0-9a-f]{8,64}\.gem$/)
       expect(default_bundle_path("cache", "mygem-1.0-x86_64-linux.gem")).not_to exist
+      expect(lockfile).to match(/^    mygem \(1\.0-[0-9a-f]{8,64}\) x86_64-linux$/)
+    end
+  end
+
+  it "resolves a content-addressed binary from the local cache after a lockfile round-trip" do
+    simulate_platform "x86_64-linux" do
+      build_repo2 do
+        build_gem "mygem", "1.0" do |s|
+          s.platform = Gem::Platform.new("x86_64-linux")
+          s.write "lib/mygem.rb", "MYGEM = '1.0 not_content_addressed'"
+        end
+      end
+
+      build_gem "mygem", "1.0", ruby_abi: current_abi, path: gem_repo2("gems") do |s|
+        s.platform = Gem::Platform.new("x86_64-linux")
+        s.required_ruby_version = "~> #{current_abi}.0"
+        s.write "lib/mygem.rb", "MYGEM = '1.0 content_addressed'"
+      end
+
+      install_gemfile <<~G, artifice: "compact_index_v2", env: { "BUNDLER_SPEC_GEM_REPO" => gem_repo2.to_s }
+        source "https://gem.repo2"
+
+        gem "mygem"
+      G
+
+      cached_file = Dir[default_bundle_path("cache", "mygem-1.0-*.gem").to_s].first
+      FileUtils.mkdir_p(bundled_app("vendor/cache"))
+      FileUtils.cp(cached_file, bundled_app("vendor/cache"))
+
+      gem_dir = Dir[default_bundle_path("gems", "mygem-1.0-*").to_s].first
+      pristine_system_gems
+      bundle "install --local"
+
+      expect(the_bundle).to include_gems "mygem 1.0 content_addressed"
+      expect(Dir[default_bundle_path("gems", "mygem-1.0-*").to_s].first).to eq(gem_dir)
     end
   end
 

spec/other/ext_spec.rb

@@ -53,3 +53,13 @@
     end
   end
 end
+
+RSpec.describe Bundler::LazySpecification do
+  describe "#to_lock" do
+    it "appends the content address after the platform lock name when set" do
+      spec = Bundler::LazySpecification.new("mygem", v("1.0"), "x86_64-linux", nil, content_address: "abcdef1234")
+
+      expect(spec.to_lock).to eq("    mygem (1.0-abcdef1234) x86_64-linux\n")
+    end
+  end
+end