From 904351248fbf259a57921a2cc3d3c9c2abc72d43 Mon Sep 17 00:00:00 2001 From: Andreas Ronge Date: Sat, 8 Aug 2026 10:38:18 +0200 Subject: [PATCH 1/4] fix(compiler): register ExDNA Mix task --- lib/mix/tasks/compile.ex_dna.ex | 27 ++++++++++++++++++++++++++ test/mix/tasks/compile_ex_dna_test.exs | 13 +++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 lib/mix/tasks/compile.ex_dna.ex create mode 100644 test/mix/tasks/compile_ex_dna_test.exs diff --git a/lib/mix/tasks/compile.ex_dna.ex b/lib/mix/tasks/compile.ex_dna.ex new file mode 100644 index 0000000..b70a72f --- /dev/null +++ b/lib/mix/tasks/compile.ex_dna.ex @@ -0,0 +1,27 @@ +defmodule Mix.Tasks.Compile.ExDna do + @shortdoc "Detect code duplication incrementally" + @moduledoc """ + Runs ExDNA clone detection as part of `mix compile`. + + Configure the compiler in `mix.exs`: + + compilers: Mix.compilers() ++ [:ex_dna] + """ + + use Mix.Task.Compiler + + alias ExDNA.Cache + + @impl true + def run(argv), do: ExDNA.Compiler.run(argv) + + @impl true + def manifests, do: [Cache.default_path()] + + @impl true + def clean do + cache_path = Cache.default_path() + File.rm(cache_path) + :ok + end +end diff --git a/test/mix/tasks/compile_ex_dna_test.exs b/test/mix/tasks/compile_ex_dna_test.exs new file mode 100644 index 0000000..6391fa5 --- /dev/null +++ b/test/mix/tasks/compile_ex_dna_test.exs @@ -0,0 +1,13 @@ +defmodule Mix.Tasks.Compile.ExDnaTest do + use ExUnit.Case, async: true + + alias Mix.Tasks.Compile.ExDna + + test "is registered under the documented compiler name" do + assert Mix.Task.get("compile.ex_dna") == ExDna + end + + test "declares its cache manifest" do + assert ExDna.manifests() == [".ex_dna_cache"] + end +end From 137bc5aa583d49e2ffc3de8c62a32148571f6342 Mon Sep 17 00:00:00 2001 From: Andreas Ronge Date: Sat, 8 Aug 2026 10:40:09 +0200 Subject: [PATCH 2/4] perf(cache): reuse complete unchanged analyses Replace the costly fragment cache with source digests and cached clone results shared by the compiler and opt-in CLI analysis. Changed sources use normal full detection, while unchanged runs return the previous result without detection or cache rewrites. Verified with mix ci and PtcRunner's 578-file duplication scope. --- CHANGELOG.md | 14 ++++ README.md | 16 +++- lib/ex_dna.ex | 14 +++- lib/ex_dna/cache.ex | 103 ++++++++++-------------- lib/ex_dna/compiler.ex | 114 +++++--------------------- lib/ex_dna/config.ex | 17 +++- lib/ex_dna/incremental.ex | 41 ++++++++++ lib/mix/tasks/compile.ex_dna.ex | 6 +- lib/mix/tasks/ex_dna.ex | 5 +- mix.exs | 2 +- test/ex_dna/cache_test.exs | 114 ++++++++++---------------- test/ex_dna/compiler_test.exs | 136 +++++++++++++------------------- test/ex_dna/config_test.exs | 8 +- test/ex_dna_test.exs | 32 ++++++++ test/mix/tasks/ex_dna_test.exs | 28 +++++++ 15 files changed, 330 insertions(+), 320 deletions(-) create mode 100644 lib/ex_dna/incremental.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index d5cf7b6..860a24a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## Unreleased + +### Fixed + +- **Incremental compiler registration** — `compilers: Mix.compilers() ++ + [:ex_dna]` now resolves the documented `compile.ex_dna` task. +- **Cache invalidation** — source digests detect same-timestamp edits, additions, + and removals, while all output-shaping detection options invalidate results. + +### Performance + +- **Complete-result cache** — unchanged compiler and opt-in CLI runs reuse the + prior clone groups without global detection or a large fragment-cache rewrite. + ## 1.5.4 ### Changed diff --git a/README.md b/README.md index 3ba698d..edddbe6 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ macro, or a behaviour callback. - **Cross-file grouping** — `actions/ ↔ tools/ (6 clones, 298 nodes)` instead of listing each pair - **Credo-style suppression comments** — suppress known/intentional duplicates -- **Incremental `Mix.Task.Compiler`** — reuses cached fingerprints for unchanged files +- **Incremental `Mix.Task.Compiler`** — reuses complete results when sources are unchanged - **LSP server** — pushes clone diagnostics to your editor alongside [Expert](https://github.com/elixir-lang/expert) or ElixirLS - **Credo integration** — drop-in replacement for `DuplicatedCode`, reuses @@ -67,6 +67,7 @@ mix ex_dna --min-fuzzy-mass 80 # require larger Type-III candidates mix ex_dna --min-mass 50 # fewer, larger clones mix ex_dna --max-clones 10 # fail only above budget mix ex_dna --format json # machine-readable +mix ex_dna --cache --format json # reuse unchanged analysis results mix ex_dna --format html # browsable report mix ex_dna --format sarif # GitHub Code Scanning ``` @@ -192,8 +193,17 @@ def project do end ``` -Only changed files are re-analyzed. Cache is stored in `.ex_dna_cache` (add to -`.gitignore`). +Source digests validate the cached result. When no source changed, ExDNA returns +the previous clone groups without rewriting the cache or rerunning detection. +When a source changes, ExDNA reruns normal analysis and replaces the result. +The cache is stored in `.ex_dna_cache` (add it to `.gitignore`). + +The same cache is available to standalone checks while preserving all output +formats and explicit paths: + +```bash +mix ex_dna lib/ test/ --cache --format json +``` ## Editor integration diff --git a/lib/ex_dna.ex b/lib/ex_dna.ex index b91f3f0..eb8bb54 100644 --- a/lib/ex_dna.ex +++ b/lib/ex_dna.ex @@ -34,7 +34,7 @@ defmodule ExDNA do See the README for the full option reference. """ - alias ExDNA.{Config, Detection, Report} + alias ExDNA.{Config, Detection, Incremental, Report} @type path_or_paths :: String.t() | [String.t()] @@ -51,6 +51,8 @@ defmodule ExDNA do * `:min_similarity` — similarity threshold 0.0–1.0 (default: `#{Config.default(:min_similarity)}`) * `:ignore` — list of glob patterns to exclude * `:reporters` — list of reporter modules (default: `[ExDNA.Reporter.Console]`) + * `:cache` — reuse incremental analysis results when sources are unchanged + * `:cache_path` — incremental cache location (default: `.ex_dna_cache`) ## Examples @@ -74,11 +76,19 @@ defmodule ExDNA do defp do_analyze(opts) do config = Config.new(opts) - {elapsed_us, {clones, files_analyzed}} = :timer.tc(fn -> Detection.Detector.run(config) end) + {elapsed_us, {clones, files_analyzed}} = + :timer.tc(fn -> analyze_with_config(config) end) Report.new(clones, config, detection_time_ms: div(elapsed_us, 1000), files_analyzed: files_analyzed ) end + + defp analyze_with_config(%Config{cache: true} = config) do + {_status, clones, files_analyzed} = Incremental.run(config) + {clones, files_analyzed} + end + + defp analyze_with_config(config), do: Detection.Detector.run(config) end diff --git a/lib/ex_dna/cache.ex b/lib/ex_dna/cache.ex index 36af7bf..ff05096 100644 --- a/lib/ex_dna/cache.ex +++ b/lib/ex_dna/cache.ex @@ -1,103 +1,82 @@ defmodule ExDNA.Cache do @moduledoc """ - Persistent cache for fingerprinted AST fragments. + Persistent cache for complete clone-analysis results. - Stores `%{file_path => %{mtime: integer(), fragments: term(), ast: Macro.t() | nil}}` - to disk. On subsequent runs, only files whose mtime has changed need to be - re-parsed and fingerprinted. - - The cache is invalidated when config fields that affect fingerprint output change. + A lightweight source-digest map validates the cached clone groups without + relying on filesystem timestamp resolution. When any source or output-shaping + configuration changes, callers rerun normal analysis and replace the result. """ - @cache_version 4 + @cache_version 5 - @type entry :: %{mtime: integer(), fragments: term(), ast: Macro.t() | nil} - @type entries :: %{String.t() => entry()} + @type state :: %{ + digests: %{String.t() => binary() | nil}, + clones: [ExDNA.Detection.Clone.t()] | nil + } @doc """ - Default cache file path (relative to the project root). + Default cache file path relative to the project root. """ @spec default_path :: String.t() def default_path, do: ".ex_dna_cache" @doc """ - Read cached entries from disk. Returns an empty map if the file is missing, - corrupt, config has changed, or was written by an incompatible cache version. + Read cached source digests and clone groups. + + Returns an empty state if the file is missing, corrupt, configured for a + different analysis, or written by an incompatible cache version. """ - @spec read(String.t(), binary()) :: entries() + @spec read(String.t(), binary()) :: state() def read(path \\ default_path(), config_hash \\ <<>>) do with {:ok, binary} <- File.read(path), - {:ok, {@cache_version, ^config_hash, entries}} <- safe_binary_to_term(binary) do - entries + {:ok, {@cache_version, ^config_hash, digests, clones}} <- safe_binary_to_term(binary) do + %{digests: digests, clones: clones} else - _ -> %{} + _error -> %{digests: %{}, clones: nil} end end @doc """ - Write entries to the cache file. + Write source digests and complete clone groups to the cache. """ - @spec write(entries(), String.t(), binary()) :: :ok | {:error, term()} - def write(entries, path \\ default_path(), config_hash \\ <<>>) do - binary = :erlang.term_to_binary({@cache_version, config_hash, entries}, [:compressed]) + @spec write(state(), String.t(), binary()) :: :ok | {:error, term()} + def write(state, path \\ default_path(), config_hash \\ <<>>) do + binary = + :erlang.term_to_binary( + {@cache_version, config_hash, state.digests, state.clones}, + [:compressed] + ) + File.write(path, binary) end @doc """ - Compute a fingerprint of config fields that affect cached fragments. + Compute a fingerprint of configuration fields that affect analysis output. """ @spec config_hash(ExDNA.Config.t()) :: binary() def config_hash(config) do - {config.min_mass, config.literal_mode, config.normalize_pipes, config.excluded_macros, - config.ignored_attributes, config.max_window_size, config.max_module_forms} + {config.min_mass, config.min_occurrences, config.min_similarity, config.literal_mode, + config.normalize_pipes, config.excluded_macros, config.ignored_attributes, + config.max_window_size, config.max_module_forms, config.mass_tolerance, + config.min_fuzzy_mass, config.parse_timeout} |> :erlang.term_to_binary() |> then(&:erlang.md5/1) end @doc """ - Return the subset of `files` whose mtime differs from the cached value - (or that aren't in the cache at all). - """ - @spec stale_files([String.t()], entries()) :: [String.t()] - def stale_files(files, cached_entries) do - Enum.filter(files, fn file -> - case Map.fetch(cached_entries, file) do - {:ok, %{mtime: cached_mtime}} -> file_mtime(file) != cached_mtime - :error -> true - end - end) - end - - @doc """ - Merge fresh fragments into the cache, dropping entries for files - that no longer exist on disk. + Compute SHA-256 source digests for a file list. """ - @spec merge(entries(), entries(), [String.t()]) :: entries() - def merge(cached, fresh_by_file, all_files) do - valid_set = MapSet.new(all_files) - - merged = - Map.merge(cached, fresh_by_file, fn _file, _old, new -> new end) - - Map.filter(merged, fn {file, _} -> MapSet.member?(valid_set, file) end) - end - - @doc """ - Build a cache entry for a single file with its current mtime. - """ - @spec build_entry(String.t(), term(), Macro.t() | nil) :: entry() - def build_entry(file, fragments, ast \\ nil) do - %{mtime: file_mtime(file), fragments: fragments, ast: ast} - end + @spec source_digests([String.t()]) :: %{String.t() => binary() | nil} + def source_digests(files), do: Map.new(files, &{&1, file_digest(&1)}) @doc """ - Get the modification time of a file as a POSIX timestamp. + Get a SHA-256 digest of a source file, or `nil` when it cannot be read. """ - @spec file_mtime(String.t()) :: integer() - def file_mtime(file) do - case File.stat(file, time: :posix) do - {:ok, %{mtime: mtime}} -> mtime - {:error, _} -> 0 + @spec file_digest(String.t()) :: binary() | nil + def file_digest(file) do + case File.read(file) do + {:ok, source} -> :crypto.hash(:sha256, source) + {:error, _reason} -> nil end end diff --git a/lib/ex_dna/compiler.ex b/lib/ex_dna/compiler.ex index c8cecee..32425de 100644 --- a/lib/ex_dna/compiler.ex +++ b/lib/ex_dna/compiler.ex @@ -2,9 +2,9 @@ defmodule ExDNA.Compiler do @moduledoc """ A `Mix.Task.Compiler` that runs clone detection incrementally. - On first run every source file is parsed, fingerprinted, and cached. - On subsequent compilations only files that changed (by mtime) are - re-analyzed; the rest is loaded from the cache. + On first run every source file is analyzed and the clone result is cached. + On subsequent compilations source digests determine whether that result can + be returned immediately or normal analysis needs to run again. ## Setup @@ -19,103 +19,27 @@ defmodule ExDNA.Compiler do use Mix.Task.Compiler - alias ExDNA.{Cache, Config} - alias ExDNA.Detection.{Detector, Pipeline} + alias ExDNA.{Config, Incremental} @impl true - def run(_argv) do + def run(argv) do config = Config.new([]) - cache_path = Cache.default_path() - cfg_hash = Cache.config_hash(config) - cached = Cache.read(cache_path, cfg_hash) - - files = Pipeline.collect_files(config) - stale = Cache.stale_files(files, cached) - - fresh_entries = - stale - |> Task.async_stream( - fn file -> - with {:ok, source} <- File.read(file), - {:ok, ast} <- Pipeline.parse_with_timeout(source, file, config.parse_timeout) do - fragments = fingerprint_sets(ast, file, config) - {file, Cache.build_entry(file, fragments, ast)} - else - _ -> {file, Cache.build_entry(file, [], nil)} - end - end, - max_concurrency: System.schedulers_online(), - ordered: false - ) - |> Map.new(fn {:ok, result} -> result end) - - merged = Cache.merge(cached, fresh_entries, files) - Cache.write(merged, cache_path, cfg_hash) - - file_ast_pairs = ast_pairs(merged) - type_i_fragments = cached_fragments(merged, :type_i) - type_ii_fragments = cached_fragments(merged, :type_ii) - - clones = - Detector.run_from_fragments(config, type_i_fragments, type_ii_fragments, file_ast_pairs) - - diagnostics = - Enum.flat_map(clones, fn clone -> - Enum.map(clone.fragments, fn frag -> - %Mix.Task.Compiler.Diagnostic{ - file: frag.file, - position: frag.line, - message: "Code clone detected (#{clone.type}, mass: #{clone.mass})", - severity: :warning, - compiler_name: "ExDNA" - } - end) - end) - - {:ok, diagnostics} - end - - defp fingerprint_sets(ast, file, config) do - %{ - type_i: - Pipeline.fingerprint_ast( - ast, - file, - fingerprint_config(config, literal_mode: :keep, normalize_variables: false) - ), - type_ii: - Pipeline.fingerprint_ast( - ast, - file, - fingerprint_config(config, - literal_mode: config.literal_mode, - normalize_variables: true - ) - ) - } - end - - defp ast_pairs(entries) do - Enum.flat_map(entries, fn {file, entry} -> - case entry do - %{ast: ast} when ast != nil -> [{file, ast}] - _ -> [] - end - end) + force? = "--force" in argv or "--force-ex_dna" in argv + {status, clones, _files_analyzed} = Incremental.run(config, force: force?) + {status, diagnostics(clones)} end - defp cached_fragments(entries, type) do - Enum.flat_map(entries, fn {_file, entry} -> - case entry do - %{fragments: %{^type => fragments}} -> fragments - _ -> [] - end + defp diagnostics(clones) do + Enum.flat_map(clones, fn clone -> + Enum.map(clone.fragments, fn fragment -> + %Mix.Task.Compiler.Diagnostic{ + file: fragment.file, + position: fragment.line, + message: "Code clone detected (#{clone.type}, mass: #{clone.mass})", + severity: :warning, + compiler_name: "ExDNA" + } + end) end) end - - defp fingerprint_config(config, overrides) do - config - |> Map.from_struct() - |> Map.merge(Map.new(overrides)) - end end diff --git a/lib/ex_dna/config.ex b/lib/ex_dna/config.ex index 619c7b7..6524d15 100644 --- a/lib/ex_dna/config.ex +++ b/lib/ex_dna/config.ex @@ -54,7 +54,9 @@ defmodule ExDNA.Config do excluded_macros: [], ignored_attributes: Map.keys(Module.reserved_attributes()), parse_timeout: 5_000, - output_file: nil + output_file: nil, + cache: false, + cache_path: ".ex_dna_cache" } defstruct Map.keys(@defaults) @@ -76,7 +78,9 @@ defmodule ExDNA.Config do excluded_macros: [atom()], ignored_attributes: [atom()], parse_timeout: pos_integer(), - output_file: String.t() | nil + output_file: String.t() | nil, + cache: boolean(), + cache_path: String.t() } @spec new(keyword()) :: t() @@ -105,6 +109,15 @@ defmodule ExDNA.Config do validate_float_range_exclusive_min!(:mass_tolerance, config.mass_tolerance, 0.0, 1.0) validate_optional_pos_int!(:min_fuzzy_mass, config.min_fuzzy_mass) + unless is_boolean(config.cache) do + raise ArgumentError, "cache must be a boolean, got: #{inspect(config.cache)}" + end + + unless is_binary(config.cache_path) and config.cache_path != "" do + raise ArgumentError, + "cache_path must be a non-empty string, got: #{inspect(config.cache_path)}" + end + unless config.literal_mode in [:keep, :abstract] do raise ArgumentError, "literal_mode must be :keep or :abstract, got: #{inspect(config.literal_mode)}" diff --git a/lib/ex_dna/incremental.ex b/lib/ex_dna/incremental.ex new file mode 100644 index 0000000..8f098e5 --- /dev/null +++ b/lib/ex_dna/incremental.ex @@ -0,0 +1,41 @@ +defmodule ExDNA.Incremental do + @moduledoc """ + Cached clone analysis shared by the Mix compiler and opt-in CLI cache. + + Unchanged runs return the previously detected clone groups without rerunning + detection. When any source changes, normal full analysis refreshes the result; + this avoids a large fragment cache whose serialization cost exceeded the work + it saved. + """ + + alias ExDNA.{Cache, Config} + alias ExDNA.Detection.Detector + alias ExDNA.Detection.Pipeline + + @type status :: :ok | :noop + @type result :: {status(), [ExDNA.Detection.Clone.t()], non_neg_integer()} + + @doc """ + Run cached analysis for `config`. + + Pass `:force` to refresh analysis even when every source digest is current, + and `:cache_path` to override the configured cache location. + """ + @spec run(Config.t(), keyword()) :: result() + def run(%Config{} = config, opts \\ []) do + cache_path = Keyword.get(opts, :cache_path, config.cache_path) + force? = Keyword.get(opts, :force, false) + config_hash = Cache.config_hash(config) + files = Pipeline.collect_files(config) + current_digests = Cache.source_digests(files) + cached = Cache.read(cache_path, config_hash) + + if not force? and cached.clones != nil and cached.digests == current_digests do + {:noop, cached.clones, length(files)} + else + {clones, files_analyzed} = Detector.run(config) + :ok = Cache.write(%{digests: current_digests, clones: clones}, cache_path, config_hash) + {:ok, clones, files_analyzed} + end + end +end diff --git a/lib/mix/tasks/compile.ex_dna.ex b/lib/mix/tasks/compile.ex_dna.ex index b70a72f..82edfd7 100644 --- a/lib/mix/tasks/compile.ex_dna.ex +++ b/lib/mix/tasks/compile.ex_dna.ex @@ -10,17 +10,17 @@ defmodule Mix.Tasks.Compile.ExDna do use Mix.Task.Compiler - alias ExDNA.Cache + alias ExDNA.Config @impl true def run(argv), do: ExDNA.Compiler.run(argv) @impl true - def manifests, do: [Cache.default_path()] + def manifests, do: [Config.new([]).cache_path] @impl true def clean do - cache_path = Cache.default_path() + cache_path = Config.new([]).cache_path File.rm(cache_path) :ok end diff --git a/lib/mix/tasks/ex_dna.ex b/lib/mix/tasks/ex_dna.ex index 38b6754..fece238 100644 --- a/lib/mix/tasks/ex_dna.ex +++ b/lib/mix/tasks/ex_dna.ex @@ -34,6 +34,7 @@ defmodule Mix.Tasks.ExDna do * `--output` — output file for `html` or `sarif` reports * `--max-clones` — maximum allowed clones. Exits with code 1 only when exceeded. Useful for gradual adoption in brownfield projects. + * `--cache` — reuse cached clone results when all source digests are unchanged. Exits with code 1 if clones are found (or exceed `--max-clones`). """ @@ -61,7 +62,8 @@ defmodule Mix.Tasks.ExDna do ignore: :keep, format: :string, output: :string, - max_clones: :integer + max_clones: :integer, + cache: :boolean ], aliases: [m: :min_mass, o: :min_occurrences, s: :min_similarity, i: :ignore, f: :format] ) @@ -122,6 +124,7 @@ defmodule Mix.Tasks.ExDna do |> Options.maybe_put(:max_module_forms, Keyword.get(opts, :max_module_forms)) |> Options.maybe_put(:mass_tolerance, Keyword.get(opts, :mass_tolerance)) |> Options.maybe_put(:min_fuzzy_mass, Keyword.get(opts, :min_fuzzy_mass)) + |> Options.maybe_put(:cache, Options.explicit_value(opts, :cache)) |> Options.maybe_put(:excluded_macros, Options.excluded_macros(opts)) |> Options.maybe_put(:ignored_attributes, Options.ignored_attributes(opts)) |> Options.maybe_put(:output_file, Keyword.get(opts, :output)) diff --git a/mix.exs b/mix.exs index 90dcda6..abce215 100644 --- a/mix.exs +++ b/mix.exs @@ -26,7 +26,7 @@ defmodule ExDNA.MixProject do def application do [ - extra_applications: [:logger] + extra_applications: [:crypto, :logger] ] end diff --git a/test/ex_dna/cache_test.exs b/test/ex_dna/cache_test.exs index 7ac7163..4343423 100644 --- a/test/ex_dna/cache_test.exs +++ b/test/ex_dna/cache_test.exs @@ -1,7 +1,7 @@ defmodule ExDNA.CacheTest do use ExUnit.Case, async: true - alias ExDNA.Cache + alias ExDNA.{Cache, Config} @moduletag :tmp_dir setup %{tmp_dir: tmp_dir} do @@ -9,105 +9,77 @@ defmodule ExDNA.CacheTest do {:ok, cache_path: cache_path} end - describe "write/2 and read/1" do - test "round-trips entries through disk", %{cache_path: path} do - entries = %{ - "lib/foo.ex" => %{mtime: 1_700_000_000, fragments: [%{hash: "abc", mass: 10}]}, - "lib/bar.ex" => %{mtime: 1_700_000_001, fragments: []} - } + describe "write/3 and read/2" do + test "round-trips source digests and clone results", %{cache_path: path} do + state = %{digests: %{"lib/foo.ex" => <<1>>}, clones: [%{type: :type_i}]} - assert :ok = Cache.write(entries, path) - assert Cache.read(path) == entries + assert :ok = Cache.write(state, path) + assert Cache.read(path) == state end - test "returns empty map when file does not exist", %{cache_path: path} do - assert Cache.read(path) == %{} + test "returns empty state when the file does not exist", %{cache_path: path} do + assert Cache.read(path) == %{digests: %{}, clones: nil} end - test "returns empty map when file is corrupt", %{cache_path: path} do + test "returns empty state when the file is corrupt", %{cache_path: path} do File.write!(path, "not a valid term") - assert Cache.read(path) == %{} + assert Cache.read(path) == %{digests: %{}, clones: nil} end - test "returns empty map when cache version mismatches", %{cache_path: path} do - binary = :erlang.term_to_binary({999, %{"a.ex" => %{mtime: 0, fragments: []}}}) + test "returns empty state when cache version mismatches", %{cache_path: path} do + binary = :erlang.term_to_binary({999, <<>>, %{}, []}) File.write!(path, binary) - assert Cache.read(path) == %{} + assert Cache.read(path) == %{digests: %{}, clones: nil} end - end - describe "stale_files/2" do - test "marks missing files as stale" do - cached = %{"lib/known.ex" => %{mtime: 1_700_000_000, fragments: []}} - assert Cache.stale_files(["lib/new.ex"], cached) == ["lib/new.ex"] + test "invalidates output-shaping config changes", %{cache_path: path} do + original = Config.new(reporters: []) + changed = Config.new(reporters: [], min_occurrences: 3) + state = %{digests: %{}, clones: []} + + assert :ok = Cache.write(state, path, Cache.config_hash(original)) + assert Cache.read(path, Cache.config_hash(original)) == state + assert Cache.read(path, Cache.config_hash(changed)) == %{digests: %{}, clones: nil} end + end - test "marks files with changed mtime as stale", %{tmp_dir: dir} do + describe "source_digests/1" do + test "changes with contents even when mtime is unchanged", %{tmp_dir: dir} do file = Path.join(dir, "changed.ex") File.write!(file, "defmodule A, do: nil") - mtime = Cache.file_mtime(file) - - cached = %{file => %{mtime: mtime - 1, fragments: []}} - assert Cache.stale_files([file], cached) == [file] - end + mtime = File.stat!(file, time: :posix).mtime + original = Cache.source_digests([file]) - test "keeps files with matching mtime", %{tmp_dir: dir} do - file = Path.join(dir, "fresh.ex") File.write!(file, "defmodule B, do: nil") - mtime = Cache.file_mtime(file) + File.touch!(file, mtime) - cached = %{file => %{mtime: mtime, fragments: []}} - assert Cache.stale_files([file], cached) == [] + refute Cache.source_digests([file]) == original end - end - describe "merge/3" do - test "overwrites cached entries with fresh ones" do - cached = %{"a.ex" => %{mtime: 1, fragments: [%{hash: "old"}]}} - fresh = %{"a.ex" => %{mtime: 2, fragments: [%{hash: "new"}]}} - - merged = Cache.merge(cached, fresh, ["a.ex"]) - assert merged["a.ex"].fragments == [%{hash: "new"}] - end + test "captures additions and removals", %{tmp_dir: dir} do + first = Path.join(dir, "first.ex") + second = Path.join(dir, "second.ex") + File.write!(first, "defmodule First, do: nil") + File.write!(second, "defmodule Second, do: nil") - test "drops entries for files no longer in the file list" do - cached = %{ - "a.ex" => %{mtime: 1, fragments: []}, - "deleted.ex" => %{mtime: 1, fragments: []} - } + assert Map.keys(Cache.source_digests([first, second])) |> Enum.sort() == + Enum.sort([first, second]) - merged = Cache.merge(cached, %{}, ["a.ex"]) - assert Map.keys(merged) == ["a.ex"] - end - - test "preserves unchanged cached entries" do - cached = %{"a.ex" => %{mtime: 1, fragments: [%{hash: "kept"}]}} - merged = Cache.merge(cached, %{}, ["a.ex"]) - assert merged == cached + assert Map.keys(Cache.source_digests([first])) == [first] end end - describe "build_entry/2" do - test "captures current mtime", %{tmp_dir: dir} do - file = Path.join(dir, "entry.ex") - File.write!(file, "defmodule E, do: nil") - - entry = Cache.build_entry(file, [%{hash: "h1"}]) - assert entry.mtime == Cache.file_mtime(file) - assert entry.fragments == [%{hash: "h1"}] - end - end - - describe "file_mtime/1" do - test "returns posix timestamp for existing file", %{tmp_dir: dir} do + describe "file_digest/1" do + test "returns a digest for an existing file", %{tmp_dir: dir} do file = Path.join(dir, "exists.ex") File.write!(file, "ok") - assert is_integer(Cache.file_mtime(file)) - assert Cache.file_mtime(file) > 0 + + assert is_binary(Cache.file_digest(file)) + assert byte_size(Cache.file_digest(file)) == 32 end - test "returns 0 for missing file" do - assert Cache.file_mtime("/nonexistent/path.ex") == 0 + test "returns nil for a missing file" do + assert Cache.file_digest("/nonexistent/path.ex") == nil end end end diff --git a/test/ex_dna/compiler_test.exs b/test/ex_dna/compiler_test.exs index f2f7505..778109f 100644 --- a/test/ex_dna/compiler_test.exs +++ b/test/ex_dna/compiler_test.exs @@ -1,96 +1,68 @@ defmodule ExDNA.CompilerTest do - use ExUnit.Case, async: false + use ExUnit.Case, async: true - alias ExDNA.{Cache, Config} - alias ExDNA.Detection.{Detector, Pipeline} + alias ExDNA.{Cache, Config, Incremental} - setup do - dir = - Path.join(System.tmp_dir!(), "ex_dna_compiler_test_#{:erlang.unique_integer([:positive])}") + @moduletag :tmp_dir + setup %{tmp_dir: dir} do + cache_path = Path.join(dir, ".ex_dna_cache") + write_duplicate_files(dir) + + config = Config.new(paths: [dir], reporters: [], min_mass: 5) + {:ok, cache_path: cache_path, config: config, dir: dir} + end - File.mkdir_p!(dir) + test "an unchanged analysis returns cached clone groups", %{ + cache_path: cache_path, + config: config + } do + assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert clones != [] - cache_path = Path.join(dir, ".ex_dna_cache") + assert {:noop, ^clones, 2} = Incremental.run(config, cache_path: cache_path) + end + + test "same-mtime content changes refresh analysis", %{ + cache_path: cache_path, + config: config, + dir: dir + } do + assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert clones != [] - on_exit(fn -> File.rm_rf!(dir) end) + file = Path.join(dir, "dup_b.ex") + mtime = File.stat!(file, time: :posix).mtime + File.write!(file, "defmodule Unique, do: nil") + File.touch!(file, mtime) - {:ok, dir: dir, cache_path: cache_path} + assert {:ok, [], 2} = Incremental.run(config, cache_path: cache_path) end - describe "incremental pipeline with cache" do - test "cached ASTs enable full Type-I/II/III detection", %{dir: dir, cache_path: cache_path} do - write_duplicate_files(dir) - - config = Config.new(paths: [dir], reporters: [], min_mass: 5) - files = Pipeline.collect_files(config) - - fresh_entries = - Map.new(files, fn file -> - with {:ok, source} <- File.read(file), - {:ok, ast} <- Pipeline.parse_with_timeout(source, file, config.parse_timeout) do - fragments = %{ - type_i: - Pipeline.fingerprint_ast( - ast, - file, - config - |> Map.from_struct() - |> Map.merge(%{literal_mode: :keep, normalize_variables: false}) - ), - type_ii: - Pipeline.fingerprint_ast( - ast, - file, - config - |> Map.from_struct() - |> Map.merge(%{normalize_variables: true}) - ) - } - - {file, Cache.build_entry(file, fragments, ast)} - end - end) - - Cache.write(fresh_entries, cache_path) - - cached = Cache.read(cache_path) - - file_ast_pairs = - Enum.flat_map(cached, fn {file, entry} -> - case entry do - %{ast: ast} when ast != nil -> [{file, ast}] - _ -> [] - end - end) - - type_i_fragments = Enum.flat_map(cached, fn {_file, entry} -> entry.fragments.type_i end) - type_ii_fragments = Enum.flat_map(cached, fn {_file, entry} -> entry.fragments.type_ii end) - - clones = - Detector.run_from_fragments(config, type_i_fragments, type_ii_fragments, file_ast_pairs) - - assert clones != [] - end + test "removed files refresh analysis", %{cache_path: cache_path, config: config, dir: dir} do + assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert clones != [] - test "cache entries include ASTs", %{dir: dir} do - file = Path.join(dir, "test.ex") + File.rm!(Path.join(dir, "dup_b.ex")) - File.write!(file, """ - defmodule Test do - def foo(x), do: x + 1 - end - """) + assert {:ok, [], 1} = Incremental.run(config, cache_path: cache_path) + end - config = Config.new(paths: [dir], reporters: [], min_mass: 5) + test "force refreshes an unchanged analysis", %{cache_path: cache_path, config: config} do + assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert {:ok, refreshed, 2} = Incremental.run(config, cache_path: cache_path, force: true) - {:ok, source} = File.read(file) - {:ok, ast} = Pipeline.parse_with_timeout(source, file, config.parse_timeout) - frags = Pipeline.fingerprint_ast(ast, file, config) + assert clone_signatures(refreshed) == clone_signatures(clones) + end - entry = Cache.build_entry(file, %{type_i: frags, type_ii: frags}, ast) - assert entry.ast != nil - assert is_tuple(entry.ast) - end + test "cache stores clone results and source digests", %{ + cache_path: cache_path, + config: config + } do + assert {:ok, _clones, 2} = Incremental.run(config, cache_path: cache_path) + + cached = Cache.read(cache_path, Cache.config_hash(config)) + assert cached.clones != [] + assert map_size(cached.digests) == 2 end defp write_duplicate_files(dir) do @@ -108,4 +80,10 @@ defmodule ExDNA.CompilerTest do """) end end + + defp clone_signatures(clones) do + clones + |> Enum.map(&{&1.type, &1.mass}) + |> Enum.sort() + end end diff --git a/test/ex_dna/config_test.exs b/test/ex_dna/config_test.exs index 2ec902c..b84d0f9 100644 --- a/test/ex_dna/config_test.exs +++ b/test/ex_dna/config_test.exs @@ -16,6 +16,8 @@ defmodule ExDNA.ConfigTest do assert config.max_module_forms == 200 assert config.min_fuzzy_mass == nil assert config.output_file == nil + assert config.cache == false + assert config.cache_path == ".ex_dna_cache" end test "overrides defaults with provided options" do @@ -26,7 +28,9 @@ defmodule ExDNA.ConfigTest do excluded_macros: [:@, :schema], max_module_forms: 300, min_fuzzy_mass: 80, - output_file: "report.html" + output_file: "report.html", + cache: true, + cache_path: "tmp/cache" ) assert config.min_mass == 10 @@ -35,6 +39,8 @@ defmodule ExDNA.ConfigTest do assert config.max_module_forms == 300 assert config.min_fuzzy_mass == 80 assert config.output_file == "report.html" + assert config.cache == true + assert config.cache_path == "tmp/cache" end test "loads config file when present" do diff --git a/test/ex_dna_test.exs b/test/ex_dna_test.exs index cb2956b..351a2ec 100644 --- a/test/ex_dna_test.exs +++ b/test/ex_dna_test.exs @@ -50,5 +50,37 @@ defmodule ExDNATest do report = ExDNA.analyze(paths: [dir], min_mass: 5, reporters: []) assert %ExDNA.Report{} = report end + + test "cache preserves the normal report", %{dir: dir} do + for name <- ~w(a.ex b.ex) do + File.write!(Path.join(dir, name), """ + defmodule #{name |> Path.rootname() |> Macro.camelize()} do + def duplicated(value) do + value + |> Enum.map(&to_string/1) + |> Enum.sort() + end + end + """) + end + + cache_path = Path.join(dir, ".ex_dna_cache") + + opts = [ + paths: [dir], + reporters: [], + min_mass: 5, + cache: true, + cache_path: cache_path + ] + + first = ExDNA.analyze(opts) + second = ExDNA.analyze(opts) + + assert first.clones != [] + assert second.clones == first.clones + assert second.stats.files_analyzed == first.stats.files_analyzed + assert File.regular?(cache_path) + end end end diff --git a/test/mix/tasks/ex_dna_test.exs b/test/mix/tasks/ex_dna_test.exs index e6e750e..a315ac2 100644 --- a/test/mix/tasks/ex_dna_test.exs +++ b/test/mix/tasks/ex_dna_test.exs @@ -69,6 +69,34 @@ defmodule Mix.Tasks.ExDnaTest do assert Exception.message(error) =~ "Invalid format" end + test "cached JSON preserves clone groups and explicit paths", %{dir: dir} do + write_duplicate_pair(dir) + + {first, second} = + File.cd!(dir, fn -> + argv = [ + "--cache", + "--format", + "json", + "--min-mass", + "5", + "--max-clones", + "10", + dir + ] + + {capture_io(fn -> ExDna.run(argv) end), capture_io(fn -> ExDna.run(argv) end)} + end) + + first = Jason.decode!(first) + second = Jason.decode!(second) + + assert first["clones"] != [] + assert second["clones"] == first["clones"] + assert second["stats"]["files_analyzed"] == 2 + assert File.regular?(Path.join(dir, ".ex_dna_cache")) + end + test "raises a Mix error for invalid explain option" do error = assert_raise Error, fn -> From 98221751e1b4457c50fdf4d376ab641e91567bb1 Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Mon, 10 Aug 2026 21:52:02 +0200 Subject: [PATCH 3/4] fix(cache): tolerate unavailable cache paths --- lib/ex_dna/cache.ex | 4 +++- lib/ex_dna/incremental.ex | 2 +- test/ex_dna/cache_test.exs | 8 ++++++++ test/ex_dna/compiler_test.exs | 8 ++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/ex_dna/cache.ex b/lib/ex_dna/cache.ex index ff05096..62b8ab3 100644 --- a/lib/ex_dna/cache.ex +++ b/lib/ex_dna/cache.ex @@ -47,7 +47,9 @@ defmodule ExDNA.Cache do [:compressed] ) - File.write(path, binary) + with :ok <- File.mkdir_p(Path.dirname(path)) do + File.write(path, binary) + end end @doc """ diff --git a/lib/ex_dna/incremental.ex b/lib/ex_dna/incremental.ex index 8f098e5..2b014aa 100644 --- a/lib/ex_dna/incremental.ex +++ b/lib/ex_dna/incremental.ex @@ -34,7 +34,7 @@ defmodule ExDNA.Incremental do {:noop, cached.clones, length(files)} else {clones, files_analyzed} = Detector.run(config) - :ok = Cache.write(%{digests: current_digests, clones: clones}, cache_path, config_hash) + Cache.write(%{digests: current_digests, clones: clones}, cache_path, config_hash) {:ok, clones, files_analyzed} end end diff --git a/test/ex_dna/cache_test.exs b/test/ex_dna/cache_test.exs index 4343423..e0c89a0 100644 --- a/test/ex_dna/cache_test.exs +++ b/test/ex_dna/cache_test.exs @@ -17,6 +17,14 @@ defmodule ExDNA.CacheTest do assert Cache.read(path) == state end + test "creates the cache directory", %{tmp_dir: dir} do + path = Path.join([dir, "nested", "cache", "results"]) + state = %{digests: %{}, clones: []} + + assert :ok = Cache.write(state, path) + assert Cache.read(path) == state + end + test "returns empty state when the file does not exist", %{cache_path: path} do assert Cache.read(path) == %{digests: %{}, clones: nil} end diff --git a/test/ex_dna/compiler_test.exs b/test/ex_dna/compiler_test.exs index 778109f..cf63569 100644 --- a/test/ex_dna/compiler_test.exs +++ b/test/ex_dna/compiler_test.exs @@ -65,6 +65,14 @@ defmodule ExDNA.CompilerTest do assert map_size(cached.digests) == 2 end + test "cache write failures do not fail analysis", %{config: config, dir: dir} do + cache_path = Path.join(dir, "cache_directory") + File.mkdir_p!(cache_path) + + assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert clones != [] + end + defp write_duplicate_files(dir) do for name <- ~w(dup_a.ex dup_b.ex) do File.write!(Path.join(dir, name), """ From cac0c45a514dae8b1e08da8e9192102d6e979c63 Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Tue, 11 Aug 2026 10:49:38 +0200 Subject: [PATCH 4/4] refactor(detection): name cached detector explicitly --- lib/ex_dna.ex | 5 ++-- lib/ex_dna/compiler.ex | 5 ++-- .../cached_detector.ex} | 13 +++++----- .../cached_detector_test.exs} | 25 ++++++++++--------- 4 files changed, 25 insertions(+), 23 deletions(-) rename lib/ex_dna/{incremental.ex => detection/cached_detector.ex} (75%) rename test/ex_dna/{compiler_test.exs => detection/cached_detector_test.exs} (69%) diff --git a/lib/ex_dna.ex b/lib/ex_dna.ex index eb8bb54..01d2c2e 100644 --- a/lib/ex_dna.ex +++ b/lib/ex_dna.ex @@ -34,7 +34,8 @@ defmodule ExDNA do See the README for the full option reference. """ - alias ExDNA.{Config, Detection, Incremental, Report} + alias ExDNA.{Config, Detection, Report} + alias ExDNA.Detection.CachedDetector @type path_or_paths :: String.t() | [String.t()] @@ -86,7 +87,7 @@ defmodule ExDNA do end defp analyze_with_config(%Config{cache: true} = config) do - {_status, clones, files_analyzed} = Incremental.run(config) + {_status, clones, files_analyzed} = CachedDetector.run(config) {clones, files_analyzed} end diff --git a/lib/ex_dna/compiler.ex b/lib/ex_dna/compiler.ex index 32425de..1d1a5f5 100644 --- a/lib/ex_dna/compiler.ex +++ b/lib/ex_dna/compiler.ex @@ -19,13 +19,14 @@ defmodule ExDNA.Compiler do use Mix.Task.Compiler - alias ExDNA.{Config, Incremental} + alias ExDNA.Config + alias ExDNA.Detection.CachedDetector @impl true def run(argv) do config = Config.new([]) force? = "--force" in argv or "--force-ex_dna" in argv - {status, clones, _files_analyzed} = Incremental.run(config, force: force?) + {status, clones, _files_analyzed} = CachedDetector.run(config, force: force?) {status, diagnostics(clones)} end diff --git a/lib/ex_dna/incremental.ex b/lib/ex_dna/detection/cached_detector.ex similarity index 75% rename from lib/ex_dna/incremental.ex rename to lib/ex_dna/detection/cached_detector.ex index 2b014aa..5fa980b 100644 --- a/lib/ex_dna/incremental.ex +++ b/lib/ex_dna/detection/cached_detector.ex @@ -1,24 +1,23 @@ -defmodule ExDNA.Incremental do +defmodule ExDNA.Detection.CachedDetector do @moduledoc """ - Cached clone analysis shared by the Mix compiler and opt-in CLI cache. + Adds complete-result caching to clone detection. Unchanged runs return the previously detected clone groups without rerunning - detection. When any source changes, normal full analysis refreshes the result; + detection. When any source changes, normal full detection refreshes the result; this avoids a large fragment cache whose serialization cost exceeded the work it saved. """ alias ExDNA.{Cache, Config} - alias ExDNA.Detection.Detector - alias ExDNA.Detection.Pipeline + alias ExDNA.Detection.{Detector, Pipeline} @type status :: :ok | :noop @type result :: {status(), [ExDNA.Detection.Clone.t()], non_neg_integer()} @doc """ - Run cached analysis for `config`. + Run cached detection for `config`. - Pass `:force` to refresh analysis even when every source digest is current, + Pass `:force` to refresh detection even when every source digest is current, and `:cache_path` to override the configured cache location. """ @spec run(Config.t(), keyword()) :: result() diff --git a/test/ex_dna/compiler_test.exs b/test/ex_dna/detection/cached_detector_test.exs similarity index 69% rename from test/ex_dna/compiler_test.exs rename to test/ex_dna/detection/cached_detector_test.exs index cf63569..83570ce 100644 --- a/test/ex_dna/compiler_test.exs +++ b/test/ex_dna/detection/cached_detector_test.exs @@ -1,7 +1,8 @@ -defmodule ExDNA.CompilerTest do +defmodule ExDNA.Detection.CachedDetectorTest do use ExUnit.Case, async: true - alias ExDNA.{Cache, Config, Incremental} + alias ExDNA.{Cache, Config} + alias ExDNA.Detection.CachedDetector @moduletag :tmp_dir setup %{tmp_dir: dir} do @@ -16,10 +17,10 @@ defmodule ExDNA.CompilerTest do cache_path: cache_path, config: config } do - assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert {:ok, clones, 2} = CachedDetector.run(config, cache_path: cache_path) assert clones != [] - assert {:noop, ^clones, 2} = Incremental.run(config, cache_path: cache_path) + assert {:noop, ^clones, 2} = CachedDetector.run(config, cache_path: cache_path) end test "same-mtime content changes refresh analysis", %{ @@ -27,7 +28,7 @@ defmodule ExDNA.CompilerTest do config: config, dir: dir } do - assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert {:ok, clones, 2} = CachedDetector.run(config, cache_path: cache_path) assert clones != [] file = Path.join(dir, "dup_b.ex") @@ -35,21 +36,21 @@ defmodule ExDNA.CompilerTest do File.write!(file, "defmodule Unique, do: nil") File.touch!(file, mtime) - assert {:ok, [], 2} = Incremental.run(config, cache_path: cache_path) + assert {:ok, [], 2} = CachedDetector.run(config, cache_path: cache_path) end test "removed files refresh analysis", %{cache_path: cache_path, config: config, dir: dir} do - assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert {:ok, clones, 2} = CachedDetector.run(config, cache_path: cache_path) assert clones != [] File.rm!(Path.join(dir, "dup_b.ex")) - assert {:ok, [], 1} = Incremental.run(config, cache_path: cache_path) + assert {:ok, [], 1} = CachedDetector.run(config, cache_path: cache_path) end test "force refreshes an unchanged analysis", %{cache_path: cache_path, config: config} do - assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) - assert {:ok, refreshed, 2} = Incremental.run(config, cache_path: cache_path, force: true) + assert {:ok, clones, 2} = CachedDetector.run(config, cache_path: cache_path) + assert {:ok, refreshed, 2} = CachedDetector.run(config, cache_path: cache_path, force: true) assert clone_signatures(refreshed) == clone_signatures(clones) end @@ -58,7 +59,7 @@ defmodule ExDNA.CompilerTest do cache_path: cache_path, config: config } do - assert {:ok, _clones, 2} = Incremental.run(config, cache_path: cache_path) + assert {:ok, _clones, 2} = CachedDetector.run(config, cache_path: cache_path) cached = Cache.read(cache_path, Cache.config_hash(config)) assert cached.clones != [] @@ -69,7 +70,7 @@ defmodule ExDNA.CompilerTest do cache_path = Path.join(dir, "cache_directory") File.mkdir_p!(cache_path) - assert {:ok, clones, 2} = Incremental.run(config, cache_path: cache_path) + assert {:ok, clones, 2} = CachedDetector.run(config, cache_path: cache_path) assert clones != [] end