Fix incremental compiler registration and cache wall-clock performance

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## Unreleased
+
+### Fixed
+
+- **Incremental compiler registration** — `compilers: Mix.compilers() ++
+  [:ex_dna]` now resolves the documented `compile.ex_dna` task.
+- **Cache invalidation** — source digests detect same-timestamp edits, additions,
+  and removals, while all output-shaping detection options invalidate results.
+
+### Performance
+
+- **Complete-result cache** — unchanged compiler and opt-in CLI runs reuse the
+  prior clone groups without global detection or a large fragment-cache rewrite.
+
 ## 1.5.4
 
 ### Changed

README.md

@@ -36,7 +36,7 @@ macro, or a behaviour callback.
 - **Cross-file grouping** — `actions/ ↔ tools/ (6 clones, 298 nodes)`
   instead of listing each pair
 - **Credo-style suppression comments** — suppress known/intentional duplicates
-- **Incremental `Mix.Task.Compiler`** — reuses cached fingerprints for unchanged files
+- **Incremental `Mix.Task.Compiler`** — reuses complete results when sources are unchanged
 - **LSP server** — pushes clone diagnostics to your editor alongside
   [Expert](https://github.com/elixir-lang/expert) or ElixirLS
 - **Credo integration** — drop-in replacement for `DuplicatedCode`, reuses
@@ -67,6 +67,7 @@ mix ex_dna --min-fuzzy-mass 80          # require larger Type-III candidates
 mix ex_dna --min-mass 50                # fewer, larger clones
 mix ex_dna --max-clones 10              # fail only above budget
 mix ex_dna --format json                # machine-readable
+mix ex_dna --cache --format json        # reuse unchanged analysis results
 mix ex_dna --format html                # browsable report
 mix ex_dna --format sarif               # GitHub Code Scanning
 ```
@@ -192,8 +193,17 @@ def project do
 end
 ```
 
-Only changed files are re-analyzed. Cache is stored in `.ex_dna_cache` (add to
-`.gitignore`).
+Source digests validate the cached result. When no source changed, ExDNA returns
+the previous clone groups without rewriting the cache or rerunning detection.
+When a source changes, ExDNA reruns normal analysis and replaces the result.
+The cache is stored in `.ex_dna_cache` (add it to `.gitignore`).
+
+The same cache is available to standalone checks while preserving all output
+formats and explicit paths:
+
+```bash
+mix ex_dna lib/ test/ --cache --format json
+```
 
 ## Editor integration
 

lib/ex_dna.ex

@@ -35,6 +35,7 @@ defmodule ExDNA do
   """
 
   alias ExDNA.{Config, Detection, Report}
+  alias ExDNA.Detection.CachedDetector
 
   @type path_or_paths :: String.t() | [String.t()]
 
@@ -51,6 +52,8 @@ defmodule ExDNA do
     * `:min_similarity` — similarity threshold 0.0–1.0 (default: `#{Config.default(:min_similarity)}`)
     * `:ignore` — list of glob patterns to exclude
     * `:reporters` — list of reporter modules (default: `[ExDNA.Reporter.Console]`)
+    * `:cache` — reuse incremental analysis results when sources are unchanged
+    * `:cache_path` — incremental cache location (default: `.ex_dna_cache`)
 
   ## Examples
 
@@ -74,11 +77,19 @@ defmodule ExDNA do
   defp do_analyze(opts) do
     config = Config.new(opts)
 
-    {elapsed_us, {clones, files_analyzed}} = :timer.tc(fn -> Detection.Detector.run(config) end)
+    {elapsed_us, {clones, files_analyzed}} =
+      :timer.tc(fn -> analyze_with_config(config) end)
 
     Report.new(clones, config,
       detection_time_ms: div(elapsed_us, 1000),
       files_analyzed: files_analyzed
     )
   end
+
+  defp analyze_with_config(%Config{cache: true} = config) do
+    {_status, clones, files_analyzed} = CachedDetector.run(config)
+    {clones, files_analyzed}
+  end
+
+  defp analyze_with_config(config), do: Detection.Detector.run(config)
 end

lib/ex_dna/cache.ex

@@ -1,103 +1,84 @@
 defmodule ExDNA.Cache do
   @moduledoc """
-  Persistent cache for fingerprinted AST fragments.
+  Persistent cache for complete clone-analysis results.
 
-  Stores `%{file_path => %{mtime: integer(), fragments: term(), ast: Macro.t() | nil}}`
-  to disk. On subsequent runs, only files whose mtime has changed need to be
-  re-parsed and fingerprinted.
-
-  The cache is invalidated when config fields that affect fingerprint output change.
+  A lightweight source-digest map validates the cached clone groups without
+  relying on filesystem timestamp resolution. When any source or output-shaping
+  configuration changes, callers rerun normal analysis and replace the result.
   """
 
-  @cache_version 4
+  @cache_version 5
 
-  @type entry :: %{mtime: integer(), fragments: term(), ast: Macro.t() | nil}
-  @type entries :: %{String.t() => entry()}
+  @type state :: %{
+          digests: %{String.t() => binary() | nil},
+          clones: [ExDNA.Detection.Clone.t()] | nil
+        }
 
   @doc """
-  Default cache file path (relative to the project root).
+  Default cache file path relative to the project root.
   """
   @spec default_path :: String.t()
   def default_path, do: ".ex_dna_cache"
 
   @doc """
-  Read cached entries from disk. Returns an empty map if the file is missing,
-  corrupt, config has changed, or was written by an incompatible cache version.
+  Read cached source digests and clone groups.
+
+  Returns an empty state if the file is missing, corrupt, configured for a
+  different analysis, or written by an incompatible cache version.
   """
-  @spec read(String.t(), binary()) :: entries()
+  @spec read(String.t(), binary()) :: state()
   def read(path \\ default_path(), config_hash \\ <<>>) do
     with {:ok, binary} <- File.read(path),
-         {:ok, {@cache_version, ^config_hash, entries}} <- safe_binary_to_term(binary) do
-      entries
+         {:ok, {@cache_version, ^config_hash, digests, clones}} <- safe_binary_to_term(binary) do
+      %{digests: digests, clones: clones}
     else
-      _ -> %{}
+      _error -> %{digests: %{}, clones: nil}
     end
   end
 
   @doc """
-  Write entries to the cache file.
+  Write source digests and complete clone groups to the cache.
   """
-  @spec write(entries(), String.t(), binary()) :: :ok | {:error, term()}
-  def write(entries, path \\ default_path(), config_hash \\ <<>>) do
-    binary = :erlang.term_to_binary({@cache_version, config_hash, entries}, [:compressed])
-    File.write(path, binary)
+  @spec write(state(), String.t(), binary()) :: :ok | {:error, term()}
+  def write(state, path \\ default_path(), config_hash \\ <<>>) do
+    binary =
+      :erlang.term_to_binary(
+        {@cache_version, config_hash, state.digests, state.clones},
+        [:compressed]
+      )
+
+    with :ok <- File.mkdir_p(Path.dirname(path)) do
+      File.write(path, binary)
+    end
   end
 
   @doc """
-  Compute a fingerprint of config fields that affect cached fragments.
+  Compute a fingerprint of configuration fields that affect analysis output.
   """
   @spec config_hash(ExDNA.Config.t()) :: binary()
   def config_hash(config) do
-    {config.min_mass, config.literal_mode, config.normalize_pipes, config.excluded_macros,
-     config.ignored_attributes, config.max_window_size, config.max_module_forms}
+    {config.min_mass, config.min_occurrences, config.min_similarity, config.literal_mode,
+     config.normalize_pipes, config.excluded_macros, config.ignored_attributes,
+     config.max_window_size, config.max_module_forms, config.mass_tolerance,
+     config.min_fuzzy_mass, config.parse_timeout}
     |> :erlang.term_to_binary()
     |> then(&:erlang.md5/1)
   end
 
   @doc """
-  Return the subset of `files` whose mtime differs from the cached value
-  (or that aren't in the cache at all).
-  """
-  @spec stale_files([String.t()], entries()) :: [String.t()]
-  def stale_files(files, cached_entries) do
-    Enum.filter(files, fn file ->
-      case Map.fetch(cached_entries, file) do
-        {:ok, %{mtime: cached_mtime}} -> file_mtime(file) != cached_mtime
-        :error -> true
-      end
-    end)
-  end
-
-  @doc """
-  Merge fresh fragments into the cache, dropping entries for files
-  that no longer exist on disk.
+  Compute SHA-256 source digests for a file list.
   """
-  @spec merge(entries(), entries(), [String.t()]) :: entries()
-  def merge(cached, fresh_by_file, all_files) do
-    valid_set = MapSet.new(all_files)
-
-    merged =
-      Map.merge(cached, fresh_by_file, fn _file, _old, new -> new end)
-
-    Map.filter(merged, fn {file, _} -> MapSet.member?(valid_set, file) end)
-  end
-
-  @doc """
-  Build a cache entry for a single file with its current mtime.
-  """
-  @spec build_entry(String.t(), term(), Macro.t() | nil) :: entry()
-  def build_entry(file, fragments, ast \\ nil) do
-    %{mtime: file_mtime(file), fragments: fragments, ast: ast}
-  end
+  @spec source_digests([String.t()]) :: %{String.t() => binary() | nil}
+  def source_digests(files), do: Map.new(files, &{&1, file_digest(&1)})
 
   @doc """
-  Get the modification time of a file as a POSIX timestamp.
+  Get a SHA-256 digest of a source file, or `nil` when it cannot be read.
   """
-  @spec file_mtime(String.t()) :: integer()
-  def file_mtime(file) do
-    case File.stat(file, time: :posix) do
-      {:ok, %{mtime: mtime}} -> mtime
-      {:error, _} -> 0
+  @spec file_digest(String.t()) :: binary() | nil
+  def file_digest(file) do
+    case File.read(file) do
+      {:ok, source} -> :crypto.hash(:sha256, source)
+      {:error, _reason} -> nil
     end
   end
 

lib/ex_dna/compiler.ex

@@ -2,9 +2,9 @@ defmodule ExDNA.Compiler do
   @moduledoc """
   A `Mix.Task.Compiler` that runs clone detection incrementally.
 
-  On first run every source file is parsed, fingerprinted, and cached.
-  On subsequent compilations only files that changed (by mtime) are
-  re-analyzed; the rest is loaded from the cache.
+  On first run every source file is analyzed and the clone result is cached.
+  On subsequent compilations source digests determine whether that result can
+  be returned immediately or normal analysis needs to run again.
 
   ## Setup
 
@@ -19,103 +19,28 @@ defmodule ExDNA.Compiler do
 
   use Mix.Task.Compiler
 
-  alias ExDNA.{Cache, Config}
-  alias ExDNA.Detection.{Detector, Pipeline}
+  alias ExDNA.Config
+  alias ExDNA.Detection.CachedDetector
 
   @impl true
-  def run(_argv) do
+  def run(argv) do
     config = Config.new([])
-    cache_path = Cache.default_path()
-    cfg_hash = Cache.config_hash(config)
-    cached = Cache.read(cache_path, cfg_hash)
-
-    files = Pipeline.collect_files(config)
-    stale = Cache.stale_files(files, cached)
-
-    fresh_entries =
-      stale
-      |> Task.async_stream(
-        fn file ->
-          with {:ok, source} <- File.read(file),
-               {:ok, ast} <- Pipeline.parse_with_timeout(source, file, config.parse_timeout) do
-            fragments = fingerprint_sets(ast, file, config)
-            {file, Cache.build_entry(file, fragments, ast)}
-          else
-            _ -> {file, Cache.build_entry(file, [], nil)}
-          end
-        end,
-        max_concurrency: System.schedulers_online(),
-        ordered: false
-      )
-      |> Map.new(fn {:ok, result} -> result end)
-
-    merged = Cache.merge(cached, fresh_entries, files)
-    Cache.write(merged, cache_path, cfg_hash)
-
-    file_ast_pairs = ast_pairs(merged)
-    type_i_fragments = cached_fragments(merged, :type_i)
-    type_ii_fragments = cached_fragments(merged, :type_ii)
-
-    clones =
-      Detector.run_from_fragments(config, type_i_fragments, type_ii_fragments, file_ast_pairs)
-
-    diagnostics =
-      Enum.flat_map(clones, fn clone ->
-        Enum.map(clone.fragments, fn frag ->
-          %Mix.Task.Compiler.Diagnostic{
-            file: frag.file,
-            position: frag.line,
-            message: "Code clone detected (#{clone.type}, mass: #{clone.mass})",
-            severity: :warning,
-            compiler_name: "ExDNA"
-          }
-        end)
-      end)
-
-    {:ok, diagnostics}
-  end
-
-  defp fingerprint_sets(ast, file, config) do
-    %{
-      type_i:
-        Pipeline.fingerprint_ast(
-          ast,
-          file,
-          fingerprint_config(config, literal_mode: :keep, normalize_variables: false)
-        ),
-      type_ii:
-        Pipeline.fingerprint_ast(
-          ast,
-          file,
-          fingerprint_config(config,
-            literal_mode: config.literal_mode,
-            normalize_variables: true
-          )
-        )
-    }
-  end
-
-  defp ast_pairs(entries) do
-    Enum.flat_map(entries, fn {file, entry} ->
-      case entry do
-        %{ast: ast} when ast != nil -> [{file, ast}]
-        _ -> []
-      end
-    end)
+    force? = "--force" in argv or "--force-ex_dna" in argv
+    {status, clones, _files_analyzed} = CachedDetector.run(config, force: force?)
+    {status, diagnostics(clones)}
   end
 
-  defp cached_fragments(entries, type) do
-    Enum.flat_map(entries, fn {_file, entry} ->
-      case entry do
-        %{fragments: %{^type => fragments}} -> fragments
-        _ -> []
-      end
+  defp diagnostics(clones) do
+    Enum.flat_map(clones, fn clone ->
+      Enum.map(clone.fragments, fn fragment ->
+        %Mix.Task.Compiler.Diagnostic{
+          file: fragment.file,
+          position: fragment.line,
+          message: "Code clone detected (#{clone.type}, mass: #{clone.mass})",
+          severity: :warning,
+          compiler_name: "ExDNA"
+        }
+      end)
     end)
   end
-
-  defp fingerprint_config(config, overrides) do
-    config
-    |> Map.from_struct()
-    |> Map.merge(Map.new(overrides))
-  end
 end