Dynamically map known root paths in cache files

core/api/src/mill/api/JsonFormatters.scala

@@ -23,8 +23,8 @@ trait JsonFormatters {
 
   implicit val pathReadWrite: RW[os.Path] = upickle.readwriter[String]
     .bimap[os.Path](
-      _.toString,
-      os.Path(_)
+      p => MappedRoots.encodeKnownRootsInPath(p),
+      s => os.Path(MappedRoots.decodeKnownRootsInPath(s))
     )
 
   implicit val relPathRW: RW[os.RelPath] = upickle.readwriter[String]

core/api/src/mill/api/MappedRoots.scala

@@ -0,0 +1,95 @@
+package mill.api
+
+import mill.api.internal.NamedParameterOnlyDummy
+import mill.constants.PathVars
+
+import scala.annotation.unused
+import scala.util.DynamicVariable
+
+type MappedRoots = Seq[(key: String, path: os.Path)]
+
+object MappedRoots extends MappedRootsImpl
+
+trait MappedRootsImpl {
+
+  private val rootMapping: DynamicVariable[MappedRoots] = DynamicVariable(Seq())
+
+  def get: MappedRoots = rootMapping.value
+
+  def toMap: Map[String, os.Path] = get.map(m => (m.key, m.path)).toMap
+
+  def withMillDefaults[T](
+      @unused t: NamedParameterOnlyDummy = new NamedParameterOnlyDummy,
+      outPath: os.Path,
+      workspacePath: os.Path = BuildCtx.workspaceRoot,
+      homePath: os.Path = os.home
+  )(thunk: => T): T = withMapping(
+    Seq(
+      ("MILL_OUT", outPath),
+      ("WORKSPACE", workspacePath),
+      // TODO: add coursier here
+      ("HOME", homePath)
+    )
+  )(thunk)
+
+  def withMapping[T](mapping: MappedRoots)(thunk: => T): T = withMapping(_ => mapping)(thunk)
+
+  def withMapping[T](mapping: MappedRoots => MappedRoots)(thunk: => T): T = {
+    val newMapping = mapping(rootMapping.value)
+    var seenKeys = Set[String]()
+    var seenPaths = Set[os.Path]()
+    newMapping.foreach { case m =>
+      require(!m.key.startsWith("$"), "Key must not start with a `$`.")
+      require(m.key != PathVars.ROOT, s"Invalid key, '${PathVars.ROOT}' is a reserved key name.")
+      require(
+        !seenKeys.contains(m.key),
+        s"Key must be unique, but '${m.key}' was given multiple times."
+      )
+      require(
+        !seenPaths.contains(m.path),
+        s"Paths must be unique, but '${m.path}' was given multiple times."
+      )
+      seenKeys += m.key
+      seenPaths += m.path
+    }
+    rootMapping.withValue(newMapping)(thunk)
+  }
+
+  def encodeKnownRootsInPath(p: os.Path): String = {
+    MappedRoots.get.collectFirst {
+      case rep if p.startsWith(rep.path) =>
+        s"$$${rep.key}${
+            if (p != rep.path) {
+              s"/${p.subRelativeTo(rep.path).toString()}"
+            } else ""
+          }"
+    }.getOrElse(p.toString)
+  }
+
+  def decodeKnownRootsInPath(encoded: String): String = {
+    if (encoded.startsWith("$")) {
+      val offset = 1 // "$".length
+      MappedRoots.get.collectFirst {
+        case mapping if encoded.startsWith(mapping.key, offset) =>
+          s"${mapping.path.toString}${encoded.substring(mapping.key.length + offset)}"
+      }.getOrElse(encoded)
+    } else {
+      encoded
+    }
+  }
+
+  /**
+   * Use this to assert at runtime, that a root path with the given `key` is defined.
+   * @throws NoSuchElementException when no path was mapped under the given `key`.
+   */
+  def requireMappedPaths(key: String*): Unit = {
+    val map = toMap
+    for {
+      singleKey <- key
+    } {
+      if (!map.contains(singleKey))
+        throw new NoSuchElementException(s"No root path mapping defined for '${key}'")
+    }
+  }
+
+}

core/api/src/mill/api/PathRef.scala

@@ -10,11 +10,12 @@ import java.util.concurrent.ConcurrentHashMap
 import scala.annotation.nowarn
 import scala.language.implicitConversions
 import scala.util.DynamicVariable
+import scala.util.hashing.MurmurHash3
 
 /**
- * A wrapper around `os.Path` that calculates it's hashcode based
- * on the contents of the filesystem underneath it. Used to ensure filesystem
- * changes can bust caches which are keyed off hashcodes.
+ * A wrapper around `os.Path` that calculates a `sig` (which ends up in the [[hashCode]])
+ * based on the contents of the filesystem underneath it.
+ * Used to ensure filesystem changes can bust caches which are keyed off hashcodes.
  */
 case class PathRef private[mill] (
     path: os.Path,
@@ -24,6 +25,18 @@ case class PathRef private[mill] (
 ) extends PathRefApi {
   private[mill] def javaPath = path.toNIO
 
+  /**
+   * The path with common mapped path roots replaced, to make it relocatable.
+   * See [[MappedRoots]].
+   */
+  private val mappedPath: String = MappedRoots.encodeKnownRootsInPath(path)
+
+  /**
+   * Apply the current contextual path mapping to this PathRef.
+   * Updates [[mappedPath]] but does not recalculate the [[sig]].
+   */
+  def remap: PathRef = PathRef(path, quick, sig, revalidate)
+
   def recomputeSig(): Int = PathRef.apply(path, quick).sig
   def validate(): Boolean = recomputeSig() == sig
 
@@ -38,16 +51,33 @@ case class PathRef private[mill] (
   def withRevalidate(revalidate: PathRef.Revalidate): PathRef = copy(revalidate = revalidate)
   def withRevalidateOnce: PathRef = copy(revalidate = PathRef.Revalidate.Once)
 
-  override def toString: String = {
+  private def toStringPrefix: String = {
     val quick = if (this.quick) "qref:" else "ref:"
+
     val valid = revalidate match {
       case PathRef.Revalidate.Never => "v0:"
       case PathRef.Revalidate.Once => "v1:"
       case PathRef.Revalidate.Always => "vn:"
     }
     val sig = String.format("%08x", this.sig: Integer)
-    quick + valid + sig + ":" + path.toString()
+    s"${quick}${valid}${sig}:"
+  }
+
+  override def toString: String = {
+    toStringPrefix + path.toString()
   }
+
+  // Instead of using `path` we need to use `mappedPath` to make the hashcode stable as cache key
+  override def hashCode(): Int = {
+    var h = MurmurHash3.productSeed
+    h = MurmurHash3.mix(h, "PathRef".hashCode)
+    h = MurmurHash3.mix(h, mappedPath.hashCode)
+    h = MurmurHash3.mix(h, quick.##)
+    h = MurmurHash3.mix(h, sig.##)
+    h = MurmurHash3.mix(h, revalidate.##)
+    MurmurHash3.finalizeHash(h, 4)
+  }
+
 }
 
 object PathRef {
@@ -195,12 +225,12 @@ object PathRef {
   implicit def jsonFormatter: RW[PathRef] = upickle.readwriter[String].bimap[PathRef](
     p => {
       storeSerializedPaths(p)
-      p.toString()
+      p.toStringPrefix + MappedRoots.encodeKnownRootsInPath(p.path)
     },
     {
-      case s"$prefix:$valid0:$hex:$pathString" if prefix == "ref" || prefix == "qref" =>
+      case s"$prefix:$valid0:$hex:$pathVal" if prefix == "ref" || prefix == "qref" =>
 
-        val path = os.Path(pathString)
+        val path = os.Path(MappedRoots.decodeKnownRootsInPath(pathVal))
         val quick = prefix match {
           case "qref" => true
           case "ref" => false
@@ -219,7 +249,7 @@ object PathRef {
         pr
       case s =>
         mill.api.BuildCtx.withFilesystemCheckerDisabled(
-          PathRef(os.Path(s, currentOverrideModulePath.value))
+          PathRef(os.Path(MappedRoots.decodeKnownRootsInPath(s), currentOverrideModulePath.value))
         )
     }
   )

core/api/test/src/mill/api/MappedRootsTests.scala

@@ -0,0 +1,48 @@
+package mill.api
+
+import utest.*
+
+import java.nio.file.Files
+import mill.api.{MappedRoots => MR}
+
+object MappedRootsTests extends TestSuite {
+  val tests: Tests = Tests {
+    test("encode") {
+      withTmpDir { tmpDir =>
+        val workspaceDir = tmpDir / "workspace"
+        val outDir = workspaceDir / "out"
+        MR.withMillDefaults(outPath = outDir, workspacePath = workspaceDir) {
+
+          def check(path: os.Path, encContains: Seq[String], containsNot: Seq[String]) = {
+            val enc = MR.encodeKnownRootsInPath(path)
+            val dec = MR.decodeKnownRootsInPath(enc)
+            assert(path.toString == dec)
+            encContains.foreach(s => assert(enc.containsSlice(s)))
+            containsNot.foreach(s => assert(!enc.containsSlice(s)))
+
+            path -> enc
+          }
+
+          val file1 = tmpDir / "file1"
+          val file2 = workspaceDir / "file2"
+          val file3 = outDir / "file3"
+
+          Seq(
+            "mapping" -> MR.get,
+            check(file1, Seq(file1.toString), Seq("$WORKSPACE", "$MILL_OUT")),
+            check(file2, Seq("$WORKSPACE/file2"), Seq("$MILL_OUT")),
+            check(file3, Seq("$MILL_OUT/file3"), Seq("$WORKSPACE"))
+          )
+        }
+      }
+    }
+  }
+
+  private def withTmpDir[T](body: os.Path => T): T = {
+    val tmpDir = os.Path(Files.createTempDirectory(""))
+    val res = body(tmpDir)
+    os.remove.all(tmpDir)
+    res
+  }
+
+}

core/api/test/src/mill/api/PathRefTests.scala

@@ -19,6 +19,7 @@ object PathRefTests extends TestSuite {
         val sig2 = PathRef(file, quick).sig
         assert(sig1 != sig2)
       }
+
       test("qref") - check(quick = true)
       test("ref") - check(quick = false)
     }
@@ -43,12 +44,18 @@ object PathRefTests extends TestSuite {
           val file = tmpDir / "foo.txt"
           val content = "hello"
           os.write.over(file, content)
-          Files.setPosixFilePermissions(file.wrapped, PosixFilePermissions.fromString("rw-rw----"))
+          Files.setPosixFilePermissions(
+            file.wrapped,
+            PosixFilePermissions.fromString("rw-rw----")
+          )
           val rwSig = PathRef(file, quick).sig
           val rwSigb = PathRef(file, quick).sig
           assert(rwSig == rwSigb)
 
-          Files.setPosixFilePermissions(file.wrapped, PosixFilePermissions.fromString("rwxrw----"))
+          Files.setPosixFilePermissions(
+            file.wrapped,
+            PosixFilePermissions.fromString("rwxrw----")
+          )
           val rwxSig = PathRef(file, quick).sig
 
           assert(rwSig != rwxSig)
@@ -76,6 +83,7 @@ object PathRefTests extends TestSuite {
         val sig2 = PathRef(tmpDir, quick).sig
         assert(sig1 == sig2)
       }
+
       test("qref") - check(quick = true)
       test("ref") - check(quick = false)
     }

core/constants/src/mill/constants/PathVars.java

@@ -0,0 +1,22 @@
+package mill.constants;
+
+/**
+ * Central place containing all the path variables that Mill uses in <code>PathRef</code> or <code>os.Path</code>.
+ */
+public interface PathVars {
+
+  /**
+   * Output directory where Mill workers' state and Mill tasks output should be
+   * written to
+   */
+  String MILL_OUT = "MILL_OUT";
+
+  /**
+   * The Mill project workspace root directory.
+   */
+  String WORKSPACE = "WORKSPACE";
+
+  String HOME = "HOME";
+
+  String ROOT = "ROOT";
+}

core/exec/src/mill/exec/Execution.scala

@@ -54,24 +54,24 @@ private[mill] case class Execution(
       offline: Boolean,
       enableTicker: Boolean
   ) = this(
-    baseLogger,
-    new JsonArrayLogger.Profile(os.Path(outPath) / millProfile),
-    os.Path(workspace),
-    os.Path(outPath),
-    os.Path(externalOutPath),
-    rootModule,
-    classLoaderSigHash,
-    classLoaderIdentityHash,
-    workerCache,
-    env,
-    failFast,
-    ec,
-    codeSignatures,
-    systemExit,
-    exclusiveSystemStreams,
-    getEvaluator,
-    offline,
-    enableTicker
+    baseLogger = baseLogger,
+    profileLogger = new JsonArrayLogger.Profile(os.Path(outPath) / millProfile),
+    workspace = os.Path(workspace),
+    outPath = os.Path(outPath),
+    externalOutPath = os.Path(externalOutPath),
+    rootModule = rootModule,
+    classLoaderSigHash = classLoaderSigHash,
+    classLoaderIdentityHash = classLoaderIdentityHash,
+    workerCache = workerCache,
+    env = env,
+    failFast = failFast,
+    ec = ec,
+    codeSignatures = codeSignatures,
+    systemExit = systemExit,
+    exclusiveSystemStreams = exclusiveSystemStreams,
+    getEvaluator = getEvaluator,
+    offline = offline,
+    enableTicker = enableTicker
   )
 
   def withBaseLogger(newBaseLogger: Logger) = this.copy(baseLogger = newBaseLogger)

core/exec/src/mill/exec/GroupExecution.scala

@@ -83,7 +83,7 @@ trait GroupExecution {
       executionContext: mill.api.TaskCtx.Fork.Api,
       exclusive: Boolean,
       upstreamPathRefs: Seq[PathRef]
-  ): GroupExecution.Results = {
+  ): GroupExecution.Results = MappedRoots.withMillDefaults(outPath = outPath) {
 
     val inputsHash = {
       val externalInputsHash = MurmurHash3.orderedHash(
@@ -242,8 +242,8 @@ trait GroupExecution {
                   newEvaluated = newEvaluated.toSeq,
                   cached = if (labelled.isInstanceOf[Task.Input[?]]) null else false,
                   inputsHash = inputsHash,
-                  previousInputsHash = cached.map(_._1).getOrElse(-1),
-                  valueHashChanged = !cached.map(_._3).contains(valueHash),
+                  previousInputsHash = cached.map(_.inputHash).getOrElse(-1),
+                  valueHashChanged = !cached.map(_.valueHash).contains(valueHash),
                   serializedPaths = serializedPaths
                 )
             }
@@ -453,7 +453,7 @@ trait GroupExecution {
       inputsHash: Int,
       labelled: Task.Named[?],
       paths: ExecutionPaths
-  ): Option[(Int, Option[(Val, Seq[PathRef])], Int)] = {
+  ): Option[(inputHash: Int, valOpt: Option[(Val, Seq[PathRef])], valueHash: Int)] = {
     for {
       cached <-
         try Some(upickle.read[Cached](paths.meta.toIO, trace = false))
@@ -618,7 +618,7 @@ object GroupExecution {
       classLoader: ClassLoader
   )(t: => T): T = {
     // Tasks must be allowed to write to upstream worker's dest folders, because
-    // the point of workers is to manualy manage long-lived state which includes
+    // the point of workers is to manually manage long-lived state which includes
     // state on disk.
     val validWriteDests =
       deps.collect { case n: Task.Worker[?] =>