[WIP] Map known root paths in cache files

core/api/src/mill/api/JsonFormatters.scala

@@ -23,8 +23,8 @@ trait JsonFormatters {
 
   implicit val pathReadWrite: RW[os.Path] = upickle.readwriter[String]
     .bimap[os.Path](
-      _.toString,
-      os.Path(_)
+      p => PathRef.encodeKnownRootsInPath(p),
+      s => os.Path(PathRef.decodeKnownRootsInPath(s))
     )
 
   implicit val relPathRW: RW[os.RelPath] = upickle.readwriter[String]

core/api/src/mill/api/MillTaskHash.scala

@@ -0,0 +1,5 @@
+package mill.api
+
+trait MillTaskHash {
+  def millCacheHash: Int
+}

core/api/src/mill/api/PathRef.scala

@@ -10,6 +10,7 @@ import java.util.concurrent.ConcurrentHashMap
 import scala.annotation.nowarn
 import scala.language.implicitConversions
 import scala.util.DynamicVariable
+import scala.util.hashing.MurmurHash3
 
 /**
  * A wrapper around `os.Path` that calculates it's hashcode based
@@ -24,6 +25,8 @@ case class PathRef private[mill] (
 ) extends PathRefApi {
   private[mill] def javaPath = path.toNIO
 
+  private[mill] val pathVal: String = PathRef.encodeKnownRootsInPath(path)
+
   def recomputeSig(): Int = PathRef.apply(path, quick).sig
   def validate(): Boolean = recomputeSig() == sig
 
@@ -38,16 +41,33 @@ case class PathRef private[mill] (
   def withRevalidate(revalidate: PathRef.Revalidate): PathRef = copy(revalidate = revalidate)
   def withRevalidateOnce: PathRef = copy(revalidate = PathRef.Revalidate.Once)
 
-  override def toString: String = {
+  private def toStringPrefix = {
     val quick = if (this.quick) "qref:" else "ref:"
+
     val valid = revalidate match {
       case PathRef.Revalidate.Never => "v0:"
       case PathRef.Revalidate.Once => "v1:"
       case PathRef.Revalidate.Always => "vn:"
     }
     val sig = String.format("%08x", this.sig: Integer)
-    quick + valid + sig + ":" + path.toString()
+    quick + valid + sig + ":"
+  }
+
+  override def toString: String = {
+    toStringPrefix + path.toString()
+  }
+
+  // Instead of using `path` we need to use `pathVal`, to make the hashcode stable as cache key
+  override def hashCode(): Int = {
+    var h = MurmurHash3.productSeed
+    h = MurmurHash3.mix(h, "PathRef".hashCode)
+    h = MurmurHash3.mix(h, pathVal.hashCode)
+    h = MurmurHash3.mix(h, quick.##)
+    h = MurmurHash3.mix(h, sig.##)
+    h = MurmurHash3.mix(h, revalidate.##)
+    MurmurHash3.finalizeHash(h, 4)
   }
+
 }
 
 object PathRef {
@@ -189,18 +209,60 @@ object PathRef {
     }
   }
 
+  private[mill] val outPathOverride: DynamicVariable[Option[os.Path]] = DynamicVariable(None)
+
+  private[api] type KnownRoots = Seq[(replacement: String, root: os.Path)]
+
+  private[api] def knownRoots: KnownRoots = {
+    // order is important!
+    Seq(
+      (
+        "$MILL_OUT",
+        outPathOverride.value.getOrElse(
+          throw RuntimeException("Can't substitute $MILL_OUT, output path is not configured.")
+        )
+      ),
+      ("$WORKSPACE", BuildCtx.workspaceRoot),
+      // TODO: add coursier here
+      ("$HOME", os.home)
+    )
+  }
+
+  private[api] def encodeKnownRootsInPath(p: os.Path): String = {
+    // TODO: Do we need to check for '$' and mask it ?
+    knownRoots.collectFirst {
+      case rep if p.startsWith(rep.root) =>
+        s"${rep.replacement}${
+            if (p != rep.root) {
+              s"/${p.subRelativeTo(rep.root).toString()}"
+            } else ""
+          }"
+    }.getOrElse(p.toString)
+  }
+
+  private[api] def decodeKnownRootsInPath(encoded: String): String = {
+    if (encoded.startsWith("$")) {
+      knownRoots.collectFirst {
+        case rep if encoded.startsWith(rep.replacement) =>
+          s"${rep.root.toString}${encoded.substring(rep.replacement.length)}"
+      }.getOrElse(encoded)
+    } else {
+      encoded
+    }
+  }
+
   /**
    * Default JSON formatter for [[PathRef]].
    */
   implicit def jsonFormatter: RW[PathRef] = upickle.readwriter[String].bimap[PathRef](
     p => {
       storeSerializedPaths(p)
-      p.toString()
+      p.toStringPrefix + p.pathVal
     },
     {
-      case s"$prefix:$valid0:$hex:$pathString" if prefix == "ref" || prefix == "qref" =>
+      case s"$prefix:$valid0:$hex:$pathVal" if prefix == "ref" || prefix == "qref" =>
 
-        val path = os.Path(pathString)
+        val path = os.Path(decodeKnownRootsInPath(pathVal))
         val quick = prefix match {
           case "qref" => true
           case "ref" => false
@@ -219,7 +281,7 @@ object PathRef {
         pr
       case s =>
         mill.api.BuildCtx.withFilesystemCheckerDisabled(
-          PathRef(os.Path(s, currentOverrideModulePath.value))
+          PathRef(os.Path(decodeKnownRootsInPath(s), currentOverrideModulePath.value))
         )
     }
   )

core/api/test/src/mill/api/PathRefTests.scala

@@ -10,28 +10,32 @@ object PathRefTests extends TestSuite {
   val tests: Tests = Tests {
     test("sig") {
       def check(quick: Boolean) = withTmpDir { tmpDir =>
-        val file = tmpDir / "foo.txt"
-        os.write.over(file, "hello")
-        val sig1 = PathRef(file, quick).sig
-        val sig1b = PathRef(file, quick).sig
-        assert(sig1 == sig1b)
-        os.write.over(file, "hello world")
-        val sig2 = PathRef(file, quick).sig
-        assert(sig1 != sig2)
+        PathRef.outPathOverride.withValue(Some(tmpDir / "out")) {
+          val file = tmpDir / "foo.txt"
+          os.write.over(file, "hello")
+          val sig1 = PathRef(file, quick).sig
+          val sig1b = PathRef(file, quick).sig
+          assert(sig1 == sig1b)
+          os.write.over(file, "hello world")
+          val sig2 = PathRef(file, quick).sig
+          assert(sig1 != sig2)
+        }
       }
       test("qref") - check(quick = true)
       test("ref") - check(quick = false)
     }
 
     test("same-sig-other-file") {
       def check(quick: Boolean) = withTmpDir { tmpDir =>
-        val file = tmpDir / "foo.txt"
-        os.write.over(file, "hello")
-        val sig1 = PathRef(file, quick).sig
-        val file2 = tmpDir / "bar.txt"
-        os.copy(file, file2)
-        val sig1b = PathRef(file2, quick).sig
-        assert(sig1 == sig1b)
+        PathRef.outPathOverride.withValue(Some(tmpDir / "out")) {
+          val file = tmpDir / "foo.txt"
+          os.write.over(file, "hello")
+          val sig1 = PathRef(file, quick).sig
+          val file2 = tmpDir / "bar.txt"
+          os.copy(file, file2)
+          val sig1b = PathRef(file2, quick).sig
+          assert(sig1 == sig1b)
+        }
       }
 //      test("qref") - check(quick = true)
       test("ref") - check(quick = false)
@@ -40,18 +44,26 @@ object PathRefTests extends TestSuite {
     test("perms") {
       def check(quick: Boolean) =
         if (isPosixFs()) withTmpDir { tmpDir =>
-          val file = tmpDir / "foo.txt"
-          val content = "hello"
-          os.write.over(file, content)
-          Files.setPosixFilePermissions(file.wrapped, PosixFilePermissions.fromString("rw-rw----"))
-          val rwSig = PathRef(file, quick).sig
-          val rwSigb = PathRef(file, quick).sig
-          assert(rwSig == rwSigb)
-
-          Files.setPosixFilePermissions(file.wrapped, PosixFilePermissions.fromString("rwxrw----"))
-          val rwxSig = PathRef(file, quick).sig
-
-          assert(rwSig != rwxSig)
+          PathRef.outPathOverride.withValue(Some(tmpDir / "out")) {
+            val file = tmpDir / "foo.txt"
+            val content = "hello"
+            os.write.over(file, content)
+            Files.setPosixFilePermissions(
+              file.wrapped,
+              PosixFilePermissions.fromString("rw-rw----")
+            )
+            val rwSig = PathRef(file, quick).sig
+            val rwSigb = PathRef(file, quick).sig
+            assert(rwSig == rwSigb)
+
+            Files.setPosixFilePermissions(
+              file.wrapped,
+              PosixFilePermissions.fromString("rwxrw----")
+            )
+            val rwxSig = PathRef(file, quick).sig
+
+            assert(rwSig != rwxSig)
+          }
         }
         else "Test Skipped on non-POSIX host"
 
@@ -61,47 +73,86 @@ object PathRefTests extends TestSuite {
 
     test("symlinks") {
       def check(quick: Boolean) = withTmpDir { tmpDir =>
-        // invalid symlink
-        os.symlink(tmpDir / "nolink", tmpDir / "nonexistant")
+        PathRef.outPathOverride.withValue(Some(tmpDir / "out")) {
+          // invalid symlink
+          os.symlink(tmpDir / "nolink", tmpDir / "nonexistant")
 
-        // symlink to empty dir
-        os.symlink(tmpDir / "emptylink", tmpDir / "empty")
-        os.makeDir(tmpDir / "empty")
+          // symlink to empty dir
+          os.symlink(tmpDir / "emptylink", tmpDir / "empty")
+          os.makeDir(tmpDir / "empty")
 
-        // recursive symlinks
-        os.symlink(tmpDir / "rlink1", tmpDir / "rlink2")
-        os.symlink(tmpDir / "rlink2", tmpDir / "rlink1")
+          // recursive symlinks
+          os.symlink(tmpDir / "rlink1", tmpDir / "rlink2")
+          os.symlink(tmpDir / "rlink2", tmpDir / "rlink1")
 
-        val sig1 = PathRef(tmpDir, quick).sig
-        val sig2 = PathRef(tmpDir, quick).sig
-        assert(sig1 == sig2)
+          val sig1 = PathRef(tmpDir, quick).sig
+          val sig2 = PathRef(tmpDir, quick).sig
+          assert(sig1 == sig2)
+        }
       }
       test("qref") - check(quick = true)
       test("ref") - check(quick = false)
     }
 
     test("json") {
-      def check(quick: Boolean) = withTmpDir { tmpDir =>
-        val file = tmpDir / "foo.txt"
-        os.write(file, "hello")
-        val pr = PathRef(file, quick)
-        val prFile = pr.path.toString().replace("\\", "\\\\")
-        val json = upickle.write(pr)
-        if (quick) {
-          assert(json.startsWith(""""qref:v0:"""))
-          assert(json.endsWith(s""":${prFile}""""))
-        } else {
-          val hash = if (Properties.isWin) "86df6a6a" else "4c7ef487"
-          val expected = s""""ref:v0:${hash}:${prFile}""""
-          assert(json == expected)
+      def check(quick: Boolean) = withTmpDir { outDir =>
+        PathRef.outPathOverride.withValue(Some(outDir)) {
+          withTmpDir { tmpDir =>
+            val file = tmpDir / "foo.txt"
+            os.write(file, "hello")
+            val pr = PathRef(file, quick)
+            val prFile =
+              pr.path.toString().replace(outDir.toString(), "$MILL_OUT").replace("\\", "\\\\")
+            val json = upickle.write(pr)
+            if (quick) {
+              assert(json.startsWith(""""qref:v0:"""))
+              assert(json.endsWith(s""":${prFile}""""))
+            } else {
+              val hash = if (Properties.isWin) "86df6a6a" else "4c7ef487"
+              val expected = s""""ref:v0:${hash}:${prFile}""""
+              assert(json == expected)
+            }
+            val pr1 = upickle.read[PathRef](json)
+            assert(pr == pr1)
+          }
         }
-        val pr1 = upickle.read[PathRef](json)
-        assert(pr == pr1)
       }
 
       test("qref") - check(quick = true)
       test("ref") - check(quick = false)
     }
+
+    test("encode") {
+      withTmpDir { tmpDir =>
+        val workspaceDir = tmpDir / "workspace"
+        BuildCtx.workspaceRoot0.withValue(workspaceDir) {
+          val outDir = workspaceDir / "out"
+          PathRef.outPathOverride.withValue(Some(outDir)) {
+
+            def check(path: os.Path, contains: Seq[String], containsNot: Seq[String]) = {
+              val enc = PathRef.encodeKnownRootsInPath(path)
+              val dec = PathRef.decodeKnownRootsInPath(enc)
+              assert(path.toString == dec)
+              contains.foreach(s => enc.containsSlice(s))
+              containsNot.foreach(s => !enc.containsSlice(s))
+
+              path -> enc
+            }
+
+            val file1 = tmpDir / "file1"
+            val file2 = workspaceDir / "file2"
+            val file3 = outDir / "file3"
+
+            Seq(
+              "mapping" -> PathRef.knownRoots,
+              check(file1, Seq("ref:v0:", file1.toString), Seq("$WORKSPACE", "$MILL_OUT")),
+              check(file2, Seq("ref:v0:", "$WORKSPACE/file2"), Seq("$MILL_OUT")),
+              check(file3, Seq("ref:v0:", "$MILL_OUT/file3"), Seq("$WORKSPACE"))
+            )
+          }
+        }
+      }
+    }
   }
 
   private def withTmpDir[T](body: os.Path => T): T = {

core/exec/src/mill/exec/GroupExecution.scala

@@ -60,7 +60,7 @@ trait GroupExecution {
       executionContext: mill.api.TaskCtx.Fork.Api,
       exclusive: Boolean,
       upstreamPathRefs: Seq[PathRef]
-  ): GroupExecution.Results = {
+  ): GroupExecution.Results = PathRef.outPathOverride.withValue(Some(outPath)) {
 
     val externalInputsHash = MurmurHash3.orderedHash(
       group.flatMap(_.inputs).filter(!group.contains(_))
@@ -104,7 +104,7 @@ trait GroupExecution {
           case single if labelled.ctx.enclosingModule.buildOverrides.contains(single) =>
             val jsonData = labelled.ctx.enclosingModule.buildOverrides(single)
 
-            import collection.JavaConverters._
+            import scala.jdk.CollectionConverters.*
             def rec(x: ujson.Value): ujson.Value = x match {
               case ujson.Str(s) => mill.constants.Util.interpolateEnvVars(s, envWithPwd.asJava)
               case ujson.Arr(xs) => ujson.Arr(xs.map(rec))
@@ -214,8 +214,8 @@ trait GroupExecution {
                   newEvaluated.toSeq,
                   cached = if (labelled.isInstanceOf[Task.Input[?]]) null else false,
                   inputsHash,
-                  cached.map(_._1).getOrElse(-1),
-                  !cached.map(_._3).contains(valueHash),
+                  cached.map(_.inputHash).getOrElse(-1),
+                  !cached.map(_.valueHash).contains(valueHash),
                   serializedPaths
                 )
             }
@@ -424,7 +424,7 @@ trait GroupExecution {
       inputsHash: Int,
       labelled: Task.Named[?],
       paths: ExecutionPaths
-  ): Option[(Int, Option[(Val, Seq[PathRef])], Int)] = {
+  ): Option[(inputHash: Int, valOpt: Option[(Val, Seq[PathRef])], valueHash: Int)] = {
     for {
       cached <-
         try Some(upickle.read[Cached](paths.meta.toIO, trace = false))
@@ -589,7 +589,7 @@ object GroupExecution {
       classLoader: ClassLoader
   )(t: => T): T = {
     // Tasks must be allowed to write to upstream worker's dest folders, because
-    // the point of workers is to manualy manage long-lived state which includes
+    // the point of workers is to manually manage long-lived state which includes
     // state on disk.
     val validWriteDests =
       deps.collect { case n: Task.Worker[?] =>

example/androidlib/java/1-hello-world/build.mill

@@ -64,7 +64,7 @@ object app extends AndroidAppModule {
 /** Usage
 
 > ./mill show app.androidApk
-".../out/app/androidApk.dest/app.apk"
+"...$MILL_OUT/app/androidApk.dest/app.apk"
 
 */