feat: allow to pass scalac options using files (#3012)

* feat: allow to pass args using files

* read arg files to process options

* correctly split args from file

* fix test

* generate ref docs

Author: kasiaMarek

.gitignore

@@ -13,4 +13,5 @@ target/
 
 # ignore vim backup files
 *.sw[op]
+.DS_Store
 

modules/cli/src/main/scala/scala/cli/commands/ScalaCommand.scala

@@ -172,7 +172,7 @@ abstract class ScalaCommand[T <: HasGlobalOptions](implicit myParser: Parser[T],
     import scala.cli.commands.shared.ScalacOptions.YScriptRunnerOption
     val logger = options.global.logging.logger
     sharedOptions(options).foreach { so =>
-      val scalacOpts = so.scalac.scalacOption.toScalacOptShadowingSeq
+      val scalacOpts = so.scalacOptions.toScalacOptShadowingSeq
       if scalacOpts.keys.contains(ScalacOpt(YScriptRunnerOption)) then
         logger.message(
           LegacyScalaOptions.yScriptRunnerWarning(scalacOpts.getOption(YScriptRunnerOption))
@@ -188,7 +188,7 @@ abstract class ScalaCommand[T <: HasGlobalOptions](implicit myParser: Parser[T],
   def maybePrintSimpleScalacOutput(options: T, buildOptions: BuildOptions): Unit =
     for {
       shared <- sharedOptions(options)
-      scalacOptions        = shared.scalac.scalacOption
+      scalacOptions        = shared.scalacOptions
       updatedScalacOptions = scalacOptions.withScalacExtraOptions(shared.scalacExtra)
       if updatedScalacOptions.exists(ScalacOptions.ScalacPrintOptions)
       logger = shared.logger

modules/cli/src/main/scala/scala/cli/commands/shared/ArgSplitter.scala

@@ -0,0 +1,35 @@
+package scala.cli.commands.shared
+
+import scala.annotation.tailrec
+import scala.collection.mutable.ListBuffer
+
+object ArgSplitter {
+  def splitToArgs(input: String) = {
+    val iter        = input.iterator
+    val accumulator = new ListBuffer[String]
+
+    @tailrec
+    def takeWhile(
+      test: Char => Boolean,
+      acc: List[Char] = Nil,
+      prevWasEscape: Boolean = false
+    ): String =
+      iter.nextOption() match
+        case Some(c) if !prevWasEscape && test(c) => acc.reverse.mkString
+        case None                                 => acc.reverse.mkString
+        case Some('\\') =>
+          takeWhile(test, '\\' :: acc, prevWasEscape = true)
+        case Some(c) =>
+          takeWhile(test, c :: acc, prevWasEscape = false)
+
+    while (iter.hasNext)
+      iter.next() match
+        case c if c.isSpaceChar || c == '\n' || c == '\r' =>
+        case c @ ('\'' | '"') => accumulator += s"$c${takeWhile(_ == c)}$c"
+        case c =>
+          accumulator += s"$c${takeWhile(c => c.isSpaceChar || c == '\n' || c == '\r')}"
+
+    accumulator.result()
+  }
+
+}

modules/cli/src/main/scala/scala/cli/commands/shared/ScalacOptions.scala

@@ -2,7 +2,7 @@ package scala.cli.commands.shared
 
 import caseapp.*
 import caseapp.core.Scala3Helpers.*
-import caseapp.core.parser.{Argument, NilParser, StandardArgument}
+import caseapp.core.parser.{Argument, ConsParser, NilParser, StandardArgument}
 import caseapp.core.util.Formatter
 import caseapp.core.{Arg, Error}
 import com.github.plokhotnyuk.jsoniter_scala.core.*
@@ -12,14 +12,16 @@ import scala.cli.commands.tags
 
 // format: off
 final case class ScalacOptions(
+  @Recurse
+  argsFiles: List[ArgFileOption] = Nil,
   @Group(HelpGroup.Scala.toString)
   @HelpMessage("Add a scalac option")
   @ValueDescription("option")
   @Name("O")
   @Name("scala-opt")
   @Name("scala-option")
   @Tag(tags.must)
-    scalacOption: List[String] = Nil
+    scalacOption: List[String] = Nil,
 )
 // format: on
 
@@ -72,8 +74,9 @@ object ScalacOptions {
 
   /** This includes all the scalac options which are redirected to native Scala CLI options. */
   val ScalaCliRedirectedOptions = Set(
-    "-classpath", // redirected to --extra-jars
-    "-d"          // redirected to --compilation-output
+    "-classpath",
+    "-cp", // redirected to --extra-jars
+    "-d"   // redirected to --compilation-output
   )
   val ScalacDeprecatedOptions: Set[String] = Set(
     YScriptRunnerOption // old 'scala' runner specific, no longer supported
@@ -115,11 +118,46 @@ object ScalacOptions {
     }
 
   implicit lazy val parser: Parser[ScalacOptions] = {
-    val baseParser =
-      scalacOptionsArgument ::
-        NilParser
-    baseParser.to[ScalacOptions]
+    val baseParser = scalacOptionsArgument :: NilParser
+    implicit val p = ArgFileOption.parser
+    baseParser.addAll[List[ArgFileOption]].to[ScalacOptions]
   }
+
   implicit lazy val help: Help[ScalacOptions]                = Help.derive
   implicit lazy val jsonCodec: JsonValueCodec[ScalacOptions] = JsonCodecMaker.make
 }
+
+case class ArgFileOption(file: String) extends AnyVal
+
+object ArgFileOption {
+  val arg = Arg(
+    name = Name("args-file"),
+    valueDescription = Some(ValueDescription("@arguments-file")),
+    helpMessage = Some(HelpMessage("File with scalac options.")),
+    group = Some(Group("Scala")),
+    origin = Some("ScalacOptions")
+  )
+  implicit lazy val parser: Parser[List[ArgFileOption]] = new Parser[List[ArgFileOption]] {
+    type D = List[ArgFileOption] *: EmptyTuple
+
+    override def withDefaultOrigin(origin: String): Parser[List[ArgFileOption]] = this
+
+    override def init: D = Nil *: EmptyTuple
+
+    override def step(args: List[String], index: Int, d: D, nameFormatter: Formatter[Name])
+      : Either[(core.Error, Arg, List[String]), Option[(D, Arg, List[String])]] =
+      args match
+        case head :: rest if head.startsWith("@") =>
+          val newD = (ArgFileOption(head.stripPrefix("@")) :: d._1) *: EmptyTuple
+          Right(Some(newD, arg, rest))
+        case _ => Right(None)
+
+    override def get(
+      d: D,
+      nameFormatter: Formatter[Name]
+    ): Either[core.Error, List[ArgFileOption]] = Right(d.head)
+
+    override def args: Seq[Arg] = Seq(arg)
+
+  }
+}

modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala

@@ -2,7 +2,9 @@ package scala.cli.commands.shared
 
 import bloop.rifle.BloopRifleConfig
 import caseapp.*
+import caseapp.core.Arg
 import caseapp.core.help.Help
+import caseapp.core.util.Formatter
 import com.github.plokhotnyuk.jsoniter_scala.core.*
 import com.github.plokhotnyuk.jsoniter_scala.macros.*
 import coursier.cache.FileCache
@@ -290,13 +292,20 @@ final case class SharedOptions(
     )
   }
 
+  lazy val scalacOptionsFromFiles: List[String] =
+    scalac.argsFiles.flatMap(argFile =>
+      ArgSplitter.splitToArgs(os.read(os.Path(argFile.file, os.pwd)))
+    )
+
+  def scalacOptions: List[String] = scalac.scalacOption ++ scalacOptionsFromFiles
+
   def buildOptions(
     enableJmh: Boolean = false,
     jmhVersion: Option[String] = None,
     ignoreErrors: Boolean = false
   ): Either[BuildException, bo.BuildOptions] = either {
-    val releaseOpt = scalac.scalacOption.getScalacOption("-release")
-    val targetOpt  = scalac.scalacOption.getScalacPrefixOption("-target")
+    val releaseOpt = scalacOptions.getScalacOption("-release")
+    val targetOpt  = scalacOptions.getScalacPrefixOption("-target")
     jvm.jvm -> (releaseOpt.toSeq ++ targetOpt) match {
       case (Some(j), compilerTargets) if compilerTargets.exists(_ != j) =>
         val compilerTargetsString = compilerTargets.distinct.mkString(", ")
@@ -380,8 +389,7 @@ final case class SharedOptions(
           semanticDbTargetRoot = semanticDbOptions.semanticDbTargetRoot.map(os.Path(_, os.pwd)),
           semanticDbSourceRoot = semanticDbOptions.semanticDbSourceRoot.map(os.Path(_, os.pwd))
         ),
-        scalacOptions = scalac
-          .scalacOption
+        scalacOptions = scalacOptions
           .withScalacExtraOptions(scalacExtra)
           .toScalacOptShadowingSeq
           .filterNonRedirected
@@ -465,7 +473,9 @@ final case class SharedOptions(
   }
 
   def extraJarsAndClassPath: List[os.Path] =
-    (extraJars ++ scalac.scalacOption.getScalacOption("-classpath"))
+    (extraJars ++ scalacOptions.getScalacOption("-classpath") ++ scalacOptions.getScalacOption(
+      "-cp"
+    ))
       .extractedClassPath
 
   def extraClasspathWasPassed: Boolean = extraJarsAndClassPath.exists(!_.hasSourceJarSuffix)
@@ -635,6 +645,7 @@ final case class SharedOptions(
 }
 
 object SharedOptions {
+  import ArgFileOption.parser
   implicit lazy val parser: Parser[SharedOptions]            = Parser.derive
   implicit lazy val help: Help[SharedOptions]                = Help.derive
   implicit lazy val jsonCodec: JsonValueCodec[SharedOptions] = JsonCodecMaker.make

modules/cli/src/main/scala/scala/cli/commands/util/BuildCommandHelpers.scala

@@ -22,7 +22,7 @@ trait BuildCommandHelpers { self: ScalaCommand[_] =>
       */
     def copyOutput(sharedOptions: SharedOptions): Unit =
       sharedOptions.compilationOutput.filter(_.nonEmpty)
-        .orElse(sharedOptions.scalac.scalacOption.getScalacOption("-d"))
+        .orElse(sharedOptions.scalacOptions.getScalacOption("-d"))
         .filter(_.nonEmpty)
         .map(os.Path(_, Os.pwd)).foreach { output =>
           os.copy(

modules/cli/src/test/scala/cli/tests/ArgSplitterTest.scala

@@ -0,0 +1,17 @@
+package cli.tests
+
+import scala.cli.commands.shared.ArgSplitter
+
+class ArgSplitterTest extends munit.FunSuite {
+
+  test("test scalac options are split correctly") {
+    val args = List(
+      List("-arg", "-other-arg"),
+      List("-yet-another-arg", "dir/path\\ with\\ space", "'another arg with space'"),
+      List("\"yet another arg with space\"")
+    )
+    val input = args.map(_.mkString("   ", " ", "")).mkString(" ", "\n", "")
+    assertEquals(ArgSplitter.splitToArgs(input), args.flatten)
+  }
+
+}

modules/integration/src/test/scala/scala/cli/integration/ArgsFileTests.scala

@@ -0,0 +1,105 @@
+package scala.cli.integration
+
+import scala.util.Properties
+
+class ArgsFileTests extends ScalaCliSuite {
+
+  val forOption = List(true, false)
+
+  for (useServer <- forOption) test(
+    s"pass scalac options using arguments file ${if (useServer) "with bloop" else "without bloop"}"
+  ) {
+    val fileName   = "Simple.sc"
+    val serverArgs = if (useServer) Nil else List("--server=false")
+    val inputs = TestInputs(
+      os.rel / "args.txt" -> """|-release
+                                |8""".stripMargin,
+      os.rel / fileName ->
+        s"""|import java.net.http.HttpClient
+            |
+            |println("Hello :)")
+            |""".stripMargin
+    )
+
+    inputs.fromRoot { root =>
+      val res = os.proc(TestUtil.cli, serverArgs, "@args.txt", fileName).call(
+        cwd = root,
+        check = false,
+        stderr = os.Pipe
+      )
+      assert(res.exitCode == 1)
+
+      val compilationError = res.err.text()
+      assert(compilationError.contains("Compilation failed"))
+    }
+  }
+
+  if (!Properties.isWin)
+    test("pass scalac options using arguments file in shebang script") {
+      val inputs = TestInputs(
+        os.rel / "args.txt" -> """|-release 8""".stripMargin,
+        os.rel / "script-with-shebang" ->
+          s"""|#!/usr/bin/env -S ${TestUtil.cli.mkString(" ")} shebang @args.txt
+              |import java.net.http.HttpClient
+              |
+              |println("Hello :)")
+              |""".stripMargin
+      )
+
+      inputs.fromRoot { root =>
+        os.perms.set(root / "script-with-shebang", os.PermSet.fromString("rwx------"))
+        val res = os.proc("./script-with-shebang").call(cwd = root, check = false, stderr = os.Pipe)
+        assert(res.exitCode == 1)
+
+        val compilationError = res.err.text()
+        assert(compilationError.contains("Compilation failed"))
+      }
+    }
+
+  test("multiple args files") {
+    val preCompileDir = "PreCompileDir"
+    val runDir        = "RunDir"
+
+    val preCompiledInput = "Message.scala"
+    val mainInput        = "Main.scala"
+
+    val expectedOutput = "Hello"
+
+    val outputDir = os.rel / "out"
+
+    TestInputs(
+      os.rel / preCompileDir / preCompiledInput -> "case class Message(value: String)",
+      os.rel / runDir / mainInput -> s"""object Main extends App { println(Message("$expectedOutput").value) }""",
+      os.rel / runDir / "args.txt" -> s"""|-d
+                                          |$outputDir""".stripMargin,
+      os.rel / runDir / "args2.txt" -> s"""|-cp
+                                           |${os.rel / os.up / preCompileDir / outputDir}""".stripMargin
+    ).fromRoot { (root: os.Path) =>
+
+      os.proc(
+        TestUtil.cli,
+        "compile",
+        "--scala-opt",
+        "-d",
+        "--scala-opt",
+        outputDir.toString,
+        preCompiledInput
+      ).call(cwd = root / preCompileDir, stderr = os.Pipe)
+      assert((root / preCompileDir / outputDir / "Message.class").toNIO.toFile().exists())
+
+      val compileOutput = root / runDir / outputDir
+      os.makeDir.all(compileOutput)
+      val runRes = os.proc(
+        TestUtil.cli,
+        "run",
+        "@args.txt",
+        "--server=false",
+        "@args2.txt",
+        mainInput
+      ).call(cwd = root / runDir, stderr = os.Pipe)
+      assert(runRes.out.trim() == expectedOutput)
+      assert((compileOutput / "Main.class").toNIO.toFile().exists())
+    }
+  }
+
+}

website/docs/reference/cli-options.md

@@ -1408,6 +1408,10 @@ Available in commands:
 
 <!-- Automatically generated, DO NOT EDIT MANUALLY -->
 
+### `--args-file`
+
+File with scalac options.
+
 ### `--scalac-option`
 
 Aliases: `-O`, `--scala-opt`, `--scala-option`

website/docs/reference/scala-command/cli-options.md

@@ -914,6 +914,12 @@ Available in commands:
 
 <!-- Automatically generated, DO NOT EDIT MANUALLY -->
 
+### `--args-file`
+
+`IMPLEMENTATION specific` per Scala Runner specification
+
+File with scalac options.
+
 ### `--scalac-option`
 
 Aliases: `-O`, `--scala-opt`, `--scala-option`