fix file watching for real

last file watching update didn’t work well enough. This now
- rips out barbary watch service as it seems buggy crashing the jvm
- make cbt exclusively write files to watch to a file
- uses fswatch instead watching all files in that file

Author: cvogt

DEVELOPER_GUIDE.txt

@@ -29,7 +29,7 @@ compatibility/      Java interfaces that all CBT versions are source compatible
 nailgun_launcher/   Self-contained helper that allows using Nailgun with minimal permanent classpath. (Is this actually needed?)
 realpath/           Self-contained realpath source code to correctly figure our CBTs home directory. (Open for replacement ideas.)
 stage1/             CBT's code that only relies only on Scala/Java built-ins. Contains a Maven resolver to download libs for stage2.
-stage2/             CBT's code that requires additional libs, e.g. barbary watchservice.
+stage2/             CBT's code that requires additional libs, e.g. jgit
 test/               Unit tests that can serve as example builds
 sonatype.login      Sonatype credentials for deployment. Not in git obviously.
 

README.md

@@ -203,7 +203,8 @@ As you can see it prints `asdf`. Adding tasks is that easy.
 
 ### Triggering tasks on file-changes
 
-When you call a task, you can prefix it with `loop`.
+When you call a task, you can prefix it with `loop`. You need to
+have fswatch install (e.g. via `brew install fswatch`).
 CBT then watches the source files, the build files and even CBT's own
 source code and re-runs the task when anything changes. If necessary,
 this forces CBT to re-build itself, the project's dependencies and the project itself.
@@ -220,6 +221,11 @@ you managed to change windows back from your editor to the shell.
 
 Try changing the build file and see how CBT reacts to it as well.
 
+To also clear the screen on each run use:
+```
+$ cbt loop clear run
+```
+
 ### Adding tests
 
 The simplest way to add tests is putting a few assertions into the previously

build/build.scala

@@ -10,7 +10,6 @@ class Build(val context: Context) extends Shared with PublishLocal{
   // FIXME: somehow consolidate this with cbt's own boot-strapping from source.
   override def dependencies = {
     super.dependencies ++ Resolver(mavenCentral).bind(
-      MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
       MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"),
       ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5")
     )

cbt

@@ -170,6 +170,17 @@ if [  "$1" = "direct" ]; then
 	use_nailgun=1
 	shift
 fi
+loop=1
+if [ "$1" == "loop" ]; then
+	loop=0
+	shift
+fi
+clearScreen=1
+if [ "$1" == "clear" ]; then
+	clearScreen=0
+	shift
+fi
+
 if [ $nailgun_installed -eq 1 ] || [ "$1" = "publishSigned" ]; then
 	use_nailgun=1
 fi
@@ -184,7 +195,8 @@ stage1 () {
 	log "Checking for changes in cbt/nailgun_launcher" "$@"
 	NAILGUN_INDICATOR=$NAILGUN$TARGET../classes.last-success
 	changed=1
-	for file in "$NAILGUN"/*.java; do
+	NAILGUN_SOURCES=("$NAILGUN"*.java)
+	for file in "${NAILGUN_SOURCES[@]}"; do
 		if [ "$file" -nt "$NAILGUN_INDICATOR" ]; then changed=0; fi
 	done
 	exitCode=0
@@ -194,7 +206,8 @@ stage1 () {
 		#rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files
 		echo "Compiling cbt/nailgun_launcher" 1>&2
 		COMPILE_TIME=$(date +%YY%mm%dd%HH%MM.%SS|sed "s/[YmdHMS]//g")
-		javac -Xlint:deprecation -Xlint:unchecked -d "$NAILGUN$TARGET" "$NAILGUN"*.java
+		#echo javac -Xlint:deprecation -Xlint:unchecked -d "$NAILGUN$TARGET" "${NAILGUN_SOURCES[@]}"
+		javac -Xlint:deprecation -Xlint:unchecked -d "$NAILGUN$TARGET" "${NAILGUN_SOURCES[@]}"
 		exitCode=$?
 		if [ $exitCode -eq 0 ]; then
 			touch -t "$COMPILE_TIME" "$NAILGUN_INDICATOR"
@@ -214,7 +227,7 @@ stage1 () {
 			log "Running JVM directly" "$@"
 			options=($JAVA_OPTS)
 			# JVM options to improve startup time. See https://github.com/cvogt/cbt/pull/262
-			java "${options[@]}" $DEBUG -Xmx6072m -Xss10M -XX:MaxJavaStackTraceDepth=-1 -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none -cp "$NAILGUN$TARGET" cbt.NailgunLauncher "$(time_taken)" "$CWD" "$@"
+			java "${options[@]}" $DEBUG -Xmx6072m -Xss10M -XX:MaxJavaStackTraceDepth=-1 -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none -cp "$NAILGUN$TARGET" cbt.NailgunLauncher "$(time_taken)" "$CWD" "$loop" "$@"
 			exitCode=$?
 		else
 			log "Running via background process (nailgun)" "$@"
@@ -238,54 +251,47 @@ stage1 () {
 				sleep 0.3
 			done
 			log "Running CBT via Nailgun." "$@"
-			$NG cbt.NailgunLauncher "$(time_taken)" "$CWD" "$@"
+			$NG cbt.NailgunLauncher "$(time_taken)" "$CWD" "$loop" "$@"
 			exitCode=$?
 		fi
 		log "Done running CBT." "$@"
 	fi
 }
 
 
-loop=1
-case "$1" in
-	"loop") loop=0
-esac
-case "$2" in
-	"loop") loop=0
-esac
-
-CBT_SIGNALS_LOOPING=253
 USER_PRESSED_CTRL_C=130
 
 CBT_LOOP_FILE="$CWD/target/.cbt-loop.tmp"
+if [ $loop -eq 0 ]; then
+	which fswatch >/dev/null 2>/dev/null
+	export fswatch_installed=$?
+	if [ ! $fswatch_installed -eq 0 ]; then
+		echo "please install fswatch to use cbt loop, e.g. via brew install fswatch"
+		exit 1
+	fi
+fi
 while true; do
+	if [ $clearScreen -eq 0 ]; then
+		clear
+	fi
+	if [ -f "$CBT_LOOP_FILE" ]; then
+		rm "$CBT_LOOP_FILE"
+	fi
 	stage1 "$@"
 	if [ ! $loop -eq 0 ] || [ $exitCode -eq $USER_PRESSED_CTRL_C ]; then
 		log "not looping, exiting" "$@"
 		break
 	else
-		if [ ! $exitCode -eq $CBT_SIGNALS_LOOPING ]; then
-			log "exitCode $exitCode" "$@"
-			which fswatch >/dev/null 2>/dev/null
-			export fswatch_installed=$?
-			if [ -f "$CBT_LOOP_FILE" ]; then
-				if [ $fswatch_installed -eq 0 ]; then
-					# fswatch allows looping over CBT's sources itself
-					log "fswatch found. looping." "$@"
-					files=($(cat "$CBT_LOOP_FILE"))
-					fswatch --one-event "${files[@]}"
-				else
-					log "fswatch not installed, stopping cbt" "$@"
-					break
-				fi
-			else
-				log "no $CBT_LOOP_FILE file, stopping cbt" "$@"
-				break
-			fi
+		files=
+		if [ -f "$CBT_LOOP_FILE" ]; then
+			files=($(cat "$CBT_LOOP_FILE"))
+			#rm "$CBT_LOOP_FILE"
 		fi
+		echo ""
+		echo "Watching for file changes... (ctrl+c short press for loop, long press for abort)"
+		#echo fswatch --one-event "${NAILGUN_SOURCES[@]}" "${files[@]}"
+		fswatch --one-event "${NAILGUN_SOURCES[@]}" "${files[@]}"
 	fi
-
-	echo "======= Restarting CBT =======" 1>&2
 done
 
 log "Exiting CBT" "$@"

compatibility/BuildInterface.java

@@ -6,7 +6,10 @@ public interface BuildInterface extends Dependency{
   public default BuildInterface finalBuild(File current){
     return finalBuild(); // legacy forwarder
   }
-  public abstract File[] triggerLoopFilesArray(); // needed for watching files across composed builds
+  @Deprecated
+  public default File[] triggerLoopFilesArray(){
+    return new File[0];
+  };
 
   // deprecated methods, which clients are still allowed to implement, but not required
   public abstract BuildInterface finalBuild(); // needed to propagage through build builds. Maybe we can get rid of this.

compatibility/Context.java

@@ -21,6 +21,9 @@ public default long start(){
   public default File workingDirectory(){
     return projectDirectory();
   };
+  public default boolean loop(){
+    return false;
+  };
 
   // methods that exist for longer which every CBT version in use should have by now, no default values needed
   public abstract File cwd(); // REPLACE by something that allows to run cbt on some other directly
@@ -32,9 +35,6 @@ public default File workingDirectory(){
   public abstract File cbtRootHome(); // REMOVE
   public abstract File compatibilityTarget(); // maybe replace this with search in the classloader for it?
   public abstract BuildInterface parentBuildOrNull();
-  public default File[] triggerLoopFilesArray(){
-    return new File[0]; // REMOVE default value on next compatibility breaking release
-  }
 
   // deprecated methods
   @java.lang.Deprecated

doc/cbt-developer/version-compatibility.md

@@ -25,6 +25,9 @@ However there are more things that can break compatibility when changed:
 - communication between versions via reflection in particular
   - how the TrapSecurityManager of each CBT version talks to the
     installed TrapSecurityManager via reflection
+- communication via the file system
+  - cache folder location any layout
+  - .cbt-loop.tmp file
 
 ## How to detect accidental breakages?
 

nailgun_launcher/NailgunLauncher.java

@@ -6,6 +6,8 @@
 import java.util.*;
 import static cbt.Stage0Lib.*;
 import static java.io.File.pathSeparator;
+import java.nio.file.*;
+import static java.nio.file.Files.write;
 
 /**
  * This launcher allows to start the JVM without loading anything else permanently into its
@@ -33,7 +35,9 @@ public static Object getBuild( Object context ) throws Throwable{
       ((File) get(context, "compatibilityTarget")).toString() + "/",
       new ClassLoaderCache(
         (HashMap) get(context, "persistentCache")
-      )
+      ),
+      (File) get(context, "cwd"),
+      (Boolean) get(context, "loop")
     );
     return
       res
@@ -97,18 +101,21 @@ public static void main( String[] args ) throws Throwable {
     String nailgunTarget = CBT_HOME + "/" + NAILGUN + TARGET;
     long nailgunLauncherLastModified = new File( nailgunTarget + "../classes.last-success" ).lastModified();
 
+    File cwd = new File(args[1]);
+    boolean loop = args[2].equals("0");
+
     BuildStage1Result res = buildStage1(
-      nailgunLauncherLastModified, start, cache, CBT_HOME, compatibilityTarget, classLoaderCache
+      nailgunLauncherLastModified, start, cache, CBT_HOME, compatibilityTarget, classLoaderCache, cwd, loop
     );
 
     try{
       int exitCode = (int) res
         .classLoader
         .loadClass("cbt.Stage1")
         .getMethod(
-          "run", String[].class, File.class, File.class, BuildStage1Result.class, Map.class
+          "run", String[].class, File.class, File.class, boolean.class, BuildStage1Result.class, Map.class
         ).invoke(
-          null, (Object) args, new File(cache), new File(CBT_HOME), res, classLoaderCache.hashMap
+          null, (Object) args, new File(cache), new File(CBT_HOME), loop, res, classLoaderCache.hashMap
         );
 
       System.exit( exitCode );
@@ -132,7 +139,7 @@ public static Throwable unwrapInvocationTargetException(Throwable e){
 
   public static BuildStage1Result buildStage1(
     final long lastModified, final long start, final String cache, final String cbtHome,
-    final String compatibilityTarget, final ClassLoaderCache classLoaderCache
+    final String compatibilityTarget, final ClassLoaderCache classLoaderCache, File cwd, boolean loop
   ) throws Throwable {
     _assert(TARGET != null, "environment variable TARGET not defined");
     String nailgunSources = cbtHome + "/" + NAILGUN;
@@ -142,6 +149,10 @@ public static BuildStage1Result buildStage1(
     File compatibilitySources = new File(cbtHome + "/compatibility");
     String mavenCache = cache + "maven";
     String mavenUrl = "https://repo1.maven.org/maven2";
+    File loopFile = new File(cwd + "/target/.cbt-loop.tmp");
+    if(loop){
+      loopFile.getParentFile().mkdirs();
+    }
 
     ClassLoader rootClassLoader = new CbtURLClassLoader( new URL[]{}, ClassLoader.getSystemClassLoader().getParent() ); // wrap for caching
     EarlyDependencies earlyDeps = new EarlyDependencies(mavenCache, mavenUrl, classLoaderCache, rootClassLoader);
@@ -166,7 +177,16 @@ public static BuildStage1Result buildStage1(
         }
       }
 
-      compatibilityLastModified = compile( 0L, "", compatibilityTarget, earlyDeps, compatibilitySourceFiles);
+      if(loop){
+        File[] _compatibilitySourceFiles = new File[compatibilitySourceFiles.size()];
+        compatibilitySourceFiles.toArray(_compatibilitySourceFiles);
+        write(
+          loopFile.toPath(),
+          (mkString( "\n", _compatibilitySourceFiles ) + "\n").getBytes(),
+          StandardOpenOption.CREATE
+        );
+      }
+      compatibilityLastModified = compile( 0L, "", compatibilityTarget, earlyDeps, compatibilitySourceFiles) ;
 
       if( !classLoaderCache.containsKey( compatibilityTarget, compatibilityLastModified ) ){
         classLoaderCache.put( compatibilityTarget, classLoader(compatibilityTarget, rootClassLoader), compatibilityLastModified );
@@ -197,6 +217,17 @@ public static BuildStage1Result buildStage1(
       lastModified,
       Math.max( lastModified, compatibilityLastModified )
     );
+
+    if(loop){
+      File[] _stage1SourceFiles = new File[stage1SourceFiles.size()];
+      stage1SourceFiles.toArray(_stage1SourceFiles);
+      write(
+        loopFile.toPath(),
+        (mkString( "\n", _stage1SourceFiles ) + "\n").getBytes(),
+        StandardOpenOption.APPEND
+      );
+    }
+
     final long stage1LastModified = compile(
       stage0LastModified, stage1Classpath, stage1Target, earlyDeps, stage1SourceFiles
     );

stage1/ContextImplementation.scala

@@ -16,7 +16,7 @@ class ContextImplementation(
   override val cbtRootHome: File,
   override val compatibilityTarget: File,
   override val parentBuildOrNull: BuildInterface,
-  override val triggerLoopFilesArray: Array[File]
+  override val loop: Boolean
 ) extends Context{
   @deprecated("this method is replaced by workingDirectory","")
   def projectDirectory = workingDirectory

stage1/Stage1.scala

@@ -42,7 +42,8 @@ class Stage2Args(
   val cache: File,
   val cbtHome: File,
   val compatibilityTarget: File,
-  val stage2sourceFiles: Seq[File]
+  val stage2sourceFiles: Seq[File],
+  val loop: Boolean
 )(
   implicit val transientCache: java.util.Map[AnyRef,AnyRef], val classLoaderCache: ClassLoaderCache, val logger: Logger
 ){
@@ -60,7 +61,9 @@ object Stage1{
     val (_, cbtLastModified, classLoader) = buildStage2(
       buildStage1,
       context.cbtHome,
-      context.cache
+      context.cache,
+      context.cwd,
+      context.loop
     )(context.transientCache, new ClassLoaderCache( context.persistentCache ), logger)
 
     classLoader
@@ -75,7 +78,7 @@ object Stage1{
   }
 
   def buildStage2(
-    buildStage1: BuildStage1Result, cbtHome: File, cache: File
+    buildStage1: BuildStage1Result, cbtHome: File, cache: File, cwd: File, loop: Boolean
   )(
     implicit transientCache: java.util.Map[AnyRef,AnyRef], classLoaderCache: ClassLoaderCache, logger: Logger
   ): (Seq[File], Long, ClassLoader) = {
@@ -99,6 +102,8 @@ object Stage1{
     )
 
     logger.stage1("Compiling stage2 if necessary")
+    if(loop)
+      lib.addLoopFiles( cwd, stage2sourceFiles.toSet )
     val Some( stage2LastModified ) = compile(
       buildStage1.stage1LastModified,
       stage2sourceFiles, stage2Target, stage2StatusFile,
@@ -158,6 +163,7 @@ object Stage1{
     _args: Array[String],
     cache: File,
     cbtHome: File,
+    loop: Boolean,
     buildStage1: BuildStage1Result,
     persistentCache: java.util.Map[AnyRef,AnyRef]
   ): Int = {
@@ -168,17 +174,19 @@ object Stage1{
     implicit val transientCache: java.util.Map[AnyRef,AnyRef] = new java.util.HashMap
     implicit val classLoaderCache = new ClassLoaderCache( persistentCache )
 
-    val (stage2sourceFiles, stage2LastModified, classLoader) = buildStage2( buildStage1, cbtHome, cache )
+    val cwd = new File( args.args(0) )
+    val (stage2sourceFiles, stage2LastModified, classLoader) = buildStage2( buildStage1, cbtHome, cache, cwd, loop )
 
     val stage2Args = new Stage2Args(
-      new File( args.args(0) ),
-      args.args.drop(1).toVector,
+      cwd,
+      args.args.drop(2).toVector,
       // launcher changes cause entire nailgun restart, so no need for them here
       stage2LastModified = stage2LastModified,
       cache,
       cbtHome,
       new File(buildStage1.compatibilityClasspath),
-      stage2sourceFiles
+      stage2sourceFiles,
+      loop
     )
 
     logger.stage1(s"Run Stage2")