Bump Google Closure Compiler & Library

Google Closure Compiler is now latest. GCL snapshot is from December.

Address various minor breaking changes in GCC & GCL.

GCL removed some long outstanding type detection helpers - goog/isFunction, goog/isString, goog/isArray. For isFunction add a new js-fn? macro which uses `typeOf` operator. For isString rely on string? now. for isArray, isNumber use goog/typeOf

Throwable from goog common Java lib is now gone. Just print the Throwable for now.

More GCL libs are now written in ES6. This broke self-parity because cljs.closure/transpile did not respect `:language-out`. Port GCC Transpiler implementation so we have more control.

Self-parity build now uses :language-out :es6 and passes.

Add tests for new transpile helpers.

GCL now has a new import mechanism only for types `goog.requireType`. This signals to GCC that the file is required but only for type checking. Update cljs.js-deps so that `:require-type` is included when parsing JS files.

Update cljs.closure/js-dependencies so that `:require-types` deps are inputs to the build so that GCC can type check.

Add JS file parsing tests that check for `:require-types` in both the JS index and for individual file parsing.

Update the CI script to adopt -M over -A.

Author: swannodette

.github/workflows/test.yaml

@@ -44,7 +44,7 @@ jobs:
             ${{ runner.os }}-jsc
 
       - name: Build tests
-        run: clojure -A:runtime.test.build
+        run: clojure -M:runtime.test.build
 
       - name: Install JSC
         run: ./ci/install_jsc.sh
@@ -86,7 +86,7 @@ jobs:
             ${{ runner.os }}-${{ env.cache-name }}-
 
       - name: Build tests
-        run: clojure -A:selfhost.test.build
+        run: clojure -M:selfhost.test.build
 
       - name: Run tests
         run: |
@@ -125,7 +125,7 @@ jobs:
             ${{ runner.os }}-${{ env.cache-name }}-
 
       - name: Build tests
-        run: clojure -A:selfparity.test.build
+        run: clojure -M:selfparity.test.build
 
       - name: Run tests
         run: |
@@ -164,7 +164,7 @@ jobs:
             ${{ runner.os }}-${{ env.cache-name }}-
 
       - name: Run tests
-        run: clojure -A:compiler.test:compiler.test.run
+        run: clojure -M:compiler.test:compiler.test.run
 
   # CLI Tests
   cli-test:
@@ -205,5 +205,5 @@ jobs:
       - name: Run tests
         run: |
           unset JAVA_TOOL_OPTIONS
-          clojure -A:cli.test.run | tee test-out.txt
+          clojure -M:cli.test.run | tee test-out.txt
           grep -qxF '0 failures, 0 errors.' test-out.txt

bin/cljsc

@@ -13,7 +13,7 @@ if ! test "$(ls -A "$CLOJURESCRIPT_HOME/lib" 2>/dev/null)"; then
 fi
 
 CLJSC_CP=''
-for next in lib/*: src/main/clojure: src/main/cljs: src/test/cljs; do
+for next in lib/*: src/main/clojure: src/main/cljs: src/test/cljs: src/test/self; do
   CLJSC_CP="${CLJSC_CP}${CLOJURESCRIPT_HOME}/${next}"
 done
 

deps.edn

@@ -1,11 +1,11 @@
 {:paths ["src/main/clojure" "src/main/cljs" "resources"]
  :deps
- {com.google.javascript/closure-compiler-unshaded {:mvn/version "v20200315"}
+ {com.google.javascript/closure-compiler-unshaded {:mvn/version "v20210202"}
   com.cognitect/transit-clj {:mvn/version "0.8.309"}
   org.clojure/clojure {:mvn/version "1.10.0"}
   org.clojure/core.specs.alpha {:mvn/version "0.1.24"}
   org.clojure/data.json {:mvn/version "0.2.6"}
-  org.clojure/google-closure-library {:mvn/version "0.0-20191016-6ae1f72f"}
+  org.clojure/google-closure-library {:mvn/version "0.0-20201211-3e6c510d"}
   org.clojure/spec.alpha {:mvn/version "0.1.143"}
   org.clojure/tools.reader {:mvn/version "1.3.3"}
   org.clojure/test.check {:mvn/version "0.10.0-alpha3"}}

pom.template.xml

@@ -30,12 +30,12 @@
         <dependency>
             <groupId>com.google.javascript</groupId>
             <artifactId>closure-compiler-unshaded</artifactId>
-            <version>v20200315</version>
+            <version>v20210202</version>
         </dependency>
         <dependency>
             <groupId>org.clojure</groupId>
             <artifactId>google-closure-library</artifactId>
-            <version>0.0-20191016-6ae1f72f</version>
+            <version>0.0-20201211-3e6c510d</version>
         </dependency>
         <dependency>
             <groupId>org.clojure</groupId>

project.clj

@@ -8,15 +8,15 @@
   :source-paths ["src/main/clojure" "src/main/cljs"]
   :resource-paths ["src/main/cljs" "resources"]
   :test-paths ["src/test/clojure" "src/test/cljs" "src/test/self" "src/test/cljs_build" "src/test/cljs_cp"]
-  :dependencies [[org.clojure/clojure "1.10.0-alpha4"]
+  :dependencies [[org.clojure/clojure "1.10.0"]
                  [org.clojure/spec.alpha "0.1.143"]
                  [org.clojure/core.specs.alpha "0.1.24"]
                  [org.clojure/data.json "0.2.6"]
                  [org.clojure/tools.reader "1.3.3"]
                  [org.clojure/test.check "0.10.0-alpha3" :scope "test"]
                  [com.cognitect/transit-clj "0.8.309"]
-                 [org.clojure/google-closure-library "0.0-20191016-6ae1f72f"]
-                 [com.google.javascript/closure-compiler-unshaded "v20200315"]]
+                 [org.clojure/google-closure-library "0.0-20201211-3e6c510d"]
+                 [com.google.javascript/closure-compiler-unshaded "v20210202"]]
   :profiles {:1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]}
              :uberjar {:aot :all :main cljs.main}
              :closure-snapshot {:dependencies [[com.google.javascript/closure-compiler-unshaded "1.0-SNAPSHOT"]]}}

resources/self_parity_test.edn

@@ -1,5 +1,8 @@
 {:optimizations :none
  :main self-parity.test
+ :language-in :es6
+ :language-out :es6
+ :verbose true
  :output-to "builds/out-self-parity/main.js"
  :output-dir "builds/out-self-parity"
  :cache-analysis-format :edn

script/bootstrap

@@ -5,10 +5,10 @@ set -e
 CLOJURE_RELEASE="1.9.0"
 SPEC_ALPHA_RELEASE="0.1.143"
 CORE_SPECS_ALPHA_RELEASE="0.1.24"
-CLOSURE_RELEASE="20200315"
+CLOSURE_RELEASE="20210202"
 DJSON_RELEASE="0.2.6"
 TRANSIT_RELEASE="0.8.309"
-GCLOSURE_LIB_RELEASE="0.0-20191016-6ae1f72f"
+GCLOSURE_LIB_RELEASE="0.0-20201211-3e6c510d"
 TREADER_RELEASE="1.3.3"
 TEST_CHECK_RELEASE="0.10.0-alpha3"
 

script/test-self-parity

@@ -15,7 +15,7 @@ mkdir -p builds/out-self-parity/clojure/test
 mv clojure/template.clj builds/out-self-parity/clojure
 mv clojure/test builds/out-self-parity/clojure
 
-if ! bin/cljsc src/test/self/self_parity "{:optimizations :none :output-to \"builds/out-self-parity/main.js\" :output-dir \"builds/out-self-parity\" :main self-parity.test :target :nodejs}"; then
+if ! bin/cljsc src/test/self/self_parity "{:optimizations :simple :language-in :es6 :language-out :es5 :output-to \"builds/out-self-parity/main.js\" :output-dir \"builds/out-self-parity\" :main self-parity.test :target :nodejs}"; then
   >&2 echo ClojureScript compilation failed
   exit 1
 fi;

src/main/cljs/cljs/core.cljs

@@ -273,7 +273,7 @@
 (defn ^boolean string?
   "Returns true if x is a JavaScript string."
   [x]
-  (goog/isString x))
+  (identical? "string" (goog/typeOf x)))
 
 (defn char?
   "Returns true if x is a JavaScript string of length one."
@@ -2026,7 +2026,7 @@ reduces them without incurring seq initialization"
 (defn fn?
   "Return true if f is a JavaScript function or satisfies the Fn protocol."
   [f]
-  (or ^boolean (goog/isFunction f) (satisfies? Fn f)))
+  (or (js-fn? f) (satisfies? Fn f)))
 
 (deftype MetaFn [afn meta]
   IMeta
@@ -2085,7 +2085,7 @@ reduces them without incurring seq initialization"
   "Returns an object of the same type and value as obj, with
   map m as its metadata."
   [o meta]
-  (if ^boolean (goog/isFunction o)
+  (if (js-fn? o)
     (MetaFn. o meta)
     (when-not (nil? o)
       (-with-meta o meta))))
@@ -6461,14 +6461,14 @@ reduces them without incurring seq initialization"
   ILookup
   (-lookup [coll k] (-lookup coll k nil))
   (-lookup [coll k not-found]
-    (if (and ^boolean (goog/isString k)
+    (if (and (string? k)
              (not (nil? (scan-array 1 k keys))))
       (unchecked-get strobj k)
       not-found))
 
   IAssociative
   (-assoc [coll k v]
-    (if ^boolean (goog/isString k)
+    (if (string? k)
         (if (or (> update-count (.-HASHMAP_THRESHOLD ObjMap))
                 (>= (alength keys) (.-HASHMAP_THRESHOLD ObjMap)))
           (obj-map->hash-map coll k v)
@@ -6484,14 +6484,14 @@ reduces them without incurring seq initialization"
         ;; non-string key. game over.
         (obj-map->hash-map coll k v)))
   (-contains-key? [coll k]
-    (if (and ^boolean (goog/isString k)
+    (if (and (string? k)
              (not (nil? (scan-array 1 k keys))))
       true
       false))
 
   IFind
   (-find [coll k]
-    (when (and ^boolean (goog/isString k)
+    (when (and (string? k)
             (not (nil? (scan-array 1 k keys))))
       (MapEntry. k (unchecked-get strobj k) nil)))
 
@@ -6510,7 +6510,7 @@ reduces them without incurring seq initialization"
 
   IMap
   (-dissoc [coll k]
-    (if (and ^boolean (goog/isString k)
+    (if (and (string? k)
              (not (nil? (scan-array 1 k keys))))
       (let [new-keys (aclone keys)
             new-strobj (obj-clone strobj keys)]
@@ -6624,7 +6624,7 @@ reduces them without incurring seq initialization"
   (cond
     (keyword? k) (array-index-of-keyword? arr k)
 
-    (or ^boolean (goog/isString k) (number? k))
+    (or (string? k) (number? k))
     (array-index-of-identical? arr k)
 
     (symbol? k) (array-index-of-symbol? arr k)
@@ -10264,12 +10264,12 @@ reduces them without incurring seq initialization"
         (array? obj)
         (pr-sequential-writer writer pr-writer "#js [" " " "]" opts obj)
 
-        ^boolean (goog/isString obj)
+        (string? obj)
         (if (:readably opts)
           (-write writer (quote-string obj))
           (-write writer obj))
 
-        ^boolean (goog/isFunction obj)
+        (js-fn? obj)
         (let [name (.-name obj)
               name (if (or (nil? name) (gstring/isEmpty name))
                      "Function"

src/main/clojure/cljs/closure.clj

@@ -37,18 +37,18 @@
                                          CompilerInput CompilerInput$ModuleType DependencyOptions
                                          CompilerOptions$LanguageMode SourceMap$Format
                                          SourceMap$DetailLevel ClosureCodingConvention SourceFile
-                                         Result JSError CheckLevel DiagnosticGroups
-                                         CommandLineRunner AnonymousFunctionNamingPolicy
-                                         JSModule SourceMap VariableMap PrintStreamErrorManager]
+                                         Result JSError CheckLevel DiagnosticGroup DiagnosticGroups
+                                         CommandLineRunner
+                                         JSModule SourceMap VariableMap PrintStreamErrorManager DiagnosticType
+                                         VariableRenamingPolicy PropertyRenamingPolicy]
            [com.google.javascript.jscomp.bundle Transpiler]
            [com.google.javascript.jscomp.deps ClosureBundler ModuleLoader$ResolutionMode ModuleNames
                                               SimpleDependencyInfo]
            [com.google.javascript.rhino Node]
            [java.nio.file Path Paths Files StandardWatchEventKinds WatchKey
                           WatchEvent FileVisitor FileVisitResult FileSystems]
            [java.nio.charset Charset StandardCharsets]
-           [com.sun.nio.file SensitivityWatchEventModifier]
-           [com.google.common.base Throwables]))
+           [com.sun.nio.file SensitivityWatchEventModifier]))
 
 ;; Copied from clojure.tools.gitlibs
 
@@ -162,7 +162,6 @@
    :message-descriptions DiagnosticGroups/MESSAGE_DESCRIPTIONS
    :misplaced-msg-annotation DiagnosticGroups/MISPLACED_MSG_ANNOTATION
    :misplaced-type-annotation DiagnosticGroups/MISPLACED_TYPE_ANNOTATION
-   :missing-getcssname DiagnosticGroups/MISSING_GETCSSNAME
    :missing-override DiagnosticGroups/MISSING_OVERRIDE
    :missing-polyfill DiagnosticGroups/MISSING_POLYFILL
    :missing-properties DiagnosticGroups/MISSING_PROPERTIES
@@ -175,7 +174,6 @@
    :non-standard-jsdoc DiagnosticGroups/NON_STANDARD_JSDOC
    :report-unknown-types DiagnosticGroups/REPORT_UNKNOWN_TYPES
    :strict-missing-properties DiagnosticGroups/STRICT_MISSING_PROPERTIES
-   :strict-missing-require DiagnosticGroups/STRICT_MISSING_REQUIRE
    :strict-module-dep-check DiagnosticGroups/STRICT_MODULE_DEP_CHECK
    :strict-requires DiagnosticGroups/STRICT_REQUIRES
    :suspicious-code DiagnosticGroups/SUSPICIOUS_CODE
@@ -262,15 +260,6 @@
   (when (contains? opts :pseudo-names)
     (set! (.generatePseudoNames compiler-options) (:pseudo-names opts)))
 
-  (when (contains? opts :anon-fn-naming-policy)
-    (let [policy (:anon-fn-naming-policy opts)]
-      (set! (.anonymousFunctionNaming compiler-options)
-        (case policy
-          :off AnonymousFunctionNamingPolicy/OFF
-          :unmapped AnonymousFunctionNamingPolicy/UNMAPPED
-          :mapped AnonymousFunctionNamingPolicy/MAPPED
-          (throw (util/compilation-error (IllegalArgumentException. (str "Invalid :anon-fn-naming-policy value " policy " - only :off, :unmapped, :mapped permitted"))))))))
-
   (when-let [lang-key (:language-in opts :ecmascript5)]
     (.setLanguageIn compiler-options (lang-key->lang-mode lang-key)))
 
@@ -816,7 +805,8 @@
     (if (seq requires)
       (let [node (or (get (@env/*compiler* :js-dependency-index) (first requires))
                      (deps/find-classpath-lib (first requires)))
-            new-req (remove #(contains? visited %) (:requires node))]
+            new-req (remove #(contains? visited %)
+                      (into (:requires node) (:require-types node)))]
         (recur (into (rest requires) new-req)
                (into visited new-req)
                (conj deps node)))
@@ -2052,18 +2042,45 @@
     (.appendTo bundler sb module source)
     (.toString sb)))
 
+(defn ^DiagnosticGroup es5-warnings []
+  (DiagnosticGroup.
+    (into-array DiagnosticType
+      [(DiagnosticType/error "JSC_CANNOT_CONVERT" "")])))
+
+(defn ^CompilerOptions transpile-options []
+  (doto (CompilerOptions.)
+    (.setQuoteKeywordProperties true)
+    (.setSkipNonTranspilationPasses true)
+    (.setVariableRenaming VariableRenamingPolicy/OFF)
+    (.setPropertyRenaming PropertyRenamingPolicy/OFF)
+    (.setWrapGoogModulesForWhitespaceOnly false)
+    (.setPrettyPrint true)
+    (.setSourceMapOutputPath "/dev/null")
+    (.setSourceMapIncludeSourcesContent true)
+    (.setWarningLevel (es5-warnings) CheckLevel/OFF)))
+
+(defn closure-transpile
+  "Transpile a single JavaScript file to JavaScript. Used to lower Closure
+  Library files written in more recent versions of the JavaScript standard."
+  ([rsc opts]
+   (closure-transpile (util/path rsc) (slurp rsc) opts))
+  ([path source opts]
+   (let [cc      (make-closure-compiler)
+         cc-opts (set-options opts (transpile-options))
+         externs (SourceFile/fromCode "externs.js" "function Symbol() {}")
+         source  (SourceFile/fromCode path source)
+         result  (.compile cc externs source cc-opts)]
+     ;; TODO: error handling
+     (.toSource cc))))
+
 ;; TODO: better error handling
-;; TODO: actually respect the target :language-out level
-;; currently just using default options for Transpiler
 (defn transpile
-  [{:keys [language-out] :or {language-out :es3}} rsc {:keys [module lang] :as js}]
+  [{:keys [language-out] :or {language-out :es3} :as opts} rsc {:keys [module lang] :as js}]
   (let [source  (slurp rsc)
         source' (if (and lang
                          (< (.indexOf lang-level (expand-lang-key language-out))
                             (.indexOf lang-level (expand-lang-key lang))))
-                  (let [cc (Transpiler/compilerSupplier)
-                        result (.compile cc (url->nio-path rsc) source)]
-                    (.source result))
+                  (closure-transpile (util/path rsc) source opts)
                   source)]
     (str "/*TRANSPILED*/"
       (cond-> source'
@@ -3286,7 +3303,7 @@
                     (if-let [f (opts-fn :watch-error-fn opts)]
                       (f e)
                       (binding [*out* *err*]
-                        (println (Throwables/getStackTraceAsString e)))))))
+                        (println e))))))
               (watch-all [^Path root]
                 (Files/walkFileTree root
                   (reify

src/main/clojure/cljs/compiler.cljc

@@ -210,7 +210,7 @@
      (nil? a) nil
      #?(:clj (map? a) :cljs (ana.impl/cljs-map? a)) (emit a)
      #?(:clj (seq? a) :cljs (ana.impl/cljs-seq? a)) (apply emits a)
-     #?(:clj (fn? a) :cljs ^boolean (goog/isFunction a)) (a)
+     #?(:clj (fn? a) :cljs (js-fn? a)) (a)
      :else (let [^String s (cond-> a (not (string? a)) .toString)]
              #?(:clj  (when-some [^AtomicLong gen-col *source-map-data-gen-col*]
                         (.addAndGet gen-col (.length s)))

src/main/clojure/cljs/core.cljc

@@ -42,7 +42,7 @@
                             #?@(:cljs [alias coercive-not coercive-not= coercive-= coercive-boolean
                                        truth_ js-arguments js-delete js-in js-debugger exists? divide js-mod
                                        unsafe-bit-and bit-shift-right-zero-fill mask bitpos caching-hash
-                                       defcurried rfn specify! js-this this-as implements? array js-obj
+                                       defcurried rfn specify! js-this this-as implements? array js-obj js-fn?
                                        simple-benchmark gen-apply-to js-str es6-iterable load-file* undefined?
                                        specify copy-arguments goog-define js-comment js-inline-comment
                                        unsafe-cast require-macros use-macros gen-apply-to-simple unchecked-get unchecked-set])])
@@ -982,6 +982,9 @@
 (core/defmacro string? [x]
   (bool-expr (core/list 'js* "typeof ~{} === 'string'" x)))
 
+(core/defmacro js-fn? [x]
+  (bool-expr (core/list 'js* "typeof ~{} === 'function'" x)))
+
 (core/defmacro exists?
   "Return true if argument exists, analogous to usage of typeof operator
    in JavaScript."