From 8c1425f1fb7077d8206029164c00aa09658d83e1 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Thu, 18 Sep 2025 13:58:51 +0200 Subject: [PATCH 01/24] macro evaluation POC --- .clj-kondo/config.edn | 3 +- deps.edn | 2 +- src/nextjournal/clerk/analyzer.clj | 113 ++++++++++++++--------- src/nextjournal/clerk/analyzer/impl.clj | 23 +++-- test/nextjournal/clerk/analyzer_test.clj | 14 +-- 5 files changed, 92 insertions(+), 63 deletions(-) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 42ed11e51..6e8f0b3ca 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -36,4 +36,5 @@ clojure.set clojure.walk]}}} clojure.test/deftest {:linters {:inline-def {:level :off}}} - shadow.cljs.modern/defclass {:linters {:redefined-var {:level :off}}}}} + shadow.cljs.modern/defclass {:linters {:redefined-var {:level :off}}} + nextjournal.clerk.utils/if-bb {:linters {:condition-always-true {:level :off}}}}} diff --git a/deps.edn b/deps.edn index f4030619e..91f7c0b1c 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,7 @@ {:paths ["src" "resources" "bb"] :deps {org.clojure/clojure {:mvn/version "1.11.1"} org.clojure/java.classpath {:mvn/version "1.0.0"} - babashka/fs {:mvn/version "0.5.22"} + babashka/fs {:mvn/version "0.5.27"} org.babashka/http-client {:mvn/version "0.4.23"} borkdude/edamame {:mvn/version "1.4.28"} weavejester/dependency {:mvn/version "0.2.1"} diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index c8193034c..a26bc5511 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -156,7 +156,8 @@ deps (set/union (set/difference (into #{} (map (comp symbol var->protocol)) @!deps) vars) deref-deps (when (var? form) #{(symbol form)})) - hash-fn (-> form meta :nextjournal.clerk/hash-fn)] + hash-fn (-> form meta :nextjournal.clerk/hash-fn) + macro? (-> analyzed :env :defmacro)] (cond-> {#_#_:analyzed analyzed :form form :ns-effect? (some? (some #{'clojure.core/require 'clojure.core/in-ns} deps)) @@ -170,7 +171,8 @@ (seq vars) (assoc :vars vars) (seq vars-) (assoc :vars- vars-) (seq declared) (assoc :declared declared) - var (assoc :var var)))) + var (assoc :var var) + macro? (assoc :macro macro?)))) #_(:vars (analyze '(do (declare x y) (def x 0) (def z) (def w 0)))) ;=> x y z w #_(:vars- (analyze '(do (def x 0) (declare x y) (def z) (def w 0)))) ;=> x z w @@ -496,55 +498,74 @@ (filter (comp #{:code} :type) blocks)))) +(defn transitive-deps [id analysis-info] + (let [{:keys [deps]} (get analysis-info id)] + (when deps + (into deps (mapcat #(transitive-deps % analysis-info) deps))))) + +(defn run-macros [init-state] + (let [{:keys [blocks ->analysis-info]} init-state + macro-block-ids (keep #(when (:macro %) + (:id %)) blocks) + + deps (mapcat #(transitive-deps % ->analysis-info) macro-block-ids) + all-block-ids (into (set macro-block-ids) deps) + all-blocks (filter #(contains? all-block-ids (:id %)) blocks)] + (doseq [block all-blocks] + (try + (load-string (:text block)) + (catch Throwable e + (binding [*out* *err*] + (println "Error when evaluating macro deps:" (:text block)) + (println "Namespace:" *ns*) + (println "Exception:" e))))) + (pos? (count all-blocks)))) + (defn build-graph "Analyzes the forms in the given file and builds a dependency graph of the vars. - Recursively decends into dependency vars as well as given they can be found in the classpath. + Recursively descends into dependency vars as well if they can be found in the classpath. " [doc] - (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} - (-> doc - analyze-doc - (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc))) - :counter 0 - :graph (dep/graph)))] - (let [unhashed (unhashed-deps ->analysis-info) - loc->syms (apply dissoc - (group-by find-location unhashed) - analyzed-file-set)] - (if (and (seq loc->syms) (< counter 10)) - (recur (-> (reduce (fn [g [source symbols]] - (let [jar? (or (nil? source) - (str/ends-with? source ".jar")) - gitlib-hash (and (not jar?) - (second (re-find #".gitlibs/libs/.*/(\b[0-9a-f]{5,40}\b)/" (fs/unixify source))))] - (if (or jar? gitlib-hash) - (update g :->analysis-info merge (into {} (map (juxt identity - (constantly (if source - (or (when gitlib-hash {:hash gitlib-hash}) - (hash-jar source)) - {})))) symbols)) - (-> g - (update :analyzed-file-set conj source) - (merge-analysis-info (analyze-file source)))))) - state - loc->syms) - (update :counter inc))) - (-> state - analyze-doc-deps - set-no-cache-on-redefs - make-deps-inherit-no-cache - (dissoc :analyzed-file-set :counter)))))) - -(comment - (reset! !file->analysis-cache {}) - - (def parsed (parser/parse-file {:doc? true} "src/nextjournal/clerk/webserver.clj")) - (def analysis (time (-> parsed analyze-doc build-graph))) - (-> analysis :->analysis-info keys set) - (let [{:keys [->analysis-info]} analysis] - (dissoc (group-by find-location (unhashed-deps ->analysis-info)) nil)) - (nextjournal.clerk/clear-cache!)) + (let [init-state-fn #(-> doc + analyze-doc + (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc))) + :counter 0 + :graph (dep/graph))) + init-state (init-state-fn) + ran-macros? (run-macros init-state) + init-state (if ran-macros? + (init-state-fn) + init-state)] + (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} + init-state] + (let [unhashed (unhashed-deps ->analysis-info) + loc->syms (apply dissoc + (group-by find-location unhashed) + analyzed-file-set)] + (if (and (seq loc->syms) (< counter 10)) + (recur (-> (reduce (fn [g [source symbols]] + (let [jar? (or (nil? source) + (str/ends-with? source ".jar")) + gitlib-hash (and (not jar?) + (second (re-find #".gitlibs/libs/.*/(\b[0-9a-f]{5,40}\b)/" (fs/unixify source))))] + (if (or jar? gitlib-hash) + (update g :->analysis-info merge (into {} (map (juxt identity + (constantly (if source + (or (when gitlib-hash {:hash gitlib-hash}) + (hash-jar source)) + {})))) symbols)) + (-> g + (update :analyzed-file-set conj source) + (merge-analysis-info (analyze-file source)))))) + state + loc->syms) + (update :counter inc))) + (-> state + analyze-doc-deps + set-no-cache-on-redefs + make-deps-inherit-no-cache + (dissoc :analyzed-file-set :counter))))))) #_(do (time (build-graph (parser/parse-clojure-string (slurp "notebooks/how_clerk_works.clj")))) :done) #_(do (time (build-graph (parser/parse-clojure-string (slurp "notebooks/viewer_api.clj")))) :done) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 9dcb237fd..168a189f7 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -383,8 +383,12 @@ (if (and (var? maybe-macro) (:macro (meta maybe-macro))) (do + ;; macroexpand is the macro here, we should register in the env that the var about to be def-ed is a macro (swap! *deps* conj maybe-macro) - (let [expanded (macroexpand-hook maybe-macro form env (rest form))] + (let [expanded (macroexpand-hook maybe-macro form env (rest form)) + env (if (identical? #'defmacro maybe-macro) + (assoc env :defmacro true) + env)] (analyze* env expanded))) {:op :invoke :form form @@ -447,14 +451,15 @@ (assoc-in env [:namespaces ns :mappings sym] var))) args (when-let [[_ init] (find args :init)] (assoc args :init (analyze* env init)))] - (merge {:op :def - :env env - :form form - :name sym - :doc (or (:doc args) (-> sym meta :doc)) - :children (into [:meta] (when (:init args) [:init])) - :var (get-in env [:namespaces ns :mappings sym]) - :meta {:val (meta sym)}} + (merge (cond-> {:op :def + :env env + :form form + :name sym + :doc (or (:doc args) (-> sym meta :doc)) + :children (into [:meta] (when (:init args) [:init])) + :var (get-in env [:namespaces ns :mappings sym]) + :meta {:val (meta sym)}} + (:defmacro env) (assoc :macro true)) args))) (defmethod -parse 'fn* [env [op & args :as form]] diff --git a/test/nextjournal/clerk/analyzer_test.clj b/test/nextjournal/clerk/analyzer_test.clj index f7a7b218a..21971d889 100644 --- a/test/nextjournal/clerk/analyzer_test.clj +++ b/test/nextjournal/clerk/analyzer_test.clj @@ -377,7 +377,8 @@ my-uuid")] (is (empty? (ana/unhashed-deps ->analysis-info))) (is (match? {:jar string?} (->analysis-info 'weavejester.dependency/graph))))) (testing "should establish dependencies across files" - (let [{:keys [graph]} (analyze-string (slurp "src/nextjournal/clerk.clj"))] + (let [{:keys [graph]} (binding [*ns* (find-ns 'nextjournal.clerk)] + (analyze-string (slurp "src/nextjournal/clerk.clj")))] (is (dep/depends? graph 'nextjournal.clerk/show! 'nextjournal.clerk.analyzer/hash))))) (deftest graph-nodes-with-anonymous-ids @@ -404,10 +405,11 @@ my-uuid")] (is (empty? (let [!missing-hash-store (atom [])] (reset! ana/!file->analysis-cache {}) - (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj") - ana/build-graph - (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry))) - ana/hash) + (binding [*ns* (find-ns 'nextjournal.clerk)] + (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj") + ana/build-graph + (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry))) + ana/hash)) (deref !missing-hash-store))))) (testing "known cases where missing hashes occur" @@ -428,7 +430,7 @@ my-uuid")] (is (:dep-with-missing-hash missing-hash-report))))) (deftest ->hash - (testing "notices change in depedency namespace" + (testing "notices change in dependency namespace" (let [test-var 'nextjournal.clerk.fixtures.generated.my-test-ns/hello test-string "(ns test (:require [nextjournal.clerk.fixtures.generated.my-test-ns :as my-test-ns])) (str my-test-ns/hello)" spit-with-value #(spit (format "test%s%s.clj" fs/file-separator (str/replace (namespace-munge (namespace test-var)) "." fs/file-separator )) From 8c0c39864fd920ef026a0129a927b5728a517ede Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 23 Sep 2025 12:46:59 +0200 Subject: [PATCH 02/24] with-ns-binding --- test/nextjournal/clerk/analyzer_test.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/nextjournal/clerk/analyzer_test.clj b/test/nextjournal/clerk/analyzer_test.clj index 21971d889..019aa2ee8 100644 --- a/test/nextjournal/clerk/analyzer_test.clj +++ b/test/nextjournal/clerk/analyzer_test.clj @@ -377,7 +377,7 @@ my-uuid")] (is (empty? (ana/unhashed-deps ->analysis-info))) (is (match? {:jar string?} (->analysis-info 'weavejester.dependency/graph))))) (testing "should establish dependencies across files" - (let [{:keys [graph]} (binding [*ns* (find-ns 'nextjournal.clerk)] + (let [{:keys [graph]} (with-ns-binding 'nextjournal.clerk (analyze-string (slurp "src/nextjournal/clerk.clj")))] (is (dep/depends? graph 'nextjournal.clerk/show! 'nextjournal.clerk.analyzer/hash))))) @@ -405,7 +405,7 @@ my-uuid")] (is (empty? (let [!missing-hash-store (atom [])] (reset! ana/!file->analysis-cache {}) - (binding [*ns* (find-ns 'nextjournal.clerk)] + (with-ns-binding 'nextjournal.clerk (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj") ana/build-graph (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry))) From 780f2b333af86b85b8dcb8c7410b43d32dd14386 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 23 Sep 2025 13:14:29 +0200 Subject: [PATCH 03/24] test --- src/nextjournal/clerk/analyzer.clj | 7 +++---- test/nextjournal/clerk/eval_test.clj | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index a26bc5511..105432fe9 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -164,15 +164,15 @@ :freezable? (and (not (some #{'clojure.core/intern} deps)) (<= (count vars) 1) (if (seq vars) (= var (first vars)) true)) - :no-cache? (no-cache? form (-> def-node :form second) *ns*)} + :no-cache? (no-cache? form (-> def-node :form second) *ns*) + :macro macro?} hash-fn (assoc :hash-fn hash-fn) (seq deps) (assoc :deps deps) (seq deref-deps) (assoc :deref-deps deref-deps) (seq vars) (assoc :vars vars) (seq vars-) (assoc :vars- vars-) (seq declared) (assoc :declared declared) - var (assoc :var var) - macro? (assoc :macro macro?)))) + var (assoc :var var)))) #_(:vars (analyze '(do (declare x y) (def x 0) (def z) (def w 0)))) ;=> x y z w #_(:vars- (analyze '(do (def x 0) (declare x y) (def z) (def w 0)))) ;=> x z w @@ -507,7 +507,6 @@ (let [{:keys [blocks ->analysis-info]} init-state macro-block-ids (keep #(when (:macro %) (:id %)) blocks) - deps (mapcat #(transitive-deps % ->analysis-info) macro-block-ids) all-block-ids (into (set macro-block-ids) deps) all-blocks (filter #(contains? all-block-ids (:id %)) blocks)] diff --git a/test/nextjournal/clerk/eval_test.clj b/test/nextjournal/clerk/eval_test.clj index a40f8e444..90a7838d5 100644 --- a/test/nextjournal/clerk/eval_test.clj +++ b/test/nextjournal/clerk/eval_test.clj @@ -286,3 +286,27 @@ (testing "show with ns arg" (clerk/show! 'nextjournal.clerk.fixtures.hello) (is (fs/exists? (:file (meta (resolve 'nextjournal.clerk.fixtures.hello/answer))))))) + +(deftest macro-analysis-test + (testing "macros are executed before analysis such that expressions relying on + them get properly cached and executed once" + (remove-ns 'my-random-namespace) + (remove-ns 'fixture-ns) + (clerk/clear-cache!) + (let [fixture-ns "(ns fixture-ns) (def state (atom 0))" + _ (eval/eval-string fixture-ns) + ns "(ns my-random-namepace (:require [fixture-ns])) + +(defn helper-fn [x] x) + +(defmacro my-macro [x] (helper-fn `(do ~x ~x))) + +(my-macro (swap! fixture-ns/state inc)) + +@fixture-ns/state" + first (do (eval/eval-string ns) + @@(resolve 'fixture-ns/state)) + _ (eval/eval-string fixture-ns) + second (do (eval/eval-string ns) + @@(resolve 'fixture-ns/state))] + (is (= [2 0] [first second]))))) From f552c187d8ec37bb4a20d4e5b5ab4f9ad0a5ec12 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 23 Sep 2025 15:40:39 +0200 Subject: [PATCH 04/24] fix bb --- src/nextjournal/clerk/analyzer/impl.clj | 4 +++- test/nextjournal/clerk/eval_test.clj | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 168a189f7..85230ed33 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -431,7 +431,9 @@ :ns ns :resolved-to v :type (type v)}))) - (let [meta (-> (dissoc (meta sym) :inline :inline-arities) + (let [meta (-> (dissoc (meta sym) :inline :inline-arities + ;; babashka has :macro on var symbol through defmacro + :macro) (update-vals unquote'))] (intern (ns-sym ns) (with-meta sym meta)))))) diff --git a/test/nextjournal/clerk/eval_test.clj b/test/nextjournal/clerk/eval_test.clj index 90a7838d5..d1fcb047a 100644 --- a/test/nextjournal/clerk/eval_test.clj +++ b/test/nextjournal/clerk/eval_test.clj @@ -295,7 +295,7 @@ (clerk/clear-cache!) (let [fixture-ns "(ns fixture-ns) (def state (atom 0))" _ (eval/eval-string fixture-ns) - ns "(ns my-random-namepace (:require [fixture-ns])) + ns "(ns my-random-namespace (:require [fixture-ns])) (defn helper-fn [x] x) From 36507fc70cd72982a2b92280c8e5cfb981509466 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 23 Sep 2025 16:58:30 +0200 Subject: [PATCH 05/24] debug windows --- src/nextjournal/clerk/analyzer.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index 105432fe9..f5079bb39 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -444,6 +444,7 @@ (defn var->location [var] (when-let [file (:file (meta var))] + (prn :file file) (some-> (if (fs/absolute? file) (when (fs/exists? file) (fs/relativize (fs/cwd) (fs/file file))) From ee347f6e48f06187b513f6a9492b76160622e435 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 23 Sep 2025 17:09:50 +0200 Subject: [PATCH 06/24] Fix bb windows --- src/nextjournal/clerk/analyzer.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index f5079bb39..3fbb398d2 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -445,7 +445,10 @@ (defn var->location [var] (when-let [file (:file (meta var))] (prn :file file) - (some-> (if (fs/absolute? file) + (some-> (if (try (fs/absolute? file) + ;; fs/absolute? crashes in bb on Windows due to the :file + ;; metadata containing "" + (catch Exception _ false)) (when (fs/exists? file) (fs/relativize (fs/cwd) (fs/file file))) (when-let [resource (io/resource file)] From edd517b8c59c1d1197ee35032d0d865c5b6985b9 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 23 Sep 2025 17:10:14 +0200 Subject: [PATCH 07/24] remove :file --- src/nextjournal/clerk/analyzer.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index 3fbb398d2..50802e4a5 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -444,7 +444,6 @@ (defn var->location [var] (when-let [file (:file (meta var))] - (prn :file file) (some-> (if (try (fs/absolute? file) ;; fs/absolute? crashes in bb on Windows due to the :file ;; metadata containing "" From 416f30214f41c79ab52e7518ff764ab9d7cf4338 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 23 Sep 2025 17:28:30 +0200 Subject: [PATCH 08/24] wip --- src/nextjournal/clerk/analyzer/impl.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 85230ed33..7b31857cc 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -383,7 +383,6 @@ (if (and (var? maybe-macro) (:macro (meta maybe-macro))) (do - ;; macroexpand is the macro here, we should register in the env that the var about to be def-ed is a macro (swap! *deps* conj maybe-macro) (let [expanded (macroexpand-hook maybe-macro form env (rest form)) env (if (identical? #'defmacro maybe-macro) From a445889a779af190a3532b87847c4687f063b418 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 23 Sep 2025 20:43:47 +0200 Subject: [PATCH 09/24] wip --- src/nextjournal/clerk/analyzer.clj | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index f99aed4bd..e6675ba30 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -501,10 +501,27 @@ (filter (comp #{:code} :type) blocks)))) -(defn transitive-deps [id analysis-info] - (let [{:keys [deps]} (get analysis-info id)] - (when deps - (into deps (mapcat #(transitive-deps % analysis-info) deps))))) +(defn transitive-deps + ([id analysis-info] + (loop [seen #{} + deps #{id} + res #{}] + (if (seq deps) + (let [dep (first deps)] + (if (contains? seen dep) + (recur seen (rest deps) res) + (let [{new-deps :deps} (get analysis-info dep) + seen (conj seen dep) + deps (concat (rest deps) new-deps) + res (into res deps)] + (recur seen deps res)))) + res)))) + +#_(transitive-deps id analysis-info) + +#_(transitive-deps :main {:main {:deps [:main :other]} + :other {:deps [:another]} + :another {:deps [:another-one :another :main]}}) (defn run-macros [init-state] (let [{:keys [blocks ->analysis-info]} init-state @@ -535,7 +552,7 @@ :counter 0 :graph (dep/graph))) init-state (init-state-fn) - ran-macros? (run-macros init-state) + ran-macros? false #_(run-macros init-state) init-state (if ran-macros? (init-state-fn) init-state)] From 76c9947d6e3eee2c7a74229d6c687c018021e60a Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Mon, 29 Sep 2025 16:39:33 +0200 Subject: [PATCH 10/24] wip [skip ci] --- src/nextjournal/clerk/analyzer.clj | 24 ++++++++++++------- src/nextjournal/clerk/analyzer/impl.clj | 2 ++ test/nextjournal/clerk/eval_test.clj | 32 +++++++++++++++++++------ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index e6675ba30..5566a4723 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -552,10 +552,11 @@ :counter 0 :graph (dep/graph))) init-state (init-state-fn) - ran-macros? false #_(run-macros init-state) + ran-macros? (run-macros init-state) init-state (if ran-macros? (init-state-fn) init-state)] + ;; #dbg (def istate1 init-state) (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} init-state] (let [unhashed (unhashed-deps ->analysis-info) loc->syms (apply dissoc @@ -648,14 +649,19 @@ (record-missing-hash-fn (assoc codeblock :dep-with-missing-hash dep-with-missing-hash :graph-node graph-node :ns ns)))) - (binding [*print-length* nil] - (let [form-with-deps-sorted - (-> hashed-deps - (conj (if form - (-> form remove-type-meta canonicalize-form pr-str) - hash)) - (into (map str) vars))] - (sha1-base58 (pr-str form-with-deps-sorted)))))) + (let [res (binding [*print-length* nil] + (let [form-with-deps-sorted + (-> hashed-deps + (conj (if form + (-> form remove-type-meta canonicalize-form pr-str) + hash)) + (into (map str) vars))] + (sha1-base58 (pr-str form-with-deps-sorted))))] + ;; (when (= '(def a1 (do (swap! fixture-ns/state inc) (attempt1 9999))) + ;; form) + ;; #dbg (def all [res form deps hashed-deps])) + res) + )) #_(hash-codeblock {} {:graph (dep/graph)} {}) #_(hash-codeblock {} {:graph (dep/graph)} {:hash "foo"}) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 7b31857cc..66297fbb5 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -383,6 +383,8 @@ (if (and (var? maybe-macro) (:macro (meta maybe-macro))) (do + (when (= "my-random-namespace" (namespace (symbol maybe-macro))) + (prn :var maybe-macro)) (swap! *deps* conj maybe-macro) (let [expanded (macroexpand-hook maybe-macro form env (rest form)) env (if (identical? #'defmacro maybe-macro) diff --git a/test/nextjournal/clerk/eval_test.clj b/test/nextjournal/clerk/eval_test.clj index 3b6ba3a33..18e0c66ee 100644 --- a/test/nextjournal/clerk/eval_test.clj +++ b/test/nextjournal/clerk/eval_test.clj @@ -294,22 +294,40 @@ (remove-ns 'fixture-ns) (clerk/clear-cache!) (let [fixture-ns "(ns fixture-ns) (def state (atom 0))" - _ (eval/eval-string fixture-ns) + _ (load-string fixture-ns) ns "(ns my-random-namespace (:require [fixture-ns])) -(defn helper-fn [x] x) +;; this function should be run during macro pre-analysis +(defn helper-fn-compile-time [x] x) -(defmacro my-macro [x] (helper-fn `(do ~x ~x))) +(defmacro attempt1 + [x] + (helper-fn-compile-time + `(try + (do (helper-fn-runtime ~x)) + (catch Exception e# e#)))) -(my-macro (swap! fixture-ns/state inc)) +(defn helper-fn-runtime [x] x) + +;; a1 has dependency on helper-fn-runtime, but this isn't clear when we don't macro-expand before analysis +;; if we don't, then a1 gets a different hash compared to the next time and we get twice the side effects +(def a1 (do + (swap! fixture-ns/state inc) + (rand-int (attempt1 9999)))) @fixture-ns/state" + _ (prn :first-eval) first (do (eval/eval-string ns) @@(resolve 'fixture-ns/state)) - _ (eval/eval-string fixture-ns) + first-rand @(resolve 'my-random-namespace/a1) + _ (prn :second-eval) second (do (eval/eval-string ns) - @@(resolve 'fixture-ns/state))] - (is (= [2 0] [first second]))))) + @@(resolve 'fixture-ns/state)) + second-rand @(resolve 'my-random-namespace/a1)] + (is (= first-rand second-rand)) + (is (= 1 first second))))) + +#_@(resolve 'my-random-namespace/a1) (deftest issue-741-can-eval-quoted-regex-test (is (match? {:blocks [{:type :code, From 4cd0d8ec707988104083b19bcd400686701b0ddc Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 30 Sep 2025 15:04:00 +0200 Subject: [PATCH 11/24] fail [skip ci] --- notebooks/my_random_namespace.clj | 27 +++++++++++++++++++++++++ src/nextjournal/clerk/analyzer.clj | 21 +++++++++++-------- src/nextjournal/clerk/analyzer/impl.clj | 13 +++++++++--- test/nextjournal/clerk/eval_test.clj | 7 ++++--- 4 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 notebooks/my_random_namespace.clj diff --git a/notebooks/my_random_namespace.clj b/notebooks/my_random_namespace.clj new file mode 100644 index 000000000..b8bde2771 --- /dev/null +++ b/notebooks/my_random_namespace.clj @@ -0,0 +1,27 @@ +(ns my-random-namespace + (:require [fixture-ns])) + +;; this function should be run during macro pre-analysis +(defn helper-fn-compile-time [x] x) + +(defn helper-fn-runtime [x] x) + +(defmacro attempt1 + [x] + (helper-fn-compile-time + `(try + (do (helper-fn-runtime ~x)) + (catch Exception e# e#)))) + +;; a1 has dependency on helper-fn-runtime, but this isn't clear when we don't macro-expand before analysis +;; if we don't, then a1 gets a different hash compared to the next time and we get twice the side effects +(def a1 (do + (swap! fixture-ns/state inc) + (rand-int (attempt1 9999)))) + +@fixture-ns/state + +#_(do (reset! fixture-ns/state 0) + (remove-ns 'my-random-namespace) + (nextjournal.clerk/clear-cache!) + (create-ns 'my-random-namespace)) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index 5566a4723..c744929b0 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -338,6 +338,7 @@ (:file doc) (assoc :file (:file doc))) block+analysis (add-block-id (merge block form-analysis))] (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*` + (prn :eval form) (eval form)) (-> state (store-info block+analysis) @@ -532,6 +533,7 @@ all-blocks (filter #(contains? all-block-ids (:id %)) blocks)] (doseq [block all-blocks] (try + (println "loading" (:text block)) (load-string (:text block)) (catch Throwable e (binding [*out* *err*] @@ -553,6 +555,7 @@ :graph (dep/graph))) init-state (init-state-fn) ran-macros? (run-macros init-state) + ;; _ (prn :ran-macros? ran-macros?) init-state (if ran-macros? (init-state-fn) init-state)] @@ -580,11 +583,12 @@ state loc->syms) (update :counter inc))) - (-> state - analyze-doc-deps - set-no-cache-on-redefs - make-deps-inherit-no-cache - (dissoc :analyzed-file-set :counter))))))) + (let [res (-> state + analyze-doc-deps + set-no-cache-on-redefs + make-deps-inherit-no-cache + (dissoc :analyzed-file-set :counter))] + res)))))) (comment (reset! !file->analysis-cache {}) @@ -657,9 +661,10 @@ hash)) (into (map str) vars))] (sha1-base58 (pr-str form-with-deps-sorted))))] - ;; (when (= '(def a1 (do (swap! fixture-ns/state inc) (attempt1 9999))) - ;; form) - ;; #dbg (def all [res form deps hashed-deps])) + (when (= '(def a1 (do (swap! fixture-ns/state inc) (rand-int (attempt1 9999)))) + form) + (prn :deps-x-hashes (sort (zipmap deps (map ->hash deps)))) + (prn :res res)) res) )) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 66297fbb5..be06550ea 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -38,9 +38,15 @@ (let [local? (and (simple-symbol? sym) (contains? (:locals env) sym))] (when-not local? - (when (symbol? sym) + (when (symbol? sym) ;; TODO: we already checkd for symbol? (let [sym-ns (when-let [ns (namespace sym)] (symbol ns)) full-ns (resolve-ns sym-ns env)] + (let [sym-name (-> sym name symbol)] + (when (= 'attempt1 sym-name) + (prn (when (or (not sym-ns) full-ns) + (let [name (if sym-ns (-> sym name symbol) sym)] + (binding [*ns* (or full-ns ns)] + (resolve name))))))) (when (or (not sym-ns) full-ns) (let [name (if sym-ns (-> sym name symbol) sym)] (binding [*ns* (or full-ns ns)] @@ -384,7 +390,7 @@ (:macro (meta maybe-macro))) (do (when (= "my-random-namespace" (namespace (symbol maybe-macro))) - (prn :var maybe-macro)) + (prn :var maybe-macro (:macro (meta maybe-macro)))) (swap! *deps* conj maybe-macro) (let [expanded (macroexpand-hook maybe-macro form env (rest form)) env (if (identical? #'defmacro maybe-macro) @@ -436,7 +442,8 @@ ;; babashka has :macro on var symbol through defmacro :macro) (update-vals unquote'))] - (intern (ns-sym ns) (with-meta sym meta)))))) + (doto (intern (ns-sym ns) (with-meta sym meta)) + prn))))) (defmethod -parse 'def [{:keys [ns] :as env} [_ sym & expr :as form]] (let [pfn (fn diff --git a/test/nextjournal/clerk/eval_test.clj b/test/nextjournal/clerk/eval_test.clj index 18e0c66ee..16449123a 100644 --- a/test/nextjournal/clerk/eval_test.clj +++ b/test/nextjournal/clerk/eval_test.clj @@ -300,15 +300,15 @@ ;; this function should be run during macro pre-analysis (defn helper-fn-compile-time [x] x) +(defn helper-fn-runtime [x] x) + (defmacro attempt1 [x] (helper-fn-compile-time `(try - (do (helper-fn-runtime ~x)) + (do (do helper-fn-runtime ~x)) (catch Exception e# e#)))) -(defn helper-fn-runtime [x] x) - ;; a1 has dependency on helper-fn-runtime, but this isn't clear when we don't macro-expand before analysis ;; if we don't, then a1 gets a different hash compared to the next time and we get twice the side effects (def a1 (do @@ -320,6 +320,7 @@ first (do (eval/eval-string ns) @@(resolve 'fixture-ns/state)) first-rand @(resolve 'my-random-namespace/a1) + _ (prn :first-rand first-rand) _ (prn :second-eval) second (do (eval/eval-string ns) @@(resolve 'fixture-ns/state)) From 315f2425af12d6208756262e77a9eae640fa2a4b Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 30 Sep 2025 17:51:49 +0200 Subject: [PATCH 12/24] wip [skip ci] --- notebooks/my_random_namespace.clj | 32 ++++++++-------- src/nextjournal/clerk.clj | 1 + src/nextjournal/clerk/eval.clj | 15 +++++--- test/nextjournal/clerk/eval_test.clj | 55 ++++++++++++---------------- 4 files changed, 49 insertions(+), 54 deletions(-) diff --git a/notebooks/my_random_namespace.clj b/notebooks/my_random_namespace.clj index b8bde2771..ff12a3f90 100644 --- a/notebooks/my_random_namespace.clj +++ b/notebooks/my_random_namespace.clj @@ -1,27 +1,25 @@ -(ns my-random-namespace - (:require [fixture-ns])) +(ns my-random-namespace) -;; this function should be run during macro pre-analysis -(defn helper-fn-compile-time [x] x) - -(defn helper-fn-runtime [x] x) +(defn macro-helper* [x] x) (defmacro attempt1 - [x] - (helper-fn-compile-time - `(try - (do (helper-fn-runtime ~x)) - (catch Exception e# e#)))) + [& body] + `(macro-helper* (try + (do ~@body) + (catch Exception e# e#)))) -;; a1 has dependency on helper-fn-runtime, but this isn't clear when we don't macro-expand before analysis -;; if we don't, then a1 gets a different hash compared to the next time and we get twice the side effects -(def a1 (do - (swap! fixture-ns/state inc) - (rand-int (attempt1 9999)))) -@fixture-ns/state +(def a1 + (do + (println "a1") + (attempt1 (rand-int 9999)))) #_(do (reset! fixture-ns/state 0) (remove-ns 'my-random-namespace) (nextjournal.clerk/clear-cache!) (create-ns 'my-random-namespace)) + + + + + diff --git a/src/nextjournal/clerk.clj b/src/nextjournal/clerk.clj index 0a63cfd85..421c227e5 100644 --- a/src/nextjournal/clerk.clj +++ b/src/nextjournal/clerk.clj @@ -69,6 +69,7 @@ _ (reset! !last-file file) {:keys [blob->result]} @webserver/!doc {:keys [result time-ms]} (eval/time-ms (binding [paths/*build-opts* (webserver/get-build-opts)] + (prn :clerk-show-ns *ns*) (eval/+eval-results blob->result (assoc doc :set-status-fn webserver/set-status!))))] (if (:error result) (println (str "Clerk encountered an error evaluating '" file "' after " time-ms "ms.")) diff --git a/src/nextjournal/clerk/eval.clj b/src/nextjournal/clerk/eval.clj index 915bf5ed0..a6a5ce513 100644 --- a/src/nextjournal/clerk/eval.clj +++ b/src/nextjournal/clerk/eval.clj @@ -285,7 +285,6 @@ (if (cljs? parsed-doc) (process-cljs parsed-doc) (let [{:as analyzed-doc :keys [ns]} - (cond no-cache parsed-doc @@ -297,16 +296,22 @@ (do (when set-status-fn (set-status-fn {:progress 0.10 :status "Analyzing…"})) - (-> parsed-doc - (assoc :blob->result in-memory-cache) - analyzer/build-graph - analyzer/hash)))] + ;; this fixes something if I set it to the namespace of the notebook... why + (prn :nsss *ns*) + (binding [#_#_*ns* (find-ns 'clojure.core)] + (-> parsed-doc + (assoc :blob->result in-memory-cache) + analyzer/build-graph + analyzer/hash))))] (when (and (not-empty (:var->block-id analyzed-doc)) (not ns)) (throw (ex-info "namespace must be set" (select-keys analyzed-doc [:file :ns])))) (binding [*ns* ns] + (prn :ns ns) (eval-analyzed-doc analyzed-doc))))) + + (defn eval-doc "Evaluates the given `doc`." ([doc] (eval-doc {} doc)) diff --git a/test/nextjournal/clerk/eval_test.clj b/test/nextjournal/clerk/eval_test.clj index 16449123a..bbb29d2a6 100644 --- a/test/nextjournal/clerk/eval_test.clj +++ b/test/nextjournal/clerk/eval_test.clj @@ -293,40 +293,31 @@ (remove-ns 'my-random-namespace) (remove-ns 'fixture-ns) (clerk/clear-cache!) - (let [fixture-ns "(ns fixture-ns) (def state (atom 0))" - _ (load-string fixture-ns) - ns "(ns my-random-namespace (:require [fixture-ns])) - -;; this function should be run during macro pre-analysis -(defn helper-fn-compile-time [x] x) - -(defn helper-fn-runtime [x] x) + (binding [;; somehow this is necessary to make the test pass... why? + #_#_*ns* (create-ns 'my-random-namespace)] + (let [fixture-ns "(ns fixture-ns) (def state (atom 0))" + _ (load-string fixture-ns) + ns "(ns my-random-namespace) +(defn macro-helper* [x] x) (defmacro attempt1 - [x] - (helper-fn-compile-time - `(try - (do (do helper-fn-runtime ~x)) - (catch Exception e# e#)))) - -;; a1 has dependency on helper-fn-runtime, but this isn't clear when we don't macro-expand before analysis -;; if we don't, then a1 gets a different hash compared to the next time and we get twice the side effects -(def a1 (do - (swap! fixture-ns/state inc) - (rand-int (attempt1 9999)))) - -@fixture-ns/state" - _ (prn :first-eval) - first (do (eval/eval-string ns) - @@(resolve 'fixture-ns/state)) - first-rand @(resolve 'my-random-namespace/a1) - _ (prn :first-rand first-rand) - _ (prn :second-eval) - second (do (eval/eval-string ns) - @@(resolve 'fixture-ns/state)) - second-rand @(resolve 'my-random-namespace/a1)] - (is (= first-rand second-rand)) - (is (= 1 first second))))) + [& body] + `(macro-helper* (try + (do ~@body) + (catch Exception e# e#)))) + + +(def a1 + (do + (println \"a1\") + (attempt1 (rand-int 9999))))" + _ (prn :first-eval) + _ (eval/eval-string ns) + first-rand @(resolve 'my-random-namespace/a1) + _ (eval/eval-string ns) + second-rand @(resolve 'my-random-namespace/a1)] + (is (= first-rand second-rand)) + #_(is (= 1 first second)))))) #_@(resolve 'my-random-namespace/a1) From 75294d61dfbf7a4d2c621fdc9588e9a4027cfb4a Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 30 Sep 2025 20:49:02 +0200 Subject: [PATCH 13/24] wip [skip ci] --- src/nextjournal/clerk/analyzer.clj | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index c744929b0..f6c57b46d 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -533,7 +533,7 @@ all-blocks (filter #(contains? all-block-ids (:id %)) blocks)] (doseq [block all-blocks] (try - (println "loading" (:text block)) + (println "loading in namespace" *ns* (:text block)) (load-string (:text block)) (catch Throwable e (binding [*out* *err*] @@ -661,7 +661,10 @@ hash)) (into (map str) vars))] (sha1-base58 (pr-str form-with-deps-sorted))))] - (when (= '(def a1 (do (swap! fixture-ns/state inc) (rand-int (attempt1 9999)))) + (when (= '(def a1 + (do + (println "a1") + (attempt1 (rand-int 9999)))) form) (prn :deps-x-hashes (sort (zipmap deps (map ->hash deps)))) (prn :res res)) From 85bd868feda83d7fc4380b969a958fdbe4195ca6 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 14 Oct 2025 17:24:25 +0200 Subject: [PATCH 14/24] somehow not expanding when evaluated from other ns --- notebooks/my_random_namespace.clj | 4 +--- src/nextjournal/clerk/analyzer/impl.clj | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/notebooks/my_random_namespace.clj b/notebooks/my_random_namespace.clj index ff12a3f90..b00e18e4a 100644 --- a/notebooks/my_random_namespace.clj +++ b/notebooks/my_random_namespace.clj @@ -8,14 +8,12 @@ (do ~@body) (catch Exception e# e#)))) - (def a1 (do (println "a1") (attempt1 (rand-int 9999)))) -#_(do (reset! fixture-ns/state 0) - (remove-ns 'my-random-namespace) +#_(do (remove-ns 'my-random-namespace) (nextjournal.clerk/clear-cache!) (create-ns 'my-random-namespace)) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index be06550ea..61119c6e7 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -81,7 +81,10 @@ (apply #'clojure.core/definterface &form &env name sigs))) (defmethod macroexpand-hook :default [the-var &form &env args] - (apply the-var &form (:locals &env) args)) + (let [expansion (apply the-var &form (:locals &env) args)] + (when (= the-var (resolve 'my-random-namespace/attempt1)) + (prn :macroexpand expansion)) + expansion)) (defmulti -parse (fn [_env form] (and (seq? form) (first form)))) From 4354a652475af715811238e793764cb2cf0d3cbf Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 14 Oct 2025 20:50:30 +0200 Subject: [PATCH 15/24] wip --- src/nextjournal/clerk/analyzer.clj | 2 + src/nextjournal/clerk/analyzer/impl.clj | 5 ++- src/nextjournal/clerk/eval.clj | 55 ++++++++++++------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index f6c57b46d..0c09ab18c 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -548,6 +548,8 @@ Recursively descends into dependency vars as well if they can be found in the classpath. " [doc] + (def d doc) ;; NOTE: doc has :ns! + (prn :build-graph *ns* ) (let [init-state-fn #(-> doc analyze-doc (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc))) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 61119c6e7..15c85b326 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -389,11 +389,12 @@ (if (seq? form) (let [[f & args] form maybe-macro (resolve-sym f env)] + (when (= "my-random-namespace" (when (var? maybe-macro) + (-> maybe-macro symbol namespace))) + (prn :var maybe-macro (:macro (meta maybe-macro)))) (if (and (var? maybe-macro) (:macro (meta maybe-macro))) (do - (when (= "my-random-namespace" (namespace (symbol maybe-macro))) - (prn :var maybe-macro (:macro (meta maybe-macro)))) (swap! *deps* conj maybe-macro) (let [expanded (macroexpand-hook maybe-macro form env (rest form)) env (if (identical? #'defmacro maybe-macro) diff --git a/src/nextjournal/clerk/eval.clj b/src/nextjournal/clerk/eval.clj index a6a5ce513..632476307 100644 --- a/src/nextjournal/clerk/eval.clj +++ b/src/nextjournal/clerk/eval.clj @@ -281,37 +281,34 @@ (defn +eval-results "Evaluates the given `parsed-doc` using the `in-memory-cache` and augments it with the results." - [in-memory-cache {:as parsed-doc :keys [set-status-fn no-cache]}] - (if (cljs? parsed-doc) - (process-cljs parsed-doc) - (let [{:as analyzed-doc :keys [ns]} - (cond - no-cache - parsed-doc - - config/cache-disabled? - (assoc parsed-doc :no-cache true) - - :else - (do - (when set-status-fn - (set-status-fn {:progress 0.10 :status "Analyzing…"})) - ;; this fixes something if I set it to the namespace of the notebook... why - (prn :nsss *ns*) - (binding [#_#_*ns* (find-ns 'clojure.core)] - (-> parsed-doc - (assoc :blob->result in-memory-cache) - analyzer/build-graph - analyzer/hash))))] - (when (and (not-empty (:var->block-id analyzed-doc)) - (not ns)) - (throw (ex-info "namespace must be set" (select-keys analyzed-doc [:file :ns])))) - (binding [*ns* ns] - (prn :ns ns) + [in-memory-cache {:as parsed-doc :keys [ns set-status-fn no-cache]}] + (binding [*ns* ns] + (if (cljs? parsed-doc) + (process-cljs parsed-doc) + (let [{:as analyzed-doc :keys [ns]} + (cond + no-cache + parsed-doc + + config/cache-disabled? + (assoc parsed-doc :no-cache true) + + :else + (do + (when set-status-fn + (set-status-fn {:progress 0.10 :status "Analyzing…"})) + ;; this fixes something if I set it to the namespace of the notebook... why + (prn :nsss *ns*) + (binding [#_#_*ns* (find-ns 'clojure.core)] + (-> parsed-doc + (assoc :blob->result in-memory-cache) + analyzer/build-graph + analyzer/hash))))] + (when (and (not-empty (:var->block-id analyzed-doc)) + (not ns)) + (throw (ex-info "namespace must be set" (select-keys analyzed-doc [:file :ns])))) (eval-analyzed-doc analyzed-doc))))) - - (defn eval-doc "Evaluates the given `doc`." ([doc] (eval-doc {} doc)) From 250cf7dffefc1c907ba53311b34da539e5e5fae7 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 14 Oct 2025 20:53:01 +0200 Subject: [PATCH 16/24] wip --- src/nextjournal/clerk.clj | 1 - src/nextjournal/clerk/analyzer.clj | 3 --- 2 files changed, 4 deletions(-) diff --git a/src/nextjournal/clerk.clj b/src/nextjournal/clerk.clj index 421c227e5..0a63cfd85 100644 --- a/src/nextjournal/clerk.clj +++ b/src/nextjournal/clerk.clj @@ -69,7 +69,6 @@ _ (reset! !last-file file) {:keys [blob->result]} @webserver/!doc {:keys [result time-ms]} (eval/time-ms (binding [paths/*build-opts* (webserver/get-build-opts)] - (prn :clerk-show-ns *ns*) (eval/+eval-results blob->result (assoc doc :set-status-fn webserver/set-status!))))] (if (:error result) (println (str "Clerk encountered an error evaluating '" file "' after " time-ms "ms.")) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index 0c09ab18c..90a5b1d36 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -338,7 +338,6 @@ (:file doc) (assoc :file (:file doc))) block+analysis (add-block-id (merge block form-analysis))] (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*` - (prn :eval form) (eval form)) (-> state (store-info block+analysis) @@ -548,8 +547,6 @@ Recursively descends into dependency vars as well if they can be found in the classpath. " [doc] - (def d doc) ;; NOTE: doc has :ns! - (prn :build-graph *ns* ) (let [init-state-fn #(-> doc analyze-doc (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc))) From 2b5b9b9f2a6ce71f52057a0d95e54bf0558e2fcb Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 14 Oct 2025 20:54:14 +0200 Subject: [PATCH 17/24] wip --- src/nextjournal/clerk/eval.clj | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/nextjournal/clerk/eval.clj b/src/nextjournal/clerk/eval.clj index 632476307..762ac5cc5 100644 --- a/src/nextjournal/clerk/eval.clj +++ b/src/nextjournal/clerk/eval.clj @@ -298,12 +298,10 @@ (when set-status-fn (set-status-fn {:progress 0.10 :status "Analyzing…"})) ;; this fixes something if I set it to the namespace of the notebook... why - (prn :nsss *ns*) - (binding [#_#_*ns* (find-ns 'clojure.core)] - (-> parsed-doc - (assoc :blob->result in-memory-cache) - analyzer/build-graph - analyzer/hash))))] + (-> parsed-doc + (assoc :blob->result in-memory-cache) + analyzer/build-graph + analyzer/hash)))] (when (and (not-empty (:var->block-id analyzed-doc)) (not ns)) (throw (ex-info "namespace must be set" (select-keys analyzed-doc [:file :ns])))) From 01607351c0c7925bfdee258e0efce4d84425a637 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 14 Oct 2025 20:55:17 +0200 Subject: [PATCH 18/24] wip --- src/nextjournal/clerk/analyzer.clj | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index 90a5b1d36..778d5d1a9 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -554,11 +554,9 @@ :graph (dep/graph))) init-state (init-state-fn) ran-macros? (run-macros init-state) - ;; _ (prn :ran-macros? ran-macros?) init-state (if ran-macros? (init-state-fn) init-state)] - ;; #dbg (def istate1 init-state) (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} init-state] (let [unhashed (unhashed-deps ->analysis-info) loc->syms (apply dissoc From ae8512d348557037a53210b10d542dfa2d1659d9 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 14 Oct 2025 21:05:59 +0200 Subject: [PATCH 19/24] wip --- src/nextjournal/clerk/analyzer.clj | 38 +++++++++---------------- src/nextjournal/clerk/analyzer/impl.clj | 6 +--- src/nextjournal/clerk/eval.clj | 1 - test/nextjournal/clerk/eval_test.clj | 18 ++++-------- 4 files changed, 21 insertions(+), 42 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index 778d5d1a9..5d26086a5 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -532,7 +532,7 @@ all-blocks (filter #(contains? all-block-ids (:id %)) blocks)] (doseq [block all-blocks] (try - (println "loading in namespace" *ns* (:text block)) + ;; (println "loading in namespace" *ns* (:text block)) (load-string (:text block)) (catch Throwable e (binding [*out* *err*] @@ -580,12 +580,11 @@ state loc->syms) (update :counter inc))) - (let [res (-> state - analyze-doc-deps - set-no-cache-on-redefs - make-deps-inherit-no-cache - (dissoc :analyzed-file-set :counter))] - res)))))) + (-> state + analyze-doc-deps + set-no-cache-on-redefs + make-deps-inherit-no-cache + (dissoc :analyzed-file-set :counter))))))) (comment (reset! !file->analysis-cache {}) @@ -650,23 +649,14 @@ (record-missing-hash-fn (assoc codeblock :dep-with-missing-hash dep-with-missing-hash :graph-node graph-node :ns ns)))) - (let [res (binding [*print-length* nil] - (let [form-with-deps-sorted - (-> hashed-deps - (conj (if form - (-> form remove-type-meta canonicalize-form pr-str) - hash)) - (into (map str) vars))] - (sha1-base58 (pr-str form-with-deps-sorted))))] - (when (= '(def a1 - (do - (println "a1") - (attempt1 (rand-int 9999)))) - form) - (prn :deps-x-hashes (sort (zipmap deps (map ->hash deps)))) - (prn :res res)) - res) - )) + (binding [*print-length* nil] + (let [form-with-deps-sorted + (-> hashed-deps + (conj (if form + (-> form remove-type-meta canonicalize-form pr-str) + hash)) + (into (map str) vars))] + (sha1-base58 (pr-str form-with-deps-sorted)))))) #_(hash-codeblock {} {:graph (dep/graph)} {}) #_(hash-codeblock {} {:graph (dep/graph)} {:hash "foo"}) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 15c85b326..baa10e736 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -389,9 +389,6 @@ (if (seq? form) (let [[f & args] form maybe-macro (resolve-sym f env)] - (when (= "my-random-namespace" (when (var? maybe-macro) - (-> maybe-macro symbol namespace))) - (prn :var maybe-macro (:macro (meta maybe-macro)))) (if (and (var? maybe-macro) (:macro (meta maybe-macro))) (do @@ -446,8 +443,7 @@ ;; babashka has :macro on var symbol through defmacro :macro) (update-vals unquote'))] - (doto (intern (ns-sym ns) (with-meta sym meta)) - prn))))) + (intern (ns-sym ns) (with-meta sym meta)))))) (defmethod -parse 'def [{:keys [ns] :as env} [_ sym & expr :as form]] (let [pfn (fn diff --git a/src/nextjournal/clerk/eval.clj b/src/nextjournal/clerk/eval.clj index 762ac5cc5..4406d4b80 100644 --- a/src/nextjournal/clerk/eval.clj +++ b/src/nextjournal/clerk/eval.clj @@ -297,7 +297,6 @@ (do (when set-status-fn (set-status-fn {:progress 0.10 :status "Analyzing…"})) - ;; this fixes something if I set it to the namespace of the notebook... why (-> parsed-doc (assoc :blob->result in-memory-cache) analyzer/build-graph diff --git a/test/nextjournal/clerk/eval_test.clj b/test/nextjournal/clerk/eval_test.clj index bbb29d2a6..a03df9314 100644 --- a/test/nextjournal/clerk/eval_test.clj +++ b/test/nextjournal/clerk/eval_test.clj @@ -293,11 +293,7 @@ (remove-ns 'my-random-namespace) (remove-ns 'fixture-ns) (clerk/clear-cache!) - (binding [;; somehow this is necessary to make the test pass... why? - #_#_*ns* (create-ns 'my-random-namespace)] - (let [fixture-ns "(ns fixture-ns) (def state (atom 0))" - _ (load-string fixture-ns) - ns "(ns my-random-namespace) + (let [ns "(ns my-random-namespace) (defn macro-helper* [x] x) (defmacro attempt1 @@ -311,13 +307,11 @@ (do (println \"a1\") (attempt1 (rand-int 9999))))" - _ (prn :first-eval) - _ (eval/eval-string ns) - first-rand @(resolve 'my-random-namespace/a1) - _ (eval/eval-string ns) - second-rand @(resolve 'my-random-namespace/a1)] - (is (= first-rand second-rand)) - #_(is (= 1 first second)))))) + _ (eval/eval-string ns) + first-rand @(resolve 'my-random-namespace/a1) + _ (eval/eval-string ns) + second-rand @(resolve 'my-random-namespace/a1)] + (is (= first-rand second-rand))))) #_@(resolve 'my-random-namespace/a1) From f188eb3551dd7b7aa3c24f0cd8ab046836b41cfd Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Tue, 14 Oct 2025 21:07:11 +0200 Subject: [PATCH 20/24] simplify --- src/nextjournal/clerk/analyzer/impl.clj | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index baa10e736..3c5a0e0b4 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -81,10 +81,7 @@ (apply #'clojure.core/definterface &form &env name sigs))) (defmethod macroexpand-hook :default [the-var &form &env args] - (let [expansion (apply the-var &form (:locals &env) args)] - (when (= the-var (resolve 'my-random-namespace/attempt1)) - (prn :macroexpand expansion)) - expansion)) + (apply the-var &form (:locals &env) args)) (defmulti -parse (fn [_env form] (and (seq? form) (first form)))) From 9ad61ff0710dcc21e3513094b9eb87adc9a26cb4 Mon Sep 17 00:00:00 2001 From: Martin Kavalar Date: Wed, 15 Oct 2025 18:06:49 +1100 Subject: [PATCH 21/24] Try removig with-ns-bindings from test --- test/nextjournal/clerk/analyzer_test.clj | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/nextjournal/clerk/analyzer_test.clj b/test/nextjournal/clerk/analyzer_test.clj index ba4d8c6b1..92a765b76 100644 --- a/test/nextjournal/clerk/analyzer_test.clj +++ b/test/nextjournal/clerk/analyzer_test.clj @@ -389,8 +389,7 @@ my-uuid")] (is (empty? (ana/unhashed-deps ->analysis-info))) (is (match? {:jar string?} (->analysis-info 'weavejester.dependency/graph))))) (testing "should establish dependencies across files" - (let [{:keys [graph]} (with-ns-binding 'nextjournal.clerk - (analyze-string (slurp "src/nextjournal/clerk.clj")))] + (let [{:keys [graph]} (analyze-string (slurp "src/nextjournal/clerk.clj"))] (is (dep/depends? graph 'nextjournal.clerk/show! 'nextjournal.clerk.analyzer/hash))))) (deftest graph-nodes-with-anonymous-ids @@ -417,11 +416,10 @@ my-uuid")] (is (empty? (let [!missing-hash-store (atom [])] (reset! ana/!file->analysis-cache {}) - (with-ns-binding 'nextjournal.clerk - (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj") - ana/build-graph - (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry))) - ana/hash)) + (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj") + ana/build-graph + (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry))) + ana/hash) (deref !missing-hash-store))))) (testing "known cases where missing hashes occur" From e682ef8004f947affb8db4504189617ba8eeb427 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Wed, 15 Oct 2025 10:05:49 +0200 Subject: [PATCH 22/24] fix error output in missing hashes test --- src/nextjournal/clerk/analyzer.clj | 77 +++++++++++++++--------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index 5d26086a5..f8547ae70 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -547,44 +547,45 @@ Recursively descends into dependency vars as well if they can be found in the classpath. " [doc] - (let [init-state-fn #(-> doc - analyze-doc - (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc))) - :counter 0 - :graph (dep/graph))) - init-state (init-state-fn) - ran-macros? (run-macros init-state) - init-state (if ran-macros? - (init-state-fn) - init-state)] - (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} init-state] - (let [unhashed (unhashed-deps ->analysis-info) - loc->syms (apply dissoc - (group-by find-location unhashed) - analyzed-file-set)] - (if (and (seq loc->syms) (< counter 10)) - (recur (-> (reduce (fn [g [source symbols]] - (let [jar? (or (nil? source) - (str/ends-with? source ".jar")) - gitlib-hash (and (not jar?) - (second (re-find #".gitlibs/libs/.*/(\b[0-9a-f]{5,40}\b)/" (fs/unixify source))))] - (if (or jar? gitlib-hash) - (update g :->analysis-info merge (into {} (map (juxt identity - (constantly (if source - (or (when gitlib-hash {:hash gitlib-hash}) - (hash-jar source)) - {})))) symbols)) - (-> g - (update :analyzed-file-set conj source) - (merge-analysis-info (analyze-file source)))))) - state - loc->syms) - (update :counter inc))) - (-> state - analyze-doc-deps - set-no-cache-on-redefs - make-deps-inherit-no-cache - (dissoc :analyzed-file-set :counter))))))) + (binding [*ns* (:ns doc)] + (let [init-state-fn #(-> doc + analyze-doc + (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc))) + :counter 0 + :graph (dep/graph))) + init-state (init-state-fn) + ran-macros? (run-macros init-state) + init-state (if ran-macros? + (init-state-fn) + init-state)] + (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} init-state] + (let [unhashed (unhashed-deps ->analysis-info) + loc->syms (apply dissoc + (group-by find-location unhashed) + analyzed-file-set)] + (if (and (seq loc->syms) (< counter 10)) + (recur (-> (reduce (fn [g [source symbols]] + (let [jar? (or (nil? source) + (str/ends-with? source ".jar")) + gitlib-hash (and (not jar?) + (second (re-find #".gitlibs/libs/.*/(\b[0-9a-f]{5,40}\b)/" (fs/unixify source))))] + (if (or jar? gitlib-hash) + (update g :->analysis-info merge (into {} (map (juxt identity + (constantly (if source + (or (when gitlib-hash {:hash gitlib-hash}) + (hash-jar source)) + {})))) symbols)) + (-> g + (update :analyzed-file-set conj source) + (merge-analysis-info (analyze-file source)))))) + state + loc->syms) + (update :counter inc))) + (-> state + analyze-doc-deps + set-no-cache-on-redefs + make-deps-inherit-no-cache + (dissoc :analyzed-file-set :counter)))))))) (comment (reset! !file->analysis-cache {}) From a39b86c89a569db0d96754351ceb57ea8e1cc4a8 Mon Sep 17 00:00:00 2001 From: Martin Kavalar Date: Wed, 15 Oct 2025 21:52:38 +1100 Subject: [PATCH 23/24] Move comment to where it now applies --- src/nextjournal/clerk/analyzer.clj | 2 +- src/nextjournal/clerk/parser.cljc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index f8547ae70..516fc4c29 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -337,7 +337,7 @@ (let [{:as form-analysis :keys [ns-effect? form]} (cond-> (analyze (:form block)) (:file doc) (assoc :file (:file doc))) block+analysis (add-block-id (merge block form-analysis))] - (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*` + (when ns-effect? (eval form)) (-> state (store-info block+analysis) diff --git a/src/nextjournal/clerk/parser.cljc b/src/nextjournal/clerk/parser.cljc index bdf337f53..4ff8a7c9b 100644 --- a/src/nextjournal/clerk/parser.cljc +++ b/src/nextjournal/clerk/parser.cljc @@ -472,7 +472,7 @@ :text nstring :form (add-loc opts loc form) :loc loc}] - (when (ns? form) + (when (ns? form) ;; needs to run before setting doc `:ns` via `*ns*` (eval form)) (cond-> (-> state (assoc :add-comment-on-line? true) From b0c5a3eedf8af74dc063b7a7adaf0e38d8e83f00 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Wed, 15 Oct 2025 19:25:49 +0200 Subject: [PATCH 24/24] Remove cruft --- notebooks/my_random_namespace.clj | 23 ----------------------- src/nextjournal/clerk/analyzer/impl.clj | 6 ------ 2 files changed, 29 deletions(-) delete mode 100644 notebooks/my_random_namespace.clj diff --git a/notebooks/my_random_namespace.clj b/notebooks/my_random_namespace.clj deleted file mode 100644 index b00e18e4a..000000000 --- a/notebooks/my_random_namespace.clj +++ /dev/null @@ -1,23 +0,0 @@ -(ns my-random-namespace) - -(defn macro-helper* [x] x) - -(defmacro attempt1 - [& body] - `(macro-helper* (try - (do ~@body) - (catch Exception e# e#)))) - -(def a1 - (do - (println "a1") - (attempt1 (rand-int 9999)))) - -#_(do (remove-ns 'my-random-namespace) - (nextjournal.clerk/clear-cache!) - (create-ns 'my-random-namespace)) - - - - - diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 3c5a0e0b4..99762ca85 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -41,12 +41,6 @@ (when (symbol? sym) ;; TODO: we already checkd for symbol? (let [sym-ns (when-let [ns (namespace sym)] (symbol ns)) full-ns (resolve-ns sym-ns env)] - (let [sym-name (-> sym name symbol)] - (when (= 'attempt1 sym-name) - (prn (when (or (not sym-ns) full-ns) - (let [name (if sym-ns (-> sym name symbol) sym)] - (binding [*ns* (or full-ns ns)] - (resolve name))))))) (when (or (not sym-ns) full-ns) (let [name (if sym-ns (-> sym name symbol) sym)] (binding [*ns* (or full-ns ns)]