diff --git a/notebooks/my_random_namespace.clj b/notebooks/my_random_namespace.clj new file mode 100644 index 000000000..ff12a3f90 --- /dev/null +++ b/notebooks/my_random_namespace.clj @@ -0,0 +1,25 @@ +(ns my-random-namespace) + +(defn macro-helper* [x] x) + +(defmacro attempt1 + [& body] + `(macro-helper* (try + (do ~@body) + (catch Exception e# e#)))) + + +(def a1 + (do + (println "a1") + (attempt1 (rand-int 9999)))) + +#_(do (reset! fixture-ns/state 0) + (remove-ns 'my-random-namespace) + (nextjournal.clerk/clear-cache!) + (create-ns 'my-random-namespace)) + + + + + diff --git a/src/nextjournal/clerk.clj b/src/nextjournal/clerk.clj index 0a63cfd85..421c227e5 100644 --- a/src/nextjournal/clerk.clj +++ b/src/nextjournal/clerk.clj @@ -69,6 +69,7 @@ _ (reset! !last-file file) {:keys [blob->result]} @webserver/!doc {:keys [result time-ms]} (eval/time-ms (binding [paths/*build-opts* (webserver/get-build-opts)] + (prn :clerk-show-ns *ns*) (eval/+eval-results blob->result (assoc doc :set-status-fn webserver/set-status!))))] (if (:error result) (println (str "Clerk encountered an error evaluating '" file "' after " time-ms "ms.")) diff --git a/src/nextjournal/clerk/analyzer.clj b/src/nextjournal/clerk/analyzer.clj index 490de5a2a..f6c57b46d 100644 --- a/src/nextjournal/clerk/analyzer.clj +++ b/src/nextjournal/clerk/analyzer.clj @@ -156,14 +156,16 @@ deps (set/union (set/difference (into #{} (map (comp symbol var->protocol)) @!deps) vars) deref-deps (when (var? form) #{(symbol form)})) - hash-fn (-> form meta :nextjournal.clerk/hash-fn)] + hash-fn (-> form meta :nextjournal.clerk/hash-fn) + macro? (-> analyzed :env :defmacro)] (cond-> {#_#_:analyzed analyzed :form form :ns-effect? (some? (some #{'clojure.core/require 'clojure.core/in-ns} deps)) :freezable? (and (not (some #{'clojure.core/intern} deps)) (<= (count vars) 1) (if (seq vars) (= var (first vars)) true)) - :no-cache? (no-cache? form (-> def-node :form second) *ns*)} + :no-cache? (no-cache? form (-> def-node :form second) *ns*) + :macro macro?} hash-fn (assoc :hash-fn hash-fn) (seq deps) (assoc :deps deps) (seq deref-deps) (assoc :deref-deps deref-deps) @@ -336,6 +338,7 @@ (:file doc) (assoc :file (:file doc))) block+analysis (add-block-id (merge block form-analysis))] (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*` + (prn :eval form) (eval form)) (-> state (store-info block+analysis) @@ -442,7 +445,10 @@ (defn var->location [var] (when-let [file (:file (meta var))] - (some-> (if (fs/absolute? file) + (some-> (if (try (fs/absolute? file) + ;; fs/absolute? crashes in bb on Windows due to the :file + ;; metadata containing "" + (catch Exception _ false)) (when (fs/exists? file) (fs/relativize (fs/cwd) (fs/file file))) (when-let [resource (io/resource file)] @@ -496,45 +502,93 @@ (filter (comp #{:code} :type) blocks)))) +(defn transitive-deps + ([id analysis-info] + (loop [seen #{} + deps #{id} + res #{}] + (if (seq deps) + (let [dep (first deps)] + (if (contains? seen dep) + (recur seen (rest deps) res) + (let [{new-deps :deps} (get analysis-info dep) + seen (conj seen dep) + deps (concat (rest deps) new-deps) + res (into res deps)] + (recur seen deps res)))) + res)))) + +#_(transitive-deps id analysis-info) + +#_(transitive-deps :main {:main {:deps [:main :other]} + :other {:deps [:another]} + :another {:deps [:another-one :another :main]}}) + +(defn run-macros [init-state] + (let [{:keys [blocks ->analysis-info]} init-state + macro-block-ids (keep #(when (:macro %) + (:id %)) blocks) + deps (mapcat #(transitive-deps % ->analysis-info) macro-block-ids) + all-block-ids (into (set macro-block-ids) deps) + all-blocks (filter #(contains? all-block-ids (:id %)) blocks)] + (doseq [block all-blocks] + (try + (println "loading in namespace" *ns* (:text block)) + (load-string (:text block)) + (catch Throwable e + (binding [*out* *err*] + (println "Error when evaluating macro deps:" (:text block)) + (println "Namespace:" *ns*) + (println "Exception:" e))))) + (pos? (count all-blocks)))) + (defn build-graph "Analyzes the forms in the given file and builds a dependency graph of the vars. - Recursively decends into dependency vars as well as given they can be found in the classpath. + Recursively descends into dependency vars as well if they can be found in the classpath. " [doc] - (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} - (-> doc - analyze-doc - (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc))) - :counter 0 - :graph (dep/graph)))] - (let [unhashed (unhashed-deps ->analysis-info) - loc->syms (apply dissoc - (group-by find-location unhashed) - analyzed-file-set)] - (if (and (seq loc->syms) (< counter 10)) - (recur (-> (reduce (fn [g [source symbols]] - (let [jar? (or (nil? source) - (str/ends-with? source ".jar")) - gitlib-hash (and (not jar?) - (second (re-find #".gitlibs/libs/.*/(\b[0-9a-f]{5,40}\b)/" (fs/unixify source))))] - (if (or jar? gitlib-hash) - (update g :->analysis-info merge (into {} (map (juxt identity - (constantly (if source - (or (when gitlib-hash {:hash gitlib-hash}) - (hash-jar source)) - {})))) symbols)) - (-> g - (update :analyzed-file-set conj source) - (merge-analysis-info (analyze-file source)))))) - state - loc->syms) - (update :counter inc))) - (-> state - analyze-doc-deps - set-no-cache-on-redefs - make-deps-inherit-no-cache - (dissoc :analyzed-file-set :counter)))))) + (let [init-state-fn #(-> doc + analyze-doc + (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc))) + :counter 0 + :graph (dep/graph))) + init-state (init-state-fn) + ran-macros? (run-macros init-state) + ;; _ (prn :ran-macros? ran-macros?) + init-state (if ran-macros? + (init-state-fn) + init-state)] + ;; #dbg (def istate1 init-state) + (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} init-state] + (let [unhashed (unhashed-deps ->analysis-info) + loc->syms (apply dissoc + (group-by find-location unhashed) + analyzed-file-set)] + (if (and (seq loc->syms) (< counter 10)) + (recur (-> (reduce (fn [g [source symbols]] + (let [jar? (or (nil? source) + (str/ends-with? source ".jar")) + gitlib-hash (and (not jar?) + (second (re-find #".gitlibs/libs/.*/(\b[0-9a-f]{5,40}\b)/" (fs/unixify source))))] + (if (or jar? gitlib-hash) + (update g :->analysis-info merge (into {} (map (juxt identity + (constantly (if source + (or (when gitlib-hash {:hash gitlib-hash}) + (hash-jar source)) + {})))) symbols)) + (-> g + (update :analyzed-file-set conj source) + (merge-analysis-info (analyze-file source)))))) + state + loc->syms) + (update :counter inc))) + (let [res (-> state + analyze-doc-deps + set-no-cache-on-redefs + make-deps-inherit-no-cache + (dissoc :analyzed-file-set :counter))] + res)))))) (comment (reset! !file->analysis-cache {}) @@ -599,14 +653,23 @@ (record-missing-hash-fn (assoc codeblock :dep-with-missing-hash dep-with-missing-hash :graph-node graph-node :ns ns)))) - (binding [*print-length* nil] - (let [form-with-deps-sorted - (-> hashed-deps - (conj (if form - (-> form remove-type-meta canonicalize-form pr-str) - hash)) - (into (map str) vars))] - (sha1-base58 (pr-str form-with-deps-sorted)))))) + (let [res (binding [*print-length* nil] + (let [form-with-deps-sorted + (-> hashed-deps + (conj (if form + (-> form remove-type-meta canonicalize-form pr-str) + hash)) + (into (map str) vars))] + (sha1-base58 (pr-str form-with-deps-sorted))))] + (when (= '(def a1 + (do + (println "a1") + (attempt1 (rand-int 9999)))) + form) + (prn :deps-x-hashes (sort (zipmap deps (map ->hash deps)))) + (prn :res res)) + res) + )) #_(hash-codeblock {} {:graph (dep/graph)} {}) #_(hash-codeblock {} {:graph (dep/graph)} {:hash "foo"}) diff --git a/src/nextjournal/clerk/analyzer/impl.clj b/src/nextjournal/clerk/analyzer/impl.clj index 9dcb237fd..be06550ea 100644 --- a/src/nextjournal/clerk/analyzer/impl.clj +++ b/src/nextjournal/clerk/analyzer/impl.clj @@ -38,9 +38,15 @@ (let [local? (and (simple-symbol? sym) (contains? (:locals env) sym))] (when-not local? - (when (symbol? sym) + (when (symbol? sym) ;; TODO: we already checkd for symbol? (let [sym-ns (when-let [ns (namespace sym)] (symbol ns)) full-ns (resolve-ns sym-ns env)] + (let [sym-name (-> sym name symbol)] + (when (= 'attempt1 sym-name) + (prn (when (or (not sym-ns) full-ns) + (let [name (if sym-ns (-> sym name symbol) sym)] + (binding [*ns* (or full-ns ns)] + (resolve name))))))) (when (or (not sym-ns) full-ns) (let [name (if sym-ns (-> sym name symbol) sym)] (binding [*ns* (or full-ns ns)] @@ -383,8 +389,13 @@ (if (and (var? maybe-macro) (:macro (meta maybe-macro))) (do + (when (= "my-random-namespace" (namespace (symbol maybe-macro))) + (prn :var maybe-macro (:macro (meta maybe-macro)))) (swap! *deps* conj maybe-macro) - (let [expanded (macroexpand-hook maybe-macro form env (rest form))] + (let [expanded (macroexpand-hook maybe-macro form env (rest form)) + env (if (identical? #'defmacro maybe-macro) + (assoc env :defmacro true) + env)] (analyze* env expanded))) {:op :invoke :form form @@ -427,9 +438,12 @@ :ns ns :resolved-to v :type (type v)}))) - (let [meta (-> (dissoc (meta sym) :inline :inline-arities) + (let [meta (-> (dissoc (meta sym) :inline :inline-arities + ;; babashka has :macro on var symbol through defmacro + :macro) (update-vals unquote'))] - (intern (ns-sym ns) (with-meta sym meta)))))) + (doto (intern (ns-sym ns) (with-meta sym meta)) + prn))))) (defmethod -parse 'def [{:keys [ns] :as env} [_ sym & expr :as form]] (let [pfn (fn @@ -447,14 +461,15 @@ (assoc-in env [:namespaces ns :mappings sym] var))) args (when-let [[_ init] (find args :init)] (assoc args :init (analyze* env init)))] - (merge {:op :def - :env env - :form form - :name sym - :doc (or (:doc args) (-> sym meta :doc)) - :children (into [:meta] (when (:init args) [:init])) - :var (get-in env [:namespaces ns :mappings sym]) - :meta {:val (meta sym)}} + (merge (cond-> {:op :def + :env env + :form form + :name sym + :doc (or (:doc args) (-> sym meta :doc)) + :children (into [:meta] (when (:init args) [:init])) + :var (get-in env [:namespaces ns :mappings sym]) + :meta {:val (meta sym)}} + (:defmacro env) (assoc :macro true)) args))) (defmethod -parse 'fn* [env [op & args :as form]] diff --git a/src/nextjournal/clerk/eval.clj b/src/nextjournal/clerk/eval.clj index 915bf5ed0..a6a5ce513 100644 --- a/src/nextjournal/clerk/eval.clj +++ b/src/nextjournal/clerk/eval.clj @@ -285,7 +285,6 @@ (if (cljs? parsed-doc) (process-cljs parsed-doc) (let [{:as analyzed-doc :keys [ns]} - (cond no-cache parsed-doc @@ -297,16 +296,22 @@ (do (when set-status-fn (set-status-fn {:progress 0.10 :status "Analyzing…"})) - (-> parsed-doc - (assoc :blob->result in-memory-cache) - analyzer/build-graph - analyzer/hash)))] + ;; this fixes something if I set it to the namespace of the notebook... why + (prn :nsss *ns*) + (binding [#_#_*ns* (find-ns 'clojure.core)] + (-> parsed-doc + (assoc :blob->result in-memory-cache) + analyzer/build-graph + analyzer/hash))))] (when (and (not-empty (:var->block-id analyzed-doc)) (not ns)) (throw (ex-info "namespace must be set" (select-keys analyzed-doc [:file :ns])))) (binding [*ns* ns] + (prn :ns ns) (eval-analyzed-doc analyzed-doc))))) + + (defn eval-doc "Evaluates the given `doc`." ([doc] (eval-doc {} doc)) diff --git a/test/nextjournal/clerk/analyzer_test.clj b/test/nextjournal/clerk/analyzer_test.clj index 92a765b76..ba4d8c6b1 100644 --- a/test/nextjournal/clerk/analyzer_test.clj +++ b/test/nextjournal/clerk/analyzer_test.clj @@ -389,7 +389,8 @@ my-uuid")] (is (empty? (ana/unhashed-deps ->analysis-info))) (is (match? {:jar string?} (->analysis-info 'weavejester.dependency/graph))))) (testing "should establish dependencies across files" - (let [{:keys [graph]} (analyze-string (slurp "src/nextjournal/clerk.clj"))] + (let [{:keys [graph]} (with-ns-binding 'nextjournal.clerk + (analyze-string (slurp "src/nextjournal/clerk.clj")))] (is (dep/depends? graph 'nextjournal.clerk/show! 'nextjournal.clerk.analyzer/hash))))) (deftest graph-nodes-with-anonymous-ids @@ -416,10 +417,11 @@ my-uuid")] (is (empty? (let [!missing-hash-store (atom [])] (reset! ana/!file->analysis-cache {}) - (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj") - ana/build-graph - (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry))) - ana/hash) + (with-ns-binding 'nextjournal.clerk + (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj") + ana/build-graph + (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry))) + ana/hash)) (deref !missing-hash-store))))) (testing "known cases where missing hashes occur" diff --git a/test/nextjournal/clerk/eval_test.clj b/test/nextjournal/clerk/eval_test.clj index 4446e0d9c..bbb29d2a6 100644 --- a/test/nextjournal/clerk/eval_test.clj +++ b/test/nextjournal/clerk/eval_test.clj @@ -287,6 +287,40 @@ (clerk/show! 'nextjournal.clerk.fixtures.hello) (is (fs/exists? (:file (meta (resolve 'nextjournal.clerk.fixtures.hello/answer))))))) +(deftest macro-analysis-test + (testing "macros are executed before analysis such that expressions relying on + them get properly cached and executed once" + (remove-ns 'my-random-namespace) + (remove-ns 'fixture-ns) + (clerk/clear-cache!) + (binding [;; somehow this is necessary to make the test pass... why? + #_#_*ns* (create-ns 'my-random-namespace)] + (let [fixture-ns "(ns fixture-ns) (def state (atom 0))" + _ (load-string fixture-ns) + ns "(ns my-random-namespace) +(defn macro-helper* [x] x) + +(defmacro attempt1 + [& body] + `(macro-helper* (try + (do ~@body) + (catch Exception e# e#)))) + + +(def a1 + (do + (println \"a1\") + (attempt1 (rand-int 9999))))" + _ (prn :first-eval) + _ (eval/eval-string ns) + first-rand @(resolve 'my-random-namespace/a1) + _ (eval/eval-string ns) + second-rand @(resolve 'my-random-namespace/a1)] + (is (= first-rand second-rand)) + #_(is (= 1 first second)))))) + +#_@(resolve 'my-random-namespace/a1) + (deftest issue-741-can-eval-quoted-regex-test (is (match? {:blocks [{:type :code, :result {:nextjournal/value "foo"}}]}