Fix #746: support macros defined in notebooks

notebooks/my_random_namespace.clj

@@ -0,0 +1,25 @@
+(ns my-random-namespace)
+
+(defn macro-helper* [x] x)
+
+(defmacro attempt1
+  [& body]
+  `(macro-helper* (try
+                    (do ~@body)
+                    (catch Exception e# e#))))
+
+
+(def a1
+  (do
+    (println "a1")
+    (attempt1 (rand-int 9999))))
+
+#_(do (reset! fixture-ns/state 0)
+      (remove-ns 'my-random-namespace)
+      (nextjournal.clerk/clear-cache!)
+      (create-ns 'my-random-namespace))
+
+
+
+
+

src/nextjournal/clerk.clj

@@ -69,6 +69,7 @@
              _ (reset! !last-file file)
              {:keys [blob->result]} @webserver/!doc
              {:keys [result time-ms]} (eval/time-ms (binding [paths/*build-opts* (webserver/get-build-opts)]
+                                                      (prn :clerk-show-ns *ns*)
                                                       (eval/+eval-results blob->result (assoc doc :set-status-fn webserver/set-status!))))]
          (if (:error result)
            (println (str "Clerk encountered an error evaluating '" file "' after " time-ms "ms."))

src/nextjournal/clerk/analyzer.clj

@@ -156,14 +156,16 @@
         deps (set/union (set/difference (into #{} (map (comp symbol var->protocol)) @!deps) vars)
                         deref-deps
                         (when (var? form) #{(symbol form)}))
-        hash-fn (-> form meta :nextjournal.clerk/hash-fn)]
+        hash-fn (-> form meta :nextjournal.clerk/hash-fn)
+        macro? (-> analyzed :env :defmacro)]
     (cond-> {#_#_:analyzed analyzed
              :form form
              :ns-effect? (some? (some #{'clojure.core/require 'clojure.core/in-ns} deps))
              :freezable? (and (not (some #{'clojure.core/intern} deps))
                               (<= (count vars) 1)
                               (if (seq vars) (= var (first vars)) true))
-             :no-cache? (no-cache? form (-> def-node :form second) *ns*)}
+             :no-cache? (no-cache? form (-> def-node :form second) *ns*)
+             :macro macro?}
       hash-fn (assoc :hash-fn hash-fn)
       (seq deps) (assoc :deps deps)
       (seq deref-deps) (assoc :deref-deps deref-deps)
@@ -336,6 +338,7 @@
                                                                                (:file doc) (assoc :file (:file doc)))
                                  block+analysis (add-block-id (merge block form-analysis))]
                              (when ns-effect? ;; needs to run before setting doc `:ns` via `*ns*`
+                               (prn :eval form)
                                (eval form))
                              (-> state
                                  (store-info block+analysis)
@@ -442,7 +445,10 @@
 
 (defn var->location [var]
   (when-let [file (:file (meta var))]
-    (some-> (if (fs/absolute? file)
+    (some-> (if (try (fs/absolute? file)
+                     ;; fs/absolute? crashes in bb on Windows due to the :file
+                     ;; metadata containing "<expr>"
+                     (catch Exception _ false))
               (when (fs/exists? file)
                 (fs/relativize (fs/cwd) (fs/file file)))
               (when-let [resource (io/resource file)]
@@ -496,45 +502,93 @@
                 (filter (comp #{:code} :type)
                         blocks))))
 
+(defn transitive-deps
+  ([id analysis-info]
+   (loop [seen #{}
+          deps #{id}
+          res #{}]
+     (if (seq deps)
+       (let [dep (first deps)]
+         (if (contains? seen dep)
+           (recur seen (rest deps) res)
+           (let [{new-deps :deps} (get analysis-info dep)
+                 seen (conj seen dep)
+                 deps (concat (rest deps) new-deps)
+                 res (into res deps)]
+             (recur seen deps res))))
+       res))))
+
+#_(transitive-deps id analysis-info)
+
+#_(transitive-deps :main {:main {:deps [:main :other]}
+                          :other {:deps [:another]}
+                          :another {:deps [:another-one :another :main]}})
+
+(defn run-macros [init-state]
+  (let [{:keys [blocks ->analysis-info]} init-state
+        macro-block-ids (keep #(when (:macro %)
+                                 (:id %)) blocks)
+        deps (mapcat #(transitive-deps % ->analysis-info) macro-block-ids)
+        all-block-ids (into (set macro-block-ids) deps)
+        all-blocks (filter #(contains? all-block-ids (:id %)) blocks)]
+    (doseq [block all-blocks]
+      (try
+        (println "loading in namespace" *ns* (:text block))
+        (load-string (:text block))
+        (catch Throwable e
+          (binding [*out* *err*]
+            (println "Error when evaluating macro deps:" (:text block))
+            (println "Namespace:" *ns*)
+            (println "Exception:" e)))))
+    (pos? (count all-blocks))))
+
 (defn build-graph
   "Analyzes the forms in the given file and builds a dependency graph of the vars.
 
-  Recursively decends into dependency vars as well as given they can be found in the classpath.
+  Recursively descends into dependency vars as well if they can be found in the classpath.
   "
   [doc]
-  (loop [{:as state :keys [->analysis-info analyzed-file-set counter]}
-         (-> doc
-             analyze-doc
-             (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc)))
-                    :counter 0
-                    :graph (dep/graph)))]
-    (let [unhashed (unhashed-deps ->analysis-info)
-          loc->syms (apply dissoc
-                           (group-by find-location unhashed)
-                           analyzed-file-set)]
-      (if (and (seq loc->syms) (< counter 10))
-        (recur (-> (reduce (fn [g [source symbols]]
-                             (let [jar? (or (nil? source)
-                                            (str/ends-with? source ".jar"))
-                                   gitlib-hash (and (not jar?)
-                                                    (second (re-find #".gitlibs/libs/.*/(\b[0-9a-f]{5,40}\b)/" (fs/unixify source))))]
-                               (if (or jar? gitlib-hash)
-                                 (update g :->analysis-info merge (into {} (map (juxt identity
-                                                                                      (constantly (if source
-                                                                                                    (or (when gitlib-hash {:hash gitlib-hash})
-                                                                                                        (hash-jar source))
-                                                                                                    {})))) symbols))
-                                 (-> g
-                                     (update :analyzed-file-set conj source)
-                                     (merge-analysis-info (analyze-file source))))))
-                           state
-                           loc->syms)
-                   (update :counter inc)))
-        (-> state
-            analyze-doc-deps
-            set-no-cache-on-redefs
-            make-deps-inherit-no-cache
-            (dissoc :analyzed-file-set :counter))))))
+  (let [init-state-fn #(-> doc
+                           analyze-doc
+                           (assoc :analyzed-file-set (cond-> #{} (:file doc) (conj (:file doc)))
+                                  :counter 0
+                                  :graph (dep/graph)))
+        init-state (init-state-fn)
+        ran-macros? (run-macros init-state)
+        ;; _ (prn :ran-macros? ran-macros?)
+        init-state (if ran-macros?
+                     (init-state-fn)
+                     init-state)]
+    ;; #dbg (def istate1 init-state)
+    (loop [{:as state :keys [->analysis-info analyzed-file-set counter]} init-state]
+      (let [unhashed (unhashed-deps ->analysis-info)
+            loc->syms (apply dissoc
+                             (group-by find-location unhashed)
+                             analyzed-file-set)]
+        (if (and (seq loc->syms) (< counter 10))
+          (recur (-> (reduce (fn [g [source symbols]]
+                               (let [jar? (or (nil? source)
+                                              (str/ends-with? source ".jar"))
+                                     gitlib-hash (and (not jar?)
+                                                      (second (re-find #".gitlibs/libs/.*/(\b[0-9a-f]{5,40}\b)/" (fs/unixify source))))]
+                                 (if (or jar? gitlib-hash)
+                                   (update g :->analysis-info merge (into {} (map (juxt identity
+                                                                                        (constantly (if source
+                                                                                                      (or (when gitlib-hash {:hash gitlib-hash})
+                                                                                                          (hash-jar source))
+                                                                                                      {})))) symbols))
+                                   (-> g
+                                       (update :analyzed-file-set conj source)
+                                       (merge-analysis-info (analyze-file source))))))
+                             state
+                             loc->syms)
+                     (update :counter inc)))
+          (let [res (-> state
+                        analyze-doc-deps
+                        set-no-cache-on-redefs
+                        make-deps-inherit-no-cache
+                        (dissoc :analyzed-file-set :counter))]
+            res))))))
 
 (comment
   (reset! !file->analysis-cache {})
@@ -599,14 +653,23 @@
         (record-missing-hash-fn (assoc codeblock
                                        :dep-with-missing-hash dep-with-missing-hash
                                        :graph-node graph-node :ns ns))))
-    (binding [*print-length* nil]
-      (let [form-with-deps-sorted
-            (-> hashed-deps
-                (conj (if form
-                        (-> form remove-type-meta canonicalize-form pr-str)
-                        hash))
-                (into (map str) vars))]
-        (sha1-base58 (pr-str form-with-deps-sorted))))))
+    (let [res (binding [*print-length* nil]
+                (let [form-with-deps-sorted
+                      (-> hashed-deps
+                          (conj (if form
+                                  (-> form remove-type-meta canonicalize-form pr-str)
+                                  hash))
+                          (into (map str) vars))]
+                  (sha1-base58 (pr-str form-with-deps-sorted))))]
+      (when (= '(def a1
+                  (do
+                    (println "a1")
+                    (attempt1 (rand-int 9999))))
+               form)
+        (prn :deps-x-hashes (sort (zipmap deps (map ->hash deps))))
+        (prn :res res))
+      res)
+    ))
 
 #_(hash-codeblock {} {:graph (dep/graph)} {})
 #_(hash-codeblock {} {:graph (dep/graph)} {:hash "foo"})

src/nextjournal/clerk/analyzer/impl.clj

@@ -38,9 +38,15 @@
     (let [local? (and (simple-symbol? sym)
                       (contains? (:locals env) sym))]
       (when-not local?
-        (when (symbol? sym)
+        (when (symbol? sym) ;; TODO: we already checkd for symbol?
           (let [sym-ns  (when-let [ns (namespace sym)] (symbol ns))
                 full-ns (resolve-ns sym-ns env)]
+            (let [sym-name (-> sym name symbol)]
+              (when (= 'attempt1 sym-name)
+                (prn (when (or (not sym-ns) full-ns)
+                       (let [name (if sym-ns (-> sym name symbol) sym)]
+                         (binding [*ns* (or full-ns ns)]
+                           (resolve name)))))))
             (when (or (not sym-ns) full-ns)
               (let [name (if sym-ns (-> sym name symbol) sym)]
                 (binding [*ns* (or full-ns ns)]
@@ -383,8 +389,13 @@
       (if (and (var? maybe-macro)
                (:macro (meta maybe-macro)))
         (do
+          (when (= "my-random-namespace" (namespace (symbol maybe-macro)))
+            (prn :var maybe-macro (:macro (meta maybe-macro))))
           (swap! *deps* conj maybe-macro)
-          (let [expanded (macroexpand-hook maybe-macro form env (rest form))]
+          (let [expanded (macroexpand-hook maybe-macro form env (rest form))
+                env (if (identical? #'defmacro maybe-macro)
+                      (assoc env :defmacro true)
+                      env)]
             (analyze* env expanded)))
         {:op       :invoke
          :form     form
@@ -427,9 +438,12 @@
                                :ns          ns
                                :resolved-to v
                                :type        (type v)})))
-      (let [meta (-> (dissoc (meta sym) :inline :inline-arities)
+      (let [meta (-> (dissoc (meta sym) :inline :inline-arities
+                             ;; babashka has :macro on var symbol through defmacro
+                             :macro)
                      (update-vals unquote'))]
-        (intern (ns-sym ns) (with-meta sym meta))))))
+        (doto (intern (ns-sym ns) (with-meta sym meta))
+          prn)))))
 
 (defmethod -parse 'def [{:keys [ns] :as env} [_ sym & expr :as form]]
   (let [pfn  (fn
@@ -447,14 +461,15 @@
                 (assoc-in env [:namespaces ns :mappings sym] var)))
         args (when-let [[_ init] (find args :init)]
                (assoc args :init (analyze* env init)))]
-    (merge {:op       :def
-            :env      env
-            :form     form
-            :name     sym
-            :doc      (or (:doc args) (-> sym meta :doc))
-            :children (into [:meta] (when (:init args) [:init]))
-            :var (get-in env [:namespaces ns :mappings sym])
-            :meta {:val (meta sym)}}
+    (merge (cond-> {:op       :def
+                    :env      env
+                    :form     form
+                    :name     sym
+                    :doc      (or (:doc args) (-> sym meta :doc))
+                    :children (into [:meta] (when (:init args) [:init]))
+                    :var (get-in env [:namespaces ns :mappings sym])
+                    :meta {:val (meta sym)}}
+             (:defmacro env) (assoc :macro true))
            args)))
 
 (defmethod -parse 'fn* [env [op & args :as form]]

src/nextjournal/clerk/eval.clj

@@ -285,7 +285,6 @@
   (if (cljs? parsed-doc)
     (process-cljs parsed-doc)
     (let [{:as analyzed-doc :keys [ns]}
-
           (cond
             no-cache
             parsed-doc
@@ -297,16 +296,22 @@
             (do
               (when set-status-fn
                 (set-status-fn {:progress 0.10 :status "Analyzing…"}))
-              (-> parsed-doc
-                  (assoc :blob->result in-memory-cache)
-                  analyzer/build-graph
-                  analyzer/hash)))]
+              ;; this fixes something if I set it to the namespace of the notebook... why
+              (prn :nsss *ns*)
+              (binding [#_#_*ns* (find-ns 'clojure.core)]
+                (-> parsed-doc
+                    (assoc :blob->result in-memory-cache)
+                    analyzer/build-graph
+                    analyzer/hash))))]
       (when (and (not-empty (:var->block-id analyzed-doc))
                  (not ns))
         (throw (ex-info "namespace must be set" (select-keys analyzed-doc [:file :ns]))))
       (binding [*ns* ns]
+        (prn :ns ns)
         (eval-analyzed-doc analyzed-doc)))))
 
+
+
 (defn eval-doc
   "Evaluates the given `doc`."
   ([doc] (eval-doc {} doc))

test/nextjournal/clerk/analyzer_test.clj

@@ -389,7 +389,8 @@ my-uuid")]
       (is (empty? (ana/unhashed-deps ->analysis-info)))
       (is (match? {:jar string?} (->analysis-info 'weavejester.dependency/graph)))))
   (testing "should establish dependencies across files"
-    (let [{:keys [graph]} (analyze-string (slurp "src/nextjournal/clerk.clj"))]
+    (let [{:keys [graph]} (with-ns-binding 'nextjournal.clerk
+                            (analyze-string (slurp "src/nextjournal/clerk.clj")))]
       (is (dep/depends? graph 'nextjournal.clerk/show! 'nextjournal.clerk.analyzer/hash)))))
 
 (deftest graph-nodes-with-anonymous-ids
@@ -416,10 +417,11 @@ my-uuid")]
     (is (empty?
          (let [!missing-hash-store (atom [])]
            (reset! ana/!file->analysis-cache {})
-           (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj")
-               ana/build-graph
-               (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry)))
-               ana/hash)
+           (with-ns-binding 'nextjournal.clerk
+             (-> (parser/parse-file {:doc? true} "src/nextjournal/clerk.clj")
+                 ana/build-graph
+                 (assoc :record-missing-hash-fn (fn [report-entry] (swap! !missing-hash-store conj report-entry)))
+                 ana/hash))
            (deref !missing-hash-store)))))
 
   (testing "known cases where missing hashes occur"