Develop

MLproject

@@ -11,34 +11,26 @@ entry_points:
     command: "python generation.py --num_samples {num_samples} --num_frames {num_frames} --exposure_time {exposure_time}"
   analysis1:
     parameters:
-      generated_data: {type: string, default: "/tmp/foobar"}
-      num_samples: {type: int, default: 1}
-      num_frames: {type: int, default: 5}
+      generation: {type: string, default: ""}
       max_sigma: {type: int, default: 4}
       min_sigma: {type: int, default: 1}
       threshold: {type: float, default: 50.0}
       overlap: {type: float, default: 0.5}
-      interval: {type: float, default: 33.0e-3}
-    command: "python analysis1.py --generated_data {generated_data} --num_samples {num_samples} --num_frames {num_frames} --min_sigma {min_sigma} --max_sigma {max_sigma} --threshold {threshold} --overlap {overlap} --interval {interval}"
+    command: "python analysis1.py --generation {generation} --min_sigma {min_sigma} --max_sigma {max_sigma} --threshold {threshold} --overlap {overlap}"
   analysis2:
     parameters:
-      generated_data: {type: string, default: "/tmp/foobar"}
-      num_samples: {type: int, default: 1}
-      num_frames: {type: int, default: 5}
-      threshold: {type: float, default: 50.0}
-      interval: {type: float, default: 33.0e-3}
-    command: "python analysis2.py --generated_data {generated_data} --num_samples {num_samples} --num_frames {num_frames} --threshold {threshold} --interval {interval}"
+      generation: {type: string, default: ""}
+      analysis1: {type: string, default: ""}
+      max_distance: {type: float, default: 50.0}
+    command: "python analysis2.py --generation {generation} --analysis1 {analysis1} --max_distance {max_distance}"
   evaluation1:
     parameters:
-      generated_data: {type: string, default: "/tmp/foobar"}
-      num_samples: {type: int, default: 1}
-      num_frames: {type: int, default: 5}
-      threshold: {type: float, default: 50.0}
-    command: "python evaluation1.py --generated_data {generated_data} --num_samples {num_samples} --num_frames {num_frames} --threshold {threshold}"
+      generation: {type: string, default: ""}
+      analysis1: {type: string, default: ""}
+    command: "python evaluation1.py --generation {generation} --analysis1 {analysis1}"
   main:
     parameters:
       num_samples: {type: int, default: 1}
       num_frames: {type: int, default: 5}
       threshold: {type: float, default: 50.0}
     command: "python main.py --num_samples {num_samples} --num_frames {num_frames} --threshold {threshold}"
-

analysis1.py

@@ -1,67 +1,56 @@
-# -*- coding: utf-8 -*-
-"""analysis1.ipynb
-
-Automatically generated by Colaboratory.
-
-Original file is located at
-    https://colab.research.google.com/github/ecell/bioimage_workflows/blob/master/analysis1.ipynb
-"""
-
 import argparse
+import pathlib
+
+import mlflow
+from mlflow import log_metric, log_param, log_artifacts
 
+entrypoint = "analysis1"
 parser = argparse.ArgumentParser(description='analysis1 step')
-parser.add_argument('--generated_data', type=str, default="/tmp/foobar")
-parser.add_argument('--num_samples', type=int, default=1)
-parser.add_argument('--num_frames', type=int, default=5)
+parser.add_argument('--generation', type=str, default="")
 parser.add_argument('--min_sigma', type=int, default=1)
 parser.add_argument('--max_sigma', type=int, default=4)
 parser.add_argument('--threshold', type=float, default=50.0)
 parser.add_argument('--overlap', type=float, default=0.5)
-parser.add_argument('--interval', type=float, default=33.0e-3)
-
 args = parser.parse_args()
 
-import mlflow
-mlflow.start_run(run_name="analysis1")
+active_run = mlflow.start_run()
+mlflow.set_tag("mlflow.runName", entrypoint)
 
-generated_data = args.generated_data
-num_samples = args.num_samples
-num_frames = args.num_frames
+generation = args.generation
 min_sigma = args.min_sigma
 max_sigma = args.max_sigma
 threshold = args.threshold
 overlap = args.overlap
-interval = args.interval
 
-from mlflow import log_metric, log_param, log_artifacts
-log_param("num_frames", num_frames)
-log_param("num_samples", num_samples)
-log_param("min_sigma", min_sigma)
-log_param("max_sigma", max_sigma)
-log_param("threshold", threshold)
-log_param("overlap", overlap)
-log_param("interval", interval)
+for key, value in vars(args).items():
+    log_param(key, value)
 
-nproc = 1
+client = mlflow.tracking.MlflowClient()
+generation_run = client.get_run(generation)
+num_samples = int(generation_run.data.params["num_samples"])
+num_frames = int(generation_run.data.params["num_frames"])
+interval = float(generation_run.data.params["interval"])
+generation_artifacts = pathlib.Path(client.download_artifacts(generation, "."))
+
+import tempfile
+artifacts = pathlib.Path(tempfile.mkdtemp()) / "artifacts"
+artifacts.mkdir(parents=True, exist_ok=True)
+
+#XXX: HERE
 
 import numpy
 timepoints = numpy.linspace(0, interval * num_frames, num_frames + 1)
 
-import pathlib
-inputpath = pathlib.Path(generated_data.replace("file://", ""))
-artifacts = pathlib.Path(generated_data.replace("file://", ""))
-artifacts.mkdir(parents=True, exist_ok=True)
-
 import scopyon
 
 import warnings
 warnings.simplefilter('ignore', RuntimeWarning)
 
 for i in range(num_samples):
-    imgs = [scopyon.Image(data) for data in numpy.load(inputpath / f"images{i:03d}.npy")]
+    imgs = [scopyon.Image(data) for data in numpy.load(generation_artifacts / f"images{i:03d}.npy")]
     spots = [
         scopyon.analysis.spot_detection(
-            img.as_array(), processes=nproc,
+            img.as_array(),
             min_sigma=min_sigma, max_sigma=max_sigma, threshold=threshold, overlap=overlap)
         for img in imgs]
 
@@ -70,13 +59,21 @@
         spots_.extend(([t] + list(row) for row in data))
     spots_ = numpy.array(spots_)
     numpy.save(artifacts / f"spots{i:03d}.npy", spots_)
-
+
+    r = 6
+    shapes = [dict(x=spot[0], y=spot[1], sigma=r, color='red')
+              for spot in spots[0]]
+    imgs[0].save(artifacts / f"spots{i:03d}_000.png", shapes=shapes)
+
     print("{} spots are detected in {} frames.".format(len(spots_), len(imgs)))
+    log_metric("num_spots", len(spots_))
 
 warnings.resetwarnings()
 
-#!ls ./artifacts
+#XXX: THERE
 
-#log_artifacts("./artifacts")
-log_artifacts(generated_data)
+log_artifacts(str(artifacts))
 mlflow.end_run()
+
+import shutil
+shutil.rmtree(str(artifacts))

analysis2.py

@@ -1,52 +1,50 @@
-# -*- coding: utf-8 -*-
-"""analysis2.ipynb
-
-Automatically generated by Colaboratory.
-
-Original file is located at
-    https://colab.research.google.com/github/ecell/bioimage_workflows/blob/master/analysis2.ipynb
-"""
-
 import argparse
+import pathlib
+
+import mlflow
+from mlflow import log_metric, log_param, log_artifacts
 
+entrypoint = "analysis2"
 parser = argparse.ArgumentParser(description='analysis2 step')
-parser.add_argument('--generated_data', type=str, default="/tmp/foobar")
-parser.add_argument('--num_samples', type=int, default=1)
-parser.add_argument('--num_frames', type=int, default=5)
-parser.add_argument('--threshold', type=float, default=50.0)
-parser.add_argument('--interval', type=float, default=33.0e-3)
+parser.add_argument('--generation', type=str, default="")
+parser.add_argument('--analysis1', type=str, default="")
+parser.add_argument('--seed', type=int, default=123)
+parser.add_argument('--max_distance', type=float, default=50.0)
 args = parser.parse_args()
 
-import mlflow
-mlflow.start_run(run_name="analysis2")
+active_run = mlflow.start_run()
+mlflow.set_tag("mlflow.runName", entrypoint)
 
-generated_data = args.generated_data
-num_samples = args.num_samples
-num_frames = args.num_frames
-threshold = args.threshold
-interval = args.interval
+generation = args.generation
+analysis1 = args.analysis1
+seed = args.seed
+max_distance = args.max_distance
 
-seed = 123
+for key, value in vars(args).items():
+    log_param(key, value)
 
-from mlflow import log_metric, log_param, log_artifacts
-log_param("num_frames", num_frames)
-log_param("num_samples", num_samples)
-log_param("interval", interval)
-log_param("seed", seed)
-log_param("threshold", threshold)
+client = mlflow.tracking.MlflowClient()
+generation_run = client.get_run(generation)
+analysis1_run = client.get_run(analysis1)
+num_samples = int(generation_run.data.params["num_samples"])
+num_frames = int(generation_run.data.params["num_frames"])
+interval = float(generation_run.data.params["interval"])
+generation_artifacts = pathlib.Path(client.download_artifacts(generation, "."))
+analysis1_artifacts = pathlib.Path(client.download_artifacts(analysis1, "."))
 
-import pathlib
-inputpath = pathlib.Path("generated_data")
-artifacts = pathlib.Path("generated_data")
+import tempfile
+artifacts = pathlib.Path(tempfile.mkdtemp()) / "artifacts"
 artifacts.mkdir(parents=True, exist_ok=True)
 
+#XXX: HERE
+
 import scopyon
-config = scopyon.Configuration(filename=inputpath / "config.yaml")
+config = scopyon.Configuration(filename=generation_artifacts / "config.yaml")
 pixel_length = config.default.detector.pixel_length / config.default.magnification
 
 import numpy
 
-def trace_spots(spots, threshold=numpy.inf, ndim=2):
+def trace_spots(spots, max_distance=numpy.inf, ndim=2):
     observation_vec = []
     lengths = []
     for i in range(len(spots[0])):
@@ -55,7 +53,7 @@ def trace_spots(spots, threshold=numpy.inf, ndim=2):
             displacements = numpy.power(spots[j][:, : ndim] - spots[j - 1][iprev, : ndim], 2).sum(axis=1)
             inext = displacements.argmin()
             displacement = numpy.sqrt(displacements[inext])
-            if displacement > threshold:
+            if displacement > max_distance:
                 if j > 1:
                     lengths.append(j - 1)
                 break
@@ -70,7 +68,7 @@ def trace_spots(spots, threshold=numpy.inf, ndim=2):
 lengths = []
 ndim = 2
 for i in range(num_samples):
-    spots_ = numpy.load(inputpath / f"spots{i:03d}.npy")
+    spots_ = numpy.load(analysis1_artifacts / f"spots{i:03d}.npy")
     t = spots_[0, 0]
     spots = [[spots_[0, 1: ]]]
     for row in spots_[1: ]:
@@ -84,7 +82,7 @@ def trace_spots(spots, threshold=numpy.inf, ndim=2):
         spots[-1] = numpy.asarray(spots[-1])
     # print(spots)
 
-    observation_vec_, lengths_ = trace_spots(spots, threshold=threshold, ndim=ndim)
+    observation_vec_, lengths_ = trace_spots(spots, max_distance=max_distance, ndim=ndim)
     observation_vec.extend(observation_vec_)
     lengths.extend(lengths_)
 observation_vec = numpy.array(observation_vec)
@@ -95,20 +93,16 @@ def trace_spots(spots, threshold=numpy.inf, ndim=2):
 from plotly.subplots import make_subplots
 
 fig = make_subplots(rows=1, cols=2, subplot_titles=['Square Displacement', 'Intensity'])
-
 fig.add_trace(go.Histogram(x=observation_vec[:, 0], nbinsx=30, histnorm='probability'), row=1, col=1)
-
 fig.add_trace(go.Histogram(x=observation_vec[:, 1], nbinsx=30, histnorm='probability'), row=1, col=2)
-
 fig.update_layout(barmode='overlay')
 fig.update_traces(opacity=0.75, showlegend=False)
-#fig.show()
-fig.write_image(generated_data + "/analysis2_1.png")
+# fig.show()
+fig.write_image(str(artifacts / "histogram1.png"))
 
 from scopyon.analysis import PTHMM
 
 rng = numpy.random.RandomState(seed)
-
 model = PTHMM(n_diffusivities=3, n_oligomers=1, n_iter=100, random_state=rng)
 model.fit(observation_vec, lengths)
 
@@ -132,18 +126,19 @@ def trace_spots(spots, threshold=numpy.inf, ndim=2):
     expected_vec[sum(lengths[: i]): sum(lengths[: i + 1])] = X_
 
 fig = make_subplots(rows=1, cols=2, subplot_titles=['Square Displacement', 'Intensity'])
-
 fig.add_trace(go.Histogram(x=observation_vec[:, 0], nbinsx=30, histnorm='probability density'), row=1, col=1)
 fig.add_trace(go.Histogram(x=expected_vec[:, 0], nbinsx=30, histnorm='probability density'), row=1, col=1)
-
 fig.add_trace(go.Histogram(x=observation_vec[:, 1], nbinsx=30, histnorm='probability density'), row=1, col=2)
 fig.add_trace(go.Histogram(x=expected_vec[:, 1], nbinsx=30, histnorm='probability density'), row=1, col=2)
-
 fig.update_layout(barmode='overlay')
 fig.update_traces(opacity=0.75, showlegend=False)
-#fig.show()
-fig.write_image(generated_data + "/analysis2_2.png")
+# fig.show()
+fig.write_image(str(artifacts / "histogram2.png"))
 
-#log_artifacts("./artifacts")
-log_artifacts(generated_data)
+#XXX: THERE
+
+log_artifacts(str(artifacts))
 mlflow.end_run()
+
+import shutil
+shutil.rmtree(str(artifacts))