Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
73b5b20
refactor: large changes to plan workflow
lispandfound Jul 24, 2025
3044815
refactor: block based templating
lispandfound Jul 28, 2025
fdb7e1f
do not bind KO matrices into hf-sim path
lispandfound Aug 6, 2025
706e376
add check-srf and check-domain stages
lispandfound Aug 6, 2025
cf3db3b
complete transition of the last stages
lispandfound Aug 6, 2025
5407ffe
allow for resource-less dependencies
lispandfound Aug 6, 2025
0ce83ae
add lf-to-xarray stage
lispandfound Aug 6, 2025
192ec96
re-add file tree resource printing
lispandfound Aug 6, 2025
ac74b80
fix workflow stage definitions
lispandfound Aug 6, 2025
0828559
Merge branch 'pegasus' into plan_workflow_enhancement
lispandfound Aug 6, 2025
0c4aea3
docs(plan-workflow): add documentation for most modules
lispandfound Aug 11, 2025
06af20b
docs(plan-workflow): document parameter class
lispandfound Aug 11, 2025
dd21d46
docs(plan-workflow): format plan_workflow docstring
lispandfound Aug 11, 2025
6c5b64c
docs(plan-workflow): document enum members.
lispandfound Aug 11, 2025
4eda0f4
refactor(plan-workflow): centralise environment management
lispandfound Aug 12, 2025
1872312
feat(plan-workflow): allow for custom environment definitions
lispandfound Aug 12, 2025
139f1b2
docs(plan-workflow): fix dfs_tree_cover documentation
lispandfound Aug 12, 2025
ae39332
refactor(plan-workflow): break out entrypoint into smaller chunks
lispandfound Aug 12, 2025
7ad641c
deps(workflow): remove pyvis dependency
lispandfound Aug 12, 2025
4e0da80
docs(plan-workflow): add top-level module documentation
lispandfound Aug 12, 2025
c963d39
docs(plan-workflow): add missing stage config documentation
lispandfound Aug 12, 2025
8d1b1d8
fix(hf-sim): fix copy-paste error
lispandfound Aug 12, 2025
36e2eb2
fix(check-srf): fix check-srf invocation
lispandfound Aug 12, 2025
1160078
fix(copy-domain-parameters): fix realisation references
lispandfound Aug 12, 2025
8c644c3
fix(plan-workflow): add extra settings for per-host config
lispandfound Aug 12, 2025
497992c
fix(stages): update bb and hf stages to latest format
lispandfound Aug 12, 2025
7db767c
fix(create-e3d-par): remove unneeded EMOD3D path
lispandfound Aug 12, 2025
4984124
fix(plan-workflow): use the correct generator syntax
lispandfound Aug 12, 2025
7ead5fc
fix(plan-workflow): ruff fixes
lispandfound Aug 12, 2025
3a76511
fix(plan-workflow): use yield from rather than return
lispandfound Aug 12, 2025
028cf9a
docs(plan-workflow): document the host_file argument
lispandfound Aug 12, 2025
a64f9a0
fix(nesi): platform is not a directive
lispandfound Aug 12, 2025
955b5b5
docs(plan-workflow): fix argument documentation ordering
lispandfound Aug 12, 2025
9cc41e3
fix(emod3d): pass missing `par=` parameter
lispandfound Aug 12, 2025
f6c7e7d
fix(bb-sim): use correct stations.vs30 location
lispandfound Aug 12, 2025
af6cc91
fix(hf-sim): fix stations location
lispandfound Aug 12, 2025
edf750e
feat(workflow): containerise emod3d
lispandfound Aug 13, 2025
413f1eb
fix(plan-workflow): add copy-input stage
lispandfound Aug 14, 2025
e8b4c30
fix(generate-station-coordinates): generate station coordinates in th…
lispandfound Aug 14, 2025
7357645
refactor: use apptainer bind paths for better DRY command line handlers
lispandfound Aug 15, 2025
4d72adc
simplify event-sample split
lispandfound Aug 15, 2025
9ce5730
update stage environments
lispandfound Aug 15, 2025
4808d10
remove event parameters
lispandfound Aug 15, 2025
357b19f
fix job inheritance
lispandfound Aug 15, 2025
4a72b3d
remove missing generate-model-coordinates script
lispandfound Aug 15, 2025
53425a8
update container definition
lispandfound Aug 15, 2025
768673c
move NZVM to correct directory
lispandfound Aug 15, 2025
de69856
generate station files in event directory
lispandfound Aug 15, 2025
0be065a
fix: cache and speed-up velocity model download
lispandfound Aug 15, 2025
0ee1c50
fix: copy don't clone velocity model
lispandfound Aug 15, 2025
87c6822
fix(rch): use multiple cores for velocity model generation
lispandfound Aug 26, 2025
9a79b37
fix(cylc): strip trailing slash from TASK_SHARE and EVENT_SHARE
lispandfound Aug 26, 2025
22d7f37
copy to more than stations
lispandfound Aug 28, 2025
0ce26c8
set emod3d environment variable
lispandfound Aug 28, 2025
443ae51
use srun instead of apptainer
lispandfound Aug 28, 2025
21fbfb5
fix emod3d directory and paths
lispandfound Aug 28, 2025
383f5c5
fix(lf-to-xarray): use correct lf path
lispandfound Aug 28, 2025
d67c02b
fix(create-e3d-par): isolate container
lispandfound Aug 28, 2025
be8f51c
fix(emod3d): correctly set environment variables
lispandfound Aug 28, 2025
d9ddab5
fix(rch): emod3d pre-script fixes
lispandfound Oct 1, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 17 additions & 15 deletions container/runner.def
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@ From: ubuntu:latest

%setup
[ -d /tmp/EMOD3D ] || git clone --depth 1 [email protected]:ucgmsim/EMOD3D.git /tmp/EMOD3D

[ -d /tmp/Velocity-Model ] || wget "https://www.dropbox.com/scl/fi/mzf5t4tdnv3qsgtbpx824/Velocity-Model.tar.zst?rlkey=df1uej7hd7xd18jerh357gy53&st=e9n8z0aw&dl=1" -O /tmp/Velocity-Model.tar.zst && cd /tmp && tar xf /tmp/Velocity-Model.tar.zst
%files
/tmp/EMOD3D /EMOD3D

/tmp/Velocity-Model /Velocity-Model
%post
apt-get update && apt-get install -y \
build-essential \
cmake \
curl \
gdal-bin \
gfortran \
ghostscript \
Expand All @@ -23,34 +24,35 @@ From: ubuntu:latest
libfftw3-dev \
libopenmpi-dev \
libpapi-dev \
openmpi-bin \
openmpi-common \
python-is-python3 \
python3-pip \
sqlite3 \
wget

wget "https://www.dropbox.com/scl/fi/b4m9g32x1j70ti69zyybq/genslip_velocity_model.vmod?rlkey=yxurhrdn8uvqh33hsb0inhsq0&st=o77aaccp&dl=0" -O /genslip_velocity_model.vmod
wget "https://www.dropbox.com/scl/fi/qakchhg6qu23aclcb02ye/Cant1D_v2-midQ_leer.1d?rlkey=wn1a6nnol2eaf5n6qsy5t1nym&st=8yahw7id&dl=0" -O /Cant1D_v2-midQ_leer.1d
wget "https://www.dropbox.com/scl/fi/tw1m15owloi0eo8h3u1kw/geoNet_stats-2023-06-28.vs30?rlkey=plu90zg0ti4lc2nvf51zu9xmg&st=iz8c2ls3&dl=0" -O /stations.vs30
wget "https://www.dropbox.com/scl/fi/bb852b1f0rly6cfvs9p9b/geoNet_stats-2023-06-28.ll?rlkey=3y8k5qviy52nbksfdkm1n92yh&st=605h60u6&dl=0" -O /stations.ll


curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
. $HOME/.cargo/env

cd /EMOD3D && \
mkdir build && \
cd build && \
cmake .. && \
make -j $(nproc) genslip_v5.6.2 srf2stoch hb_high_binmod_v6.0.3 hb_high_binmod_v5.4.5

cmake -DHF_TIME_WINDOW=off -DUSE_FFTW=on -DCMAKE_C_COMPILER=mpicc -DCMAKE_Fotran_COMPILER=mpicc .. && \
make -j $(nproc) genslip_v5.6.2 srf2stoch generic_slip2srf hb_high_binmod_v6.0.3 hb_high_binmod_v5.4.5 emod3d-mpi_v3.0.13
cd /
# Remove all folders in EMOD3D except the tools folder
find /EMOD3D -mindepth 1 -maxdepth 1 ! -name "tools" -exec rm -rf \{\} \;


git clone --depth 1 https://github.com/ucgmsim/Velocity-Model.git /Velocity-Model && \
cd /Velocity-Model && \
mkdir build && \
cd build && \
cmake ../ -DCMAKE_C_FLAGS="-fopenmp" -DCMAKE_CXX_FLAGS="-fopenmp" && \
make -j $(nproc)
cmake ../ -DCMAKE_C_FLAGS="-fopenmp" -DCMAKE_CXX_FLAGS="-fopenmp" && \
make -j $(nproc) && \
mv NZVM ..

pip install "git+https://github.com/ucgmsim/workflow@pegasus" --break-system-packages
pip install "git+https://github.com/ucgmsim/workflow@bb_sim_new" --break-system-packages

%environment
export PATH="$PATH:/EMOD3D/tools:/Velocity-Model"
export PATH="$PATH:/root/.cargo/bin:/EMOD3D/tools:/Velocity-Model"
189 changes: 90 additions & 99 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,52 +10,46 @@ readme = "README.md"
requires-python = ">=3.12"
dynamic = ["version"]
dependencies = [
# UCGMSim Dependencies
"im_calculation @ git+https://github.com/ucgmsim/IM_calculation.git",
"nshmdb @ git+https://github.com/ucgmsim/NSHM2022DB.git",
"oq_wrapper @ git+https://github.com/ucgmsim/Empirical_Engine.git#subdirectory=oq_wrapper",
"pygmt_helper @ git+https://github.com/ucgmsim/pygmt_helper.git",
"qcore @ git+https://github.com/ucgmsim/qcore.git",
"source_modelling @ git+https://github.com/ucgmsim/source_modelling.git",
"velocity_modelling @ git+https://github.com/ucgmsim/velocity_modelling.git",

# Data Formats
"geopandas",
"h5py",
"pandas[parquet, hdf5]",
"pyyaml",
"xarray[io]",

# Workflow Planner
"jinja2",
"networkx",
"printree",
"pyvis",

# Numerics
"numexpr",
"numpy",
"scipy",
"shapely",

# CLI
"tqdm",
"typer",

# Misc.
"requests", # For gcmt-to-realisation
"schema", # For loading realisations
"structlog", # Logging.
# UCGMSim Dependencies
"im_calculation @ git+https://github.com/ucgmsim/IM_calculation.git",
"nshmdb @ git+https://github.com/ucgmsim/NSHM2022DB.git",
"oq_wrapper @ git+https://github.com/ucgmsim/Empirical_Engine.git#subdirectory=oq_wrapper",
"pygmt_helper @ git+https://github.com/ucgmsim/pygmt_helper.git",
"qcore @ git+https://github.com/ucgmsim/qcore.git",
"source_modelling @ git+https://github.com/ucgmsim/source_modelling.git",
"velocity_modelling @ git+https://github.com/ucgmsim/velocity_modelling.git",

# Data Formats
"geopandas",
"h5py",
"pandas[parquet, hdf5]",
"pyyaml",
"xarray[io]",

# Workflow Planner
"jinja2",
"networkx",
"printree",

# Numerics
"numexpr",
"numpy",
"scipy",
"shapely",

# CLI
"tqdm",
"typer",

# Misc.
"requests", # For gcmt-to-realisation
"schema", # For loading realisations
"structlog", # Logging.
]

[project.optional-dependencies]
test = [
"pytest",
]
dev = [
"ruff",
"deptry"
]
test = ["pytest"]
dev = ["ruff", "deptry"]

[tool.deptry]
pep621_dev_dependency_groups = ["test", "dev"]
Expand All @@ -67,7 +61,6 @@ realisation-to-srf = "workflow.scripts.realisation_to_srf:app"
generate-velocity-model-parameters = "workflow.scripts.generate_velocity_model_parameters:app"
generate-velocity-model = "workflow.scripts.generate_velocity_model:app"
generate-station-coordinates = "workflow.scripts.generate_station_coordinates:app"
generate-model-coordinates = "workflow.scripts.generate_model_coordinates:app"
generate-rupture-propagation = "workflow.scripts.generate_rupture_propagation:app"
copy-domain-parameters = "workflow.scripts.copy_velocity_model_parameters:app"
create-e3d-par = "workflow.scripts.create_e3d_par:app"
Expand All @@ -86,40 +79,38 @@ import-realisation = "workflow.scripts.import_realisation:app"
workflow = "workflow"




[tool.setuptools_scm]


[tool.ruff.lint]
extend-select = [
# isort imports
"I",
# Use r'\s+' rather than '\s+'
"W605",
# All the naming errors, like using camel case for function names.
"N",
# Missing docstrings in classes, methods, and functions
"D101",
"D102",
"D103",
"D105",
"D107",
# Use f-string instead of a format call
"UP032",
# Standard library import is deprecated
"UP035",
# Missing function argument type-annotation
"ANN001",
"ANN002",
"ANN003",
"ANN201",
"ANN202",
"ANN204",
"ANN205",
"ANN206",
# Using except without specifying an exception type to catch
"BLE001",
# isort imports
"I",
# Use r'\s+' rather than '\s+'
"W605",
# All the naming errors, like using camel case for function names.
"N",
# Missing docstrings in classes, methods, and functions
"D101",
"D102",
"D103",
"D105",
"D107",
# Use f-string instead of a format call
"UP032",
# Standard library import is deprecated
"UP035",
# Missing function argument type-annotation
"ANN001",
"ANN002",
"ANN003",
"ANN201",
"ANN202",
"ANN204",
"ANN205",
"ANN206",
# Using except without specifying an exception type to catch
"BLE001",
]
ignore = ["D104"]

Expand All @@ -128,15 +119,15 @@ convention = "numpy"

[tool.ruff.lint.isort]
known-first-party = [
"source_modelling",
"workflow",
"pygmt_helper",
"qcore",
"empirical",
"nshmdb",
"IM",
"mera",
"velocity_modelling",
"source_modelling",
"workflow",
"pygmt_helper",
"qcore",
"empirical",
"nshmdb",
"IM",
"mera",
"velocity_modelling",
]

[tool.ruff.lint.per-file-ignores]
Expand All @@ -150,24 +141,24 @@ omit = ["workflow/scripts/*.py"]

[tool.numpydoc_validation]
checks = [
"GL05",
"GL08",
"GL10",
"PR01",
"PR02",
"PR03",
"PR04",
"PR05",
"PR06",
"PR07",
"RT01",
"RT02",
"RT03",
"RT04",
"YD01",
"GL05",
"GL08",
"GL10",
"PR01",
"PR02",
"PR03",
"PR04",
"PR05",
"PR06",
"PR07",
"RT01",
"RT02",
"RT03",
"RT04",
"YD01",
]
# remember to use single quotes for regex in TOML
exclude = [ # don't report on objects that match any of these regex
'\.undocumented_method$',
'\.__repr__$',
'\.undocumented_method$',
'\.__repr__$',
]
Loading