SefariaExport/entrypoint.sh at master · Otzaria/SefariaExport · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
#!/bin/bash
set -e

umask 022

echo "=== Sefaria Export Pipeline ==="
echo "MongoDB: $MONGO_HOST:$MONGO_PORT"
echo "Database: $MONGO_DB_NAME"
echo ""

# Wait for MongoDB
echo "Waiting for MongoDB..."
./11_wait_for_mongodb.sh

# Handle timestamp: use env if provided, otherwise compute
if [ -z "${TS_STAMP:-}" ]; then
  ./01_compute_timestamp.sh
  export TS_STAMP="$(cat ts.txt)"
else
  export TS_STAMP
fi
echo "Using timestamp: $TS_STAMP"

# Run the export pipeline
echo "Starting export pipeline..."

./04_download_small_dump.sh
./05_clone_sefaria_project.sh
./06_install_build_deps.sh || true
./07_pip_install_requirements.sh || ./08_fallback_built_google_re2.sh
./09_create_exports_dir.sh
./10_create_local_settings.sh
./12_restore_db_from_dump.sh
./13_check_export_module.sh
./14_run_exports.sh
./15_verify_exports.sh
./16_drop_db.sh
./17a_remove_english_in_exports.sh
./17b_flatten_hebrew_in_exports.sh
./17_build_combined_archive.sh
./18_split_archive.sh

# Move archives to output directory (mapped as volume)
mkdir -p /app/output
shopt -s nullglob
ARCHIVES=( /app/sefaria-exports-*.tar.zst* )
if [ "${#ARCHIVES[@]}" -gt 0 ]; then
  mv "${ARCHIVES[@]}" /app/output/
  OUTPUT_ARCHIVES=( /app/output/sefaria-exports-*.tar.zst* )
  chmod a+r "${OUTPUT_ARCHIVES[@]}"
fi

echo ""
echo "=== Export complete! ==="
echo "Archives available in /app/output"
ls -lah /app/output/