Constrain memory used by cron-run scripts

By default, the JVM will use a heap of 960GB on the current instance
size we are using if Xmx isn't provided. This ensures that the scripts
will use only 512GB, and will fully allocate it on startup to fail-fast
if it isn't available.

This also sets the jobs to exit if they OOM, and dump a heap in that
case.

This is to help prevent memory contention with the server process.

scripts/build-search-index

@@ -7,14 +7,16 @@ set -e
 
 dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
+source "$dir/set_default_java_opts.bash"
+
 cd "$dir/.."
 
 clojars_jar=$1
 
 mkdir -p tmp
 
 index_out=tmp/index-out.txt
-java -cp "$clojars_jar" clojure.main -m clojars.tools.build-search-index production | tee "$index_out"
+java "${JAVA_OPTS[@]}" -cp "$clojars_jar" clojure.main -m clojars.tools.build-search-index production | tee "$index_out"
 
 index_path=$(grep "index-path:" "$index_out" | sed -n 's/index-path: \(.*\)/\1/p')
 

scripts/combine-cdn-logs

@@ -11,11 +11,15 @@ if [ -z "$DATE" ]; then
     DATE=$(date --date='1 day ago' +%Y%m%d)
 fi
 
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+source "$dir/set_default_java_opts.bash"
+
 mkdir -p tmp
 
 S3_LOG_BUCKET=clojars-fastly-logs
 
-java -cp "$clojars_jar" clojure.main -m clojars.tools.combine-cdn-logs \
+java "${JAVA_OPTS[@]}" -cp "$clojars_jar" clojure.main -m clojars.tools.combine-cdn-logs \
      "$S3_LOG_BUCKET" "$DATE" "$output_file"
 
 if [ -s "$output_file" ]; then

scripts/maven-index-repo

@@ -12,6 +12,10 @@ repo=$4
 CLOJARS_ENVIRONMENT=production
 S3_REPO_BUCKET=clojars-repo-production
 
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+source "$dir/set_default_java_opts.bash"
+
 # synchronize the repo bucket locally
 # we only need the existing index, the poms, and maven-metadata.xml files
 aws s3 sync "s3://$S3_REPO_BUCKET/" "$repo" \
@@ -21,7 +25,7 @@ aws s3 sync "s3://$S3_REPO_BUCKET/" "$repo" \
     --include ".index/*"
 
 # generate the index shards
-java -jar $indexer_jar -n clojars -i $index_dir -d $repo/.index -r $repo -s -q -t min -l -c -k 50
+java "${JAVA_OPTS[@]}" -jar $indexer_jar -n clojars -i $index_dir -d $repo/.index -r $repo -s -q -t min -l -c -k 50
 
 # remove old sums locally (the indexer removes old shards itself, but doesn't remove the sums for those shards)
 shopt -s extglob
@@ -35,11 +39,11 @@ done
 popd
 
 # upload new shards and sums to s3
-java -cp $clojars_jar clojure.main -m clojars.tools.upload-repo-s3 $repo \
+java "${JAVA_OPTS[@]}" -cp $clojars_jar clojure.main -m clojars.tools.upload-repo-s3 $repo \
      $S3_REPO_BUCKET .index :gen-index 2> /dev/null
 
 # remove old shards and sums from s3
-java -cp $clojars_jar clojure.main -m clojars.tools.remove-deletions-s3 $repo \
+java "${JAVA_OPTS[@]}" -cp $clojars_jar clojure.main -m clojars.tools.remove-deletions-s3 $repo \
      $S3_REPO_BUCKET \
      .index 2> /dev/null
 

scripts/set_default_java_opts.bash

@@ -0,0 +1,9 @@
+# Shsred java options for util scripts
+
+JAVA_OPTS=(
+    -XX:+HeapDumpOnOutOfMemoryError
+    -XX:+ExitOnOutOfMemoryError
+    -Xmx512m
+    -Xms512m
+    -Daws.region=us-east-2
+)

scripts/update-pom-list

@@ -6,6 +6,10 @@ set -e
 
 clojars_jar=$1
 
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+source "$dir/set_default_java_opts.bash"
+
 mkdir -p tmp
 
-java -cp $clojars_jar clojure.main -m clojars.tools.generate-feeds tmp production 2> /dev/null
+java "${JAVA_OPTS[@]}" -cp $clojars_jar clojure.main -m clojars.tools.generate-feeds tmp production 2> /dev/null

scripts/update-stats

@@ -6,6 +6,8 @@ set -e
 
 dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
+source "$dir/set_default_java_opts.bash"
+
 clojars_jar=$1
 
 date=$2
@@ -28,8 +30,8 @@ downloads_all_new=tmp/all-new.edn
 # grab the latest all.edn from s3
 aws s3 cp "s3://${S3_STATS_BUCKET}/all.edn" "$downloads_all_old"
 
-cat "$cdn_logfile" | java -cp "$clojars_jar" clojure.main -m clojars.tools.process-stats > "$downloads_date"
-java -cp "$clojars_jar" clojure.main -m clojars.tools.merge-stats "$downloads_all_old" "$downloads_date" > "$downloads_all_new"
+cat "$cdn_logfile" | java "${JAVA_OPTS[@]}" -cp "$clojars_jar" clojure.main -m clojars.tools.process-stats > "$downloads_date"
+java "${JAVA_OPTS[@]}" -cp "$clojars_jar" clojure.main -m clojars.tools.merge-stats "$downloads_all_old" "$downloads_date" > "$downloads_all_new"
 
 # upload the new stats to the s3 bucket
 aws s3 cp --no-progress --content-type 'application/edn' --acl public-read "$downloads_date" "s3://${S3_STATS_BUCKET}/"