Bump ElastAlert to ElastAlert2 (2.20.0), Python to 3.12 for support Elasticsearch 7.x, 8.x (drop 6.x)

add-blobs.sh

@@ -17,17 +17,18 @@ blob_download() {
   fi
 }
 
-blob_download python2.7 https://www.python.org/ftp/python/2.7.15/Python-2.7.15.tgz Python-2.7.15.tgz
+blob_download python3.12 https://www.python.org/ftp/python/3.12.8/Python-3.12.8.tgz Python-3.12.8.tgz
 blob_download libffi https://buildpacks.cloudfoundry.org/dependencies/manual-binaries/python/libffi-3.2.1-linux-x64-5f5bf32c.tgz libffi-3.2.1.tgz
+blob_download rust https://static.rust-lang.org/dist/rust-1.83.0-x86_64-unknown-linux-gnu.tar.xz rust-1.83.0-x86_64-unknown-linux-gnu.tar.xz
 
-pip download -d elastalert --no-binary :all: elastalert==0.1.35
-pip download -d elastalert --no-binary :all: python-magic
-pip download -d elastalert --no-binary :all: future
-curl -L -J https://files.pythonhosted.org/packages/ab/10/817237669677f568238bb26760fe373b3b0be200cac309e0035389beff9a/thehive4py-1.6.0-py2-none-any.whl -o elastalert/thehive4py-1.6.0-py2-none-any.whl
-rm -f elastalert/elastalert-*.tar.gz
-curl -L -J https://files.pythonhosted.org/packages/59/47/d5c3c0b687c9e4c81b75eacee1b3cd29f0a101c92ebebd4a41464b61c622/elastalert-0.2.0b2.tar.gz -o elastalert/elastalert-0.2.0b2.tar.gz
-for f in $(ls elastalert/*.tar.gz elastalert/*.whl);do 
-  bosh add-blob --dir=${DIR} ${f} ${f}
+pip download -d elastalert --no-binary :all: "elastalert2==2.20.0"
+pip download -d elastalert --no-binary :all: "setuptools>=40.8.0" "setuptools_scm<8.0,>=6.4" "flit_core>=3.3" "hatchling" "calver" "hatch-vcs" "hatch-fancy-pypi-readme" "wheel" "Cython" "poetry-core" "maturin<2.0,>=1.2" "setuptools-rust>=1.4.0" "mypy<=1.14.0,>=1.4.1" "types-psutil" "types-setuptools" "expandvars"
+rm -f elastalert/pillow-*.tar.gz
+pip wheel -w elastalert pillow
+for f in $(ls elastalert/*.tar.gz elastalert/*.whl);do
+  if [ ! -f ${DIR}/blobs/${f} ];then
+    bosh add-blob --dir=${DIR} ${f} ${f}
+  fi
 done
 
 # export BOSH_INSTALL_TARGET=bosh

jobs/elastalert-silence/templates/bin/run.sh

@@ -8,10 +8,10 @@ TMP_DIR=/var/vcap/sys/tmp/elastalert
 PIDFILE=$RUN_DIR/elastalert.pid
 RULE_DIR=<%= link("elastalert").p("elastalert.rules_folder") %>
 
-export PATH=/var/vcap/packages/python2.7/bin:/var/vcap/packages/elastalert/bin:$PATH
+export PATH=/var/vcap/packages/python3.12/bin:/var/vcap/packages/elastalert/bin:$PATH
 export CONFIG_DIR=/var/vcap/jobs/elastalert/config
-export PYTHONPATH=/var/vcap/packages/elastalert/lib/python2.7/site-packages/
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/vcap/packages/python2.7/lib
+export PYTHONPATH=/var/vcap/packages/elastalert/lib/python3.12/site-packages/
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/vcap/packages/python3.12/lib
 
 source $JOB_DIR/bin/ctl_utils.sh
 

jobs/elastalert/spec

@@ -16,7 +16,7 @@ consumes:
 
 packages:
 - elastalert
-- python2.7
+- python3.12
 
 properties:
   elastalert.rules_folder:

jobs/elastalert/templates/bin/ctl.sh

@@ -8,10 +8,10 @@ JOB_DIR=/var/vcap/jobs/elastalert
 TMP_DIR=/var/vcap/sys/tmp/elastalert
 PIDFILE=$RUN_DIR/elastalert.pid
 
-export PATH=/var/vcap/packages/python2.7/bin:/var/vcap/packages/elastalert/bin:$PATH
+export PATH=/var/vcap/packages/python3.12/bin:/var/vcap/packages/elastalert/bin:$PATH
 export CONFIG_DIR=/var/vcap/jobs/elastalert/config
-export PYTHONPATH=/var/vcap/packages/elastalert/lib/python2.7/site-packages/
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/vcap/packages/python2.7/lib
+export PYTHONPATH=/var/vcap/packages/elastalert/lib/python3.12/site-packages/
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/vcap/packages/python3.12/lib
 
 source $JOB_DIR/bin/ctl_utils.sh
 

jobs/elastalert/templates/bin/elastalert-test-rule.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 set -e
 
-export PATH=/var/vcap/packages/python2.7/bin:/var/vcap/packages/elastalert/bin:$PATH
+export PATH=/var/vcap/packages/python3.12/bin:/var/vcap/packages/elastalert/bin:$PATH
 export CONFIG_DIR=/var/vcap/jobs/elastalert/config
-export PYTHONPATH=/var/vcap/packages/elastalert/lib/python2.7/site-packages/
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/vcap/packages/python2.7/lib
+export PYTHONPATH=/var/vcap/packages/elastalert/lib/python3.12/site-packages/
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/vcap/packages/python3.12/lib
 
 RULE_DIR=<%= p('elastalert.rules_folder') %>
 

packages/elastalert/packaging

@@ -2,18 +2,25 @@ set -e
 
 
 LIBFFI_VERSION=3.2.1
+RUST_VERSION=1.83.0
+RUST_ARCH_DISTRO=x86_64-unknown-linux-gnu
 
 tar xzf libffi/libffi-${LIBFFI_VERSION}.tgz -C libffi/
 rm -f libffi/libffi-${LIBFFI_VERSION}.tgz
 sed -i "s|/app/.cloudfoundry/vendor|$(pwd)/libffi|" libffi/lib/pkgconfig/libffi.pc
 ln -s $(pwd)/libffi/lib/pkgconfig/libffi.pc /usr/share/pkgconfig/
 
 # workaround for Xenial stemcell
-cp -a $PWD/libffi/lib/libffi-${LIBFFI_VERSION}/include/*.h /var/vcap/packages/python2.7/include/python2.7/
-cp -a $PWD/libffi/lib/libffi.*  /var/vcap/packages/python2.7/lib/
+cp -a $PWD/libffi/lib/libffi-${LIBFFI_VERSION}/include/*.h /var/vcap/packages/python3.12/include/python3.12/
+cp -a $PWD/libffi/lib/libffi.*  /var/vcap/packages/python3.12/lib/
 
-export PATH=/var/vcap/packages/python2.7/bin:$PATH
-export LD_LIBRARY_PATH=/var/vcap/packages/python2.7/lib:$LD_LIBRARY_PATH
-export C_INCLUDE_PATH=/var/vcap/packages/python2.7/include:$C_INCLUDE_PATH
+export PATH=/var/vcap/packages/python3.12/bin:$PATH
+export LD_LIBRARY_PATH=/var/vcap/packages/python3.12/lib:$LD_LIBRARY_PATH
+export C_INCLUDE_PATH=/var/vcap/packages/python3.12/include:$C_INCLUDE_PATH
 
-pip2.7 install --no-index --no-allow-external --find-links ./elastalert/ --prefix=${BOSH_INSTALL_TARGET} elastalert
+tar Jxf rust/rust-${RUST_VERSION}-${RUST_ARCH_DISTRO}.tar.xz -C rust/
+sh rust/rust-${RUST_VERSION}-${RUST_ARCH_DISTRO}/install.sh --components="rustc,rust-std-${RUST_ARCH_DISTRO},cargo"
+rm -f rust/rust-${RUST_VERSION}-${RUST_ARCH_DISTRO}.tar.xz
+rm -rf rust/rust-${RUST_VERSION}-${RUST_ARCH_DISTRO}
+
+pip3.12 install --no-index --find-links ./elastalert/ --prefix=${BOSH_INSTALL_TARGET} elastalert2

packages/elastalert/spec

@@ -2,8 +2,10 @@
 name: elastalert
 
 dependencies:
-- python2.7
+- python3.12
 
 files:
 - elastalert/*.tar.gz
+- elastalert/*.whl
 - libffi/libffi-3.2.1.tgz
+- rust/rust-1.83.0-x86_64-unknown-linux-gnu.tar.xz

packages/python2.7/packaging → packages/python3.12/packaging

@@ -1,9 +1,9 @@
 set -e
 
 CPUS=`grep -c ^processor /proc/cpuinfo`
-PYTHON_VERSION=2.7.15
+PYTHON_VERSION=3.12.8
 
-tar xzf python2.7/Python-${PYTHON_VERSION}.tgz
+tar xzf python3.12/Python-${PYTHON_VERSION}.tgz
 
 pushd Python-${PYTHON_VERSION}
   ./configure \

packages/python3.12/spec

@@ -0,0 +1,7 @@
+---
+name: python3.12
+
+dependencies: []
+
+files:
+- python3.12/Python-3.12.8.tgz