From cb7a64741dc54f06aafb33691a1f4655820b44b5 Mon Sep 17 00:00:00 2001 From: Jan Walter Date: Tue, 14 Apr 2020 16:16:45 -0400 Subject: [PATCH 1/5] add Vagrant build environment for docker images and local java builds. add Dockerfiles and docker-compose for dockerized demo version of Herd, based on the 0.98.0 Cloudformation, tested up to 0.119.0 --- .../herd-docker-vagrant/Vagrantfile | 87 +++++++ .../docker-compose-build.sh | 88 +++++++ .../herd-docker-vagrant/docker-compose.yml | 46 ++++ .../elasticsearch/Dockerfile | 10 + .../elasticsearch/setup-elasticsearch.sh | 24 ++ .../herd-docker-vagrant/herd/Dockerfile | 26 ++ .../herd-docker-vagrant/herd/run-herd.sh | 59 +++++ .../herd-docker-vagrant/herd/setup-herd.sh | 243 ++++++++++++++++++ .../herd-docker-vagrant/herd/setup-os.sh | 24 ++ .../herd-docker-vagrant/readme.md | 62 +++++ 10 files changed, 669 insertions(+) create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/Vagrantfile create mode 100755 herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/Dockerfile create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/setup-elasticsearch.sh create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/herd/Dockerfile create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/herd/run-herd.sh create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/herd/setup-os.sh create mode 100644 herd-build/herd-scripts/herd-docker-vagrant/readme.md diff --git a/herd-build/herd-scripts/herd-docker-vagrant/Vagrantfile b/herd-build/herd-scripts/herd-docker-vagrant/Vagrantfile new file mode 100644 index 0000000000..dc042f7727 --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/Vagrantfile @@ -0,0 +1,87 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "generic/centos7" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" + config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1" + config.vm.network "forwarded_port", guest: 5443, host: 5443, host_ip: "127.0.0.1" + config.vm.network "forwarded_port", guest: 5432, host: 5432, host_ip: "127.0.0.1" + config.vm.network "forwarded_port", guest: 18630, host: 18630, host_ip: "127.0.0.1" + config.vm.network "forwarded_port", guest: 18631, host: 18631, host_ip: "127.0.0.1" + config.vm.network "forwarded_port", guest: 18632, host: 18632, host_ip: "127.0.0.1" + + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + config.vm.synced_folder ".", "/home/vagrant/external" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # Customize the amount of memory on the VM: + vb.memory = "8192" + vb.linked_clone = true + vb.cpus = 2 + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell", inline: <<-SHELL + yum update -y + yum install -y git + yum install -y yum-utils device-mapper-persistent-data lvm2 + yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + yum install -y docker-ce docker-ce-cli containerd.io docker-compose + usermod -a -G docker vagrant + systemctl enable docker + systemctl start docker + SHELL +end diff --git a/herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh b/herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh new file mode 100755 index 0000000000..bba6d2bc86 --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Copyright 2015 herd contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds herd 0.119 image + +HERD_VER="0.119.0" + +### CHANGE THESE ### +echo "**** WARNING **** CHANGE VARIABLES IN THE SCRIPT FOR YOUR S3 BUCKETS AND SQS QUEUES !!!!!" +S3BUCKET="222222:s3::::test-bucket" +INCOMING_SQSQ="22222:sqs:::::herd-incoming" +IDXUPTD_SQSQ="22222:sqs:::::herd-idxupdt" + +# here to save time +if [ ! -e herd/herd-war-$HERD_VER.war ] ; then + curl https://oss.sonatype.org/service/local/repositories/releases/content/org/finra/herd/herd-war/$HERD_VER/herd-war-$HERD_VER.war > herd/herd-war-$HERD_VER.war + +fi ; + +if [ ! -e herd-scripts-sql-$HERD_VER.jar ] ; then + curl https://oss.sonatype.org/service/local/repositories/releases/content/org/finra/herd/herd-scripts-sql/$HERD_VER/herd-scripts-sql-$HERD_VER.jar > herd-scripts-sql-$HERD_VER.jar +fi ; + +if [ ! -d herd-setul-sql ] ; then + mkdir herd-setup-sql + cd herd-setup-sql + unzip ../herd-scripts-sql-$HERD_VER.jar + # fix file orders per cloudformation script + mv herd.postgres.0.1.0.create.sql 1herd.postgres.0.1.0.create.sql + mv herd.postgres.0.1.0.refdata.sql 2herd.postgres.0.1.0.refdata.sql + mv herd.postgres.0.1.0.cnfgn.sql 3herd.postgres.0.1.0.cnfgn.sql + rm herd.postgres.clean.refdata.sql + + for f in *.upgrade.sql ; do + mv $f 4$f ; + done ; + + for f in activiti*sql ; do + mv $f 5$f ; + done ; + + for f in quartz*sql ; do + # moved to last, this borks in the current release and the initializer then stops running, so set variable + # more scripts + + echo "\set ON_ERROR_STOP 0" >> 99$f + cat $f >> 99$f + rm -f $f + done ; + + for f in elasticsearch*sql ; do + mv $f 7$f ; + done ; + + # manual additional data + + cat > 98last.sql << EOF + +DELETE FROM cnfgn WHERE cnfgn_key_nm = 's3.managed.bucket.name'; +INSERT INTO cnfgn VALUES ('s3.managed.bucket.name','${S3BUCKET}', NULL); +DELETE FROM cnfgn WHERE cnfgn_key_nm = 'herd.notification.sqs.incoming.queue.name'; +INSERT INTO cnfgn VALUES ('herd.notification.sqs.incoming.queue.name','${INCOMING_SQSQ}', NULL); +DELETE FROM cnfgn WHERE cnfgn_key_nm = 'search.index.update.sqs.queue.name'; +INSERT INTO cnfgn VALUES ('search.index.update.sqs.queue.name','${IDXUPTD_SQSQ}', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.best.fields.query.boost','100', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.phrase.prefix.query.boost','1', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.phrase.query.boost','1000', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.rest.client.hostname','herd-elasticsearch', NULL); + +EOF +fi ; +#if behind a proxy, remember --build-arg http_proxy --build-arg https_proxy + +echo "You're now ready to run 'docker-compose build' if you haven't already, or re-initialize the database." + +# FIXME: add uname check, this isn't needed unless it's on linux +# check sysctl value to make sure elasticsearch is going to be happy +echo "*********** WARNING ****************" +echo "Make sure vm.max_map_count = 262144 or Elasticsearch will not run. Current value is "`sysctl vm.max_map_count` \ No newline at end of file diff --git a/herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml b/herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml new file mode 100644 index 0000000000..46410a7151 --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml @@ -0,0 +1,46 @@ +version: '3.5' + +volumes: + herd-pgsql-data: + driver: local + herd-elasticsearch-data: + driver: local + +services: + herd-pgsql: + image: postgres:10 + restart: always + environment: + - POSTGRES_USER=finraherd + - POSTGRES_PASSWORD=Changeme123 + - POSTGRES_DB=herd + volumes: + - herd-pgsql-data:/var/lib/postgresql/data + - ./herd-setup-sql:/docker-entrypoint-initdb.d + + herd-elasticsearch: + image: herd-elasticsearch:latest + build: + context: elasticsearch/ + dockerfile: Dockerfile + volumes: + - herd-elasticsearch-data:/var/lib/elasticsearch + + herd: + image: herdsvc:latest + build: + context: herd/ + dockerfile: Dockerfile + args: + - HERD_RELEASE=0.119.0 + depends_on: + - herd-pgsql + - herd-elasticsearch + environment: + - HERD_UI_HOST=${HERD_UI_HOST} + links: + - "herd-pgsql" + - "herd-elasticsearch" + ports: + - "8080:8080" + - "5443:80" diff --git a/herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/Dockerfile b/herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/Dockerfile new file mode 100644 index 0000000000..29e914f359 --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/Dockerfile @@ -0,0 +1,10 @@ +# vim:set ft=dockerfile: +#FROM debian:stretch-slim +FROM openjdk:8-jdk + +ADD setup-elasticsearch.sh /setup/setup-elasticsearch.sh +RUN set -ex ; \ + chmod a+x /setup/setup-elasticsearch.sh ; cd /setup ; bash ./setup-elasticsearch.sh +EXPOSE 9200 9300 +USER elasticsearch +ENTRYPOINT ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/setup-elasticsearch.sh b/herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/setup-elasticsearch.sh new file mode 100644 index 0000000000..12c5636156 --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/elasticsearch/setup-elasticsearch.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2015 herd contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# setup for elasticache locally + +set -ex ; +curl https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.0.1-amd64.deb > esearch.deb ; +apt-get install -y ./esearch.deb ; +rm -f ./esearch.deb ; +echo "network.host : 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml +echo "http.port : 9200" >> /etc/elasticsearch/elasticsearch.yml +echo "node.name: node-1" >> /etc/elasticsearch/elasticsearch.yml +echo "cluster.initial_master_nodes: [\"node-1\"]" >> /etc/elasticsearch/elasticsearch.yml +# the below is in the Cloudformation, but doing it here causes things to bork +#echo "script.engine.groovy.inline.search: on" >> /etc/elasticsearch/elasticsearch.yml diff --git a/herd-build/herd-scripts/herd-docker-vagrant/herd/Dockerfile b/herd-build/herd-scripts/herd-docker-vagrant/herd/Dockerfile new file mode 100644 index 0000000000..01ee6d27b6 --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/herd/Dockerfile @@ -0,0 +1,26 @@ +# vim:set ft=dockerfile: +#FROM debian:stretch-slim +FROM tomcat:jre8 + + +ADD setup-os.sh /setup/setup-os.sh +RUN set -ex ; \ + chmod a+x /setup/setup-os.sh ; \ + cd /setup/ ; \ + bash /setup/setup-os.sh ; + +ADD setup-herd.sh /setup/setup-herd.sh +ARG HERD_RELEASE +ADD herd-war-${HERD_RELEASE}.war /herd-app.war +RUN set -ex ; \ + chmod a+x /setup/setup-herd.sh; \ + cd /setup/ ; \ + bash /setup/setup-herd.sh ; + +ADD run-herd.sh /usr/local/tomcat +RUN chmod a+x /usr/local/tomcat/run-herd.sh + +ENTRYPOINT /usr/local/tomcat/run-herd.sh + +EXPOSE 80 8080 + diff --git a/herd-build/herd-scripts/herd-docker-vagrant/herd/run-herd.sh b/herd-build/herd-scripts/herd-docker-vagrant/herd/run-herd.sh new file mode 100644 index 0000000000..b5edb7f88d --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/herd/run-herd.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright 2015 herd contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# need to start apache first and then run catalina +# the way the container did before + +# FIRST RUN + +if [ ! -e /var/www/html/configuration.json ] ; then + + echo "Determining host IP for configuration ..." + if [ -z $HERD_UI_HOST ] ; then + # check if we're in AWS, or bork + herd_ui_host=$(/usr/bin/curl http://169.254.169.254/latest/meta-data/public-hostname) + if [ -z herd_ui_host ] ; then + echo "ERROR: unable to determine host IP or name; please set HERD_UI_HOST env variable for container." + exit 2 + fi; + + else + herd_ui_host=$HERD_UI_HOST + fi; + echo "Setting UI host to $herd_ui_host" + +echo > /var/www/html/configuration.json << EOF3 +{ + 'restBaseUri': 'http://'$herd_ui_host':8080/herd-app/rest', + 'basicAuthRestBaseUri': 'basicAuthRestBaseUri', + 'helpUrl': 'helpUrl', + 'supportEmail': 'orgSupportEmail', + 'brandHeader': 'Herd-UI', + 'brandMotto': 'Locate and understand data available in HERD', + 'docTitlePrefix': 'Herd-UI', + 'useBasicAuth': 'false', + 'alertDelayInSeconds': '10', + 'trackAnalytics': 'false', + 'ga': { + 'key': 'key', + 'iv': 'iv', + 'trackingId': 'trackingId' + } +} + +EOF3 + +fi ; + + +service apache2 start +catalina.sh run diff --git a/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh b/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh new file mode 100644 index 0000000000..f5e20144ea --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh @@ -0,0 +1,243 @@ +#!/bin/bash +# Copyright 2015 herd contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# rather than deal with cumbersome docker exec, put it in a script +# and it'll work everywhere + + +herdTagVersion=${HERD_RELEASE} +herdUIVersion='0.68.0' +NexusPath='https://oss.sonatype.org/service/local/repositories/releases/content/org/finra/herd/' + +# Tomcat paths since they change based on whether we're using the apache tomcat image +# or rolled our own + +TC_HOME=/usr/local/tomcat + +set -ex ; + + +curl ${NexusPath}herd-scripts-sql/${herdTagVersion}/herd-scripts-sql-${herdTagVersion}.jar > herd-sql.jar +# clear out existing apps +rm -rf $TC_HOME/webapps/* +mv /herd-app.war $TC_HOME/webapps/ +#curl ${NexusPath}herd-war/${herdTagVersion}/herd-war-${herdTagVersion}.war > $TC_HOME/webapps/herd.war +#chown tomcat8:tomcat8 $TC_HOME/webapps/herd.war +chmod 0664 $TC_HOME/webapps/herd-app.war + +curl 'https://jdbc.postgresql.org/download/postgresql-9.4-1202.jdbc41.jar' > $TC_HOME/lib/postgresql-9.4-1202.jdbc41.jar +#chown tomcat8:tomcat8 $TC_HOME/lib/postgresql-9.4-1202.jdbc41.jar +chmod 0644 $TC_HOME/lib/postgresql-9.4-1202.jdbc41.jar + +cat > $TC_HOME/conf/context.xml << EOF + + + WEB-INF/web.xml + + + +EOF + + +cat > $TC_HOME/conf/server.xml << EOF2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +EOF2 + +/bin/sed -i '/Built In Filter Definitions/a \ +\ +CorsFilter\ +org.apache.catalina.filters.CorsFilter\ +\ +cors.allowed.origins\ +*\ +\ +\ +cors.allowed.headers\ +Content-Type,Authorization,Accept,Origin\ +\ +\ +cors.allowed.methods\ +GET,POST,PUT,DELETE,HEAD,OPTIONS\ +\ +\ +\ +CorsFilter\ +/*\ +' $TC_HOME/conf/web.xml + + +/bin/sed -i '/Options Indexes FollowSymLinks/a RewriteRule ^ index.html [L]' /etc/apache2/apache2.conf +/bin/sed -i '/Options Indexes FollowSymLinks/a RewriteRule ^ - [L]' /etc/apache2/apache2.conf +/bin/sed -i '/Options Indexes FollowSymLinks/a RewriteCond %{REQUEST_FILENAME} -d' /etc/apache2/apache2.conf +/bin/sed -i '/Options Indexes FollowSymLinks/a RewriteCond %{REQUEST_FILENAME} -f [OR]' /etc/apache2/apache2.conf +/bin/sed -i '/Options Indexes FollowSymLinks/a RewriteEngine On' /etc/apache2/apache2.conf +#sudo service apache2 start +# Deployig herd-ui +curl https://registry.npmjs.org/@herd/herd-ui-dist/-/herd-ui-dist-${herdUIVersion}.tgz | /bin/tar xz -C /tmp +mv /tmp/package/dist/* /var/www/html +# Configuring directory and file ownership for the http server +#sudo chown -R httpd:httpd /var/www +# for debian docker images, need to enable mod_rewrite +pushd /etc/apache2/mods-enabled +ln -s ../mods-available/rewrite.* . +popd +# make permissions sane +chmod 2775 /var/www +find /var/www -type d -exec chmod 2775 {} + +find /var/www -type f -exec chmod 0664 {} + +# Setup configuration.json herd-ui settings +# moved to run-herd script + + diff --git a/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-os.sh b/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-os.sh new file mode 100644 index 0000000000..f4717c232c --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-os.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2015 herd contributors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# first stage in docker image build; install all packages + +set -ex ; + +apt-get update ; +apt-get install -y curl nano git ; + + +apt-get install -y apache2 libapache2-mod-jk ; +#apt-get install -y openjdk-8-jre-headless; +#apt-get install -y tomcat8-common ; +#apt-get install -y tomcat8 ; \ No newline at end of file diff --git a/herd-build/herd-scripts/herd-docker-vagrant/readme.md b/herd-build/herd-scripts/herd-docker-vagrant/readme.md new file mode 100644 index 0000000000..40b0075bbc --- /dev/null +++ b/herd-build/herd-scripts/herd-docker-vagrant/readme.md @@ -0,0 +1,62 @@ +# Docker and Vagrant Demo Environments + +## The Vagrant Build Environment + +The environment is here to provide a consistent Docker build environment. If you have one already, +you probably don't need it, but it's here as a reference. + +Usage: + +1. Install Vagrant from https://vagrantup.com + * Windows, OSX, Linux are supported + * Requires a supported virtualization environment + * provided Vagrantfile tested with Oracle VirtualBox (FOSS Edition) on Windows and OSX +2. From the current directory, issue the command `vagrant up` +3. Once the system has finished configuring itself, connect to it using `vagrant ssh` +4. After the initial remote key approval etc, `cd external` and you should be in the same + folder this file is in +5. Follow the Docker build directions + +*Notes: The ports shared by the Vagrant box are only exposed to localhost - 127.0.0.1 by default. If you want to share the fruits of +your labors with others on your LAN you'll need to change that.* + + +## Building the Docker Images + +Because I tend to run a large number of builds back to back when trying to get a systems config +correct, I download a number of things ahead of time. Also, configuring the Dockerfiles to +minimize the number of repeat downloads to rebuilds is good practice. + +### 1. Set Environment Variables and Edit Files + +The system will want to know the UI is at, because some things like CORS headers for good +HTTP security depend on the system knowing what it's expected to be called. + +`export HERD_UI_HOST=127.0.0.1` will map the CORS to allow a simple localhost connection, and if you're doing +this just for yourself this is what you'll want. + +If you're going to drop this somewhere else, figure out its hostname or IP address and use that. + +### 2. Download Prerequisites + +`./docker-compose-build.sh` will download a number of prerequisites and arrange them so the subsequent +steps will work. + +What happens: + +The script downloads a number of things from all over, unpacks and re-organized them so it's something +that the Docker containers can ingest when they run for the first time. This is important, since the +version defined in the script defines the versions of the SQL scripts and stuff it downloads. + +### 3. Run `docker-compose` as you normally would + +Run `docker-compose build` to pull down and build the images, or if you have a lot of faith, +simply go ahead and run `docker-compose up`. + +What happens: + +The Postgresql container mounts a new directory prepared by the `docker-compose-build.sh` script and executes +all the sql for the release therein. + +The other containers download what they need to, and stand themselves up, and they use the same repositories referenced +in the AWS CloudFormation scripts. \ No newline at end of file From 177741e99cb46e147feb54427a4c004f6e56ae1b Mon Sep 17 00:00:00 2001 From: Jan Walter Date: Tue, 14 Apr 2020 16:50:44 -0400 Subject: [PATCH 2/5] update herd-ui version to .89 --- .../herd-scripts/herd-docker-vagrant/herd/setup-herd.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh b/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh index f5e20144ea..d51e6ecce9 100644 --- a/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh +++ b/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh @@ -15,7 +15,7 @@ herdTagVersion=${HERD_RELEASE} -herdUIVersion='0.68.0' +herdUIVersion='0.89.0' NexusPath='https://oss.sonatype.org/service/local/repositories/releases/content/org/finra/herd/' # Tomcat paths since they change based on whether we're using the apache tomcat image @@ -25,8 +25,6 @@ TC_HOME=/usr/local/tomcat set -ex ; - -curl ${NexusPath}herd-scripts-sql/${herdTagVersion}/herd-scripts-sql-${herdTagVersion}.jar > herd-sql.jar # clear out existing apps rm -rf $TC_HOME/webapps/* mv /herd-app.war $TC_HOME/webapps/ From 473fa72b16d49f482782767a7ffb6929646de569 Mon Sep 17 00:00:00 2001 From: Jan Walter Date: Thu, 21 May 2020 09:53:08 -0400 Subject: [PATCH 3/5] add aws keys to build for local runs, remove for aws-hosted builds --- .../herd-scripts/herd-docker-vagrant/docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml b/herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml index 46410a7151..508feab23b 100644 --- a/herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml +++ b/herd-build/herd-scripts/herd-docker-vagrant/docker-compose.yml @@ -37,7 +37,9 @@ services: - herd-pgsql - herd-elasticsearch environment: - - HERD_UI_HOST=${HERD_UI_HOST} + - AWS_ACCESS_KEY_ID + - AWS_SECRET_KEY + - HERD_UI_HOST=localhost:5443 links: - "herd-pgsql" - "herd-elasticsearch" From 8de3006a0f686fb5449ad692beb74bcdc232da76 Mon Sep 17 00:00:00 2001 From: Jan Walter Date: Thu, 21 May 2020 09:55:36 -0400 Subject: [PATCH 4/5] corrected cors to latest spec --- .../herd-docker-vagrant/herd/setup-herd.sh | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh b/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh index d51e6ecce9..dabdba6a86 100644 --- a/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh +++ b/herd-build/herd-scripts/herd-docker-vagrant/herd/setup-herd.sh @@ -193,13 +193,48 @@ cat > $TC_HOME/conf/server.xml << EOF2 EOF2 +cat > $TC_HOME/conf/tomcat-users.xml << EOF3 + + + + + + + +EOF3 + +## NOTE: CORS settings can be finicky, so if this needs to get hacked up for making +## the docker container run elsewhere other than localhost + /bin/sed -i '/Built In Filter Definitions/a \ \ CorsFilter\ org.apache.catalina.filters.CorsFilter\ \ cors.allowed.origins\ -*\ +http://localhost:5443\ \ \ cors.allowed.headers\ @@ -209,6 +244,10 @@ EOF2 cors.allowed.methods\ GET,POST,PUT,DELETE,HEAD,OPTIONS\ \ +\ +cors.support.credentials\ +true\ +\ \ \ CorsFilter\ From e51c056fcc43d9f146aa7de9036a5a1baf8718ac Mon Sep 17 00:00:00 2001 From: Jan Walter Date: Thu, 21 May 2020 12:18:45 -0400 Subject: [PATCH 5/5] some setting names changed, turn off auth in settings --- .../herd-docker-vagrant/docker-compose-build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh b/herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh index bba6d2bc86..59d272f81f 100755 --- a/herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh +++ b/herd-build/herd-scripts/herd-docker-vagrant/docker-compose-build.sh @@ -71,10 +71,17 @@ DELETE FROM cnfgn WHERE cnfgn_key_nm = 'herd.notification.sqs.incoming.queue.nam INSERT INTO cnfgn VALUES ('herd.notification.sqs.incoming.queue.name','${INCOMING_SQSQ}', NULL); DELETE FROM cnfgn WHERE cnfgn_key_nm = 'search.index.update.sqs.queue.name'; INSERT INTO cnfgn VALUES ('search.index.update.sqs.queue.name','${IDXUPTD_SQSQ}', NULL); +DELETE FROM cnfgn WHERE cnfgn_key_nm = 'security.enabled.spel.expression'; +INSERT INTO cnfgn VALUES ('security.enabled.spel.expression','false', NULL); INSERT INTO cnfgn VALUES ('elasticsearch.best.fields.query.boost','100', NULL); INSERT INTO cnfgn VALUES ('elasticsearch.phrase.prefix.query.boost','1', NULL); INSERT INTO cnfgn VALUES ('elasticsearch.phrase.query.boost','1000', NULL); INSERT INTO cnfgn VALUES ('elasticsearch.rest.client.hostname','herd-elasticsearch', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.rest.client.scheme','http', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.rest.client.port','9200', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.domain.rest.client.hostname','herd-elasticsearch', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.domain.rest.client.port','9200', NULL); +INSERT INTO cnfgn VALUES ('elasticsearch.domain.rest.client.scheme','http', NULL); EOF fi ;