METRON-2324 verify_licenses.sh and list_dependencies.sh don't properly handle maven failures (justinleet) closes #1568

Author: justinleet

.travis.yml

@@ -36,12 +36,12 @@ matrix:
   include:
     - name: Unit Tests
       script:
-        - time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true -B -V -pl '!:metron-config,!:metron-alerts'
+        - time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true -Dskip.npm -B -V
         - time mvn surefire:test@unit-tests -T 2C
 
     - name: Integration Tests
       script:
-        - time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true -B -V -pl '!:metron-config,!:metron-alerts'
+        - time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true -Dskip.npm -B -V
         - time mvn surefire:test@integration-tests
 
     - name: Alerts UI Tests

dependencies_with_url.csv

@@ -50,6 +50,7 @@ javax.xml.bind:jaxb-api:jar:2.2.2:compile,CDDL,https://jaxb.dev.java.net/
 javax.xml.bind:jaxb-api:jar:2.3.0:compile,CDDL,https://jaxb.dev.java.net/
 javax.xml.stream:stax-api:jar:1.0-2:compile,COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0,https://docs.oracle.com/javase/7/docs/api/javax/xml/stream/package-summary.html
 jline:jline:jar:0.9.94:compile,BSD,http://jline.sourceforge.net
+junit:junit:jar:4.12:compile,Eclipse Public License 1.0,http://junit.org
 net.razorvine:pyrolite:jar:4.13:compile,MIT,https://github.com/irmen/Pyrolite
 net.sf.jopt-simple:jopt-simple:jar:3.2:compile,The MIT License,http://jopt-simple.sourceforge.net
 net.sf.jopt-simple:jopt-simple:jar:4.6:compile,The MIT License,http://jopt-simple.sourceforge.net

dev-utilities/build-utils/list_dependencies.sh

@@ -16,4 +16,16 @@
 #  limitations under the License.
 #
 
-{ mvn dependency:list || { echo "ERROR:  Failed to run mvn dependency:list" ; exit 1 ; } ; mvn dependency:list -PHDP-2.5.0.0 || { echo "ERROR:  Failed to run mvn dependency:list -PHDP-2.5.0.0" ; exit 1 ; } ; } | grep "^\[INFO\]   " | awk '{print $2}' | grep -v "org.apache" | grep -v "test" | grep -v "provided" | grep -v "runtime" | grep -v ":system" |  sort | uniq
+DEPS=$(mvn dependency:list)
+rc=$?
+if [[ $rc != 0 ]]; then
+  echo "ERROR:  Failed to run mvn dependency:list"
+  DEPS=$(mvn dependency:list -PHDP-2.5.0.0)
+  rc=$?
+  if [[ $rc != 0 ]]; then
+    echo "ERROR:  Failed to run mvn dependency:list -PHDP-2.5.0.0"
+    exit $rc
+  fi
+fi
+
+echo "$DEPS" | grep "^\[INFO\]   " | awk '{print $2}' | grep -v "org.apache" | grep -v "test" | grep -v "provided" | grep -v "runtime" | grep -v ":system" | sort | uniq

dev-utilities/build-utils/verify_licenses.sh

@@ -16,4 +16,25 @@
 #  limitations under the License.
 #
 
-dev-utilities/build-utils/list_dependencies.sh | python dev-utilities/build-utils/verify_license.py ./dependencies_with_url.csv
+# Need to build before we can properly list dependencies
+echo "Building Metron"
+mvn install -T 2C -q -DskipTests=true \
+  -Dmaven.javadoc.skip=true \
+  -Dskip.npm \
+  -B -V
+
+echo "Determining dependencies"
+DEPS=$(dev-utilities/build-utils/list_dependencies.sh)
+rc=$?
+if [[ $rc != 0 ]]; then
+  echo "Failed to determine dependencies"
+  exit $rc
+fi
+echo "$DEPS" | python dev-utilities/build-utils/verify_license.py ./dependencies_with_url.csv
+rc=$?
+if [[ $rc != 0 ]]; then
+  echo "Finished with dependency issues. Please ensure all dependencies are in dependencies_with_url.csv"
+  exit $rc
+else
+  echo "Finished dependencies."
+fi

metron-interface/metron-alerts/scripts/prepend_license_header.sh

@@ -33,9 +33,11 @@ LICENSE_HEADER="/**
  * limitations under the License.
  */
  "
+
+shopt -s nullglob
 for file in ./dist/*.{js,css}
 do
-    if !(grep -Fxq "$LICENSE_HEADER" $file)
+    if ! (grep -Fxq "$LICENSE_HEADER" $file)
     then
         echo "$LICENSE_HEADER$(cat $file)" > $file
     fi

metron-interface/metron-config/scripts/prepend_license_header.sh

@@ -33,9 +33,11 @@ LICENSE_HEADER="/**
  * limitations under the License.
  */
  "
+
+shopt -s nullglob
 for file in ./dist/*.{js,css}
 do
-    if !(grep -Fxq "$LICENSE_HEADER" $file)
+    if ! (grep -Fxq "$LICENSE_HEADER" $file)
     then
         echo "$LICENSE_HEADER$(cat $file)" > $file
     fi