WIP: Split bear execution by runtime

.ci/bears.dirs.prune.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# Delete empty bear directories
+set -e
+
+dirs=$(find bears -depth -type d -and -not -name '__pycache__')
+
+for dir in $dirs; do
+  bears=$(ls $dir/*Bear.py 2>/dev/null || true)
+  subdirs=$(ls -d $dir/*/ 2>/dev/null || true)
+  if [[ -z "$bears""$subdirs" ]]; then
+    echo Removing $dir
+    rm -rf $dir
+  fi
+done

.ci/bears.prune.sh

@@ -0,0 +1,164 @@
+#!/bin/sh
+
+# Delete bears
+set -e -x
+
+if [[ "$BEARS" == "all" ]]; then
+  echo "No bears pruned."
+  exit 0
+fi
+
+if [[ -z "$BEARS" ]]; then
+  if [[ -n "$TRAVIS_LANGUAGE" ]]; then
+    BEARS=$TRAVIS_LANGUAGE
+  fi
+fi
+
+if [[ "$BEARS" == "haskell" || "$BEARS" == "ghc" ]]; then
+  BEARS=cabal
+elif [[ "$BEARS" == "ruby" ]]; then
+  BEARS=gem
+elif [[ "${BEARS/node/}" != "$BEARS" ]]; then
+  BEARS=npm
+elif [[ "$BEARS" == "r" ]]; then
+  BEARS=rscript
+elif [[ "$BEARS" == "generic" ]]; then
+  BEARS=apt-get
+fi
+
+bears=$(find bears -type f -and -name '*Bear.py' | sort)
+
+yield_result_bears=$(grep -m 1 -l 'yield Result' $bears)
+
+# unused
+# non_yield_result_bears=$(comm -23 <(ls $bears) <(ls $yield_result_bears))
+
+requirement_bears=$(grep -m 1 -l 'Requirement(' $bears)
+
+cabal_requirement_bears=$(grep -m 1 -l CabalRequirement $requirement_bears)
+gem_requirement_bears=$(grep -m 1 -l GemRequirement $requirement_bears)
+go_requirement_bears=$(grep -m 1 -l GoRequirement $requirement_bears)
+npm_requirement_bears=$(grep -m 1 -l NpmRequirement $requirement_bears)
+rscript_requirement_bears=$(grep -m 1 -l RscriptRequirement $requirement_bears)
+
+pip_requirement_bears=$(grep -m 1 -l PipRequirement $requirement_bears)
+
+non_pip_runtime_requirement_bears="$cabal_requirement_bears $gem_requirement_bears $go_requirement_bears $npm_requirement_bears $rscript_requirement_bears"
+
+other_requirement_bears=$(comm -23 <(ls $requirement_bears) <(ls $non_pip_runtime_requirement_bears $pip_requirement_bears))
+
+distribution_requirement_bears=$(grep -m 1 -l 'DistributionRequirement(' $requirement_bears)
+
+apt_get_requirement_bears=$(grep -m 1 -l 'apt_get' $distribution_requirement_bears)
+
+# Verify that DistributionRequirement is the only other Requirement subclass used
+unknown_requirement_bears=$(grep -m 1 -Pl '(?<!Distribution)Requirement\(' $other_requirement_bears || true)
+
+if [[ -n "$unknown_requirement_bears" ]]; then
+  echo "Unknown requirements for $unknown_requirement_bears"
+  exit 1
+fi
+
+pip_only_requirement_bears=$(comm -23 <(ls $pip_requirement_bears) <(ls $non_pip_runtime_requirement_bears))
+# unused:
+# pip_plus_requirement_bears=$(comm -23 <(ls $pip_requirement_bears) <(ls $pip_only_requirement_bears))
+
+no_requirement_bears=$(comm -23 <(ls $bears) <(ls $requirement_bears))
+
+clang_bears=''
+other_bears=''
+for bear in $no_requirement_bears; do
+  if [[ ${bear/Clang/} != "${bear}" ]]; then
+    clang_bears="$clang_bears $bear"
+  else
+    other_bears="$other_bears $bear"
+  fi
+done
+
+executable_linter_bears=$(grep -m 1 -l '@linter(.' $other_bears)
+executable_other_bears=$(grep -m 1 -l 'run_shell_command' $other_bears)
+
+other_bears=$(comm -23 <(ls $other_bears) <(ls $executable_linter_bears $executable_other_bears))
+
+non_yield_result_other_bears=$(comm -23 <(ls $other_bears) <(ls $yield_result_bears))
+
+if [[ -n "$non_yield_result_other_bears" ]]; then
+  echo "Unknown bear type $non_yield_result_other_bears"
+  exit 1
+fi
+
+echo Other exe $executable_linter_bears $executable_other_bears
+
+python_bears="$pip_only_requirement_bears $clang_bears $other_bears"
+
+non_python_bears=$(comm -23 <(ls $bears) <(ls $python_bears))
+
+cabal_requirement_bears="$cabal_requirement_bears bears/haskell/HaskellLintBear.py bears/shell/ShellCheckBear.py"
+
+dart_bears=$(ls bears/dart/*Bear.py)
+julia_bears=$(ls bears/julia/*Bear.py)
+lua_bears=$(ls bears/lua/*Bear.py)
+opam_bears="bears/java/InferBear.py"
+perl_bears=$(ls bears/perl/*Bear.py)
+perl_bears="$perl_bears bears/vhdl/VHDLLintBear.py"
+php_bears=$(ls bears/php/*Bear.py)
+
+
+java_bears=$(grep -m 1 -l 'default-jre' $apt_get_requirement_bears)
+pmd_bears=$(grep -m 1 -l 'PMD is missing' $executable_linter_bears $executable_other_bears)
+java_bears="$java_bears $pmd_bears bears/swift/TailorBear.py"
+
+apt_get_requirement_bears=$(echo $apt_get_requirement_bears | xargs -n 1 | egrep -v '(Haskell|Julia|Lua)' )
+if [[ "$DIST" == "precise" ]]; then
+  apt_get_requirement_bears=$(echo $apt_get_requirement_bears | xargs -n 1 | grep -v 'PHPCodeSniffer' )
+fi
+if [[ "$DIST" == "debian-sid" ]]; then
+  apt_get_requirement_bears=$(echo $apt_get_requirement_bears | xargs -n 1 | egrep -v '(CSharp|PerlCritic|PHPCodeSniffer|PHPLintBear)' )
+fi
+
+remove_bears=''
+
+echo Removing bears not desirable for $BEARS
+
+if [[ $BEARS == "python" ]]; then
+  # The test for generate_package depends on non-Python bears
+  remove_bears="$non_python_bears"
+elif [[ $BEARS == "cabal" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $cabal_requirement_bears))
+elif [[ $BEARS == "dart" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $dart_bears))
+elif [[ $BEARS == "gem" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $gem_requirement_bears))
+elif [[ $BEARS == "go" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $go_requirement_bears))
+elif [[ $BEARS == "java" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $java_bears))
+elif [[ $BEARS == "julia" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $julia_bears))
+elif [[ $BEARS == "opam" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $opam_bears))
+elif [[ $BEARS == "lua" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $lua_bears))
+elif [[ $BEARS == "perl" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $perl_bears))
+elif [[ $BEARS == "php" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $php_bears))
+elif [[ $BEARS == "npm" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $npm_requirement_bears))
+elif [[ $BEARS == "rscript" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $rscript_requirement_bears))
+elif [[ $BEARS == "apt-get" ]]; then
+  remove_bears=$(comm -23 <(ls $bears) <(ls $apt_get_requirement_bears))
+fi
+
+if [[ -n "$remove_bears" ]]; then
+  # generate_package.py only needs testing with BEARS=all
+  remove_bears="bears/generate_package.py $remove_bears"
+fi
+
+for bear in $remove_bears; do
+  echo Removing $bear
+  rm $bear
+done
+
+source .ci/bears.dirs.prune.sh

.ci/deps.apt-new.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e -x
+
+if [[ "$DIST" != "" && "$DIST" != "precise" && "$DIST" != "trusty" && "$DIST" != "debian-sid" ]]; then
+  echo "deb http://archive.ubuntu.com/ubuntu/ $DIST main universe" | sudo tee -a /etc/apt/sources.list.d/$DIST.list > /dev/null
+  echo "deb http://archive.ubuntu.com/ubuntu/ $DIST-updates main universe" | sudo tee -a /etc/apt/sources.list.d/$DIST.list > /dev/null
+  echo "deb http://archive.ubuntu.com/ubuntu/ $DIST-backports main universe" | sudo tee -a /etc/apt/sources.list.d/$DIST.list > /dev/null
+  sudo apt-get -y update
+  sudo apt-get -y --no-install-recommends install chktex cppcheck default-jre flawfinder indent libperl-critic-perl libxml2-utils mono-mcs php7.0-cli php-codesniffer verilator
+fi
+
+# Change environment for flawfinder from python to python2
+if [[ -n "$(which flawfinder)" ]]; then
+  sed -e '1s/.*/#!\/usr\/bin\/env python2/' $(which flawfinder) > $HOME/bin/flawfinder;
+  chmod a+x $HOME/bin/flawfinder;
+fi

.ci/deps.go.sh

@@ -9,3 +9,6 @@ gometalinter --install
 go get -u sourcegraph.com/sqs/goreturns
 
 go get -u github.com/BurntSushi/toml/cmd/tomlv
+
+# other approach
+# grep ImportPath Godeps/Godeps.json | cut -d '"' -f 4 | grep '[a-z]' | xargs go get;

.ci/deps.opam.sh

@@ -27,4 +27,6 @@ if [ ! -e ~/infer-linux64-v0.7.0/infer/bin ]; then
   opam pin add --yes --no-action infer .
   opam install --deps-only --yes infer
   ./build-infer.sh java
+
+  cd -
 fi

.ci/deps.perl.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x
+
+# VHDL Bakalint Installation
+if [ ! -e ~/.local/bin/bakalint.pl ]; then
+  BAKALINT_VERSION=0.4.0
+  wget "http://downloads.sourceforge.net/project/fpgalibre/bakalint/0.4.0/bakalint-0.4.0.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Ffpgalibre%2Ffiles%2Fbakalint%2F0.4.0%2F&ts=1461844926&use_mirror=netcologne" -O ~/bl.tar.gz
+  tar xf ~/bl.tar.gz -C ~/
+  mv ~/bakalint-$BAKALINT_VERSION/bakalint.pl ~/.local/bin/
+fi

.ci/deps.python.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -e -x
+
+echo pip3.4 is $(which pip3.4)
+echo python3.4 is $(which python3.4)
+if [[ -n "$(which pip3.4)" ]]; then
+  ln -sf $(which pip3.4) $HOME/bin/pip3
+fi
+if [[ -n "$(which python3.4)" ]]; then
+  ln -sf $(which python3.4) $HOME/bin/python3
+fi
+
+echo pip3.5 is $(which pip3.5)
+echo python3.5 is $(which python3.5)
+if [[ -n "$(which pip3.5)" ]]; then
+  ln -sf $(which pip3.5) $HOME/bin/pip3
+fi
+if [[ -n "$(which python3.5)" ]]; then
+  ln -sf $(which python3.5) $HOME/bin/python3
+fi
+
+echo pip3 is $(which pip3)
+echo python3 is $(which python3)
+if [[ -n "$(which python3)" ]]; then
+  ln -sf $(which python3) $HOME/bin/python
+fi
+
+python --version
+# Needed as some hosts may be providing python 3.2
+wget https://bootstrap.pypa.io/get-pip.py
+python get-pip.py --user
+rm get-pip.py
+cp $HOME/.local/bin/pip $HOME/.local/bin/pip3
+cp $HOME/.local/bin/pip $HOME/bin/pip
+cp $HOME/.local/bin/pip $HOME/bin/pip3
+pip --version

.ci/deps.sh

@@ -24,14 +24,6 @@ if ! dartanalyzer -v &> /dev/null; then
   cp -rp ~/dart-sdk/* ~/.local/
 fi
 
-# VHDL Bakalint Installation
-if [ ! -e ~/.local/bin/bakalint.pl ]; then
-  BAKALINT_VERSION=0.4.0
-  wget "http://downloads.sourceforge.net/project/fpgalibre/bakalint/0.4.0/bakalint-0.4.0.tar.gz?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Ffpgalibre%2Ffiles%2Fbakalint%2F0.4.0%2F&ts=1461844926&use_mirror=netcologne" -O ~/bl.tar.gz
-  tar xf ~/bl.tar.gz -C ~/
-  mv ~/bakalint-$BAKALINT_VERSION/bakalint.pl ~/.local/bin/
-fi
-
 # elm-format Installation
 if [ ! -e ~/.local/bin/elm-format ]; then
   curl -fsSL -o elm-format.tgz https://github.com/avh4/elm-format/releases/download/0.5.2-alpha/elm-format-0.17-0.5.2-alpha-linux-x64.tgz

.ci/tests.prune.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Delete tests for bears that have been removed
+set -e
+
+bear_tests=$(find tests -type f -and -name '*Test.py')
+
+for test in tests/generate_packageTest.py $bear_tests; do
+  bear=${test/Test.py/.py}
+  bear=${bear/tests/bears}
+  dir=$(dirname $bear)
+  if [[ ! -d $dir ]]; then
+    echo Removing $test
+    rm -f $test
+  elif [[ ! -f $bear ]]; then
+    echo Removing $test
+    rm -f $test
+  fi
+done

.moban.dt/bears-setup.py.jj2

@@ -16,4 +16,16 @@ extras_require = {
 required += [req for req in bear_required
              if not any(req.startswith(ignore)
                         for ignore in ignore_requirements)]
+
+python_bears_deps = True
+
+config = configparser.ConfigParser()
+config.read('setup.cfg')
+if 'setup:custom' in config.sections():
+    if config['setup:custom']['python-bears'] == 'False':
+        python_bears_deps = False
+
+if python_bears_deps:
+    required += [req for req in bear_required
+                 if not req.startswith('language-check')]
 {% endblock %}