Merge branch 'master' into slxu-kill-query-at-aborted-downloading

Author: dhalperi

build.gradle

@@ -138,7 +138,7 @@ dependencies {
   compile "net.sf.supercsv:super-csv:2.2.0"
 
   // Expression compiler
-  compile "org.codehaus.janino:janino:2.7.5"
+  compile "org.codehaus.janino:janino:2.7.6"
 
   runtime "monetdb:monetdb-jdbc:2.11"
   runtime "mysql:mysql-connector-java:5.1.31"

src/edu/washington/escience/myria/MyriaConstants.java

@@ -4,8 +4,8 @@
 
 /**
  * This class holds the constants for the Myria execution.
- * 
- * 
+ *
+ *
  */
 public final class MyriaConstants {
   /**
@@ -270,6 +270,12 @@ public final class MyriaConstants {
    */
   public static final int MAXIMUM_NUM_SUBQUERIES = 100;
 
+  /**
+   * Default imports for janino. Modules imported here can be used in expressions.
+   */
+  public static final String[] DEFAULT_JANINO_IMPORTS =
+    { "com.google.common.hash.Hashing", "java.nio.charset.Charset" };
+
   /** Private constructor to disallow building utility class. */
   private MyriaConstants() {
   }

src/edu/washington/escience/myria/expression/ExpressionOperator.java

@@ -1,5 +1,5 @@
 /**
- * 
+ *
  */
 package edu.washington.escience.myria.expression;
 
@@ -26,10 +26,10 @@
     @Type(name = "ABS", value = AbsExpression.class), @Type(name = "CAST", value = CastExpression.class),
     @Type(name = "CEIL", value = CeilExpression.class), @Type(name = "COS", value = CosExpression.class),
     @Type(name = "FLOOR", value = FloorExpression.class), @Type(name = "LEN", value = LenExpression.class),
-    @Type(name = "LOG", value = LogExpression.class), @Type(name = "NEG", value = NegateExpression.class),
-    @Type(name = "NOT", value = NotExpression.class), @Type(name = "SIN", value = SinExpression.class),
-    @Type(name = "SQRT", value = SqrtExpression.class), @Type(name = "TAN", value = TanExpression.class),
-    @Type(name = "UPPER", value = ToUpperCaseExpression.class),
+    @Type(name = "LOG", value = LogExpression.class), @Type(name = "MD5", value = HashMd5Expression.class),
+    @Type(name = "NEG", value = NegateExpression.class), @Type(name = "NOT", value = NotExpression.class),
+    @Type(name = "SIN", value = SinExpression.class), @Type(name = "SQRT", value = SqrtExpression.class),
+    @Type(name = "TAN", value = TanExpression.class), @Type(name = "UPPER", value = ToUpperCaseExpression.class),
     /* Binary */
     @Type(name = "AND", value = AndExpression.class), @Type(name = "DIVIDE", value = DivideExpression.class),
     @Type(name = "EQ", value = EqualsExpression.class), @Type(name = "GREATER", value = GreaterExpression.class),

src/edu/washington/escience/myria/expression/HashMd5Expression.java

@@ -0,0 +1,55 @@
+package edu.washington.escience.myria.expression;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import edu.washington.escience.myria.Type;
+import edu.washington.escience.myria.expression.evaluate.ExpressionOperatorParameter;
+
+/**
+ * Return the a hash representation as long of the operand.
+ */
+public class HashMd5Expression extends UnaryExpression {
+  /***/
+  private static final long serialVersionUID = 1L;
+
+  /**
+   * This is not really unused, it's used automagically by Jackson deserialization.
+   */
+  @SuppressWarnings("unused")
+  private HashMd5Expression() {
+    super();
+  }
+
+  /**
+   * Compute MD5 hash code.
+   *
+   * @param operand the operand.
+   */
+  public HashMd5Expression(final ExpressionOperator operand) {
+    super(operand);
+  }
+
+  @Override
+  public Type getOutputType(final ExpressionOperatorParameter parameters) {
+    Type operandType = getOperand().getOutputType(parameters);
+    ImmutableList<Type> validTypes = ImmutableList.of(Type.STRING_TYPE, Type.LONG_TYPE, Type.INT_TYPE);
+    int operandIdx = validTypes.indexOf(operandType);
+    Preconditions.checkArgument(operandIdx != -1, "%s cannot handle operand [%s] of Type %s", getClass()
+        .getSimpleName(), getOperand(), operandType);
+    return Type.LONG_TYPE;
+  }
+
+  @Override
+  public String getJavaString(final ExpressionOperatorParameter parameters) {
+    Type operandType = getOperand().getOutputType(parameters);
+    if (operandType == Type.LONG_TYPE) {
+      return getFunctionCallUnaryString("Hashing.md5().hashLong", parameters).concat(".asLong()");
+    } else if (operandType == Type.INT_TYPE) {
+      return getFunctionCallUnaryString("Hashing.md5().hashInt", parameters).concat(".asLong()");
+    } else {
+      return new StringBuilder("Hashing.md5().hashString(").append(getOperand().getJavaString(parameters)).append(
+          ", Charset.defaultCharset()).asLong()").toString();
+    }
+  }
+}

src/edu/washington/escience/myria/expression/ToUpperCaseExpression.java

@@ -1,7 +1,6 @@
 package edu.washington.escience.myria.expression;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
 
 import edu.washington.escience.myria.Type;
 import edu.washington.escience.myria.expression.evaluate.ExpressionOperatorParameter;
@@ -23,7 +22,7 @@ private ToUpperCaseExpression() {
 
   /**
    * Change all characters in a string to upper case.
-   * 
+   *
    * @param operand the operand.
    */
   public ToUpperCaseExpression(final ExpressionOperator operand) {
@@ -33,9 +32,7 @@ public ToUpperCaseExpression(final ExpressionOperator operand) {
   @Override
   public Type getOutputType(final ExpressionOperatorParameter parameters) {
     Type operandType = getOperand().getOutputType(parameters);
-    ImmutableList<Type> validTypes = ImmutableList.of(Type.STRING_TYPE);
-    int operandIdx = validTypes.indexOf(operandType);
-    Preconditions.checkArgument(operandIdx != -1, "%s cannot handle operand [%s] of Type %s", getClass()
+    Preconditions.checkArgument(operandType == Type.STRING_TYPE, "%s cannot handle operand [%s] of Type %s", getClass()
         .getSimpleName(), getOperand(), operandType);
     return Type.STRING_TYPE;
   }

src/edu/washington/escience/myria/expression/evaluate/BooleanEvaluator.java

@@ -3,11 +3,12 @@
 import java.lang.reflect.InvocationTargetException;
 
 import org.codehaus.commons.compiler.CompilerFactoryFactory;
-import org.codehaus.commons.compiler.IScriptEvaluator;
+import org.codehaus.commons.compiler.IExpressionEvaluator;
 
 import com.google.common.base.Preconditions;
 
 import edu.washington.escience.myria.DbException;
+import edu.washington.escience.myria.MyriaConstants;
 import edu.washington.escience.myria.Type;
 import edu.washington.escience.myria.expression.Expression;
 import edu.washington.escience.myria.storage.TupleBatch;
@@ -23,7 +24,7 @@ public class BooleanEvaluator extends Evaluator {
 
   /**
    * Default constructor.
-   * 
+   *
    * @param expression the expression for the evaluator
    * @param parameters parameters that are passed to the expression
    */
@@ -34,13 +35,15 @@ public BooleanEvaluator(final Expression expression, final ExpressionOperatorPar
 
   /**
    * Compiles the {@link #javaExpression}.
-   * 
+   *
    * @throws DbException compilation failed
    */
   @Override
   public void compile() throws DbException {
     try {
-      IScriptEvaluator se = CompilerFactoryFactory.getDefaultCompilerFactory().newExpressionEvaluator();
+      IExpressionEvaluator se = CompilerFactoryFactory.getDefaultCompilerFactory().newExpressionEvaluator();
+
+      se.setDefaultImports(MyriaConstants.DEFAULT_JANINO_IMPORTS);
 
       evaluator =
           (BooleanEvalInterface) se.createFastEvaluator(getJavaExpression(), BooleanEvalInterface.class, new String[] {
@@ -52,7 +55,7 @@ public void compile() throws DbException {
 
   /**
    * Evaluates the {@link #getJavaExpression()} using the {@link #evaluator}.
-   * 
+   *
    * @param tb a tuple batch
    * @param rowId the row that should be used for input data
    * @return the result from the evaluation

src/edu/washington/escience/myria/expression/evaluate/ConstEvalInterface.java

@@ -8,6 +8,7 @@
 import com.google.common.base.Preconditions;
 
 import edu.washington.escience.myria.DbException;
+import edu.washington.escience.myria.MyriaConstants;
 import edu.washington.escience.myria.Type;
 import edu.washington.escience.myria.column.Column;
 import edu.washington.escience.myria.column.ConstantValueColumn;
@@ -33,7 +34,7 @@ public final class ConstantEvaluator extends GenericEvaluator {
 
   /**
    * Default constructor.
-   * 
+   *
    * @param expression the expression for the evaluator
    * @param parameters parameters that are passed to the expression
    * @throws DbException if there is an error compiling the expression
@@ -50,8 +51,14 @@ public ConstantEvaluator(final Expression expression, final ExpressionOperatorPa
     } catch (Exception e) {
       throw new DbException("Error when generating Java expression " + this, e);
     }
+
+    evaluator = new ExpressionEvaluator();
+    evaluator.setParameters(new String[] {}, new Class<?>[] {});
+    evaluator.setDefaultImports(MyriaConstants.DEFAULT_JANINO_IMPORTS);
+
     try {
-      evaluator = new ExpressionEvaluator(java, type.toJavaType(), new String[] {}, new Class<?>[] {});
+      evaluator.setExpressionType(type.toJavaType());
+      evaluator.cook(java);
       value = evaluator.evaluate(NO_ARGS);
     } catch (CompileException e) {
       throw new DbException("Error when compiling expression " + java, e);
@@ -76,7 +83,7 @@ public void compile() {
 
   /**
    * Evaluates the {@link #getJavaExpression()} using the {@link #evaluator}.
-   * 
+   *
    * @return the result from the evaluation
    */
   public Object eval() {

src/edu/washington/escience/myria/expression/evaluate/ConstantEvaluator.java

@@ -11,12 +11,11 @@ public interface EvalInterface {
   /**
    * The interface for applying expressions. We only need a reference to the tuple batch and a row id. The variables
    * will be fetched from the tuple buffer using the rowId provided in {@link VariableExpression}.
-   * 
+   *
    * @param tb a tuple batch
    * @param rowId the row in the tb that should be used.
    * @param result the result column that the value should be appended to
    * @param state optional state that is passed during evaluation
-   * @return the result from the evaluation
    */
-  Object evaluate(final ReadableTable tb, final int rowId, final WritableColumn result, final ReadableTable state);
+  void evaluate(final ReadableTable tb, final int rowId, final WritableColumn result, final ReadableTable state);
 }

src/edu/washington/escience/myria/expression/evaluate/EvalInterface.java

@@ -4,11 +4,12 @@
 
 import org.codehaus.commons.compiler.CompileException;
 import org.codehaus.commons.compiler.CompilerFactoryFactory;
-import org.codehaus.commons.compiler.IScriptEvaluator;
+import org.codehaus.commons.compiler.IExpressionEvaluator;
 
 import com.google.common.base.Preconditions;
 
 import edu.washington.escience.myria.DbException;
+import edu.washington.escience.myria.MyriaConstants;
 import edu.washington.escience.myria.Type;
 import edu.washington.escience.myria.column.Column;
 import edu.washington.escience.myria.column.builder.ColumnBuilder;
@@ -39,7 +40,7 @@ public class GenericEvaluator extends Evaluator {
 
   /**
    * Default constructor.
-   * 
+   *
    * @param expression the expression for the evaluator
    * @param parameters parameters that are passed to the expression
    */
@@ -49,7 +50,7 @@ public GenericEvaluator(final Expression expression, final ExpressionOperatorPar
 
   /**
    * Compiles the {@link #javaExpression}.
-   * 
+   *
    * @throws DbException compilation failed
    */
   @Override
@@ -58,14 +59,17 @@ public void compile() throws DbException {
         "This expression does not need to be compiled.");
 
     String javaExpression = getJavaExpression();
-    IScriptEvaluator se;
+    IExpressionEvaluator se;
     try {
       se = CompilerFactoryFactory.getDefaultCompilerFactory().newExpressionEvaluator();
     } catch (Exception e) {
       LOGGER.error("Could not create expression evaluator", e);
       throw new DbException("Could not create expression evaluator", e);
     }
 
+    se.setExpressionType(Void.TYPE);
+    se.setDefaultImports(MyriaConstants.DEFAULT_JANINO_IMPORTS);
+
     try {
       evaluator =
           (EvalInterface) se.createFastEvaluator(javaExpression, EvalInterface.class, new String[] {
@@ -79,7 +83,7 @@ public void compile() throws DbException {
   /**
    * Evaluates the {@link #getJavaExpression()} using the {@link #evaluator}. Prefer to use
    * {@link #evaluateColumn(TupleBatch)} as it can copy data without evaluating the expression.
-   * 
+   *
    * @param tb a tuple batch
    * @param rowIdx the row that should be used for input data
    * @param result the column that the result should be appended to
@@ -109,7 +113,7 @@ public String getJavaExpression() {
   /**
    * Evaluate an expression over an entire TupleBatch and return the column of results. This method cannot take state
    * into consideration.
-   * 
+   *
    * @param tb the tuples to be input to this expression
    * @return a column containing the result of evaluating this expression on the entire TupleBatch
    * @throws InvocationTargetException exception thrown from janino

src/edu/washington/escience/myria/expression/evaluate/GenericEvaluator.java

@@ -27,6 +27,7 @@
 import edu.washington.escience.myria.expression.GreaterExpression;
 import edu.washington.escience.myria.expression.GreaterThanExpression;
 import edu.washington.escience.myria.expression.GreaterThanOrEqualsExpression;
+import edu.washington.escience.myria.expression.HashMd5Expression;
 import edu.washington.escience.myria.expression.IntDivideExpression;
 import edu.washington.escience.myria.expression.LenExpression;
 import edu.washington.escience.myria.expression.LessThanExpression;
@@ -78,6 +79,7 @@ public void testJsonMapping() throws IOException {
     CeilExpression ceil = new CeilExpression(constant);
     FloorExpression floor = new FloorExpression(constant);
     LogExpression log = new LogExpression(constant);
+    HashMd5Expression md5 = new HashMd5Expression(constant);
     NegateExpression negate = new NegateExpression(constant);
     CosExpression cos = new CosExpression(constant);
     SinExpression sin = new SinExpression(constant);
@@ -86,8 +88,8 @@ public void testJsonMapping() throws IOException {
     ToUpperCaseExpression upper = new ToUpperCaseExpression(constant);
     NotExpression not = new NotExpression(constant);
     LenExpression len = new LenExpression(constant);
-    expressions.add(abs).add(ceil).add(cos).add(floor).add(log).add(negate).add(not).add(sin).add(sqrt).add(tan).add(
-        upper).add(len);
+    expressions.add(abs).add(ceil).add(cos).add(floor).add(log).add(md5).add(negate).add(not).add(sin).add(sqrt).add(
+        tan).add(upper).add(len);
 
     /* Binary */
     DivideExpression divide = new DivideExpression(constant, variable);