Implement evaluation of relational values

Relational values were supported in aggregate contexts before where they were
immediately eliminated.  This change extends the planner and evaluator to be
able to evaluate them more generally. This change comes with support for a new
method: `Parameter.getName`. While this is not itself relational, it was
impossible to use previously since the only way to get a parameter was
`Method.getAParameter`. The combination of the two is now supported.

This change required two major extensions:

- The type checker now allows relational values in many more contexts
- The planner supports evaluation of relational values as lists in some contexts

All of this ended up requiring a refactoring that I was expecting. Many values
that were previously boxed are now ref counted, as we are now constructing
closures with strange lifetimes that can't just take ownership of their inputs.

This change also includes a major fix to the integration tests. Before, errors
weren't being detected because it was cleanly returning `Err`. Now it fails
loudly with `unwrap`, which revealed that all of the tests were actually
broken (for a silly reason).

Author: travitch

doc/library.kdl

@@ -43,7 +43,7 @@ type "Callable" {
 
 type "Parameter" {
     method "getType" type="Type" status="Unimplemented"
-    method "getName" type="string" status="Unimplemented"
+    method "getName" type="string"
     method "getIndex" type="int" status="Unimplemented"
 }
 

examples/4-argument-functions.ql

@@ -0,0 +1,3 @@
+from Function f
+where count(f.getAParameter()) = 4
+select f

examples/param-name-eq.ql

@@ -0,0 +1,4 @@
+from Function f
+where
+  f.getAParameter().getName() == "buf"
+select f

src/evaluate.rs

@@ -42,9 +42,21 @@ fn evaluate_filter(target : &source_file::SourceFile, ctx : &mut EvaluationConte
             let s = (nm.extract)(ctx, target.source.as_bytes());
             Ok(Value::Constant(Constant::String_(s)))
         },
+        NodeFilter::PredicateListComputation(nm) => {
+            // We evaluate a list of predicates p as `any(p)` (i.e., it is true
+            // if any of the predicates evaluates to true)
+            let ps = (nm.extract)(ctx, target.source.as_bytes());
+            Ok(Value::Constant(Constant::Boolean(ps.iter().any(|p| *p))))
+        },
+        NodeFilter::ArgumentComputation(_c) => {
+            panic!("Not evaluation arguments yet");
+        },
         NodeFilter::ArgumentListComputation(_c) => {
             panic!("No concrete list evaluation needed yet");
         },
+        NodeFilter::StringListComputation(_c) => {
+            panic!("No concrete list evaluation needed yet");
+        },
         NodeFilter::CallableComputation(_c) => {
             panic!("Not evaluating callables yet");
         },

src/plan.rs

@@ -1,26 +1,21 @@
+use std::rc::Rc;
 use tree_sitter::Node;
 
 use crate::query::val_type::Type;
 use crate::plan::interface::*;
 use crate::source_file::SourceFile;
 
-pub struct CPPTreeInterface<'a> {
-    file : &'a SourceFile
+pub struct CPPTreeInterface {
 }
 
-impl<'a> CPPTreeInterface<'a> {
-    pub fn new(f : &'a SourceFile) -> Self {
+impl CPPTreeInterface {
+    pub fn new(_f : &SourceFile) -> Self {
         CPPTreeInterface {
-            file : f
         }
     }
 }
 
-impl<'a> TreeInterface for CPPTreeInterface<'a> {
-    fn source(&self) -> &'a SourceFile {
-        self.file
-    }
-
+impl TreeInterface for CPPTreeInterface {
     fn top_level_type(&self, t : &Type) -> Option<TopLevelMatcher> {
         match t {
             Type::Function => {
@@ -35,11 +30,12 @@ impl<'a> TreeInterface for CPPTreeInterface<'a> {
         }
     }
 
-    fn callable_arguments(&self, base : NodeMatcher<CallableRef>) -> Option<NodeMatcher<Vec<FormalArgument>>>
+    fn callable_arguments(&self, base : &NodeMatcher<CallableRef>) -> Option<NodeMatcher<Vec<FormalArgument>>>
     {
+        let x = Rc::clone(&base.extract);
         let matcher = NodeMatcher {
-            extract: Box::new(move |ctx, source| {
-                let callable_ref = (base.extract)(ctx, source);
+            extract: Rc::new(move |ctx, source| {
+                let callable_ref = x(ctx, source);
                 let node = ctx.lookup_callable(&callable_ref);
                 let mut cur = tree_sitter::QueryCursor::new();
                 let ql_query = "(parameter_declaration) @parameter";
@@ -52,11 +48,12 @@ impl<'a> TreeInterface for CPPTreeInterface<'a> {
         Some(matcher)
     }
 
-    fn callable_name(&self, base : NodeMatcher<CallableRef>) -> Option<NodeMatcher<String>>
+    fn callable_name(&self, base : &NodeMatcher<CallableRef>) -> Option<NodeMatcher<String>>
     {
+        let x = Rc::clone(&base.extract);
         let matcher = NodeMatcher {
-            extract: Box::new(move |ctx, source| {
-                let callable_ref = (base.extract)(ctx, source);
+            extract: Rc::new(move |ctx, source| {
+                let callable_ref = x(ctx, source);
                 let node = ctx.lookup_callable(&callable_ref);
                 let mut cur = tree_sitter::QueryCursor::new();
                 let ql_query = "(function_declarator (identifier) @function.name)";

src/plan/cpp.rs

@@ -8,8 +8,6 @@ pub enum PlanError {
     NonSingletonSelect(usize),
     #[error("Unsupported target for a select expression: `{0:?}`")]
     UnsupportedSelectTarget(Expr_<Typed>),
-    #[error("Unsupported type for a select expression: `{0:?}`")]
-    UnsupportedSelectType(Type),
     #[error("Unsupported type `{0:?}` for language {1:?}")]
     UnsupportedTypeForLanguage(Type, Language),
     #[error("Use of undefined variable `{0}`")]

src/plan/errors.rs

@@ -1,8 +1,8 @@
 use tree_sitter::Node;
 use std::collections::HashMap;
+use std::rc::Rc;
 
 use crate::query::val_type::Type;
-use crate::source_file::SourceFile;
 
 pub struct TopLevelMatcher {
     /// The Tree Sitter query to match the given "top-level" structure
@@ -109,13 +109,14 @@ impl<'a> EvaluationContext<'a> {
 /// FIXME: Add another field that records ranges to highlight in results (i.e.,
 /// the code that contributes to a Node being included)
 pub struct NodeMatcher<R> {
-    pub extract : Box<dyn for <'a> Fn(&'a EvaluationContext<'a>, &'a [u8]) -> R>
+    pub extract : Rc<dyn for <'a> Fn(&'a EvaluationContext<'a>, &'a [u8]) -> R>
 }
 
 /// A representation of language-level types (e.g., Java or C types)
 ///
 /// This is a type for clarity in the data model. For now it is just a `String`,
 /// but eventually we will probably want more structure
+#[derive(Clone)]
 pub struct LanguageType(String);
 
 impl LanguageType {
@@ -134,6 +135,7 @@ impl LanguageType {
 /// arguments to indicate that they are unused (e.g., C).
 ///
 /// The type is optional because some languages are untyped.
+#[derive(Clone)]
 pub struct FormalArgument {
     pub name : Option<String>,
     pub declared_type : Option<LanguageType>
@@ -148,9 +150,6 @@ pub struct FormalArgument {
 /// In cases where structure is sufficiently uniform, we should use normal
 /// functions instead.
 pub trait TreeInterface {
-    /// The underlying source file
-    fn source(&self) -> &SourceFile;
-
     /// Return a matcher for a type declared in the From clause of a QL query
     ///
     /// This can fail if the given type is not supported for the language
@@ -159,11 +158,11 @@ pub trait TreeInterface {
 
     /// A node matcher that extracts formal arguments from a callable node
     /// (e.g., a method or function)
-    fn callable_arguments(&self, node : NodeMatcher<CallableRef>) -> Option<NodeMatcher<Vec<FormalArgument>>>;
+    fn callable_arguments(&self, node : &NodeMatcher<CallableRef>) -> Option<NodeMatcher<Vec<FormalArgument>>>;
 
     /// A node matcher that extracts the name of a callable node.
     ///
     /// This is tailored to just callables because getting the name of other
     /// types of nodes requires different patterns
-    fn callable_name(&self, node : NodeMatcher<CallableRef>) -> Option<NodeMatcher<String>>;
+    fn callable_name(&self, node : &NodeMatcher<CallableRef>) -> Option<NodeMatcher<String>>;
 }

src/plan/interface.rs

@@ -1,26 +1,21 @@
+use std::rc::Rc;
 use tree_sitter::Node;
 
 use crate::query::val_type::Type;
 use crate::plan::interface::*;
 use crate::source_file::SourceFile;
 
-pub struct JavaTreeInterface<'a> {
-    file : &'a SourceFile
+pub struct JavaTreeInterface {
 }
 
-impl<'a> JavaTreeInterface<'a> {
-    pub fn new(f : &'a SourceFile) -> Self {
+impl JavaTreeInterface {
+    pub fn new(_f : &SourceFile) -> Self {
         JavaTreeInterface {
-            file : f
         }
     }
 }
 
-impl<'a> TreeInterface for JavaTreeInterface<'a> {
-    fn source(&self) -> &'a SourceFile {
-        self.file
-    }
-
+impl TreeInterface for JavaTreeInterface {
     // FIXME: If we associate the top-level query with the value of a var ref,
     // we get rid of var ref and just have a typed computation
     fn top_level_type(&self, t : &Type) -> Option<TopLevelMatcher> {
@@ -37,12 +32,13 @@ impl<'a> TreeInterface for JavaTreeInterface<'a> {
         }
     }
 
-    fn callable_arguments(&self, base : NodeMatcher<CallableRef>) ->
+    fn callable_arguments(&self, base : &NodeMatcher<CallableRef>) ->
         Option<NodeMatcher<Vec<FormalArgument>>>
     {
+        let x = Rc::clone(&base.extract);
         let matcher = NodeMatcher {
-            extract: Box::new(move |ctx, source| {
-                let callable_ref = (base.extract)(ctx, source);
+            extract: Rc::new(move |ctx, source| {
+                let callable_ref = x(ctx, source);
                 let node = ctx.lookup_callable(&callable_ref);
                 let mut cur = tree_sitter::QueryCursor::new();
                 let ql_query = "(formal_parameter) @parameter";
@@ -55,11 +51,12 @@ impl<'a> TreeInterface for JavaTreeInterface<'a> {
         Some(matcher)
     }
 
-    fn callable_name(&self, base : NodeMatcher<CallableRef>) -> Option<NodeMatcher<String>>
+    fn callable_name(&self, base : &NodeMatcher<CallableRef>) -> Option<NodeMatcher<String>>
     {
+        let x = Rc::clone(&base.extract);
         let matcher = NodeMatcher {
-            extract: Box::new(move |ctx, source| {
-                let callable_ref = (base.extract)(ctx, source);
+            extract: Rc::new(move |ctx, source| {
+                let callable_ref = x(ctx, source);
                 let node = ctx.lookup_callable(&callable_ref);
                 let mut cur = tree_sitter::QueryCursor::new();
                 let ql_query = "(method_declaration (identifier) @method.name)";

src/plan/java.rs

@@ -1,5 +1,7 @@
 use once_cell::sync::Lazy;
 use std::collections::HashMap;
+use std::rc::Rc;
+use std::sync::Arc;
 
 use crate::query::ir::*;
 use crate::query::val_type::Type;
@@ -18,15 +20,17 @@ use crate::plan::interface::{NodeMatcher, TreeInterface};
 /// 3. The typed operands of the method call
 ///
 /// The callback returns a node filter that is suitable for evaluating the method call
-pub struct Handler(pub Box<dyn for <'a> Fn(&'a Box<dyn TreeInterface + 'a>, NodeFilter, &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> + Send + Sync>);
+///
+/// FIXME: The arguments need to be run-time values
+pub struct Handler(pub Arc<dyn for <'a> Fn(Rc<dyn TreeInterface>, &'a NodeFilter, &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> + Send + Sync>);
 
-fn callable_get_name<'a>(ti : &Box<dyn TreeInterface + 'a>, base : NodeFilter, operands : &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> {
+fn callable_get_name<'a>(ti : Rc<dyn TreeInterface>, base : &'a NodeFilter, operands : &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> {
     assert!(operands.is_empty());
     // This is not necessarily an assertion, as this may not be supported for
     // the current language (which we don't know).
     match base {
         NodeFilter::CallableComputation(callable_matcher) => {
-            let name_matcher = ti.callable_name(callable_matcher)
+            let name_matcher = ti.callable_name(&callable_matcher)
                 .ok_or_else(|| PlanError::NotSupported("getNames".into(), "callable".into()))?;
             Ok(NodeFilter::StringComputation(name_matcher))
         },
@@ -37,11 +41,11 @@ fn callable_get_name<'a>(ti : &Box<dyn TreeInterface + 'a>, base : NodeFilter, o
 }
 
 /// This is a relational method that returns a *list* of parameters
-fn callable_get_a_parameter<'a>(ti : &Box<dyn TreeInterface + 'a>, base : NodeFilter, operands : &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> {
+fn callable_get_a_parameter<'a>(ti : Rc<dyn TreeInterface>, base : &'a NodeFilter, operands : &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> {
     assert!(operands.is_empty());
     match base {
         NodeFilter::CallableComputation(callable_matcher) => {
-            let arg_matcher = ti.callable_arguments(callable_matcher)
+            let arg_matcher = ti.callable_arguments(&callable_matcher)
                 .ok_or_else(|| PlanError::NotSupported("arguments".into(), "callable".into()))?;
             Ok(NodeFilter::ArgumentListComputation(arg_matcher))
         },
@@ -51,7 +55,7 @@ fn callable_get_a_parameter<'a>(ti : &Box<dyn TreeInterface + 'a>, base : NodeFi
     }
 }
 
-fn string_regexp_match<'a>(_ti : &Box<dyn TreeInterface + 'a>, base : NodeFilter, operands : &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> {
+fn string_regexp_match<'a>(_ti : Rc<dyn TreeInterface>, base : &'a NodeFilter, operands : &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> {
     assert!(operands.len() == 1);
     // The base must be a string computation (and we require that it was a literal that we were able to already compile)
     let c = match base {
@@ -66,15 +70,34 @@ fn string_regexp_match<'a>(_ti : &Box<dyn TreeInterface + 'a>, base : NodeFilter
             panic!("Invalid regex for `regexpMatch`")
         }
     };
+    let x = Rc::clone(&c.extract);
     let comp = NodeMatcher {
-        extract: Box::new(move |ctx, source| {
-                    let matched_string = (c.extract)(ctx, source);
+        extract: Rc::new(move |ctx, source| {
+                    let matched_string = x(ctx, source);
                     rx.is_match(matched_string.as_ref())
                 })
     };
     Ok(NodeFilter::Predicate(comp))
 }
 
+fn parameter_get_name<'a>(_ti : Rc<dyn TreeInterface>, base : &'a NodeFilter, operands : &'a Vec<Expr<Typed>>) -> anyhow::Result<NodeFilter> {
+    assert!(operands.is_empty());
+    match base {
+        NodeFilter::ArgumentComputation(c) => {
+            let x = Rc::clone(&c.extract);
+            let comp = NodeMatcher {
+                extract: Rc::new(move |ctx, source| {
+                    x(ctx, source).name.unwrap_or("<none>".into())
+                })
+            };
+            Ok(NodeFilter::StringComputation(comp))
+        },
+        _ => {
+            panic!("Invalid base value for Parameter.getName");
+        }
+    }
+}
+
 /// Validate the implementations of method calls against the claims in the
 /// library documentation.  The intent is that the library documentation should
 /// always correctly reflect what subset of CodeQL ql-grep supports.  Any
@@ -132,21 +155,23 @@ fn validate_library(impls : &HashMap<(Type, String), Handler>) {
 static METHOD_IMPLS: Lazy<HashMap<(Type, String), Handler>> = Lazy::new(|| {
     let mut impls = HashMap::new();
 
-    impls.insert((Type::Method, "getName".into()), Handler(Box::new(callable_get_name)));
-    impls.insert((Type::Function, "getName".into()), Handler(Box::new(callable_get_name)));
-    impls.insert((Type::Callable, "getName".into()), Handler(Box::new(callable_get_name)));
+    impls.insert((Type::Method, "getName".into()), Handler(Arc::new(callable_get_name)));
+    impls.insert((Type::Function, "getName".into()), Handler(Arc::new(callable_get_name)));
+    impls.insert((Type::Callable, "getName".into()), Handler(Arc::new(callable_get_name)));
+
+    impls.insert((Type::Method, "getAParameter".into()), Handler(Arc::new(callable_get_a_parameter)));
+    impls.insert((Type::Function, "getAParameter".into()), Handler(Arc::new(callable_get_a_parameter)));
+    impls.insert((Type::Callable, "getAParameter".into()), Handler(Arc::new(callable_get_a_parameter)));
 
-    impls.insert((Type::Method, "getAParameter".into()), Handler(Box::new(callable_get_a_parameter)));
-    impls.insert((Type::Function, "getAParameter".into()), Handler(Box::new(callable_get_a_parameter)));
-    impls.insert((Type::Callable, "getAParameter".into()), Handler(Box::new(callable_get_a_parameter)));
+    impls.insert((Type::Parameter, "getName".into()), Handler(Arc::new(parameter_get_name)));
 
-    impls.insert((Type::PrimString, "regexpMatch".into()), Handler(Box::new(string_regexp_match)));
+    impls.insert((Type::PrimString, "regexpMatch".into()), Handler(Arc::new(string_regexp_match)));
 
     validate_library(&impls);
 
     impls
 });
 
-pub fn method_impl_for(base_type : Type, method_name : &str) -> Option<&Handler> {
-    Lazy::force(&METHOD_IMPLS).get(&(base_type, method_name.into()))
+pub fn method_impl_for(base_type : Type, method_name : String) -> Option<&'static Handler> {
+    Lazy::force(&METHOD_IMPLS).get(&(base_type, method_name))
 }