test: selects use indexes

Author: tobyhede

README.md

@@ -103,8 +103,8 @@ SELECT eql_v2.add_column('users', 'encrypted_email');
 After modifying configurations, activate them by running:
 
 ```sql
-SELECT eql_v2.encrypt();
-SELECT eql_v2.activate();
+SELECT eql_v2.migrate_config();
+SELECT eql_v2.activate_config();
 ```
 
 **Important:** These functions must be run after any modifications to the configuration.
@@ -219,8 +219,8 @@ SELECT eql_v2.add_search_config(
 After adding an index, you have to activate the configuration:
 
 ```sql
-SELECT eql_v2.encrypt();
-SELECT eql_v2.activate();
+SELECT eql_v2.migrate_config();
+SELECT eql_v2.activate_config();
 ```
 
 ## Searching data with EQL

src/config/functions.sql

@@ -144,7 +144,7 @@ $$ LANGUAGE plpgsql;
 -- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt.
 --
 
-CREATE FUNCTION eql_v2.encrypt()
+CREATE FUNCTION eql_v2.migrate_config()
   RETURNS boolean
 AS $$
 	BEGIN
@@ -168,7 +168,7 @@ $$ LANGUAGE plpgsql;
 
 
 
-CREATE FUNCTION eql_v2.activate()
+CREATE FUNCTION eql_v2.activate_config()
   RETURNS boolean
 AS $$
 	BEGIN

src/encryptindex/functions_test.sql

@@ -155,7 +155,7 @@ CREATE TABLE users
 DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
-    PERFORM eql_v2.encrypt();
+    PERFORM eql_v2.migrate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
@@ -205,7 +205,7 @@ CREATE TABLE users
 DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
-    PERFORM eql_v2.encrypt();
+    PERFORM eql_v2.migrate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
@@ -256,8 +256,8 @@ DO $$
   BEGIN
     PERFORM eql_v2.add_search_config('users', 'name', 'match');
 
-    PERFORM eql_v2.encrypt(); -- need to encrypt first
-    PERFORM eql_v2.activate();
+    PERFORM eql_v2.migrate_config(); -- need to encrypt first
+    PERFORM eql_v2.activate_config();
 
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
     ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'inactive'));

src/hmac_256/functions.sql

@@ -16,6 +16,27 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
+CREATE FUNCTION eql_v2.has_hmac_256(val jsonb)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN val ? 'hm';
+  END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE FUNCTION eql_v2.has_hmac_256(val eql_v2_encrypted)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN eql_v2.has_hmac_256(val.data);
+  END;
+$$ LANGUAGE plpgsql;
+
+
+
 -- extracts hmac_256 index from an encrypted column
 
 CREATE FUNCTION eql_v2.hmac_256(val eql_v2_encrypted)

src/hmac_256/functions_test.sql

@@ -12,3 +12,15 @@ DO $$
 
   END;
 $$ LANGUAGE plpgsql;
+
+
+DO $$
+  DECLARE
+   e eql_v2_encrypted;
+  BEGIN
+    e := create_encrypted_json(1, 'hm');
+
+    ASSERT eql_v2.has_hmac_256(e);
+  END;
+$$ LANGUAGE plpgsql;
+

src/operators/operator_class.sql

@@ -10,9 +10,43 @@
 -- REQUIRE: src/operators/>.sql
 
 
+--
+-- Compare two eql_v2_encrypted values
+-- Uses `ore_block_u64_8_256` or `has_hmac_256` index terms for comparison if defined on ONE of the compared value
+--
+-- Important note: order of term operations is reversed
+-- In equality operations, `has_hmac_256` is preferred as it reduces to a text comparison and is more efficient
+-- As compare is used for ordering, `ore_block_u64_8_256` provides more complete ordering and is checked first.
+--
+--
 CREATE FUNCTION eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted)
   RETURNS integer
   IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  BEGIN
+
+    -- PERFORM eql_v2.log('eql_v2.has_hmac_256(a)', eql_v2.has_hmac_256(a)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_hmac_256(b)', eql_v2.has_hmac_256(b)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text);
+    -- PERFORM eql_v2.log('eql_v2.has_ore_block_u64_8_256(b)', eql_v2.has_ore_block_u64_8_256(b)::text);
+
+    IF eql_v2.has_ore_block_u64_8_256(a) OR eql_v2.has_ore_block_u64_8_256(b) THEN
+      RETURN eql_v2.compare_ore_block_u64_8_256(a, b);
+    END IF;
+
+    IF eql_v2.has_hmac_256(a) OR eql_v2.has_hmac_256(b) THEN
+      RETURN eql_v2.compare_hmac(a, b);
+    END IF;
+
+    RAISE 'Expected an hmac_256 (hm) or ore_block_u64_8_256 (ob) value in json: %', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+--------------------
+
+CREATE FUNCTION eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted)
+  RETURNS integer
+  IMMUTABLE STRICT PARALLEL SAFE
 AS $$
   DECLARE
     a_ore eql_v2.ore_block_u64_8_256;
@@ -38,20 +72,9 @@ AS $$
   END;
 $$ LANGUAGE plpgsql;
 
-CREATE OPERATOR FAMILY eql_v2.encrypted_operator USING btree;
-
-CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator AS
-  OPERATOR 1 <,
-  OPERATOR 2 <=,
-  OPERATOR 3 =,
-  OPERATOR 4 >=,
-  OPERATOR 5 >,
-  FUNCTION 1 eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted);
-
 
 --------------------
 
-
 CREATE FUNCTION eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted)
   RETURNS integer
   IMMUTABLE STRICT PARALLEL SAFE
@@ -80,13 +103,40 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
-CREATE OPERATOR FAMILY eql_v2.encrypted_hmac_256_operator USING btree;
+--------------------
 
-CREATE OPERATOR CLASS eql_v2.encrypted_hmac_256_operator FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_hmac_256_operator AS
+CREATE OPERATOR FAMILY eql_v2.encrypted_operator USING btree;
+
+CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator AS
   OPERATOR 1 <,
   OPERATOR 2 <=,
   OPERATOR 3 =,
   OPERATOR 4 >=,
   OPERATOR 5 >,
-  FUNCTION 1 eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted);
+  FUNCTION 1 eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted);
+
+
+--------------------
+
+-- CREATE OPERATOR FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 USING btree;
+
+-- CREATE OPERATOR CLASS eql_v2.encrypted_operator_ore_block_u64_8_256 FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator_ore_block_u64_8_256 AS
+--   OPERATOR 1 <,
+--   OPERATOR 2 <=,
+--   OPERATOR 3 =,
+--   OPERATOR 4 >=,
+--   OPERATOR 5 >,
+--   FUNCTION 1 eql_v2.compare_ore_block_u64_8_256(a eql_v2_encrypted, b eql_v2_encrypted);
+
+-- --------------------
+
+-- CREATE OPERATOR FAMILY eql_v2.encrypted_hmac_256_operator USING btree;
+
+-- CREATE OPERATOR CLASS eql_v2.encrypted_hmac_256_operator FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_hmac_256_operator AS
+--   OPERATOR 1 <,
+--   OPERATOR 2 <=,
+--   OPERATOR 3 =,
+--   OPERATOR 4 >=,
+--   OPERATOR 5 >,
+--   FUNCTION 1 eql_v2.compare_hmac(a eql_v2_encrypted, b eql_v2_encrypted);
 

src/operators/operator_class_test.sql

@@ -58,5 +58,77 @@ DO $$
   END;
 $$ LANGUAGE plpgsql;
 
+SELECT * FROM encrypted;
+
+--
+-- ORE GROUP BY
+--
+DO $$
+  DECLARE
+    ore_term eql_v2_encrypted;
+    result text;
+  BEGIN
+
+    PERFORM create_table_with_encrypted();
+
+    EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"hm\": \"abc\"}")'';' into result;
+
+    PERFORM eql_v2.log('', result);
+
+    IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+      RAISE EXCEPTION 'Unexpected Bitmap Heap Scan: %', result;
+    ELSE
+      ASSERT true;
+    END IF;
+
+    EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"ob\": \"abc\"}")'';' into result;
+
+    PERFORM eql_v2.log('', result);
+
+    IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+      RAISE EXCEPTION 'Unexpected Bitmap Heap Scan: %', result;
+    ELSE
+      ASSERT true;
+    END IF;
+
+
+    -- Add index
+    CREATE INDEX ON encrypted (e);
+
+    EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"hm\": \"abc\"}")'';' into result;
+
+    PERFORM eql_v2.log(result);
+
+    IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+      ASSERT true;
+    ELSE
+      RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result;
+    END IF;
+
+    PERFORM seed_encrypted_json();
+
+    SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+    EXECUTE format('EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = %L::eql_v2_encrypted;', ore_term) into result;
+
+    PERFORM eql_v2.log(result);
+
+    IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+      ASSERT true;
+    ELSE
+      RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result;
+    END IF;
+
+
+    -- ---
+    -- EXECUTE 'EXPLAIN ANALYZE SELECT e::jsonb FROM encrypted WHERE e = ''("{\"blah\": \"vtha\"}")'';' into result;
+
+    -- IF position('Bitmap Heap Scan on encrypted' in result) > 0 THEN
+    --   ASSERT true;
+    -- ELSE
+    --   RAISE EXCEPTION 'Expected Bitmap Heap Scan: %', result;
+    -- END IF;
+
+  END;
+$$ LANGUAGE plpgsql;
 
 SELECT drop_table_with_encrypted();

src/ore_block_u64_8_256/functions.sql

@@ -1,10 +1,10 @@
 -- REQUIRE: src/schema.sql
+-- REQUIRE: src/crypto.sql
 -- REQUIRE: src/encrypted/types.sql
 -- REQUIRE: src/encrypted/functions.sql
 -- REQUIRE: src/ore_block_u64_8_256/types.sql
 
 
-
 -- Casts a jsonb array of hex-encoded strings to the `ore_block_u64_8_256` composite type.
 -- In other words, this function takes the ORE index format sent through in the
 -- EQL payload from Proxy and decodes it as the composite type that we use for
@@ -78,6 +78,29 @@ AS $$
 $$ LANGUAGE plpgsql;
 
 
+--
+-- Checks if val contains an ore_block_u64_8_256 search term
+--
+CREATE FUNCTION eql_v2.has_ore_block_u64_8_256(val jsonb)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN val ? 'ob';
+  END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE FUNCTION eql_v2.has_ore_block_u64_8_256(val eql_v2_encrypted)
+  RETURNS boolean
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+	BEGIN
+    RETURN eql_v2.has_ore_block_u64_8_256(val.data);
+  END;
+$$ LANGUAGE plpgsql;
+
+
 -- This function uses lexicographic comparison
 
 CREATE FUNCTION eql_v2.compare_ore_block_u64_8_256(a eql_v2.ore_block_u64_8_256, b eql_v2.ore_block_u64_8_256)
@@ -97,6 +120,8 @@ AS $$
     eq boolean := true;
     unequal_block smallint := 0;
     hash_key bytea;
+    data_block bytea;
+    encrypt_block bytea;
     target_block bytea;
 
     left_block_size CONSTANT smallint := 16;
@@ -150,13 +175,18 @@ AS $$
     -- first right block is at right offset + nonce_size (ordinally indexed)
     target_block := substr(b.bytes, right_offset + 17 + (unequal_block * right_block_size), right_block_size);
 
+    data_block := substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size);
+
+    -- PERFORM eql_v2.log('substr', data_block::text);
+    -- PERFORM eql_v2.log('hash_key', hash_key::text);
+    -- PERFORM eql_v2.log('data_block', pg_typeof(data_block)::text);
+    -- PERFORM eql_v2.log('hash_key', pg_typeof(hash_key)::text);
+
+    encrypt_block := public.encrypt(data_block::bytea, hash_key::bytea, 'aes-ecb');
+
     indicator := (
       get_bit(
-        encrypt(
-          substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size),
-          hash_key,
-          'aes-ecb'
-        ),
+        encrypt_block,
         0
       ) + get_bit(target_block, get_byte(a.bytes, unequal_block))) % 2;
 

src/ore_block_u64_8_256/functions_test.sql

@@ -12,6 +12,21 @@ DO $$
   END;
 $$ LANGUAGE plpgsql;
 
+
+
+DO $$
+  DECLARE
+    ore_term eql_v2_encrypted;
+  BEGIN
+    SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+
+    ASSERT eql_v2.has_ore_block_u64_8_256(ore_term);
+
+  END;
+$$ LANGUAGE plpgsql;
+
+
+
 --
 -- ORE - ORDER BY ore_block_u64_8_256(eql_v2_encrypted)
 --