Skip to content

EQL: remove zero-downtime migration workflow #119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 29, 2025
Merged

EQL: remove zero-downtime migration workflow #119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ SELECT eql_v2.add_column('users', 'encrypted_email');

**Note:** This function allows you to encrypt and decrypt data but does not enable searchable encryption. See [Searching data with EQL](#searching-data-with-eql) for enabling searchable encryption.

<!--
NOTE: NO LONGER REQUIRED
DOCUMENTATION CAN BE UPDATED WHEN/IF ZERO DOWNTIME SUPPORT IS ADDED TO PROXY
### Activating configuration

After modifying configurations, activate them by running:
Expand All @@ -106,6 +109,7 @@ After modifying configurations, activate them by running:
SELECT eql_v2.migrate_config();
SELECT eql_v2.activate_config();
```
-->

**Important:** These functions must be run after any modifications to the configuration.

Expand Down
18 changes: 9 additions & 9 deletions src/config/config_test.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ DO $$
BEGIN

-- Add indexes
PERFORM eql_v2.add_search_config('users', 'name', 'match');
PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
ASSERT (SELECT _search_config_exists('users', 'name', 'match'));

-- Add index with cast
PERFORM eql_v2.add_search_config('users', 'name', 'unique', 'int');
PERFORM eql_v2.add_search_config('users', 'name', 'unique', 'int', migrating => true);
ASSERT (SELECT _search_config_exists('users', 'name', 'unique'));

ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c
Expand Down Expand Up @@ -60,15 +60,15 @@ DO $$
BEGIN

-- Add indexes
PERFORM eql_v2.add_search_config('users', 'name', 'match');
PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
ASSERT (SELECT _search_config_exists('users', 'name', 'match'));

ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c
WHERE c.state = 'pending' AND
c.data #> array['tables', 'users', 'name', 'indexes'] ? 'match'));

-- Add index with cast
PERFORM eql_v2.add_search_config('blah', 'vtha', 'unique', 'int');
PERFORM eql_v2.add_search_config('blah', 'vtha', 'unique', 'int', migrating => true);
ASSERT (SELECT _search_config_exists('blah', 'vtha', 'unique'));

ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c
Expand Down Expand Up @@ -107,11 +107,11 @@ $$ LANGUAGE plpgsql;

DO $$
BEGIN
PERFORM eql_v2.add_search_config('users', 'name', 'match');
PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
ASSERT (SELECT _search_config_exists('users', 'name', 'match'));

-- Pending configuration contains the path `user/name.match.option`
PERFORM eql_v2.modify_search_config('users', 'name', 'match', 'int', '{"option": "value"}'::jsonb);
PERFORM eql_v2.modify_search_config('users', 'name', 'match', 'int', '{"option": "value"}'::jsonb, migrating => true);
ASSERT (SELECT _search_config_exists('users', 'name', 'match'));

ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c
Expand Down Expand Up @@ -162,7 +162,7 @@ DO $$
BEGIN
ASSERT (SELECT _search_config_exists('users', 'blah', 'match', 'active'));

PERFORM eql_v2.add_search_config('users', 'name', 'match');
PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);

-- index added to name
ASSERT (SELECT _search_config_exists('users', 'name', 'match' ));
Expand Down Expand Up @@ -205,15 +205,15 @@ DO $$
-- reset the table
PERFORM create_table_with_encrypted();

PERFORM eql_v2.add_column('encrypted', 'e');
PERFORM eql_v2.add_column('encrypted', 'e', migrating => true);

PERFORM assert_count(
'Pending configuration was created',
'SELECT * FROM eql_v2_configuration c WHERE c.state = ''pending''',
1);


PERFORM eql_v2.remove_column('encrypted', 'e');
PERFORM eql_v2.remove_column('encrypted', 'e', migrating => true);

PERFORM assert_no_result(
'Pending configuration was removed',
Expand Down
31 changes: 25 additions & 6 deletions src/config/functions.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
-- REQUIRE: src/config/types.sql
-- REQUIRE: src/config/functions_private.sql
--
-- REQUIRE: src/encrypted/functions.sql


-- Customer-facing configuration functions
-- Depends on private functions for implemenation
--
Expand All @@ -10,7 +12,7 @@
-- Adds an index term to the configuration
--

CREATE FUNCTION eql_v2.add_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}')
CREATE FUNCTION eql_v2.add_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}', migrating boolean DEFAULT false)
RETURNS jsonb

AS $$
Expand Down Expand Up @@ -54,6 +56,13 @@ AS $$
DO UPDATE
SET data = _config;

IF NOT migrating THEN
PERFORM eql_v2.migrate_config();
PERFORM eql_v2.activate_config();
END IF;

-- PERFORM eql_v2.add_encrypted_constraint(table_name, column_name);

-- exeunt
RETURN _config;
END;
Expand Down Expand Up @@ -121,12 +130,12 @@ $$ LANGUAGE plpgsql;



CREATE FUNCTION eql_v2.modify_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}')
CREATE FUNCTION eql_v2.modify_search_config(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}', migrating boolean DEFAULT false)
RETURNS jsonb
AS $$
BEGIN
PERFORM eql_v2.remove_search_config(table_name, column_name, index_name);
RETURN eql_v2.add_search_config(table_name, column_name, index_name, cast_as, opts);
RETURN eql_v2.add_search_config(table_name, column_name, index_name, cast_as, opts, migrating);
END;
$$ LANGUAGE plpgsql;

Expand Down Expand Up @@ -200,7 +209,7 @@ $$ LANGUAGE plpgsql;



CREATE FUNCTION eql_v2.add_column(table_name text, column_name text, cast_as text DEFAULT 'text')
CREATE FUNCTION eql_v2.add_column(table_name text, column_name text, cast_as text DEFAULT 'text', migrating boolean DEFAULT false)
RETURNS jsonb
AS $$
DECLARE
Expand Down Expand Up @@ -231,6 +240,11 @@ AS $$
DO UPDATE
SET data = _config;

IF NOT migrating THEN
PERFORM eql_v2.migrate_config();
PERFORM eql_v2.activate_config();
END IF;

PERFORM eql_v2.add_encrypted_constraint(table_name, column_name);

-- exeunt
Expand All @@ -240,7 +254,7 @@ $$ LANGUAGE plpgsql;



CREATE FUNCTION eql_v2.remove_column(table_name text, column_name text)
CREATE FUNCTION eql_v2.remove_column(table_name text, column_name text, migrating boolean DEFAULT false)
RETURNS jsonb
AS $$
DECLARE
Expand Down Expand Up @@ -289,6 +303,11 @@ AS $$

PERFORM eql_v2.remove_encrypted_constraint(table_name, column_name);

IF NOT migrating THEN
PERFORM eql_v2.migrate_config();
PERFORM eql_v2.activate_config();
END IF;

-- exeunt
RETURN _config;

Expand Down
30 changes: 26 additions & 4 deletions src/encryptindex/functions_test.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,16 +154,38 @@ CREATE TABLE users
-- An encrypting config should exist
DO $$
BEGIN
PERFORM eql_v2.add_search_config('users', 'name', 'match');
PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
PERFORM eql_v2.migrate_config();

ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting'));
ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'));
END;
$$ LANGUAGE plpgsql;


-- Encrypting config without `migrating = true` is immediately active
DO $$
BEGIN
TRUNCATE TABLE eql_v2_configuration;
PERFORM eql_v2.add_search_config('users', 'name', 'match');
ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
END;
$$ LANGUAGE plpgsql;


-- migrate_config() should raise an exception when no pending configuration exists
DO $$
BEGIN
TRUNCATE TABLE eql_v2_configuration;
PERFORM eql_v2.add_search_config('users', 'name', 'match');

PERFORM assert_exception(
'eql_v2.migrate_config() should raise an exception when no pending configuration exists',
'SELECT eql_v2.migrate_config()'
);
END;
$$ LANGUAGE plpgsql;

-- -----------------------------------------------
-- With existing active config and an updated schema using a raw JSONB column
-- Start encryptindexing
Expand Down Expand Up @@ -204,7 +226,7 @@ CREATE TABLE users
-- An encrypting config should exist
DO $$
BEGIN
PERFORM eql_v2.add_search_config('users', 'name', 'match');
PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);
PERFORM eql_v2.migrate_config();

ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active'));
Expand Down Expand Up @@ -254,7 +276,7 @@ CREATE TABLE users
-- An encrypting config should exist
DO $$
BEGIN
PERFORM eql_v2.add_search_config('users', 'name', 'match');
PERFORM eql_v2.add_search_config('users', 'name', 'match', migrating => true);

PERFORM eql_v2.migrate_config(); -- need to encrypt first
PERFORM eql_v2.activate_config();
Expand Down