diff --git a/grid-proxy/internal/explorer/db/cache/README.md b/grid-proxy/internal/explorer/db/cache/README.md new file mode 100644 index 00000000..1a3b1ef9 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/README.md @@ -0,0 +1,93 @@ +# Database Setup Documentation + +This directory contains SQL scripts that set up a caching system for the TFGrid Explorer database. The system pre-computes and maintains frequently queried data to improve query performance. + +## Overview + +The setup consists of: +- **Constants**: Configuration values for resource calculations +- **Functions**: Utility functions for price calculations and data transformations +- **Views**: Base views that define the structure of cached data +- **Cache Tables**: Materialized tables storing pre-computed data +- **Triggers**: Automatic cache maintenance when source data changes +- **Error Logging**: Monitoring and debugging of cache operations + +## Cached Tables + +### `resources_cache` + +Pre-computed node resource information for fast queries. + +**Key Features:** +- Stores total, free, and used resources (HRU, MRU, SRU, CRU) in bytes +- Includes hardware information (DMI, GPUs, CPU benchmarks, network speeds) +- Contains rental information (renter, rent_contract_id) +- Tracks active contract counts +- Automatically calculates `price_usd` using a generated column + +**Resource Reservations:** +- **MRU**: Reserved amount = `max(MRU/10, 2GB)` - automatically reserved and not available for contracts +- **SRU**: Fixed reservation of 20GB - automatically reserved + +**Primary Key:** `node_id` +**Indexed on:** `farm_id` for fast farm-based queries + +### `public_ips_cache` + +Aggregated public IP information per farm. + +**Key Features:** +- Tracks total IPs assigned to each farm +- Counts free IPs (where `contract_id = 0`) +- Stores complete IP details as JSONB array (id, ip, contract_id, gateway) + +**Primary Key:** `farm_id` + +## Triggers + +Triggers automatically maintain cache tables when source data changes. Each trigger handles specific data types: + +### Node Resources + +| Trigger | Source Table | What It Does | +|---------|--------------|--------------| +| `tg_node` | `node` | On INSERT: Populates cache for new node. On DELETE: Removes node from cache. | +| `tg_node_resources_total` | `node_resources_total` | Updates total resources and adjusts free resources (handles MRU reserved amount changes). | +| `tg_contract_resources` | `contract_resources` | Updates used/free resources when contracts change (only processes 'Created'/'GracePeriod' contracts). | +| `tg_node_contract` | `node_contract` | On INSERT: Updates contract count. On UPDATE to 'Deleted': Releases contract resources and updates count. | + +### Node Metadata + +| Trigger | Source Table | What It Does | +|---------|--------------|--------------| +| `tg_node_gpu_count` | `node_gpu` | Recalculates GPU count and JSON array when GPUs are added/removed/updated. | +| `tg_rent_contract` | `rent_contract` | On INSERT: Sets renter and rent_contract_id. On UPDATE to 'Deleted': Clears rental info. | +| `tg_dmi` | `dmi` | Updates hardware information (bios, baseboard, processor, memory) in cache. | +| `tg_speed` | `speed` | Updates network speed test results (upload, download, IPv4/IPv6, TCP/UDP). | +| `tg_cpu_benchmark` | `cpu_benchmark` | Updates CPU benchmark results (single-threaded, multi-threaded, threads, workloads). | + +### Public IPs + +| Trigger | Source Table | What It Does | +|---------|--------------|--------------| +| `tg_public_ip` | `public_ip` | Updates free/total IP counts and re-aggregates IP JSON when IPs are inserted/updated/deleted or contract_id changes. | +| `tg_farm` | `farm` | On INSERT: Creates empty cache entry. On DELETE: Removes farm from cache. | + +## Error Handling + +All triggers include error handling that: +- Logs errors to `cache_errors` table for monitoring +- Raises warnings without failing the source transaction +- Includes context information (record IDs, old/new values) for debugging + +## Cache Management + +Manual cache refresh functions are available: +- `refresh_resources_cache()` - Refreshes entire resources cache +- `refresh_resources_cache_node(node_id)` - Refreshes a single node +- `refresh_public_ips_cache()` - Refreshes entire IP cache +- `refresh_public_ips_cache_farm(farm_id)` - Refreshes a single farm +- `validate_resources_cache()` - Validates cache consistency + +The cache is also automatically refreshed nightly at midnight using `pg_cron`. + diff --git a/grid-proxy/internal/explorer/db/cache/cache_management.sql b/grid-proxy/internal/explorer/db/cache/cache_management.sql new file mode 100644 index 00000000..a10c26f5 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/cache_management.sql @@ -0,0 +1,197 @@ +-- ============================================================================ +-- CACHE MANAGEMENT FUNCTIONS +-- ============================================================================ +-- Functions for manual cache refresh and validation. +-- Use these to fix inconsistencies or refresh after bulk operations. + +/* + * refresh_resources_cache_node + * + * Refreshes cache for a single node by recalculating from the view. + * Useful when cache becomes inconsistent for a specific node. + * + * @param p_node_id - Node ID to refresh + */ +CREATE OR REPLACE FUNCTION refresh_resources_cache_node(p_node_id INTEGER) RETURNS VOID AS +$$ +BEGIN + DELETE FROM resources_cache WHERE node_id = p_node_id; + + INSERT INTO resources_cache + SELECT * + FROM resources_cache_view + WHERE resources_cache_view.node_id = p_node_id; +END; +$$ LANGUAGE plpgsql; + +/* + * refresh_resources_cache + * + * Refreshes entire resources_cache table from the view. + * Use after bulk operations or when cache inconsistencies are detected. + * WARNING: This truncates the table and recalculates all rows. + */ +CREATE OR REPLACE FUNCTION refresh_resources_cache() RETURNS VOID AS +$$ +BEGIN + TRUNCATE resources_cache; + INSERT INTO resources_cache + SELECT * + FROM resources_cache_view; +END; +$$ LANGUAGE plpgsql; + +/* + * refresh_public_ips_cache_farm + * + * Refreshes cache for a single farm by recalculating IP aggregations. + * + * @param p_farm_id - Farm ID to refresh + */ +CREATE OR REPLACE FUNCTION refresh_public_ips_cache_farm(p_farm_id INTEGER) RETURNS VOID AS +$$ +BEGIN + DELETE FROM public_ips_cache WHERE farm_id = p_farm_id; + + INSERT INTO public_ips_cache + SELECT + farm.farm_id, + COALESCE(public_ip_agg.free_ips, 0), + COALESCE(public_ip_agg.total_ips, 0), + COALESCE(public_ip_agg.ips, '[]') + FROM farm + LEFT JOIN( + SELECT + p1.farm_id, + COUNT(p1.id) total_ips, + COUNT(CASE WHEN p1.contract_id = 0 THEN 1 END) free_ips, + jsonb_agg(jsonb_build_object('id', p1.id, 'ip', p1.ip, 'contract_id', p1.contract_id, 'gateway', p1.gateway)) as ips + FROM public_ip AS p1 + GROUP BY + p1.farm_id + ) public_ip_agg on public_ip_agg.farm_id = farm.id + WHERE farm.farm_id = p_farm_id; +END; +$$ LANGUAGE plpgsql; + +/* + * refresh_public_ips_cache + * + * Refreshes entire public_ips_cache table from source. + * Use after bulk IP operations or when cache inconsistencies are detected. + * WARNING: This truncates the table and recalculates all rows. + */ +CREATE OR REPLACE FUNCTION refresh_public_ips_cache() RETURNS VOID AS +$$ +BEGIN + TRUNCATE public_ips_cache; + + INSERT INTO public_ips_cache + SELECT + farm.farm_id, + COALESCE(public_ip_agg.free_ips, 0), + COALESCE(public_ip_agg.total_ips, 0), + COALESCE(public_ip_agg.ips, '[]') + FROM farm + LEFT JOIN( + SELECT + p1.farm_id, + COUNT(p1.id) total_ips, + COUNT(CASE WHEN p1.contract_id = 0 THEN 1 END) free_ips, + jsonb_agg(jsonb_build_object('id', p1.id, 'ip', p1.ip, 'contract_id', p1.contract_id, 'gateway', p1.gateway)) as ips + FROM public_ip AS p1 + GROUP BY + p1.farm_id + ) public_ip_agg on public_ip_agg.farm_id = farm.id; +END; +$$ LANGUAGE plpgsql; + +/* + * validate_resources_cache + * + * Validates cache consistency by comparing cache table with view. + * Checks all resource fields for mismatches. + * + * @returns INTEGER - Count of mismatched records (0 = cache is consistent) + */ +CREATE OR REPLACE FUNCTION validate_resources_cache() RETURNS INTEGER AS +$$ +DECLARE + mismatch_count INTEGER; +BEGIN + SELECT COUNT(*) + INTO mismatch_count + FROM resources_cache rc + FULL OUTER JOIN resources_cache_view rcv ON rc.node_id = rcv.node_id + WHERE rc.node_id IS NULL OR rcv.node_id IS NULL + OR rc.total_hru != rcv.total_hru + OR rc.total_mru != rcv.total_mru + OR rc.total_sru != rcv.total_sru + OR rc.total_cru != rcv.total_cru + OR rc.free_hru != rcv.free_hru + OR rc.free_mru != rcv.free_mru + OR rc.free_sru != rcv.free_sru + OR rc.used_hru != rcv.used_hru + OR rc.used_mru != rcv.used_mru + OR rc.used_sru != rcv.used_sru + OR rc.used_cru != rcv.used_cru + OR rc.node_contracts_count != rcv.node_contracts_count; + + RETURN mismatch_count; +END; +$$ LANGUAGE plpgsql; + +/* + * refresh_all_caches + * + * Convenience function to refresh all cache tables. + * Calls refresh_resources_cache() and refresh_public_ips_cache(). + * WARNING: This truncates and recalculates all cache tables. + */ +CREATE OR REPLACE FUNCTION refresh_all_caches() RETURNS VOID AS +$$ +BEGIN + PERFORM refresh_resources_cache(); + PERFORM refresh_public_ips_cache(); +END; +$$ LANGUAGE plpgsql; + +-- ============================================================================ +-- AUTOMATED CACHE REFRESH SCHEDULING +-- ============================================================================ +-- Schedule automatic cache refresh using pg_cron extension. +-- The cache will be refreshed daily at midnight (00:00:00). + +-- Install pg_cron extension if not already installed +CREATE EXTENSION IF NOT EXISTS pg_cron; + +-- Schedule nightly cache refresh at midnight (00:00:00) +-- Cron expression: '0 0 * * *' = every day at 00:00:00 (midnight) +-- This will call refresh_all_caches() every night +DO $$ +DECLARE + job_id INTEGER; +BEGIN + -- Check if job already exists and unschedule it (idempotent) + SELECT jobid INTO job_id + FROM cron.job + WHERE jobname = 'refresh-cache-nightly'; + + IF job_id IS NOT NULL THEN + PERFORM cron.unschedule(job_id); + RAISE NOTICE 'Removed existing cache refresh schedule'; + END IF; + + -- Schedule the nightly refresh + PERFORM cron.schedule( + 'refresh-cache-nightly', -- Job name + '0 0 * * *', -- Cron: Every day at midnight (00:00:00) + $$SELECT refresh_all_caches()$$ -- SQL to execute + ); + + RAISE NOTICE 'Scheduled cache refresh: Daily at midnight (00:00:00)'; +EXCEPTION + WHEN OTHERS THEN + RAISE WARNING 'Failed to schedule cache refresh: %. pg_cron extension may not be available or may require superuser privileges.', SQLERRM; +END $$; + diff --git a/grid-proxy/internal/explorer/db/cache/cache_tables.sql b/grid-proxy/internal/explorer/db/cache/cache_tables.sql new file mode 100644 index 00000000..f5b0142f --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/cache_tables.sql @@ -0,0 +1,113 @@ +-- ============================================================================ +-- CACHE TABLES +-- ============================================================================ +-- Materialized cache tables that store pre-computed data for fast queries. +-- These tables are automatically maintained by triggers. + +/* + * resources_cache + * + * Materialized cache table storing pre-computed node resource information. + * This table is automatically maintained by triggers when source data changes. + * + * Key features: + * - price_usd is a generated column using calc_price() function + * - All resource amounts are stored in bytes + * - GPU information stored as JSONB array + * - Indexed on node_id (primary key) and farm_id for fast lookups + */ +DROP TABLE IF EXISTS resources_cache; +CREATE TABLE IF NOT EXISTS resources_cache( + node_id INTEGER PRIMARY KEY, + farm_id INTEGER NOT NULL, + total_hru NUMERIC NOT NULL, + total_mru NUMERIC NOT NULL, + total_sru NUMERIC NOT NULL, + total_cru NUMERIC NOT NULL, + free_hru NUMERIC NOT NULL, + free_mru NUMERIC NOT NULL, + free_sru NUMERIC NOT NULL, + used_hru NUMERIC NOT NULL, + used_mru NUMERIC NOT NULL, + used_sru NUMERIC NOT NULL, + used_cru NUMERIC NOT NULL, + renter INTEGER, + rent_contract_id INTEGER, + node_contracts_count INTEGER NOT NULL, + country TEXT, + bios jsonb, + baseboard jsonb, + processor jsonb, + memory jsonb, + upload_speed numeric, + download_speed numeric, + udp_download_ipv4 numeric, + udp_upload_ipv4 numeric, + tcp_download_ipv6 numeric, + tcp_upload_ipv6 numeric, + udp_download_ipv6 numeric, + udp_upload_ipv6 numeric, + single_threaded_cpu numeric, + multi_threaded_cpu numeric, + threads_cpu integer, + workloads_cpu integer, + certified BOOLEAN, + policy_id INTEGER, + extra_fee NUMERIC, + gpus jsonb, + node_gpu_count INTEGER NOT NULL, + -- Generated column: automatically calculates price in USD + -- Converts resource amounts from bytes to GB for calc_price function + price_usd NUMERIC GENERATED ALWAYS AS ( + calc_price( + total_cru, + total_sru / get_bytes_per_gb(), -- Convert bytes to GB + total_hru / get_bytes_per_gb(), -- Convert bytes to GB + total_mru / get_bytes_per_gb(), -- Convert bytes to GB + certified, + policy_id, + extra_fee + ) + ) STORED +); + +-- Populate cache table from view +INSERT INTO resources_cache +SELECT * +FROM resources_cache_view; + +/* + * public_ips_cache + * + * Materialized cache table storing aggregated public IP information per farm. + * Tracks total IPs, free IPs (contract_id = 0), and IP details as JSONB. + * + * Automatically maintained by triggers when public_ip table changes. + */ +DROP TABLE IF EXISTS public_ips_cache; +CREATE TABLE public_ips_cache( + farm_id INTEGER PRIMARY KEY, + free_ips INTEGER NOT NULL, -- Count of IPs with contract_id = 0 + total_ips INTEGER NOT NULL, -- Total IPs assigned to farm + ips jsonb -- JSON array of all IPs with details +); + +-- Populate cache table with aggregated IP data +INSERT INTO public_ips_cache + SELECT + farm.farm_id, + COALESCE(public_ip_agg.free_ips, 0), + COALESCE(public_ip_agg.total_ips, 0), + COALESCE(public_ip_agg.ips, '[]') + FROM farm + LEFT JOIN( + SELECT + p1.farm_id, + COUNT(p1.id) total_ips, + COUNT(CASE WHEN p1.contract_id = 0 THEN 1 END) free_ips, + jsonb_agg(jsonb_build_object('id', p1.id, 'ip', p1.ip, 'contract_id', p1.contract_id, 'gateway', p1.gateway)) as ips + FROM public_ip AS p1 + GROUP BY + p1.farm_id + ) public_ip_agg on public_ip_agg.farm_id = farm.id; + diff --git a/grid-proxy/internal/explorer/db/cache/constants.sql b/grid-proxy/internal/explorer/db/cache/constants.sql new file mode 100644 index 00000000..f07b5cf7 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/constants.sql @@ -0,0 +1,116 @@ +-- ============================================================================ +-- CONSTANTS +-- ============================================================================ +-- This file defines all constant values used throughout the cache system. +-- Constants are stored in a table for easy modification and querying. + +-- Create constants table +CREATE TABLE IF NOT EXISTS cache_constants ( + constant_name TEXT PRIMARY KEY, + constant_value NUMERIC NOT NULL, + description TEXT NOT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Resource reservation constants (in bytes) +INSERT INTO cache_constants (constant_name, constant_value, description) VALUES + ('MRU_RESERVED_MIN_BYTES', 2147483648, 'MRU reserved minimum: 2GB (2^31 bytes)'), + ('MRU_RESERVED_FRACTION', 10, 'MRU reserved fraction: 10% (1/10 of total MRU)'), + ('SRU_RESERVED_BYTES', 21474836480, 'SRU fixed reservation: 20GB') +ON CONFLICT (constant_name) DO NOTHING; + +-- Storage Unit (SU) conversion factors +INSERT INTO cache_constants (constant_name, constant_value, description) VALUES + ('HRU_TO_SU_FACTOR', 1200, 'HRU to SU conversion: 1 SU = 1200 GB HRU'), + ('SRU_TO_SU_FACTOR', 200, 'SRU to SU conversion: 1 SU = 200 GB SRU') +ON CONFLICT (constant_name) DO NOTHING; + +-- Price calculation constants +INSERT INTO cache_constants (constant_name, constant_value, description) VALUES + ('CERTIFIED_MULTIPLIER', 1.25, 'Certified node premium: 25% (multiplier = 1.25)'), + ('HOURS_PER_MONTH', 720, 'Hours per month: 24 hours * 30 days = 720'), + ('USD_CONVERSION_FACTOR', 10000000, 'USD conversion: divide by 1e7 (10000000)') +ON CONFLICT (constant_name) DO NOTHING; + +-- Bytes to GB conversion factor +INSERT INTO cache_constants (constant_name, constant_value, description) VALUES + ('BYTES_PER_GB', 1073741824, 'Bytes per GB: 1024 * 1024 * 1024 = 1073741824') +ON CONFLICT (constant_name) DO NOTHING; + +-- Discount calculation constants +INSERT INTO cache_constants (constant_name, constant_value, description) VALUES + ('DISCOUNT_TIER_18X', 18, 'Discount tier: balance >= 18x cost (60% discount)'), + ('DISCOUNT_TIER_6X', 6, 'Discount tier: balance >= 6x cost (40% discount)'), + ('DISCOUNT_TIER_3X', 3, 'Discount tier: balance >= 3x cost (30% discount)'), + ('DISCOUNT_TIER_1_5X', 1.5, 'Discount tier: balance >= 1.5x cost (20% discount)'), + ('DISCOUNT_60_PCT', 0.6, 'Discount percentage: 60%'), + ('DISCOUNT_40_PCT', 0.4, 'Discount percentage: 40%'), + ('DISCOUNT_30_PCT', 0.3, 'Discount percentage: 30%'), + ('DISCOUNT_20_PCT', 0.2, 'Discount percentage: 20%') +ON CONFLICT (constant_name) DO NOTHING; + +-- Helper functions to retrieve constants +CREATE OR REPLACE FUNCTION get_cache_constant(const_name TEXT) RETURNS NUMERIC AS $$ +DECLARE + result NUMERIC; +BEGIN + SELECT constant_value INTO result + FROM cache_constants + WHERE constant_name = const_name; + + IF result IS NULL THEN + RAISE EXCEPTION 'Constant % not found', const_name; + END IF; + + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + +-- Convenience functions for commonly used constants +-- These are IMMUTABLE for use in generated columns and views +-- Values are read from cache_constants table but cached as IMMUTABLE functions for performance + +CREATE OR REPLACE FUNCTION get_mru_reserved_min_bytes() RETURNS NUMERIC AS $$ + -- MRU reserved minimum: 2GB = 2147483648 bytes + SELECT 2147483648::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION get_mru_reserved_fraction() RETURNS NUMERIC AS $$ + -- MRU reserved fraction: 10% (1/10 of total MRU) + SELECT 10::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION get_sru_reserved_bytes() RETURNS NUMERIC AS $$ + -- SRU fixed reservation: 20GB = 21474836480 bytes + SELECT 21474836480::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION get_hru_to_su_factor() RETURNS NUMERIC AS $$ + -- HRU to SU conversion: 1 SU = 1200 GB HRU + SELECT 1200::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION get_sru_to_su_factor() RETURNS NUMERIC AS $$ + -- SRU to SU conversion: 1 SU = 200 GB SRU + SELECT 200::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION get_certified_multiplier() RETURNS NUMERIC AS $$ + -- Certified node premium: 25% (multiplier = 1.25) + SELECT 1.25::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION get_hours_per_month() RETURNS NUMERIC AS $$ + -- Hours per month: 24 hours * 30 days = 720 + SELECT 720::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION get_usd_conversion_factor() RETURNS NUMERIC AS $$ + -- USD conversion: divide by 1e7 (10000000) + SELECT 10000000::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; + +CREATE OR REPLACE FUNCTION get_bytes_per_gb() RETURNS NUMERIC AS $$ + -- Bytes per GB: 1024 * 1024 * 1024 = 1073741824 + SELECT 1073741824::NUMERIC; +$$ LANGUAGE sql IMMUTABLE; diff --git a/grid-proxy/internal/explorer/db/cache/error_logging.sql b/grid-proxy/internal/explorer/db/cache/error_logging.sql new file mode 100644 index 00000000..4d1ff986 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/error_logging.sql @@ -0,0 +1,139 @@ +-- ============================================================================ +-- ERROR LOGGING +-- ============================================================================ +-- Error logging table and functions to track cache maintenance errors. +-- This allows monitoring and debugging of trigger failures without losing +-- error information. + +/* + * cache_errors + * + * Table to log errors from cache triggers and maintenance functions. + * Errors are logged here instead of just raising warnings, allowing + * for monitoring, alerting, and debugging. + */ +CREATE TABLE IF NOT EXISTS cache_errors ( + id BIGSERIAL PRIMARY KEY, + error_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + error_type TEXT NOT NULL, -- Type of error (trigger name, function name, etc.) + operation TEXT NOT NULL, -- Operation that failed (INSERT, UPDATE, DELETE, etc.) + table_name TEXT, -- Table where error occurred + record_id TEXT, -- ID of the record (can be node_id, farm_id, etc.) + error_message TEXT NOT NULL, -- Error message from SQLERRM + error_context JSONB, -- Additional context (OLD/NEW values, etc.) + resolved BOOLEAN DEFAULT FALSE, -- Whether error has been resolved + resolved_at TIMESTAMP, + resolved_by TEXT, + notes TEXT +); + +-- Note: Indexes are created in 04_indexes.sql to maintain proper ordering + +/* + * log_cache_error + * + * Helper function to log errors to cache_errors table. + * + * @param p_error_type - Type of error (e.g., 'reflect_node_changes') + * @param p_operation - Operation that failed (INSERT, UPDATE, DELETE) + * @param p_table_name - Table where error occurred + * @param p_record_id - ID of the record + * @param p_error_message - Error message + * @param p_error_context - Additional context as JSONB + */ +CREATE OR REPLACE FUNCTION log_cache_error( + p_error_type TEXT, + p_operation TEXT, + p_table_name TEXT DEFAULT NULL, + p_record_id TEXT DEFAULT NULL, + p_error_message TEXT DEFAULT NULL, + p_error_context JSONB DEFAULT NULL +) RETURNS VOID AS $$ +BEGIN + INSERT INTO cache_errors ( + error_type, + operation, + table_name, + record_id, + error_message, + error_context + ) VALUES ( + p_error_type, + p_operation, + p_table_name, + p_record_id, + COALESCE(p_error_message, SQLERRM), + p_error_context + ); +EXCEPTION + WHEN OTHERS THEN + -- If logging fails, raise warning (we don't want to fail silently) + RAISE WARNING 'Failed to log error to cache_errors: %', SQLERRM; +END; +$$ LANGUAGE plpgsql; + +/* + * get_unresolved_errors + * + * Returns count of unresolved errors, optionally filtered by type. + * + * @param p_error_type - Optional filter by error type + * @returns INTEGER - Count of unresolved errors + */ +CREATE OR REPLACE FUNCTION get_unresolved_errors(p_error_type TEXT DEFAULT NULL) RETURNS INTEGER AS $$ +DECLARE + error_count INTEGER; +BEGIN + SELECT COUNT(*) INTO error_count + FROM cache_errors + WHERE resolved = FALSE + AND (p_error_type IS NULL OR error_type = p_error_type); + + RETURN error_count; +END; +$$ LANGUAGE plpgsql; + +/* + * resolve_cache_error + * + * Marks an error as resolved. + * + * @param p_error_id - ID of error to resolve + * @param p_resolved_by - Who resolved it (optional) + * @param p_notes - Notes about resolution (optional) + */ +CREATE OR REPLACE FUNCTION resolve_cache_error( + p_error_id BIGINT, + p_resolved_by TEXT DEFAULT NULL, + p_notes TEXT DEFAULT NULL +) RETURNS VOID AS $$ +BEGIN + UPDATE cache_errors + SET resolved = TRUE, + resolved_at = CURRENT_TIMESTAMP, + resolved_by = p_resolved_by, + notes = p_notes + WHERE id = p_error_id; +END; +$$ LANGUAGE plpgsql; + +/* + * cleanup_old_errors + * + * Removes resolved errors older than specified days. + * + * @param p_days_to_keep - Number of days to keep resolved errors (default 30) + */ +CREATE OR REPLACE FUNCTION cleanup_old_errors(p_days_to_keep INTEGER DEFAULT 30) RETURNS INTEGER AS $$ +DECLARE + deleted_count INTEGER; +BEGIN + DELETE FROM cache_errors + WHERE resolved = TRUE + AND resolved_at < CURRENT_TIMESTAMP - (p_days_to_keep || ' days')::INTERVAL; + + GET DIAGNOSTICS deleted_count = ROW_COUNT; + RETURN deleted_count; +END; +$$ LANGUAGE plpgsql; + diff --git a/grid-proxy/internal/explorer/db/cache/functions.sql b/grid-proxy/internal/explorer/db/cache/functions.sql new file mode 100644 index 00000000..b0fda148 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/functions.sql @@ -0,0 +1,148 @@ +-- ============================================================================ +-- HELPER FUNCTIONS +-- ============================================================================ +-- Utility functions used for calculations and data transformations + +DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); + +/* + * convert_to_decimal + * + * Safely converts a text input to decimal, returning NULL if conversion fails. + * This prevents errors from invalid numeric strings in the data. + * + * @param v_input - Text value to convert + * @returns DECIMAL or NULL if conversion fails + */ +CREATE OR REPLACE FUNCTION convert_to_decimal(v_input TEXT) RETURNS DECIMAL AS +$$ +DECLARE + v_dec_value DECIMAL DEFAULT NULL; +BEGIN + BEGIN + v_dec_value := v_input::DECIMAL; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; + RETURN NULL; + END; + RETURN v_dec_value; +END; +$$ LANGUAGE plpgsql; + +/* + * calc_discount + * + * Calculates discount based on balance-to-cost ratio. + * Higher balance relative to cost results in larger discounts. + * + * Discount tiers: + * - 60% discount: balance >= 18x cost + * - 40% discount: balance >= 6x cost + * - 30% discount: balance >= 3x cost + * - 20% discount: balance >= 1.5x cost + * - 0% discount: otherwise + * + * @param cost - Base cost amount + * @param balance - Available balance + * @returns NUMERIC - Final cost after discount + */ +CREATE OR REPLACE FUNCTION calc_discount( + cost NUMERIC, + balance NUMERIC +) RETURNS NUMERIC AS $$ +DECLARE + discount NUMERIC; + -- Discount tier multipliers + discount_tier_18x CONSTANT NUMERIC := 18; + discount_tier_6x CONSTANT NUMERIC := 6; + discount_tier_3x CONSTANT NUMERIC := 3; + discount_tier_1_5x CONSTANT NUMERIC := 1.5; + -- Discount percentages + discount_60_pct CONSTANT NUMERIC := 0.6; + discount_40_pct CONSTANT NUMERIC := 0.4; + discount_30_pct CONSTANT NUMERIC := 0.3; + discount_20_pct CONSTANT NUMERIC := 0.2; +BEGIN + discount := ( + CASE + WHEN balance >= cost * discount_tier_18x THEN discount_60_pct + WHEN balance >= cost * discount_tier_6x THEN discount_40_pct + WHEN balance >= cost * discount_tier_3x THEN discount_30_pct + WHEN balance >= cost * discount_tier_1_5x THEN discount_20_pct + ELSE 0 + END + ); + + RETURN cost - cost * discount; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +/* + * calc_price + * + * Calculates the monthly price for node resources based on pricing policy. + * + * Price calculation: + * 1. Compute CU (Compute Unit) from CRU and MRU using the maximum of three formulas + * 2. Compute SU (Storage Unit) from HRU and SRU + * 3. Apply certified node multiplier (25% premium) + * 4. Convert to USD (divide by 1e7) + * + * @param cru - Compute Resource Units + * @param sru - Storage Resource Units (in bytes) + * @param hru - HDD Resource Units (in bytes) + * @param mru - Memory Resource Units (in bytes) + * @param certified - Whether node is certified (25% premium) + * @param policy_id - Pricing policy ID + * @param extra_fee - Additional fees + * @returns NUMERIC - Price in USD per month + */ +CREATE OR REPLACE FUNCTION calc_price( + cru NUMERIC, + sru NUMERIC, + hru NUMERIC, + mru NUMERIC, + certified BOOLEAN, + policy_id INTEGER, + extra_fee NUMERIC +) RETURNS NUMERIC AS $$ +DECLARE + su NUMERIC; + cu NUMERIC; + su_value NUMERIC; + cu_value NUMERIC; + cost_per_month NUMERIC; +BEGIN + -- Fetch pricing values from policy (optimized single query) + SELECT pricing_policy.cu->'value', pricing_policy.su->'value' + INTO cu_value, su_value + FROM pricing_policy + WHERE pricing_policy_id = policy_id; + + IF cu_value IS NULL OR su_value IS NULL THEN + RAISE EXCEPTION 'pricing values not found for policy_id: %', policy_id; + END IF; + + -- Compute CU: take the minimum of three different calculation methods + -- This ensures CU reflects the most constraining resource + cu := LEAST( + GREATEST(mru / 4, cru / 2), + GREATEST(mru / 8, cru), + GREATEST(mru / 2, cru / 4) + ); + + -- Compute SU: Storage Unit = HRU/HRU_TO_SU_FACTOR + SRU/SRU_TO_SU_FACTOR + su := (hru / get_hru_to_su_factor() + sru / get_sru_to_su_factor()); + + -- Calculate monthly cost: + -- (CU * CU_price + SU * SU_price + extra_fee) * certified_multiplier * hours_per_month + cost_per_month := (cu * cu_value + su * su_value + extra_fee) * + (CASE certified WHEN true THEN get_certified_multiplier() ELSE 1 END) * + get_hours_per_month(); + + -- Convert to USD (divide by conversion factor) + RETURN cost_per_month / get_usd_conversion_factor(); +END; +$$ LANGUAGE plpgsql STABLE; + diff --git a/grid-proxy/internal/explorer/db/helpers.sql b/grid-proxy/internal/explorer/db/cache/helpers.sql similarity index 100% rename from grid-proxy/internal/explorer/db/helpers.sql rename to grid-proxy/internal/explorer/db/cache/helpers.sql diff --git a/grid-proxy/internal/explorer/db/cache/indexes.sql b/grid-proxy/internal/explorer/db/cache/indexes.sql new file mode 100644 index 00000000..2de5db66 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/indexes.sql @@ -0,0 +1,33 @@ +-- ============================================================================ +-- INDEXES +-- ============================================================================ +-- Performance indexes to optimize cache table queries + +-- Required PostgreSQL extensions for index types +CREATE EXTENSION IF NOT EXISTS pg_trgm; -- For trigram similarity search +CREATE EXTENSION IF NOT EXISTS btree_gin; -- For GIN indexes on scalar values + +-- Indexes on source tables (used by views and triggers) +CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); +CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); +CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); +-- GIN indexes for UUID/string lookups (faster than B-tree for large datasets) +CREATE INDEX IF NOT EXISTS idx_node_contract_id ON public.node_contract USING gin(id); +CREATE INDEX IF NOT EXISTS idx_name_contract_id ON public.name_contract USING gin(id); +CREATE INDEX IF NOT EXISTS idx_rent_contract_id ON public.rent_contract USING gin(id); + +-- Indexes on cache tables (for fast queries) +CREATE INDEX IF NOT EXISTS idx_resources_cache_farm_id ON resources_cache(farm_id); +CREATE INDEX IF NOT EXISTS idx_resources_cache_node_id ON resources_cache(node_id); +CREATE INDEX IF NOT EXISTS idx_public_ips_cache_farm_id ON public_ips_cache(farm_id); + +-- Additional indexes for indexer tables +CREATE INDEX IF NOT EXISTS idx_location_id ON location USING gin(id); +CREATE INDEX IF NOT EXISTS idx_public_config_node_id ON public_config USING gin(node_id); + +-- Indexes for error logging table +CREATE INDEX IF NOT EXISTS idx_cache_errors_timestamp ON cache_errors(error_timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_cache_errors_type ON cache_errors(error_type); +CREATE INDEX IF NOT EXISTS idx_cache_errors_resolved ON cache_errors(resolved) WHERE resolved = FALSE; +CREATE INDEX IF NOT EXISTS idx_cache_errors_table ON cache_errors(table_name); + diff --git a/grid-proxy/internal/explorer/db/cache/tests/00_test_helpers.sql b/grid-proxy/internal/explorer/db/cache/tests/00_test_helpers.sql new file mode 100644 index 00000000..6cd75f7e --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/00_test_helpers.sql @@ -0,0 +1,192 @@ +-- ============================================================================ +-- TEST HELPERS +-- ============================================================================ +-- Helper functions for setting up test data and cleaning up after tests + +-- Function to create minimal test data for a node +CREATE OR REPLACE FUNCTION create_test_node( + p_node_id INTEGER, + p_farm_id INTEGER, + p_twin_id INTEGER, + p_country TEXT DEFAULT 'US', + p_certification TEXT DEFAULT NULL, + p_extra_fee NUMERIC DEFAULT 0 +) RETURNS VOID AS $$ +BEGIN + INSERT INTO node ( + id, grid_version, node_id, farm_id, twin_id, country, + created, farming_policy_id, secure, virtualized, created_at, updated_at, certification, extra_fee + ) VALUES ( + 'node-' || p_node_id::TEXT, 3, p_node_id, p_farm_id, p_twin_id, p_country, + EXTRACT(EPOCH FROM NOW())::INTEGER, 1, false, false, + EXTRACT(EPOCH FROM NOW())::NUMERIC, EXTRACT(EPOCH FROM NOW())::NUMERIC, + COALESCE(p_certification, 'Diy'), p_extra_fee + ) ON CONFLICT (id) DO NOTHING; +END; +$$ LANGUAGE plpgsql; + +-- Function to create test farm +CREATE OR REPLACE FUNCTION create_test_farm( + p_farm_id INTEGER, + p_twin_id INTEGER, + p_name TEXT DEFAULT 'Test Farm', + p_pricing_policy_id INTEGER DEFAULT 1 +) RETURNS VOID AS $$ +BEGIN + INSERT INTO farm ( + id, grid_version, farm_id, name, twin_id, pricing_policy_id, dedicated_farm, certification + ) VALUES ( + 'farm-' || p_farm_id::TEXT, 3, p_farm_id, p_name, p_twin_id, p_pricing_policy_id, false, 'Certified' + ) ON CONFLICT (id) DO NOTHING; +END; +$$ LANGUAGE plpgsql; + +-- Function to create test node_resources_total +CREATE OR REPLACE FUNCTION create_test_node_resources_total( + p_node_id VARCHAR, + p_hru NUMERIC DEFAULT 1000000000000, -- 1TB + p_mru NUMERIC DEFAULT 100000000000, -- 100GB + p_sru NUMERIC DEFAULT 100000000000, -- 100GB + p_cru NUMERIC DEFAULT 16 -- 16 cores +) RETURNS VOID AS $$ +BEGIN + INSERT INTO node_resources_total ( + id, hru, mru, sru, cru, node_id + ) VALUES ( + 'nrt-' || p_node_id, p_hru, p_mru, p_sru, p_cru, p_node_id + ) ON CONFLICT (id) DO UPDATE SET + hru = EXCLUDED.hru, + mru = EXCLUDED.mru, + sru = EXCLUDED.sru, + cru = EXCLUDED.cru; +END; +$$ LANGUAGE plpgsql; + +-- Function to create test node_contract +CREATE OR REPLACE FUNCTION create_test_node_contract( + p_contract_id VARCHAR, + p_node_id INTEGER, + p_twin_id INTEGER, + p_state TEXT DEFAULT 'Created', + p_resources_used_id VARCHAR DEFAULT NULL +) RETURNS VARCHAR AS $$ +DECLARE + v_resources_id VARCHAR; +BEGIN + IF p_resources_used_id IS NULL THEN + v_resources_id := 'cr-' || p_contract_id; + ELSE + v_resources_id := p_resources_used_id; + END IF; + + INSERT INTO node_contract ( + id, grid_version, contract_id, twin_id, node_id, deployment_data, + deployment_hash, number_of_public_i_ps, created_at, resources_used_id, state + ) VALUES ( + 'nc-' || p_contract_id, 3, p_contract_id::NUMERIC, p_twin_id, p_node_id, + '{}', 'hash', 0, EXTRACT(EPOCH FROM NOW())::NUMERIC, v_resources_id, p_state + ) ON CONFLICT (id) DO UPDATE SET + state = EXCLUDED.state, + resources_used_id = EXCLUDED.resources_used_id; + + RETURN v_resources_id; +END; +$$ LANGUAGE plpgsql; + +-- Function to create test contract_resources +CREATE OR REPLACE FUNCTION create_test_contract_resources( + p_contract_id VARCHAR, + p_hru NUMERIC DEFAULT 1000000000, -- 1GB + p_mru NUMERIC DEFAULT 1000000000, -- 1GB + p_sru NUMERIC DEFAULT 1000000000, -- 1GB + p_cru NUMERIC DEFAULT 1 -- 1 core +) RETURNS VOID AS $$ +BEGIN + INSERT INTO contract_resources ( + id, hru, mru, sru, cru, contract_id + ) VALUES ( + 'cr-' || p_contract_id, p_hru, p_mru, p_sru, p_cru, p_contract_id + ) ON CONFLICT (id) DO UPDATE SET + hru = EXCLUDED.hru, + mru = EXCLUDED.mru, + sru = EXCLUDED.sru, + cru = EXCLUDED.cru; +END; +$$ LANGUAGE plpgsql; + +-- Function to create test pricing_policy +CREATE OR REPLACE FUNCTION create_test_pricing_policy( + p_policy_id INTEGER, + p_cu_value NUMERIC DEFAULT 1000000, + p_su_value NUMERIC DEFAULT 1000000 +) RETURNS VOID AS $$ +BEGIN + INSERT INTO pricing_policy ( + id, grid_version, pricing_policy_id, name, su, cu, nu, ipu, + foundation_account, certified_sales_account, dedicated_node_discount + ) VALUES ( + 'pp-' || p_policy_id::TEXT, 3, p_policy_id, 'Test Policy', + jsonb_build_object('value', p_su_value), + jsonb_build_object('value', p_cu_value), + '{}'::jsonb, '{}'::jsonb, 'foundation', 'certified', 0 + ) ON CONFLICT (id) DO UPDATE SET + su = EXCLUDED.su, + cu = EXCLUDED.cu; +END; +$$ LANGUAGE plpgsql; + +-- Function to create test public_ip +CREATE OR REPLACE FUNCTION create_test_public_ip( + p_ip_id VARCHAR, + p_farm_id VARCHAR, + p_ip TEXT DEFAULT '1.2.3.4', + p_gateway TEXT DEFAULT '1.2.3.1', + p_contract_id NUMERIC DEFAULT 0 +) RETURNS VOID AS $$ +BEGIN + INSERT INTO public_ip ( + id, gateway, ip, contract_id, farm_id + ) VALUES ( + p_ip_id, p_gateway, p_ip, p_contract_id, p_farm_id + ) ON CONFLICT (id) DO UPDATE SET + contract_id = EXCLUDED.contract_id, + ip = EXCLUDED.ip, + gateway = EXCLUDED.gateway; +END; +$$ LANGUAGE plpgsql; + +-- Function to create test twin +CREATE OR REPLACE FUNCTION create_test_twin( + p_twin_id INTEGER, + p_account_id TEXT DEFAULT 'account123' +) RETURNS VOID AS $$ +BEGIN + INSERT INTO twin ( + id, grid_version, twin_id, account_id, relay + ) VALUES ( + 'twin-' || p_twin_id::TEXT, 3, p_twin_id, p_account_id, '' + ) ON CONFLICT (id) DO NOTHING; +END; +$$ LANGUAGE plpgsql; + +-- Function to clean up test data +CREATE OR REPLACE FUNCTION cleanup_test_data() RETURNS VOID AS $$ +BEGIN + -- Clean in reverse order of dependencies + DELETE FROM contract_resources WHERE id LIKE 'cr-%'; + DELETE FROM node_contract WHERE id LIKE 'nc-%'; + DELETE FROM node_gpu WHERE id LIKE 'gpu-%'; + DELETE FROM dmi WHERE node_twin_id IN (SELECT twin_id FROM node WHERE id LIKE 'node-%'); + DELETE FROM speed WHERE node_twin_id IN (SELECT twin_id FROM node WHERE id LIKE 'node-%'); + DELETE FROM cpu_benchmark WHERE node_twin_id IN (SELECT twin_id FROM node WHERE id LIKE 'node-%'); + DELETE FROM node_resources_total WHERE id LIKE 'nrt-%'; + DELETE FROM rent_contract WHERE id LIKE 'rc-%'; + DELETE FROM public_ip WHERE id LIKE 'ip-%'; + DELETE FROM node WHERE id LIKE 'node-%'; + DELETE FROM farm WHERE id LIKE 'farm-%'; + DELETE FROM twin WHERE id LIKE 'twin-%'; + DELETE FROM resources_cache WHERE node_id IN (SELECT node_id FROM node WHERE id LIKE 'node-%'); + DELETE FROM public_ips_cache WHERE farm_id IN (SELECT farm_id FROM farm WHERE id LIKE 'farm-%'); +END; +$$ LANGUAGE plpgsql; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/01_test_node_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/01_test_node_trigger.sql new file mode 100644 index 00000000..cfe58565 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/01_test_node_trigger.sql @@ -0,0 +1,82 @@ +-- ============================================================================ +-- TEST: reflect_node_changes Trigger +-- ============================================================================ +-- Tests for node INSERT and DELETE operations on resources_cache + +BEGIN; + +-- Load pgTAP +SELECT plan(6); + +-- Setup: Create test farm and pricing policy +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); + +-- Test 1: INSERT node should populate resources_cache +SELECT create_test_node(1001, 1001, 2001, 'US', 'Certified', 0); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); + +-- Wait for trigger to fire +SELECT pg_sleep(0.1); + +SELECT ok( + EXISTS ( + SELECT 1 FROM resources_cache WHERE node_id = 1001 + ), + 'Node INSERT should create entry in resources_cache' +); + +-- Test 2: Verify resources_cache values match expected from view +SELECT is( + (SELECT total_hru FROM resources_cache WHERE node_id = 1001), + 1000000000000, + 'resources_cache total_hru should match node_resources_total' +); + +SELECT is( + (SELECT total_mru FROM resources_cache WHERE node_id = 1001), + 100000000000, + 'resources_cache total_mru should match node_resources_total' +); + +-- Test 3: DELETE node should remove entry from resources_cache +DELETE FROM node_resources_total WHERE node_id = 'node-1001'; +DELETE FROM node WHERE id = 'node-1001'; + +-- Wait for trigger to fire +SELECT pg_sleep(0.1); + +SELECT ok( + NOT EXISTS ( + SELECT 1 FROM resources_cache WHERE node_id = 1001 + ), + 'Node DELETE should remove entry from resources_cache' +); + +-- Test 4: INSERT node with all fields populated +SELECT create_test_node(1002, 1001, 2002, 'DE', 'Certified', 100); +SELECT create_test_twin(2002); +SELECT create_test_node_resources_total('node-1002', 2000000000000, 200000000000, 200000000000, 32); + +SELECT pg_sleep(0.1); + +SELECT ok( + EXISTS ( + SELECT 1 FROM resources_cache + WHERE node_id = 1002 + AND farm_id = 1001 + AND country = 'DE' + AND certified = true + AND extra_fee = 100 + ), + 'Node INSERT should populate all fields in resources_cache' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/02_test_node_resources_total_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/02_test_node_resources_total_trigger.sql new file mode 100644 index 00000000..e4802b81 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/02_test_node_resources_total_trigger.sql @@ -0,0 +1,157 @@ +-- ============================================================================ +-- TEST: reflect_total_resources_changes Trigger +-- ============================================================================ +-- Tests for node_resources_total INSERT and UPDATE operations + +BEGIN; + +-- Load pgTAP +SELECT plan(12); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT pg_sleep(0.1); + +-- Get initial cache values +SELECT + (SELECT total_hru FROM resources_cache WHERE node_id = 1001) as initial_hru, + (SELECT total_mru FROM resources_cache WHERE node_id = 1001) as initial_mru, + (SELECT total_sru FROM resources_cache WHERE node_id = 1001) as initial_sru, + (SELECT total_cru FROM resources_cache WHERE node_id = 1001) as initial_cru, + (SELECT free_hru FROM resources_cache WHERE node_id = 1001) as initial_free_hru, + (SELECT free_mru FROM resources_cache WHERE node_id = 1001) as initial_free_mru, + (SELECT free_sru FROM resources_cache WHERE node_id = 1001) as initial_free_sru, + (SELECT used_mru FROM resources_cache WHERE node_id = 1001) as initial_used_mru +INTO TEMP initial_cache; + +-- Test 1: UPDATE total_hru should update cache +UPDATE node_resources_total +SET hru = 2000000000000 +WHERE node_id = 'node-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT total_hru FROM resources_cache WHERE node_id = 1001), + 2000000000000, + 'UPDATE total_hru should update resources_cache total_hru' +); + +-- Calculate expected free_hru change: NEW.hru - OLD.hru +SELECT is( + (SELECT free_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_free_hru FROM initial_cache) + 1000000000000, + 'UPDATE total_hru should increment free_hru by the difference' +); + +-- Test 2: UPDATE total_mru should update cache and adjust reserved amount +UPDATE node_resources_total +SET mru = 200000000000 +WHERE node_id = 'node-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT total_mru FROM resources_cache WHERE node_id = 1001), + 200000000000, + 'UPDATE total_mru should update resources_cache total_mru' +); + +-- MRU reserved calculation: GREATEST(MRU/10, 2147483648) +-- Old reserved: GREATEST(100000000000/10, 2147483648) = GREATEST(10000000000, 2147483648) = 10000000000 +-- New reserved: GREATEST(200000000000/10, 2147483648) = GREATEST(20000000000, 2147483648) = 20000000000 +-- free_mru change = (old_reserved - new_reserved) + (new_mru - old_mru) +-- = (10000000000 - 20000000000) + (200000000000 - 100000000000) +-- = -10000000000 + 100000000000 = 90000000000 +-- But we need to check the actual calculation... + +-- Test 3: UPDATE total_sru should update cache +UPDATE node_resources_total +SET sru = 200000000000 +WHERE node_id = 'node-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT total_sru FROM resources_cache WHERE node_id = 1001), + 200000000000, + 'UPDATE total_sru should update resources_cache total_sru' +); + +-- Test 4: UPDATE total_cru should update cache +UPDATE node_resources_total +SET cru = 32 +WHERE node_id = 'node-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT total_cru FROM resources_cache WHERE node_id = 1001), + 32, + 'UPDATE total_cru should update resources_cache total_cru' +); + +-- Test 5: INSERT new node_resources_total should update cache +SELECT create_test_node(1002, 1001, 2002); +SELECT create_test_twin(2002); +SELECT create_test_node_resources_total('node-1002', 500000000000, 50000000000, 50000000000, 8); +SELECT pg_sleep(0.1); + +SELECT ok( + EXISTS ( + SELECT 1 FROM resources_cache + WHERE node_id = 1002 + AND total_hru = 500000000000 + AND total_mru = 50000000000 + AND total_sru = 50000000000 + AND total_cru = 8 + ), + 'INSERT node_resources_total should update resources_cache' +); + +-- Test 6: UPDATE MRU below reserved minimum threshold +-- MRU reserved minimum is 2147483648 (2GB) +-- If MRU is 10000000000 (10GB), reserved = GREATEST(10GB/10, 2GB) = GREATEST(1GB, 2GB) = 2GB +-- If MRU is 100000000000 (100GB), reserved = GREATEST(100GB/10, 2GB) = GREATEST(10GB, 2GB) = 10GB +UPDATE node_resources_total +SET mru = 10000000000 -- 10GB, below threshold so reserved should be 2GB +WHERE node_id = 'node-1001'; + +SELECT pg_sleep(0.1); + +-- Verify used_mru includes reserved amount (at least 2GB) +SELECT ok( + (SELECT used_mru FROM resources_cache WHERE node_id = 1001) >= 2147483648, + 'used_mru should include at least minimum reserved amount (2GB)' +); + +-- Test 7: Multiple simultaneous updates +UPDATE node_resources_total +SET hru = 3000000000000, mru = 300000000000, sru = 300000000000, cru = 64 +WHERE node_id = 'node-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT total_hru FROM resources_cache WHERE node_id = 1001), + 3000000000000, + 'Multiple UPDATE should update all total fields' +); + +SELECT is( + (SELECT total_cru FROM resources_cache WHERE node_id = 1001), + 64, + 'Multiple UPDATE should update total_cru' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/03_test_contract_resources_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/03_test_contract_resources_trigger.sql new file mode 100644 index 00000000..dfdcec08 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/03_test_contract_resources_trigger.sql @@ -0,0 +1,155 @@ +-- ============================================================================ +-- TEST: reflect_contract_resources_changes Trigger +-- ============================================================================ +-- Tests for contract_resources INSERT, UPDATE, and DELETE operations + +BEGIN; + +-- Load pgTAP +SELECT plan(15); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT pg_sleep(0.1); + +-- Get initial cache values +SELECT + (SELECT free_hru FROM resources_cache WHERE node_id = 1001) as initial_free_hru, + (SELECT free_mru FROM resources_cache WHERE node_id = 1001) as initial_free_mru, + (SELECT free_sru FROM resources_cache WHERE node_id = 1001) as initial_free_sru, + (SELECT used_hru FROM resources_cache WHERE node_id = 1001) as initial_used_hru, + (SELECT used_mru FROM resources_cache WHERE node_id = 1001) as initial_used_mru, + (SELECT used_sru FROM resources_cache WHERE node_id = 1001) as initial_used_sru, + (SELECT used_cru FROM resources_cache WHERE node_id = 1001) as initial_used_cru +INTO TEMP initial_cache; + +-- Create node_contract and contract_resources +SELECT create_test_node_contract('1001', 1001, 2001, 'Created', 'cr-1001'); +SELECT create_test_contract_resources('1001', 1000000000, 2000000000, 3000000000, 2); + +SELECT pg_sleep(0.1); + +-- Test 1: INSERT contract_resources should increment used and decrement free +SELECT is( + (SELECT used_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_used_hru FROM initial_cache) + 1000000000, + 'INSERT contract_resources should increment used_hru' +); + +SELECT is( + (SELECT used_mru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_used_mru FROM initial_cache) + 2000000000, + 'INSERT contract_resources should increment used_mru' +); + +SELECT is( + (SELECT free_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_free_hru FROM initial_cache) - 1000000000, + 'INSERT contract_resources should decrement free_hru' +); + +SELECT is( + (SELECT free_mru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_free_mru FROM initial_cache) - 2000000000, + 'INSERT contract_resources should decrement free_mru' +); + +-- Test 2: UPDATE contract_resources should adjust differences +UPDATE contract_resources +SET hru = 2000000000, mru = 4000000000, sru = 5000000000, cru = 4 +WHERE id = 'cr-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT used_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_used_hru FROM initial_cache) + 2000000000, + 'UPDATE contract_resources should update used_hru to new value' +); + +SELECT is( + (SELECT used_mru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_used_mru FROM initial_cache) + 4000000000, + 'UPDATE contract_resources should update used_mru to new value' +); + +SELECT is( + (SELECT free_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_free_hru FROM initial_cache) - 2000000000, + 'UPDATE contract_resources should adjust free_hru' +); + +-- Test 3: DELETE contract_resources should decrement used and increment free +DELETE FROM contract_resources WHERE id = 'cr-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT used_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_used_hru FROM initial_cache), + 'DELETE contract_resources should decrement used_hru back to initial' +); + +SELECT is( + (SELECT used_mru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_used_mru FROM initial_cache), + 'DELETE contract_resources should decrement used_mru back to initial' +); + +SELECT is( + (SELECT free_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_free_hru FROM initial_cache), + 'DELETE contract_resources should increment free_hru back to initial' +); + +-- Test 4: Contract with 'GracePeriod' state should be counted +SELECT create_test_node_contract('1002', 1001, 2001, 'GracePeriod', 'cr-1002'); +SELECT create_test_contract_resources('1002', 500000000, 500000000, 500000000, 1); + +SELECT pg_sleep(0.1); + +SELECT ok( + (SELECT used_hru FROM resources_cache WHERE node_id = 1001) > (SELECT initial_used_hru FROM initial_cache), + 'Contract in GracePeriod state should be counted in used resources' +); + +-- Test 5: Contract with 'Deleted' state should NOT be counted +DELETE FROM contract_resources WHERE id = 'cr-1002'; +UPDATE node_contract SET state = 'Deleted' WHERE id = 'nc-1002'; +SELECT create_test_node_contract('1003', 1001, 2001, 'Deleted', 'cr-1003'); +SELECT create_test_contract_resources('1003', 1000000000, 1000000000, 1000000000, 1); + +SELECT pg_sleep(0.1); + +-- Should not affect cache because state is 'Deleted' +SELECT is( + (SELECT used_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_used_hru FROM initial_cache), + 'Contract in Deleted state should NOT affect cache' +); + +-- Test 6: Multiple contracts on same node +SELECT create_test_node_contract('1004', 1001, 2001, 'Created', 'cr-1004'); +SELECT create_test_contract_resources('1004', 1000000000, 1000000000, 1000000000, 1); +SELECT create_test_node_contract('1005', 1001, 2001, 'Created', 'cr-1005'); +SELECT create_test_contract_resources('1005', 1000000000, 1000000000, 1000000000, 1); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT used_hru FROM resources_cache WHERE node_id = 1001), + (SELECT initial_used_hru FROM initial_cache) + 2000000000, + 'Multiple contracts should sum up used resources' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/04_test_node_contract_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/04_test_node_contract_trigger.sql new file mode 100644 index 00000000..708eec7d --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/04_test_node_contract_trigger.sql @@ -0,0 +1,135 @@ +-- ============================================================================ +-- TEST: reflect_node_contract_changes Trigger +-- ============================================================================ +-- Tests for node_contract INSERT and UPDATE state to 'Deleted' + +BEGIN; + +-- Load pgTAP +SELECT plan(8); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT pg_sleep(0.1); + +-- Test 1: INSERT node_contract with 'Created' state should increment node_contracts_count +SELECT create_test_node_contract('1001', 1001, 2001, 'Created', 'cr-1001'); +SELECT create_test_contract_resources('1001', 1000000000, 2000000000, 3000000000, 2); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001), + 1, + 'INSERT node_contract with Created state should set node_contracts_count to 1' +); + +-- Test 2: INSERT another contract should increment count +SELECT create_test_node_contract('1002', 1001, 2001, 'Created', 'cr-1002'); +SELECT create_test_contract_resources('1002', 500000000, 500000000, 500000000, 1); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001), + 2, + 'INSERT multiple contracts should increment node_contracts_count' +); + +-- Test 3: INSERT contract with 'GracePeriod' state should be counted +SELECT create_test_node_contract('1003', 1001, 2001, 'GracePeriod', 'cr-1003'); +SELECT create_test_contract_resources('1003', 1000000000, 1000000000, 1000000000, 1); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001), + 3, + 'INSERT contract with GracePeriod state should be counted' +); + +-- Test 4: UPDATE contract state to 'Deleted' should decrement count and release resources +-- Get initial values before deletion +SELECT + (SELECT used_hru FROM resources_cache WHERE node_id = 1001) as before_used_hru, + (SELECT free_hru FROM resources_cache WHERE node_id = 1001) as before_free_hru, + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001) as before_count +INTO TEMP before_delete; + +UPDATE node_contract SET state = 'Deleted' WHERE id = 'nc-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001), + (SELECT before_count FROM before_delete) - 1, + 'UPDATE contract to Deleted state should decrement node_contracts_count' +); + +-- Test 5: UPDATE to 'Deleted' should release contract resources +SELECT is( + (SELECT used_hru FROM resources_cache WHERE node_id = 1001), + (SELECT before_used_hru FROM before_delete) - 1000000000, + 'UPDATE to Deleted should release contract resources (decrement used_hru)' +); + +SELECT is( + (SELECT free_hru FROM resources_cache WHERE node_id = 1001), + (SELECT before_free_hru FROM before_delete) + 1000000000, + 'UPDATE to Deleted should release contract resources (increment free_hru)' +); + +-- Test 6: UPDATE multiple contracts to 'Deleted' +UPDATE node_contract SET state = 'Deleted' WHERE id = 'nc-1002'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001), + 1, + 'Multiple contract deletions should update count correctly' +); + +-- Test 7: INSERT contract with 'Deleted' state should NOT increment count +SELECT create_test_node_contract('1004', 1001, 2001, 'Deleted', 'cr-1004'); +SELECT create_test_contract_resources('1004', 1000000000, 1000000000, 1000000000, 1); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001), + 1, + 'INSERT contract with Deleted state should NOT increment count' +); + +-- Test 8: UPDATE to non-Deleted state should not trigger resource release +-- (Only UPDATE of state column triggers, and only to 'Deleted') +SELECT create_test_node_contract('1005', 1001, 2001, 'Created', 'cr-1005'); +SELECT create_test_contract_resources('1005', 1000000000, 1000000000, 1000000000, 1); +SELECT pg_sleep(0.1); + +SELECT + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001) as count_before +INTO TEMP count_before_update; + +UPDATE node_contract SET deployment_data = '{}' WHERE id = 'nc-1005'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_contracts_count FROM resources_cache WHERE node_id = 1001), + (SELECT count_before FROM count_before_update), + 'UPDATE non-state column should not trigger count change' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/05_test_node_gpu_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/05_test_node_gpu_trigger.sql new file mode 100644 index 00000000..d7ac974f --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/05_test_node_gpu_trigger.sql @@ -0,0 +1,133 @@ +-- ============================================================================ +-- TEST: reflect_node_gpu_count_change Trigger +-- ============================================================================ +-- Tests for node_gpu INSERT, DELETE, and UPDATE operations + +BEGIN; + +-- Load pgTAP +SELECT plan(8); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT pg_sleep(0.1); + +-- Test 1: INSERT node_gpu should update gpu count and gpus array +INSERT INTO node_gpu (id, node_twin_id, vendor, device, vram, contract) +VALUES ('gpu-1', 2001, 'NVIDIA', 'RTX 3090', 24000000000, 0); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_gpu_count FROM resources_cache WHERE node_id = 1001), + 1, + 'INSERT node_gpu should set node_gpu_count to 1' +); + +SELECT ok( + (SELECT gpus FROM resources_cache WHERE node_id = 1001) IS NOT NULL, + 'INSERT node_gpu should populate gpus JSONB array' +); + +SELECT ok( + jsonb_array_length((SELECT gpus FROM resources_cache WHERE node_id = 1001)) = 1, + 'INSERT node_gpu should create gpus array with 1 element' +); + +-- Test 2: INSERT multiple GPUs should aggregate correctly +INSERT INTO node_gpu (id, node_twin_id, vendor, device, vram, contract) +VALUES ('gpu-2', 2001, 'NVIDIA', 'RTX 4090', 24000000000, 0), + ('gpu-3', 2001, 'AMD', 'RX 7900', 20000000000, 0); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_gpu_count FROM resources_cache WHERE node_id = 1001), + 3, + 'INSERT multiple GPUs should increment node_gpu_count' +); + +SELECT ok( + jsonb_array_length((SELECT gpus FROM resources_cache WHERE node_id = 1001)) = 3, + 'INSERT multiple GPUs should aggregate all in gpus array' +); + +-- Test 3: Verify GPU JSON structure +SELECT ok( + (SELECT gpus->0->>'vendor' FROM resources_cache WHERE node_id = 1001) = 'NVIDIA', + 'GPU JSON should contain vendor field' +); + +SELECT ok( + (SELECT gpus->0->>'device' FROM resources_cache WHERE node_id = 1001) = 'RTX 3090', + 'GPU JSON should contain device field' +); + +-- Test 4: DELETE node_gpu should update count and array +DELETE FROM node_gpu WHERE id = 'gpu-1'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_gpu_count FROM resources_cache WHERE node_id = 1001), + 2, + 'DELETE node_gpu should decrement node_gpu_count' +); + +SELECT ok( + jsonb_array_length((SELECT gpus FROM resources_cache WHERE node_id = 1001)) = 2, + 'DELETE node_gpu should remove from gpus array' +); + +-- Test 5: UPDATE node_gpu should re-aggregate +UPDATE node_gpu SET vram = 30000000000 WHERE id = 'gpu-2'; + +SELECT pg_sleep(0.1); + +SELECT ok( + EXISTS ( + SELECT 1 FROM resources_cache + WHERE node_id = 1001 + AND gpus::jsonb @> '[{"id": "gpu-2", "vram": 30000000000}]'::jsonb + ), + 'UPDATE node_gpu should update gpus array' +); + +-- Test 6: DELETE all GPUs should set count to 0 +DELETE FROM node_gpu WHERE node_twin_id = 2001; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT node_gpu_count FROM resources_cache WHERE node_id = 1001), + 0, + 'DELETE all GPUs should set node_gpu_count to 0' +); + +SELECT ok( + (SELECT gpus FROM resources_cache WHERE node_id = 1001) = '[]'::jsonb, + 'DELETE all GPUs should set gpus to empty array' +); + +-- Test 7: GPU with contract assigned +INSERT INTO node_gpu (id, node_twin_id, vendor, device, vram, contract) +VALUES ('gpu-4', 2001, 'NVIDIA', 'A100', 40000000000, 1001); + +SELECT pg_sleep(0.1); + +SELECT ok( + (SELECT gpus::jsonb @> '[{"contract": 1001}]'::jsonb FROM resources_cache WHERE node_id = 1001), + 'GPU with contract should be stored in gpus array' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/06_test_rent_contract_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/06_test_rent_contract_trigger.sql new file mode 100644 index 00000000..2d342472 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/06_test_rent_contract_trigger.sql @@ -0,0 +1,102 @@ +-- ============================================================================ +-- TEST: reflect_rent_contract_changes Trigger +-- ============================================================================ +-- Tests for rent_contract INSERT and UPDATE state to 'Deleted' + +BEGIN; + +-- Load pgTAP +SELECT plan(6); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_twin(3001); -- Renter twin +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT pg_sleep(0.1); + +-- Test 1: INSERT rent_contract should set renter and rent_contract_id +INSERT INTO rent_contract ( + id, grid_version, contract_id, twin_id, node_id, created_at, state +) VALUES ( + 'rc-1001', 3, 1001, 3001, 1001, EXTRACT(EPOCH FROM NOW())::NUMERIC, 'Created' +); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT renter FROM resources_cache WHERE node_id = 1001), + 3001, + 'INSERT rent_contract should set renter field' +); + +SELECT is( + (SELECT rent_contract_id FROM resources_cache WHERE node_id = 1001), + 1001, + 'INSERT rent_contract should set rent_contract_id field' +); + +-- Test 2: UPDATE rent_contract state to 'Deleted' should clear renter and rent_contract_id +UPDATE rent_contract SET state = 'Deleted' WHERE id = 'rc-1001'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT renter FROM resources_cache WHERE node_id = 1001), + NULL, + 'UPDATE rent_contract to Deleted should clear renter field' +); + +SELECT is( + (SELECT rent_contract_id FROM resources_cache WHERE node_id = 1001), + NULL, + 'UPDATE rent_contract to Deleted should clear rent_contract_id field' +); + +-- Test 3: INSERT new rent_contract should update fields again +INSERT INTO rent_contract ( + id, grid_version, contract_id, twin_id, node_id, created_at, state +) VALUES ( + 'rc-1002', 3, 1002, 3002, 1001, EXTRACT(EPOCH FROM NOW())::NUMERIC, 'Created' +); + +SELECT create_test_twin(3002); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT renter FROM resources_cache WHERE node_id = 1001), + 3002, + 'INSERT new rent_contract should update renter field' +); + +SELECT is( + (SELECT rent_contract_id FROM resources_cache WHERE node_id = 1001), + 1002, + 'INSERT new rent_contract should update rent_contract_id field' +); + +-- Test 4: UPDATE to non-Deleted state should not trigger +SELECT + (SELECT renter FROM resources_cache WHERE node_id = 1001) as renter_before +INTO TEMP renter_before; + +UPDATE rent_contract SET solution_provider_id = 1 WHERE id = 'rc-1002'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT renter FROM resources_cache WHERE node_id = 1001), + (SELECT renter_before FROM renter_before), + 'UPDATE non-state column should not trigger renter change' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/07_test_dmi_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/07_test_dmi_trigger.sql new file mode 100644 index 00000000..e6690797 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/07_test_dmi_trigger.sql @@ -0,0 +1,91 @@ +-- ============================================================================ +-- TEST: reflect_dmi_changes Trigger +-- ============================================================================ +-- Tests for dmi INSERT and UPDATE operations + +BEGIN; + +-- Load pgTAP +SELECT plan(6); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT pg_sleep(0.1); + +-- Test 1: INSERT dmi should update cache fields +INSERT INTO dmi (node_twin_id, bios, baseboard, processor, memory, updated_at) +VALUES ( + 2001, + '{"vendor": "Test", "version": "1.0"}'::jsonb, + '{"manufacturer": "TestMB"}'::jsonb, + '[{"model": "TestCPU"}]'::jsonb, + '[{"size": 16000000000}]'::jsonb, + EXTRACT(EPOCH FROM NOW())::BIGINT +); + +SELECT pg_sleep(0.1); + +SELECT ok( + (SELECT bios FROM resources_cache WHERE node_id = 1001) IS NOT NULL, + 'INSERT dmi should populate bios field' +); + +SELECT ok( + (SELECT baseboard FROM resources_cache WHERE node_id = 1001) IS NOT NULL, + 'INSERT dmi should populate baseboard field' +); + +SELECT ok( + (SELECT processor FROM resources_cache WHERE node_id = 1001) IS NOT NULL, + 'INSERT dmi should populate processor field' +); + +SELECT ok( + (SELECT memory FROM resources_cache WHERE node_id = 1001) IS NOT NULL, + 'INSERT dmi should populate memory field' +); + +-- Test 2: Verify JSON structure +SELECT is( + (SELECT bios->>'vendor' FROM resources_cache WHERE node_id = 1001), + 'Test', + 'bios JSON should contain correct vendor' +); + +SELECT is( + (SELECT baseboard->>'manufacturer' FROM resources_cache WHERE node_id = 1001), + 'TestMB', + 'baseboard JSON should contain correct manufacturer' +); + +-- Test 3: UPDATE dmi should update cache +UPDATE dmi SET + bios = '{"vendor": "Updated", "version": "2.0"}'::jsonb, + baseboard = '{"manufacturer": "UpdatedMB"}'::jsonb +WHERE node_twin_id = 2001; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT bios->>'vendor' FROM resources_cache WHERE node_id = 1001), + 'Updated', + 'UPDATE dmi should update bios field' +); + +SELECT is( + (SELECT baseboard->>'manufacturer' FROM resources_cache WHERE node_id = 1001), + 'UpdatedMB', + 'UPDATE dmi should update baseboard field' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/08_test_speed_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/08_test_speed_trigger.sql new file mode 100644 index 00000000..007955ee --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/08_test_speed_trigger.sql @@ -0,0 +1,111 @@ +-- ============================================================================ +-- TEST: reflect_speed_changes Trigger +-- ============================================================================ +-- Tests for speed INSERT and UPDATE operations + +BEGIN; + +-- Load pgTAP +SELECT plan(10); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT pg_sleep(0.1); + +-- Test 1: INSERT speed should update all speed fields +INSERT INTO speed ( + node_twin_id, upload, download, + udp_download_ipv4, udp_upload_ipv4, + tcp_download_ipv6, tcp_upload_ipv6, + udp_download_ipv6, udp_upload_ipv6, + updated_at +) VALUES ( + 2001, 100.5, 200.5, + 150.5, 160.5, + 170.5, 180.5, + 190.5, 200.5, + EXTRACT(EPOCH FROM NOW())::BIGINT +); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT upload_speed FROM resources_cache WHERE node_id = 1001), + 100.5, + 'INSERT speed should set upload_speed' +); + +SELECT is( + (SELECT download_speed FROM resources_cache WHERE node_id = 1001), + 200.5, + 'INSERT speed should set download_speed' +); + +SELECT is( + (SELECT udp_download_ipv4 FROM resources_cache WHERE node_id = 1001), + 150.5, + 'INSERT speed should set udp_download_ipv4' +); + +SELECT is( + (SELECT udp_upload_ipv4 FROM resources_cache WHERE node_id = 1001), + 160.5, + 'INSERT speed should set udp_upload_ipv4' +); + +SELECT is( + (SELECT tcp_download_ipv6 FROM resources_cache WHERE node_id = 1001), + 170.5, + 'INSERT speed should set tcp_download_ipv6' +); + +SELECT is( + (SELECT tcp_upload_ipv6 FROM resources_cache WHERE node_id = 1001), + 180.5, + 'INSERT speed should set tcp_upload_ipv6' +); + +SELECT is( + (SELECT udp_download_ipv6 FROM resources_cache WHERE node_id = 1001), + 190.5, + 'INSERT speed should set udp_download_ipv6' +); + +SELECT is( + (SELECT udp_upload_ipv6 FROM resources_cache WHERE node_id = 1001), + 200.5, + 'INSERT speed should set udp_upload_ipv6' +); + +-- Test 2: UPDATE speed should update all fields +UPDATE speed SET + upload = 300.5, + download = 400.5, + udp_download_ipv4 = 350.5 +WHERE node_twin_id = 2001; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT upload_speed FROM resources_cache WHERE node_id = 1001), + 300.5, + 'UPDATE speed should update upload_speed' +); + +SELECT is( + (SELECT download_speed FROM resources_cache WHERE node_id = 1001), + 400.5, + 'UPDATE speed should update download_speed' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/09_test_cpu_benchmark_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/09_test_cpu_benchmark_trigger.sql new file mode 100644 index 00000000..40c2e89b --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/09_test_cpu_benchmark_trigger.sql @@ -0,0 +1,80 @@ +-- ============================================================================ +-- TEST: reflect_cpu_benchmark_changes Trigger +-- ============================================================================ +-- Tests for cpu_benchmark INSERT and UPDATE operations + +BEGIN; + +-- Load pgTAP +SELECT plan(6); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT pg_sleep(0.1); + +-- Test 1: INSERT cpu_benchmark should update all CPU fields +INSERT INTO cpu_benchmark ( + node_twin_id, single_threaded, multi_threaded, threads, workloads, updated_at +) VALUES ( + 2001, 1000.5, 8000.5, 16, 8, EXTRACT(EPOCH FROM NOW())::BIGINT +); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT single_threaded_cpu FROM resources_cache WHERE node_id = 1001), + 1000.5, + 'INSERT cpu_benchmark should set single_threaded_cpu' +); + +SELECT is( + (SELECT multi_threaded_cpu FROM resources_cache WHERE node_id = 1001), + 8000.5, + 'INSERT cpu_benchmark should set multi_threaded_cpu' +); + +SELECT is( + (SELECT threads_cpu FROM resources_cache WHERE node_id = 1001), + 16, + 'INSERT cpu_benchmark should set threads_cpu' +); + +SELECT is( + (SELECT workloads_cpu FROM resources_cache WHERE node_id = 1001), + 8, + 'INSERT cpu_benchmark should set workloads_cpu' +); + +-- Test 2: UPDATE cpu_benchmark should update all fields +UPDATE cpu_benchmark SET + single_threaded = 2000.5, + multi_threaded = 16000.5, + threads = 32, + workloads = 16 +WHERE node_twin_id = 2001; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT single_threaded_cpu FROM resources_cache WHERE node_id = 1001), + 2000.5, + 'UPDATE cpu_benchmark should update single_threaded_cpu' +); + +SELECT is( + (SELECT multi_threaded_cpu FROM resources_cache WHERE node_id = 1001), + 16000.5, + 'UPDATE cpu_benchmark should update multi_threaded_cpu' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/10_test_public_ip_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/10_test_public_ip_trigger.sql new file mode 100644 index 00000000..3e983365 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/10_test_public_ip_trigger.sql @@ -0,0 +1,145 @@ +-- ============================================================================ +-- TEST: reflect_public_ip_changes Trigger +-- ============================================================================ +-- Tests for public_ip INSERT, DELETE, and UPDATE contract_id operations + +BEGIN; + +-- Load pgTAP +SELECT plan(12); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_twin(2001); +SELECT pg_sleep(0.1); + +-- Verify initial state (should have 0 IPs) +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001), + 0, + 'Initial state should have 0 free IPs' +); + +SELECT is( + (SELECT total_ips FROM public_ips_cache WHERE farm_id = 1001), + 0, + 'Initial state should have 0 total IPs' +); + +-- Test 1: INSERT public_ip with contract_id=0 should increment free_ips and total_ips +INSERT INTO public_ip (id, gateway, ip, contract_id, farm_id) +VALUES ('ip-1', '1.2.3.1', '1.2.3.4', 0, 'farm-1001'); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001), + 1, + 'INSERT public_ip with contract_id=0 should increment free_ips' +); + +SELECT is( + (SELECT total_ips FROM public_ips_cache WHERE farm_id = 1001), + 1, + 'INSERT public_ip should increment total_ips' +); + +-- Test 2: INSERT public_ip with contract_id!=0 should NOT increment free_ips +INSERT INTO public_ip (id, gateway, ip, contract_id, farm_id) +VALUES ('ip-2', '1.2.3.1', '1.2.3.5', 1001, 'farm-1001'); + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001), + 1, + 'INSERT public_ip with contract_id!=0 should NOT increment free_ips' +); + +SELECT is( + (SELECT total_ips FROM public_ips_cache WHERE farm_id = 1001), + 2, + 'INSERT public_ip should increment total_ips' +); + +-- Test 3: UPDATE contract_id from 0 to non-zero should decrement free_ips +UPDATE public_ip SET contract_id = 1002 WHERE id = 'ip-1'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001), + 0, + 'UPDATE contract_id from 0 to non-zero should decrement free_ips' +); + +-- Test 4: UPDATE contract_id from non-zero to 0 should increment free_ips +UPDATE public_ip SET contract_id = 0 WHERE id = 'ip-2'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001), + 1, + 'UPDATE contract_id from non-zero to 0 should increment free_ips' +); + +-- Test 5: DELETE public_ip with contract_id=0 should decrement free_ips and total_ips +DELETE FROM public_ip WHERE id = 'ip-2'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001), + 0, + 'DELETE public_ip with contract_id=0 should decrement free_ips' +); + +SELECT is( + (SELECT total_ips FROM public_ips_cache WHERE farm_id = 1001), + 1, + 'DELETE public_ip should decrement total_ips' +); + +-- Test 6: DELETE public_ip with contract_id!=0 should NOT decrement free_ips +DELETE FROM public_ip WHERE id = 'ip-1'; + +SELECT pg_sleep(0.1); + +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001), + 0, + 'DELETE public_ip with contract_id!=0 should NOT decrement free_ips' +); + +SELECT is( + (SELECT total_ips FROM public_ips_cache WHERE farm_id = 1001), + 0, + 'DELETE public_ip should decrement total_ips' +); + +-- Test 7: Verify IPs JSON array is updated +INSERT INTO public_ip (id, gateway, ip, contract_id, farm_id) +VALUES + ('ip-3', '1.2.3.1', '1.2.3.6', 0, 'farm-1001'), + ('ip-4', '1.2.3.1', '1.2.3.7', 1001, 'farm-1001'); + +SELECT pg_sleep(0.1); + +SELECT ok( + (SELECT ips FROM public_ips_cache WHERE farm_id = 1001) IS NOT NULL, + 'IPs JSON array should be populated' +); + +SELECT ok( + jsonb_array_length((SELECT ips FROM public_ips_cache WHERE farm_id = 1001)) = 2, + 'IPs JSON array should contain all IPs for the farm' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/11_test_farm_trigger.sql b/grid-proxy/internal/explorer/db/cache/tests/11_test_farm_trigger.sql new file mode 100644 index 00000000..0634271d --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/11_test_farm_trigger.sql @@ -0,0 +1,60 @@ +-- ============================================================================ +-- TEST: reflect_farm_changes Trigger +-- ============================================================================ +-- Tests for farm INSERT and DELETE operations + +BEGIN; + +-- Load pgTAP +SELECT plan(4); + +-- Test 1: INSERT farm should create entry in public_ips_cache +SELECT create_test_farm(1002, 2002, 'Test Farm 2', 1); +SELECT create_test_twin(2002); + +SELECT pg_sleep(0.1); + +SELECT ok( + EXISTS ( + SELECT 1 FROM public_ips_cache WHERE farm_id = 1002 + ), + 'INSERT farm should create entry in public_ips_cache' +); + +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1002), + 0, + 'INSERT farm should initialize free_ips to 0' +); + +SELECT is( + (SELECT total_ips FROM public_ips_cache WHERE farm_id = 1002), + 0, + 'INSERT farm should initialize total_ips to 0' +); + +SELECT is( + (SELECT ips FROM public_ips_cache WHERE farm_id = 1002), + '[]'::jsonb, + 'INSERT farm should initialize ips to empty array' +); + +-- Test 2: DELETE farm should remove entry from public_ips_cache +DELETE FROM farm WHERE id = 'farm-1002'; + +SELECT pg_sleep(0.1); + +SELECT ok( + NOT EXISTS ( + SELECT 1 FROM public_ips_cache WHERE farm_id = 1002 + ), + 'DELETE farm should remove entry from public_ips_cache' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/12_test_cache_refreshers.sql b/grid-proxy/internal/explorer/db/cache/tests/12_test_cache_refreshers.sql new file mode 100644 index 00000000..a87dd65e --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/12_test_cache_refreshers.sql @@ -0,0 +1,161 @@ +-- ============================================================================ +-- TEST: Cache Refresher Functions +-- ============================================================================ +-- Tests for cache refresh functions + +BEGIN; + +-- Load pgTAP +SELECT plan(10); + +-- Setup: Create test data +SELECT create_test_farm(1001, 2001, 'Test Farm 1', 1); +SELECT create_test_pricing_policy(1, 1000000, 1000000); +SELECT create_test_twin(2001); +SELECT create_test_node(1001, 1001, 2001); +SELECT create_test_node_resources_total('node-1001', 1000000000000, 100000000000, 100000000000, 16); +SELECT create_test_node_contract('1001', 1001, 2001, 'Created', 'cr-1001'); +SELECT create_test_contract_resources('1001', 1000000000, 2000000000, 3000000000, 2); +SELECT pg_sleep(0.1); + +-- Get initial cache values from view +SELECT + total_hru, total_mru, total_sru, total_cru, + free_hru, free_mru, free_sru, + used_hru, used_mru, used_sru, used_cru, + node_contracts_count +FROM resources_cache_view WHERE node_id = 1001 +INTO TEMP expected_cache; + +-- Test 1: refresh_resources_cache_node should refresh single node +-- First, corrupt the cache +UPDATE resources_cache SET total_hru = 999999999 WHERE node_id = 1001; + +SELECT refresh_resources_cache_node(1001); + +SELECT is( + (SELECT total_hru FROM resources_cache WHERE node_id = 1001), + (SELECT total_hru FROM expected_cache), + 'refresh_resources_cache_node should restore correct total_hru' +); + +SELECT is( + (SELECT free_hru FROM resources_cache WHERE node_id = 1001), + (SELECT free_hru FROM expected_cache), + 'refresh_resources_cache_node should restore correct free_hru' +); + +SELECT is( + (SELECT used_hru FROM resources_cache WHERE node_id = 1001), + (SELECT used_hru FROM expected_cache), + 'refresh_resources_cache_node should restore correct used_hru' +); + +-- Test 2: refresh_resources_cache should refresh all nodes +-- Create another node +SELECT create_test_node(1002, 1001, 2002); +SELECT create_test_twin(2002); +SELECT create_test_node_resources_total('node-1002', 2000000000000, 200000000000, 200000000000, 32); +SELECT pg_sleep(0.1); + +-- Corrupt both caches +UPDATE resources_cache SET total_hru = 111111111 WHERE node_id IN (1001, 1002); + +SELECT refresh_resources_cache(); + +SELECT ok( + (SELECT COUNT(*) FROM resources_cache) >= 2, + 'refresh_resources_cache should refresh all nodes' +); + +SELECT ok( + EXISTS ( + SELECT 1 FROM resources_cache + WHERE node_id = 1001 AND total_hru = (SELECT total_hru FROM resources_cache_view WHERE node_id = 1001) + ), + 'refresh_resources_cache should refresh node 1001 correctly' +); + +SELECT ok( + EXISTS ( + SELECT 1 FROM resources_cache + WHERE node_id = 1002 AND total_hru = (SELECT total_hru FROM resources_cache_view WHERE node_id = 1002) + ), + 'refresh_resources_cache should refresh node 1002 correctly' +); + +-- Test 3: refresh_public_ips_cache_farm should refresh single farm +SELECT create_test_public_ip('ip-1', 'farm-1001', '1.2.3.4', '1.2.3.1', 0); +SELECT pg_sleep(0.1); + +-- Get expected values +SELECT + COALESCE(COUNT(*), 0) as expected_total, + COALESCE(COUNT(CASE WHEN contract_id = 0 THEN 1 END), 0) as expected_free +FROM public_ip WHERE farm_id = 'farm-1001' +INTO TEMP expected_ips; + +-- Corrupt cache +UPDATE public_ips_cache SET free_ips = 999, total_ips = 999 WHERE farm_id = 1001; + +SELECT refresh_public_ips_cache_farm(1001); + +SELECT is( + (SELECT total_ips FROM public_ips_cache WHERE farm_id = 1001), + (SELECT expected_total FROM expected_ips), + 'refresh_public_ips_cache_farm should restore correct total_ips' +); + +SELECT is( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001), + (SELECT expected_free FROM expected_ips), + 'refresh_public_ips_cache_farm should restore correct free_ips' +); + +-- Test 4: refresh_public_ips_cache should refresh all farms +SELECT create_test_farm(1003, 2003, 'Test Farm 3', 1); +SELECT create_test_twin(2003); +SELECT create_test_public_ip('ip-2', 'farm-1003', '2.3.4.5', '2.3.4.1', 0); +SELECT pg_sleep(0.1); + +-- Corrupt cache +UPDATE public_ips_cache SET free_ips = 888 WHERE farm_id IN (1001, 1003); + +SELECT refresh_public_ips_cache(); + +SELECT ok( + (SELECT COUNT(*) FROM public_ips_cache) >= 2, + 'refresh_public_ips_cache should refresh all farms' +); + +SELECT ok( + EXISTS ( + SELECT 1 FROM public_ips_cache + WHERE farm_id = 1001 AND free_ips = (SELECT COUNT(*) FROM public_ip WHERE farm_id = 'farm-1001' AND contract_id = 0) + ), + 'refresh_public_ips_cache should refresh farm 1001 correctly' +); + +-- Test 5: refresh_all_caches should refresh both cache tables +UPDATE resources_cache SET total_hru = 777777777 WHERE node_id = 1001; +UPDATE public_ips_cache SET free_ips = 777 WHERE farm_id = 1001; + +SELECT refresh_all_caches(); + +SELECT ok( + (SELECT total_hru FROM resources_cache WHERE node_id = 1001) != 777777777, + 'refresh_all_caches should refresh resources_cache' +); + +SELECT ok( + (SELECT free_ips FROM public_ips_cache WHERE farm_id = 1001) != 777, + 'refresh_all_caches should refresh public_ips_cache' +); + +-- Cleanup +SELECT cleanup_test_data(); + +SELECT * FROM finish(); + +ROLLBACK; + diff --git a/grid-proxy/internal/explorer/db/cache/tests/README.md b/grid-proxy/internal/explorer/db/cache/tests/README.md new file mode 100644 index 00000000..6e8d2855 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/tests/README.md @@ -0,0 +1,119 @@ +# Cache Triggers and Refresh Functions Tests + +This directory contains pgTAP SQL tests for all cache triggers and cache refresher functions. + +## Prerequisites + +1. PostgreSQL with pgTAP extension installed +2. Database schema set up (all setup SQL files from parent directory) + +To install pgTAP: +```sql +CREATE EXTENSION IF NOT EXISTS pgtap; +``` + +## Test Files + +- `00_test_helpers.sql` - Helper functions for creating test data +- `01_test_node_trigger.sql` - Tests for `reflect_node_changes` trigger +- `02_test_node_resources_total_trigger.sql` - Tests for `reflect_total_resources_changes` trigger +- `03_test_contract_resources_trigger.sql` - Tests for `reflect_contract_resources_changes` trigger +- `04_test_node_contract_trigger.sql` - Tests for `reflect_node_contract_changes` trigger +- `05_test_node_gpu_trigger.sql` - Tests for `reflect_node_gpu_count_change` trigger +- `06_test_rent_contract_trigger.sql` - Tests for `reflect_rent_contract_changes` trigger +- `07_test_dmi_trigger.sql` - Tests for `reflect_dmi_changes` trigger +- `08_test_speed_trigger.sql` - Tests for `reflect_speed_changes` trigger +- `09_test_cpu_benchmark_trigger.sql` - Tests for `reflect_cpu_benchmark_changes` trigger +- `10_test_public_ip_trigger.sql` - Tests for `reflect_public_ip_changes` trigger +- `11_test_farm_trigger.sql` - Tests for `reflect_farm_changes` trigger +- `12_test_cache_refreshers.sql` - Tests for cache refresh functions + +## Running Tests + +### Using pg_prove (recommended) + +```bash +# Run all tests +pg_prove -d your_database_name -h localhost -U postgres tests/*.sql + +# Run specific test file +pg_prove -d your_database_name -h localhost -U postgres tests/01_test_node_trigger.sql +``` + +### Using psql + +```bash +# Run all tests +psql -d your_database_name -h localhost -U postgres -f tests/01_test_node_trigger.sql + +# Or run all tests in sequence +for file in tests/*.sql; do + psql -d your_database_name -h localhost -U postgres -f "$file" +done +``` + +### Using SQL directly + +```sql +-- Load helper functions first +\i tests/00_test_helpers.sql + +-- Then run individual test files +\i tests/01_test_node_trigger.sql +``` + +## Test Structure + +Each test file follows this pattern: + +1. **BEGIN** - Start transaction for isolation +2. **SELECT plan(N)** - Declare number of tests +3. **Setup** - Create test data using helper functions +4. **Tests** - Fire triggers and verify cache table values +5. **Cleanup** - Remove test data +6. **SELECT * FROM finish()** - Complete pgTAP test run +7. **ROLLBACK** - Rollback transaction + +## Test Coverage + +### Resources Cache Triggers + +- ✅ Node INSERT/DELETE +- ✅ Node resources total INSERT/UPDATE +- ✅ Contract resources INSERT/UPDATE/DELETE +- ✅ Node contract INSERT/UPDATE to Deleted +- ✅ Node GPU INSERT/DELETE/UPDATE +- ✅ Rent contract INSERT/UPDATE to Deleted +- ✅ DMI INSERT/UPDATE +- ✅ Speed INSERT/UPDATE +- ✅ CPU benchmark INSERT/UPDATE + +### Public IPs Cache Triggers + +- ✅ Public IP INSERT/DELETE/UPDATE contract_id +- ✅ Farm INSERT/DELETE + +### Cache Refreshers + +- ✅ refresh_resources_cache_node +- ✅ refresh_resources_cache +- ✅ refresh_public_ips_cache_farm +- ✅ refresh_public_ips_cache +- ✅ refresh_all_caches + +## Notes + +- Tests use transactions (BEGIN/ROLLBACK) to ensure isolation +- Helper functions create test data with predictable IDs (node-1001, farm-1001, etc.) +- Tests include `pg_sleep(0.1)` to allow triggers to complete +- All test data is cleaned up using `cleanup_test_data()` function + +## Troubleshooting + +If tests fail: + +1. Ensure pgTAP extension is installed: `CREATE EXTENSION pgtap;` +2. Ensure all setup SQL files have been run (00_constants.sql through 06_cache_management.sql) +3. Check that base tables exist (node, farm, contract_resources, etc.) +4. Verify triggers are created: `SELECT * FROM pg_trigger WHERE tgname LIKE 'tg_%';` + diff --git a/grid-proxy/internal/explorer/db/cache/triggers.sql b/grid-proxy/internal/explorer/db/cache/triggers.sql new file mode 100644 index 00000000..4f2fa2d1 --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/triggers.sql @@ -0,0 +1,670 @@ +-- ============================================================================ +-- TRIGGERS +-- ============================================================================ +-- Trigger functions and triggers that automatically maintain cache tables +-- when source data changes. Each trigger handles a specific data type. + +/* + * reflect_node_changes + * + * Maintains resources_cache when nodes are inserted or deleted. + * - INSERT: Populates cache from view for new node + * - DELETE: Removes node from cache + */ +CREATE OR REPLACE FUNCTION reflect_node_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF (TG_OP = 'INSERT') THEN + BEGIN + INSERT INTO resources_cache + SELECT * + FROM resources_cache_view + WHERE resources_cache_view.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_node_changes', + 'INSERT', + 'node', + NEW.node_id::TEXT, + SQLERRM, + jsonb_build_object('node_id', NEW.node_id, 'farm_id', NEW.farm_id) + ); + RAISE WARNING 'Error inserting resources_cache: %', SQLERRM; + END; + + ELSIF (TG_OP = 'DELETE') THEN + BEGIN + DELETE FROM resources_cache WHERE node_id = OLD.node_id; + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_node_changes', + 'DELETE', + 'node', + OLD.node_id::TEXT, + SQLERRM, + jsonb_build_object('node_id', OLD.node_id) + ); + RAISE WARNING 'Error deleting node from resources_cache: %', SQLERRM; + END; + END IF; + + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_node + AFTER INSERT OR DELETE + ON node + FOR EACH ROW EXECUTE PROCEDURE reflect_node_changes(); + +/* + * reflect_total_resources_changes + * + * Updates cache when node_resources_total changes. + * + * For MRU: Adjusts both the reserved amount (MRU/10, min 2GB) and the total change. + * - Reserved amount change: OLD.reserved - NEW.reserved + * - Total change: NEW.mru - OLD.mru + * - Combined: free_mru = free_mru + (OLD.reserved - NEW.reserved) + (NEW.mru - OLD.mru) + * + * For HRU/SRU: Simple incremental update based on total change. + */ +CREATE OR REPLACE FUNCTION reflect_total_resources_changes() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN + UPDATE resources_cache + SET + -- Update total resources + total_cru = NEW.cru, + total_mru = NEW.mru, + total_sru = NEW.sru, + total_hru = NEW.hru, + -- MRU: Adjust for reserved amount change + total change + -- Reserved amount: GREATEST(MRU/get_mru_reserved_fraction(), get_mru_reserved_min_bytes()) + free_mru = free_mru + GREATEST(CAST((OLD.mru / get_mru_reserved_fraction()) AS bigint), get_mru_reserved_min_bytes()) - + GREATEST(CAST((NEW.mru / get_mru_reserved_fraction()) AS bigint), get_mru_reserved_min_bytes()) + + (NEW.mru - COALESCE(OLD.mru, 0)), + -- HRU: Simple incremental update + free_hru = free_hru + (NEW.hru - COALESCE(OLD.hru, 0)), + -- SRU: Simple incremental update + free_sru = free_sru + (NEW.sru - COALESCE(OLD.sru, 0)), + -- MRU used: Adjust reserved amount (used includes reserved) + used_mru = used_mru - GREATEST(CAST((OLD.mru / get_mru_reserved_fraction()) AS bigint), get_mru_reserved_min_bytes()) + + GREATEST(CAST((NEW.mru / get_mru_reserved_fraction()) AS bigint), get_mru_reserved_min_bytes()) + WHERE + resources_cache.node_id = ( + SELECT node.node_id FROM node WHERE node.id = NEW.node_id + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_total_resources_changes', + TG_OP, + 'node_resources_total', + COALESCE(NEW.node_id, OLD.node_id)::TEXT, + SQLERRM, + jsonb_build_object( + 'old_mru', OLD.mru, + 'new_mru', NEW.mru, + 'old_hru', OLD.hru, + 'new_hru', NEW.hru, + 'old_sru', OLD.sru, + 'new_sru', NEW.sru + ) + ); + RAISE WARNING 'Error reflecting total_resources changes: %', SQLERRM; + END; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_node_resources_total + AFTER INSERT OR UPDATE + ON node_resources_total FOR EACH ROW + EXECUTE PROCEDURE reflect_total_resources_changes(); + + +/* + * reflect_contract_resources_changes + * + * Updates cache when contract_resources change. + * Only processes contracts in 'Created' or 'GracePeriod' states. + * + * - INSERT/UPDATE: Increments used, decrements free + * - DELETE: Decrements used, increments free + */ +CREATE OR REPLACE FUNCTION reflect_contract_resources_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF (TG_OP = 'DELETE') THEN + BEGIN + -- Handle DELETE: decrement used resources and increment free resources + UPDATE resources_cache + SET used_cru = used_cru - OLD.cru, + used_mru = used_mru - OLD.mru, + used_sru = used_sru - OLD.sru, + used_hru = used_hru - OLD.hru, + free_mru = free_mru + OLD.mru, + free_hru = free_hru + OLD.hru, + free_sru = free_sru + OLD.sru + WHERE + resources_cache.node_id = ( + SELECT node_id FROM node_contract + WHERE node_contract.id = OLD.contract_id + AND node_contract.state IN ('Created', 'GracePeriod') + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_contract_resources_changes', + 'DELETE', + 'contract_resources', + OLD.contract_id::TEXT, + SQLERRM, + jsonb_build_object('contract_id', OLD.contract_id, 'cru', OLD.cru, 'mru', OLD.mru, 'sru', OLD.sru, 'hru', OLD.hru) + ); + RAISE WARNING 'Error reflecting contract_resources DELETE: %', SQLERRM; + END; + ELSE + -- Handle INSERT and UPDATE + BEGIN + UPDATE resources_cache + SET used_cru = used_cru + (NEW.cru - COALESCE(OLD.cru, 0)), + used_mru = used_mru + (NEW.mru - COALESCE(OLD.mru, 0)), + used_sru = used_sru + (NEW.sru - COALESCE(OLD.sru, 0)), + used_hru = used_hru + (NEW.hru - COALESCE(OLD.hru, 0)), + free_mru = free_mru - (NEW.mru - COALESCE(OLD.mru, 0)), + free_hru = free_hru - (NEW.hru - COALESCE(OLD.hru, 0)), + free_sru = free_sru - (NEW.sru - COALESCE(OLD.sru, 0)) + WHERE + resources_cache.node_id = ( + SELECT node_id FROM node_contract + WHERE node_contract.id = NEW.contract_id + AND node_contract.state IN ('Created', 'GracePeriod') + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_contract_resources_changes', + TG_OP, + 'contract_resources', + NEW.contract_id::TEXT, + SQLERRM, + jsonb_build_object('contract_id', NEW.contract_id, 'cru', NEW.cru, 'mru', NEW.mru, 'sru', NEW.sru, 'hru', NEW.hru) + ); + RAISE WARNING 'Error reflecting contract_resources changes: %', SQLERRM; + END; + END IF; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_contract_resources + AFTER INSERT OR UPDATE OR DELETE ON contract_resources FOR EACH ROW + EXECUTE PROCEDURE reflect_contract_resources_changes(); + +/* + * reflect_node_contract_changes + * + * Updates cache when node_contract state changes. + * + * - INSERT: Recalculates node_contracts_count (active contracts only) + * - UPDATE to 'Deleted': Releases contract resources and updates count + * Uses row locking to prevent concurrent update conflicts + */ +CREATE OR REPLACE FUNCTION reflect_node_contract_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN + BEGIN + -- Lock cache row to prevent concurrent updates during resource release + PERFORM 1 + FROM resources_cache + WHERE node_id = NEW.node_id + FOR UPDATE; + + UPDATE resources_cache + SET + used_cru = resources_cache.used_cru - contract_resources.cru, + used_mru = resources_cache.used_mru - contract_resources.mru, + used_sru = resources_cache.used_sru - contract_resources.sru, + used_hru = resources_cache.used_hru - contract_resources.hru, + free_mru = resources_cache.free_mru + contract_resources.mru, + free_sru = resources_cache.free_sru + contract_resources.sru, + free_hru = resources_cache.free_hru + contract_resources.hru, + node_contracts_count = COALESCE(ncc.count, 0) + FROM contract_resources + LEFT JOIN + (SELECT node_id, COUNT(contract_id) as count + FROM node_contract + WHERE state IN ('Created', 'GracePeriod') + GROUP BY node_id) AS ncc + ON ncc.node_id = NEW.node_id + WHERE + contract_resources.contract_id = NEW.id + AND resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_node_contract_changes', + 'UPDATE', + 'node_contract', + NEW.id::TEXT, + SQLERRM, + jsonb_build_object('contract_id', NEW.id, 'node_id', NEW.node_id, 'state', NEW.state) + ); + RAISE WARNING 'Error reflecting node_contract updates: %', SQLERRM; + END; + + ELSIF (TG_OP = 'INSERT') THEN + BEGIN + UPDATE resources_cache + SET node_contracts_count = ( + SELECT COALESCE(COUNT(contract_id), 0) + FROM node_contract + WHERE node_id = NEW.node_id + AND state IN ('Created', 'GracePeriod') + ) + WHERE resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_node_contract_changes', + 'INSERT', + 'node_contract', + NEW.id::TEXT, + SQLERRM, + jsonb_build_object('contract_id', NEW.id, 'node_id', NEW.node_id) + ); + RAISE WARNING 'Error calculating node_contracts_count: %', SQLERRM; + END; + END IF; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_node_contract + AFTER INSERT OR UPDATE OF state ON node_contract FOR EACH ROW + EXECUTE PROCEDURE reflect_node_contract_changes(); + +/* + * reflect_node_gpu_count_change + * + * Updates GPU information in cache when node_gpu changes. + * Re-aggregates all GPUs for the node and updates count + JSON array. + * + * - INSERT/DELETE/UPDATE: Recalculates GPU count and JSON array + */ +CREATE OR REPLACE FUNCTION reflect_node_gpu_count_change() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN + UPDATE resources_cache + SET node_gpu_count = gpu.count, gpus = gpu.gpus + FROM ( + SELECT COUNT(*) AS count, + jsonb_agg( + jsonb_build_object( + 'id', id, + 'vendor', vendor, + 'device', device, + 'vram', vram, + 'contract', contract + ) + ) AS gpus + FROM node_gpu + WHERE node_twin_id = COALESCE(NEW.node_twin_id, OLD.node_twin_id) + ) AS gpu + WHERE resources_cache.node_id = ( + SELECT node_id from node where node.twin_id = COALESCE(NEW.node_twin_id, OLD.node_twin_id) + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_node_gpu_count_change', + TG_OP, + 'node_gpu', + COALESCE(NEW.id, OLD.id)::TEXT, + SQLERRM, + jsonb_build_object('node_twin_id', COALESCE(NEW.node_twin_id, OLD.node_twin_id)) + ); + RAISE WARNING 'Error updating resources_cache gpu fields: %', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_node_gpu_count + AFTER INSERT OR DELETE OR UPDATE ON node_gpu FOR EACH ROW + EXECUTE PROCEDURE reflect_node_gpu_count_change(); + +/* + * reflect_rent_contract_changes + * + * Updates rental information in cache when rent_contract changes. + * + * - INSERT: Sets renter and rent_contract_id + * - UPDATE to 'Deleted': Clears renter and rent_contract_id + */ +CREATE OR REPLACE FUNCTION reflect_rent_contract_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN + BEGIN + UPDATE resources_cache + SET renter = NULL, + rent_contract_id = NULL + WHERE + resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_rent_contract_changes', + 'UPDATE', + 'rent_contract', + NEW.contract_id::TEXT, + SQLERRM, + jsonb_build_object('contract_id', NEW.contract_id, 'node_id', NEW.node_id, 'state', NEW.state) + ); + RAISE WARNING 'Error removing resources_cache rent fields: %', SQLERRM; + END; + ELSIF (TG_OP = 'INSERT') THEN + BEGIN + UPDATE resources_cache + SET renter = NEW.twin_id, + rent_contract_id = NEW.contract_id + WHERE + resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_rent_contract_changes', + 'INSERT', + 'rent_contract', + NEW.contract_id::TEXT, + SQLERRM, + jsonb_build_object('contract_id', NEW.contract_id, 'node_id', NEW.node_id, 'twin_id', NEW.twin_id) + ); + RAISE WARNING 'Error reflecting rent_contract changes: %', SQLERRM; + END; + END IF; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_rent_contract + AFTER INSERT OR UPDATE OF state ON rent_contract FOR EACH ROW + EXECUTE PROCEDURE reflect_rent_contract_changes(); + +/* + * reflect_dmi_changes + * + * Updates DMI (hardware information) in cache when dmi table changes. + * + * - INSERT/UPDATE: Updates bios, baseboard, processor, memory fields + */ +CREATE OR REPLACE FUNCTION reflect_dmi_changes() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN + UPDATE resources_cache + SET bios = NEW.bios, + baseboard = NEW.baseboard, + processor = NEW.processor, + memory = NEW.memory + WHERE resources_cache.node_id = ( + SELECT node_id from node where node.twin_id = NEW.node_twin_id + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_dmi_changes', + TG_OP, + 'dmi', + NULL, + SQLERRM, + jsonb_build_object('node_twin_id', NEW.node_twin_id) + ); + RAISE WARNING 'Error updating resources_cache dmi fields: %', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_dmi + AFTER INSERT OR UPDATE ON dmi FOR EACH ROW + EXECUTE PROCEDURE reflect_dmi_changes(); + + +/* + * reflect_speed_changes + * + * Updates network speed test results in cache when speed table changes. + * + * - INSERT/UPDATE: Updates all speed metrics (upload, download, IPv4/IPv6, TCP/UDP) + */ +CREATE OR REPLACE FUNCTION reflect_speed_changes() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN + UPDATE resources_cache + SET upload_speed = NEW.upload, + download_speed = NEW.download, + udp_download_ipv4 = NEW.udp_download_ipv4, + udp_upload_ipv4 = NEW.udp_upload_ipv4, + tcp_download_ipv6 = NEW.tcp_download_ipv6, + tcp_upload_ipv6 = NEW.tcp_upload_ipv6, + udp_download_ipv6 = NEW.udp_download_ipv6, + udp_upload_ipv6 = NEW.udp_upload_ipv6 + WHERE resources_cache.node_id = ( + SELECT node_id from node where node.twin_id = NEW.node_twin_id + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_speed_changes', + TG_OP, + 'speed', + NULL, + SQLERRM, + jsonb_build_object('node_twin_id', NEW.node_twin_id) + ); + RAISE WARNING 'Error updating resources_cache speed fields: %', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_speed + AFTER INSERT OR UPDATE ON speed FOR EACH ROW + EXECUTE PROCEDURE reflect_speed_changes(); + +/* + * reflect_cpu_benchmark_changes + * + * Updates CPU benchmark results in cache when cpu_benchmark table changes. + * + * - INSERT/UPDATE: Updates single-threaded, multi-threaded, threads, workloads + */ +CREATE OR REPLACE FUNCTION reflect_cpu_benchmark_changes() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN + UPDATE resources_cache + SET single_threaded_cpu = NEW.single_threaded, + multi_threaded_cpu = NEW.multi_threaded, + threads_cpu = NEW.threads, + workloads_cpu = NEW.workloads + WHERE resources_cache.node_id = ( + SELECT node_id from node where node.twin_id = NEW.node_twin_id + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_cpu_benchmark_changes', + TG_OP, + 'cpu_benchmark', + NULL, + SQLERRM, + jsonb_build_object('node_twin_id', NEW.node_twin_id) + ); + RAISE WARNING 'Error updating resources_cache cpu_benchmark fields: %', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_cpu_benchmark + AFTER INSERT OR UPDATE ON cpu_benchmark FOR EACH ROW + EXECUTE PROCEDURE reflect_cpu_benchmark_changes(); + +/* + * reflect_public_ip_changes + * + * Updates public_ips_cache when public_ip changes. + * Tracks free_ips (contract_id = 0) and total_ips, re-aggregates IP JSON. + * + * Logic: + * - INSERT with contract_id=0: free_ips++, total_ips++ + * - DELETE with contract_id=0: free_ips--, total_ips-- + * - DELETE with contract_id!=0: free_ips unchanged, total_ips-- + * - UPDATE: free/reserved: free_ips--, reserved/free: free_ips++ + * - Always re-aggregates entire IP JSON array for the farm + */ +CREATE OR REPLACE FUNCTION reflect_public_ip_changes() RETURNS TRIGGER AS +$$ +BEGIN + + BEGIN + UPDATE public_ips_cache + SET free_ips = free_ips + ( + CASE + -- handles insertion/update by freeing ip + WHEN (TG_OP = 'INSERT' AND NEW.contract_id = 0) OR + (TG_OP = 'UPDATE' AND NEW.contract_id = 0 AND OLD.contract_id != 0) + THEN 1 + -- handles deletion/update by reserving ip + WHEN (TG_OP = 'DELETE' AND OLD.contract_id = 0) OR + (TG_OP = 'UPDATE' AND OLD.contract_id = 0 AND NEW.contract_id != 0) + THEN -1 + -- handles delete reserved ips (no change to free_ips) + ELSE 0 + END + ), + + total_ips = total_ips + ( + CASE + WHEN TG_OP = 'INSERT' + THEN 1 + WHEN TG_OP = 'DELETE' + THEN -1 + ELSE 0 + END + ), + + ips = ( + SELECT jsonb_agg( + jsonb_build_object( + 'id', + public_ip.id, + 'ip', + public_ip.ip, + 'contract_id', + public_ip.contract_id, + 'gateway', + public_ip.gateway + ) + ) + -- old/new farm_id are the same + from public_ip where farm_id = COALESCE(NEW.farm_id, OLD.farm_id) + ) + WHERE + public_ips_cache.farm_id = ( + SELECT farm_id FROM farm WHERE farm.id = COALESCE(NEW.farm_id, OLD.farm_id) + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_public_ip_changes', + TG_OP, + 'public_ip', + COALESCE(NEW.id, OLD.id)::TEXT, + SQLERRM, + jsonb_build_object( + 'farm_id', COALESCE(NEW.farm_id, OLD.farm_id), + 'contract_id_old', OLD.contract_id, + 'contract_id_new', NEW.contract_id + ) + ); + RAISE WARNING 'Error reflecting public_ips changes: %', SQLERRM; + END; + +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_public_ip + AFTER INSERT OR DELETE OR UPDATE OF contract_id ON public_ip FOR EACH ROW + EXECUTE PROCEDURE reflect_public_ip_changes(); + + +/* + * reflect_farm_changes + * + * Maintains public_ips_cache when farms are inserted or deleted. + * + * - INSERT: Creates empty cache entry (0 IPs) + * - DELETE: Removes farm from cache + */ +CREATE OR REPLACE FUNCTION reflect_farm_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF TG_OP = 'INSERT' THEN + BEGIN + INSERT INTO public_ips_cache VALUES( + NEW.farm_id, + 0, + 0, + '[]' + ); + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_farm_changes', + 'INSERT', + 'farm', + NEW.farm_id::TEXT, + SQLERRM, + jsonb_build_object('farm_id', NEW.farm_id) + ); + RAISE WARNING 'Error inserting public_ips_cache record: %', SQLERRM; + END; + + ELSIF (TG_OP = 'DELETE') THEN + BEGIN + DELETE FROM public_ips_cache WHERE public_ips_cache.farm_id = OLD.farm_id; + EXCEPTION + WHEN OTHERS THEN + PERFORM log_cache_error( + 'reflect_farm_changes', + 'DELETE', + 'farm', + OLD.farm_id::TEXT, + SQLERRM, + jsonb_build_object('farm_id', OLD.farm_id) + ); + RAISE WARNING 'Error deleting public_ips_cache record: %', SQLERRM; + END; + END IF; + +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_farm + AFTER INSERT OR DELETE ON farm FOR EACH ROW + EXECUTE PROCEDURE reflect_farm_changes(); + diff --git a/grid-proxy/internal/explorer/db/cache/views.sql b/grid-proxy/internal/explorer/db/cache/views.sql new file mode 100644 index 00000000..a56df75f --- /dev/null +++ b/grid-proxy/internal/explorer/db/cache/views.sql @@ -0,0 +1,122 @@ +-- ============================================================================ +-- VIEWS +-- ============================================================================ +-- Base views that define the structure of cached data. +-- These views are used to populate and validate cache tables. + +-- Clean up legacy triggers +DROP TRIGGER IF EXISTS node_added ON node; +DROP VIEW IF EXISTS resources_cache_view; + +/* + * resources_cache_view + * + * Comprehensive view aggregating all node resource information for caching. + * + * Calculates: + * - Total resources: from node_resources_total + * - Used resources: sum of contract_resources for active contracts + * - Free resources: total - used - reserved amounts + * - Reserved amounts: MRU/10 (min 2GB), SRU has fixed 20GB reservation + * - Node metadata: country, DMI info, speed tests, CPU benchmarks, GPUs + * - Contract counts: active contracts (Created, GracePeriod states only) + * - Rent info: current renter and rent contract ID + */ +CREATE OR REPLACE VIEW resources_cache_view AS +SELECT + node.node_id as node_id, + node.farm_id as farm_id, + COALESCE(node_resources_total.hru, 0) as total_hru, + COALESCE(node_resources_total.mru, 0) as total_mru, + COALESCE(node_resources_total.sru, 0) as total_sru, + COALESCE(node_resources_total.cru, 0) as total_cru, + -- Free resources = Total - Used - Reserved + -- HRU: No reserved amount + COALESCE(node_resources_total.hru, 0) - COALESCE(sum(contract_resources.hru), 0) as free_hru, + -- MRU: Reserved amount is MRU/get_mru_reserved_fraction(), minimum get_mru_reserved_min_bytes() + COALESCE(node_resources_total.mru, 0) - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / get_mru_reserved_fraction()) AS bigint), get_mru_reserved_min_bytes()) as free_mru, + -- SRU: Fixed reservation of get_sru_reserved_bytes() + COALESCE(node_resources_total.sru, 0) - COALESCE(sum(contract_resources.sru), 0) - get_sru_reserved_bytes() as free_sru, + -- Used resources from active contracts + COALESCE(sum(contract_resources.hru), 0) as used_hru, + -- MRU used includes reserved amount + COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST((node_resources_total.mru / get_mru_reserved_fraction()) AS bigint), get_mru_reserved_min_bytes()) as used_mru, + -- SRU used includes fixed reservation + COALESCE(sum(contract_resources.sru), 0) + get_sru_reserved_bytes() as used_sru, + COALESCE(sum(contract_resources.cru), 0) as used_cru, + rent_contract.twin_id as renter, + rent_contract.contract_id as rent_contract_id, + count(node_contract.contract_id) as node_contracts_count, + node.country as country, + COALESCE(dmi.bios, '{}') as bios, + COALESCE(dmi.baseboard, '{}') as baseboard, + COALESCE(dmi.processor, '[]') as processor, + COALESCE(dmi.memory, '[]') as memory, + COALESCE(speed.upload, 0) as upload_speed, + COALESCE(speed.download, 0) as download_speed, + COALESCE(speed.udp_download_ipv4, 0) as udp_download_ipv4, + COALESCE(speed.udp_upload_ipv4, 0) as udp_upload_ipv4, + COALESCE(speed.tcp_download_ipv6, 0) as tcp_download_ipv6, + COALESCE(speed.tcp_upload_ipv6, 0) as tcp_upload_ipv6, + COALESCE(speed.udp_download_ipv6, 0) as udp_download_ipv6, + COALESCE(speed.udp_upload_ipv6, 0) as udp_upload_ipv6, + COALESCE(cpu_benchmark.single_threaded, 0) as single_threaded_cpu, + COALESCE(cpu_benchmark.multi_threaded, 0) as multi_threaded_cpu, + COALESCE(cpu_benchmark.threads, 0) as threads_cpu, + COALESCE(cpu_benchmark.workloads, 0) as workloads_cpu, + CASE WHEN node.certification = 'Certified' THEN true ELSE false END as certified, + CASE WHEN farm.pricing_policy_id = 0 THEN 1 ELSE farm.pricing_policy_id END as policy_id, + COALESCE(node.extra_fee, 0) as extra_fee, + COALESCE(node_gpu_agg.gpus, '[]'), + COALESCE(node_gpu_agg.gpu_count, 0) as node_gpu_count +FROM node + LEFT JOIN node_contract ON node.node_id = node_contract.node_id AND node_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN contract_resources ON node_contract.resources_used_id = contract_resources.id + LEFT JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id + LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN speed ON node.twin_id = speed.node_twin_id + LEFT JOIN cpu_benchmark ON node.twin_id = cpu_benchmark.node_twin_id + LEFT JOIN dmi ON node.twin_id = dmi.node_twin_id + LEFT JOIN farm ON farm.farm_id = node.farm_id + -- Aggregate GPU information per node + LEFT JOIN( + SELECT + g1.node_twin_id, + COUNT(g1.id) gpu_count, + jsonb_agg(jsonb_build_object('id', g1.id, 'vendor', g1.vendor, 'vram', g1.vram, 'contract', g1.contract, 'device', g1.device)) as gpus + FROM node_gpu AS g1 + GROUP BY + g1.node_twin_id + ) node_gpu_agg on node_gpu_agg.node_twin_id = node.twin_id +GROUP BY + node.node_id, + node_resources_total.mru, + node_resources_total.sru, + node_resources_total.hru, + node_resources_total.cru, + node.farm_id, + rent_contract.contract_id, + rent_contract.twin_id, + COALESCE(node_gpu_agg.gpus, '[]'), + COALESCE(node_gpu_agg.gpu_count, 0), + node.country, + COALESCE(dmi.bios, '{}'), + COALESCE(dmi.baseboard, '{}'), + COALESCE(dmi.processor, '[]'), + COALESCE(dmi.memory, '[]'), + COALESCE(speed.upload, 0), + COALESCE(speed.download, 0), + COALESCE(speed.udp_download_ipv4, 0), + COALESCE(speed.udp_upload_ipv4, 0), + COALESCE(speed.tcp_download_ipv6, 0), + COALESCE(speed.tcp_upload_ipv6, 0), + COALESCE(speed.udp_download_ipv6, 0), + COALESCE(speed.udp_upload_ipv6, 0), + COALESCE(cpu_benchmark.single_threaded, 0), + COALESCE(cpu_benchmark.multi_threaded, 0), + COALESCE(cpu_benchmark.threads, 0), + COALESCE(cpu_benchmark.workloads, 0), + node.certification, + node.extra_fee, + farm.pricing_policy_id; + diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index ee617a6d..dabe2f25 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -32,8 +32,47 @@ var ( ErrContractNotFound = errors.New("contract not found") ) -//go:embed setup.sql -var setupFile string +//go:embed cache/constants.sql +var cacheConstants string + +//go:embed cache/error_logging.sql +var cacheErrorLogging string + +//go:embed cache/functions.sql +var cacheFunctions string + +//go:embed cache/views.sql +var cacheViews string + +//go:embed cache/cache_tables.sql +var cacheCacheTables string + +//go:embed cache/indexes.sql +var cacheIndexes string + +//go:embed cache/triggers.sql +var cacheTriggers string + +//go:embed cache/cache_management.sql +var cacheCacheManagement string + +//go:embed cache/helpers.sql +var cacheHelpers string + +// cacheFile combines all cache modules in order +var cacheFile = func() string { + return "BEGIN;\n\n" + + cacheConstants + "\n\n" + + cacheErrorLogging + "\n\n" + + cacheFunctions + "\n\n" + + cacheViews + "\n\n" + + cacheCacheTables + "\n\n" + + cacheIndexes + "\n\n" + + cacheTriggers + "\n\n" + + cacheCacheManagement + "\n\n" + + cacheHelpers + "\n\n" + + "COMMIT;" +}() // PostgresDatabase postgres db client type PostgresDatabase struct { @@ -160,8 +199,8 @@ func (d *PostgresDatabase) Initialize() error { return errors.Wrap(err, "failed to migrate indexer tables") } - if err := d.gormDB.Exec(setupFile).Error; err != nil { - return errors.Wrap(err, "failed to setup cache tables") + if err := d.gormDB.Exec(cacheFile).Error; err != nil { + return errors.Wrap(err, "failed to setup cache") } if err := d.gormDB.Exec(`ALTER TABLE node_gpu DROP CONSTRAINT IF EXISTS node_gpu_pkey;`).Error; err != nil { diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql deleted file mode 100644 index b0b4c5a6..00000000 --- a/grid-proxy/internal/explorer/db/setup.sql +++ /dev/null @@ -1,760 +0,0 @@ -BEGIN; - ----- --- Helper functions ----- -DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); - -CREATE OR REPLACE FUNCTION convert_to_decimal(v_input TEXT) RETURNS DECIMAL AS -$$ -DECLARE v_dec_value DECIMAL DEFAULT NULL; -BEGIN - BEGIN - v_dec_value := v_input:: DECIMAL; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; - RETURN NULL; - END; -RETURN v_dec_value; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION calc_discount( - cost NUMERIC, - balance NUMERIC -) RETURNS NUMERIC AS $$ - -DECLARE - discount NUMERIC; - -BEGIN - discount := ( - CASE - WHEN balance >= cost * 18 THEN 0.6 - WHEN balance >= cost * 6 THEN 0.4 - WHEN balance >= cost * 3 THEN 0.3 - WHEN balance >= cost * 1.5 THEN 0.2 - ELSE 0 - END); - - RETURN cost - cost * discount; -END; -$$ LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION calc_price( - cru NUMERIC, - sru NUMERIC, - hru NUMERIC, - mru NUMERIC, - certified BOOLEAN, - policy_id INTEGER, - extra_fee NUMERIC -) RETURNS NUMERIC AS $$ - -DECLARE - su NUMERIC; - cu NUMERIC; - su_value NUMERIC; - cu_value NUMERIC; - cost_per_month NUMERIC; - -BEGIN - SELECT pricing_policy.cu->'value' - INTO cu_value - FROM pricing_policy - WHERE pricing_policy_id = policy_id; - - SELECT pricing_policy.su->'value' - INTO su_value - FROM pricing_policy - WHERE pricing_policy_id = policy_id; - - IF cu_value IS NULL OR su_value IS NULL THEN - RAISE EXCEPTION 'pricing values not found for policy_id: %', policy_id; - END IF; - - cu := (LEAST( - GREATEST(mru / 4, cru / 2), - GREATEST(mru / 8, cru), - GREATEST(mru / 2, cru / 4) - )); - - su := (hru / 1200 + sru / 200); - - cost_per_month := (cu * cu_value + su * su_value + extra_fee) * - (CASE certified WHEN true THEN 1.25 ELSE 1 END) * - (24 * 30); - - RETURN cost_per_month / 10000000; -- 1e7 -END; -$$ LANGUAGE plpgsql IMMUTABLE; - ----- --- Clean old triggers ----- -DROP TRIGGER IF EXISTS node_added ON node; - ----- --- Resources cache table ----- -DROP VIEW IF EXISTS resources_cache_view; - -CREATE OR REPLACE VIEW resources_cache_view AS -SELECT - node.node_id as node_id, - node.farm_id as farm_id, - COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.mru, 0) as total_mru, - COALESCE(node_resources_total.sru, 0) as total_sru, - COALESCE(node_resources_total.cru, 0) as total_cru, - COALESCE(node_resources_total.hru, 0) - COALESCE(sum(contract_resources.hru), 0) as free_hru, - COALESCE(node_resources_total.mru, 0) - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, - COALESCE(node_resources_total.sru, 0) - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST( (node_resources_total.mru / 10) AS bigint), 2147483648 ) as used_mru, - COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, - COALESCE(sum(contract_resources.cru), 0) as used_cru, - rent_contract.twin_id as renter, - rent_contract.contract_id as rent_contract_id, - count(node_contract.contract_id) as node_contracts_count, - node.country as country, - COALESCE(dmi.bios, '{}') as bios, - COALESCE(dmi.baseboard, '{}') as baseboard, - COALESCE(dmi.processor, '[]') as processor, - COALESCE(dmi.memory, '[]') as memory, - COALESCE(speed.upload, 0) as upload_speed, - COALESCE(speed.download, 0) as download_speed, - COALESCE(speed.udp_download_ipv4, 0) as udp_download_ipv4, - COALESCE(speed.udp_upload_ipv4, 0) as udp_upload_ipv4, - COALESCE(speed.tcp_download_ipv6, 0) as tcp_download_ipv6, - COALESCE(speed.tcp_upload_ipv6, 0) as tcp_upload_ipv6, - COALESCE(speed.udp_download_ipv6, 0) as udp_download_ipv6, - COALESCE(speed.udp_upload_ipv6, 0) as udp_upload_ipv6, - COALESCE(cpu_benchmark.single_threaded, 0) as single_threaded_cpu, - COALESCE(cpu_benchmark.multi_threaded, 0) as multi_threaded_cpu, - COALESCE(cpu_benchmark.threads, 0) as threads_cpu, - COALESCE(cpu_benchmark.workloads, 0) as workloads_cpu, - CASE WHEN node.certification = 'Certified' THEN true ELSE false END as certified, - CASE WHEN farm.pricing_policy_id = 0 THEN 1 ELSE farm.pricing_policy_id END as policy_id, - COALESCE(node.extra_fee, 0) as extra_fee, - COALESCE(node_gpu_agg.gpus, '[]'), - COALESCE(node_gpu_agg.gpu_count, 0) as node_gpu_count -FROM node - LEFT JOIN node_contract ON node.node_id = node_contract.node_id AND node_contract.state IN ('Created', 'GracePeriod') - LEFT JOIN contract_resources ON node_contract.resources_used_id = contract_resources.id - LEFT JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id - LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') - LEFT JOIN speed ON node.twin_id = speed.node_twin_id - LEFT JOIN cpu_benchmark ON node.twin_id = cpu_benchmark.node_twin_id - LEFT JOIN dmi ON node.twin_id = dmi.node_twin_id - LEFT JOIN farm ON farm.farm_id = node.farm_id - -- join aggregated gpus table - LEFT JOIN( - SELECT - g1.node_twin_id, - COUNT(g1.id) gpu_count, - jsonb_agg(jsonb_build_object('id', g1.id, 'vendor', g1.vendor, 'vram', g1.vram, 'contract', g1.contract, 'device', g1.device)) as gpus - FROM node_gpu AS g1 - LEFT JOIN node_gpu g2 ON g1.id = g2.id - GROUP BY - g1.node_twin_id - ) node_gpu_agg on node_gpu_agg.node_twin_id = node.twin_id -GROUP BY - node.node_id, - node_resources_total.mru, - node_resources_total.sru, - node_resources_total.hru, - node_resources_total.cru, - node.farm_id, - rent_contract.contract_id, - rent_contract.twin_id, - COALESCE(node_gpu_agg.gpus, '[]'), - COALESCE(node_gpu_agg.gpu_count, 0), - node.country, - COALESCE(dmi.bios, '{}'), - COALESCE(dmi.baseboard, '{}'), - COALESCE(dmi.processor, '[]'), - COALESCE(dmi.memory, '[]'), - COALESCE(speed.upload, 0), - COALESCE(speed.download, 0), - COALESCE(speed.udp_download_ipv4, 0), - COALESCE(speed.udp_upload_ipv4, 0), - COALESCE(speed.tcp_download_ipv6, 0), - COALESCE(speed.tcp_upload_ipv6, 0), - COALESCE(speed.udp_download_ipv6, 0), - COALESCE(speed.udp_upload_ipv6, 0), - COALESCE(cpu_benchmark.single_threaded, 0), - COALESCE(cpu_benchmark.multi_threaded, 0), - COALESCE(cpu_benchmark.threads, 0), - COALESCE(cpu_benchmark.workloads, 0), - node.certification, - node.extra_fee, - farm.pricing_policy_id; - -DROP TABLE IF EXISTS resources_cache; -CREATE TABLE IF NOT EXISTS resources_cache( - node_id INTEGER PRIMARY KEY, - farm_id INTEGER NOT NULL, - total_hru NUMERIC NOT NULL, - total_mru NUMERIC NOT NULL, - total_sru NUMERIC NOT NULL, - total_cru NUMERIC NOT NULL, - free_hru NUMERIC NOT NULL, - free_mru NUMERIC NOT NULL, - free_sru NUMERIC NOT NULL, - used_hru NUMERIC NOT NULL, - used_mru NUMERIC NOT NULL, - used_sru NUMERIC NOT NULL, - used_cru NUMERIC NOT NULL, - renter INTEGER, - rent_contract_id INTEGER, - node_contracts_count INTEGER NOT NULL, - country TEXT, - bios jsonb, - baseboard jsonb, - processor jsonb, - memory jsonb, - upload_speed numeric, - download_speed numeric, - udp_download_ipv4 numeric, - udp_upload_ipv4 numeric, - tcp_download_ipv6 numeric, - tcp_upload_ipv6 numeric, - udp_download_ipv6 numeric, - udp_upload_ipv6 numeric, - single_threaded_cpu numeric, - multi_threaded_cpu numeric, - threads_cpu integer, - workloads_cpu integer, - certified BOOLEAN, - policy_id INTEGER, - extra_fee NUMERIC, - gpus jsonb, - node_gpu_count INTEGER NOT NULL, - price_usd NUMERIC GENERATED ALWAYS AS ( - calc_price( - total_cru, - total_sru / (1024*1024*1024), - total_hru / (1024*1024*1024), - total_mru / (1024*1024*1024), - certified, - policy_id, - extra_fee - ) - ) STORED - ); - -INSERT INTO resources_cache -SELECT * -FROM resources_cache_view; - - ----- --- PublicIpsCache table ----- -DROP TABLE IF EXISTS public_ips_cache; -CREATE TABLE public_ips_cache( - farm_id INTEGER PRIMARY KEY, - free_ips INTEGER NOT NULL, - total_ips INTEGER NOT NULL, - ips jsonb -); - -INSERT INTO public_ips_cache - SELECT - farm.farm_id, - COALESCE(public_ip_agg.free_ips, 0), - COALESCE(public_ip_agg.total_ips, 0), - COALESCE(public_ip_agg.ips, '[]') -FROM farm - LEFT JOIN( - SELECT - p1.farm_id, - COUNT(p1.id) total_ips, - COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips, - jsonb_agg(jsonb_build_object('id', p1.id, 'ip', p1.ip, 'contract_id', p1.contract_id, 'gateway', p1.gateway)) as ips - FROM public_ip AS p1 - LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY - p1.farm_id - ) public_ip_agg on public_ip_agg.farm_id = farm.id; - ----- --- Create Indices ----- -CREATE EXTENSION IF NOT EXISTS pg_trgm; - -CREATE EXTENSION IF NOT EXISTS btree_gin; - -CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); -CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); -CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); -CREATE INDEX IF NOT EXISTS idx_node_contract_id ON public.node_contract USING gin(id); -CREATE INDEX IF NOT EXISTS idx_name_contract_id ON public.name_contract USING gin(id); -CREATE INDEX IF NOT EXISTS idx_rent_contract_id ON public.rent_contract USING gin(id); - - -CREATE INDEX IF NOT EXISTS idx_resources_cache_farm_id ON resources_cache (farm_id); -CREATE INDEX IF NOT EXISTS idx_resources_cache_node_id ON resources_cache(node_id); -CREATE INDEX IF NOT EXISTS idx_public_ips_cache_farm_id ON public_ips_cache(farm_id); - -CREATE INDEX IF NOT EXISTS idx_location_id ON location USING gin(id); -CREATE INDEX IF NOT EXISTS idx_public_config_node_id ON public_config USING gin(node_id); - ----- ---create triggers ----- - -/* - Node Trigger: - - Insert node record > Insert new resources_cache record -*/ -CREATE OR REPLACE FUNCTION reflect_node_changes() RETURNS TRIGGER AS -$$ -BEGIN - IF (TG_OP = 'INSERT') THEN - BEGIN - INSERT INTO resources_cache - SELECT * - FROM resources_cache_view - WHERE resources_cache_view.node_id = NEW.node_id; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error inserting resources_cache: %', SQLERRM; - END; - - ELSIF (TG_OP = 'DELETE') THEN - BEGIN - DELETE FROM resources_cache WHERE node_id = OLD.node_id; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error deleting node from resources_cache: %', SQLERRM; - END; - END IF; - - RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_node - AFTER INSERT OR DELETE - ON node - FOR EACH ROW EXECUTE PROCEDURE reflect_node_changes(); - -/* - Total resources trigger - - Insert/Update node_resources_total > Update equivalent resources_cache record. - */ -CREATE OR REPLACE FUNCTION reflect_total_resources_changes() RETURNS TRIGGER AS -$$ -BEGIN - BEGIN - UPDATE resources_cache - SET - total_cru = NEW.cru, - total_mru = NEW.mru, - total_sru = NEW.sru, - total_hru = NEW.hru, - free_mru = free_mru + GREATEST(CAST((OLD.mru / 10) AS bigint), 2147483648) - - GREATEST(CAST((NEW.mru / 10) AS bigint), 2147483648) + (NEW.mru-COALESCE(OLD.mru, 0)), - free_hru = free_hru + (NEW.hru-COALESCE(OLD.hru, 0)), - free_sru = free_sru + (NEW.sru-COALESCE(OLD.sru, 0)), - used_mru = used_mru - GREATEST(CAST((OLD.mru / 10) AS bigint), 2147483648) + - GREATEST(CAST((NEW.mru / 10) AS bigint), 2147483648) - WHERE - resources_cache.node_id = ( - SELECT node.node_id FROM node WHERE node.id = New.node_id - ); - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error reflecting total_resources changes %', SQLERRM; - END; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_node_resources_total - AFTER INSERT OR UPDATE - ON node_resources_total FOR EACH ROW - EXECUTE PROCEDURE reflect_total_resources_changes(); - - -/* - Contract resources - - Insert/Update contract_resources report > update resources_cache used/free fields - */ - -CREATE OR REPLACE FUNCTION reflect_contract_resources_changes() RETURNS TRIGGER AS -$$ -BEGIN - BEGIN - UPDATE resources_cache - SET used_cru = used_cru + (NEW.cru - COALESCE(OLD.cru, 0)), - used_mru = used_mru + (NEW.mru - COALESCE(OLD.mru, 0)), - used_sru = used_sru + (NEW.sru - COALESCE(OLD.sru, 0)), - used_hru = used_hru + (NEW.hru - COALESCE(OLD.hru, 0)), - free_mru = free_mru - (NEW.mru - COALESCE(OLD.mru, 0)), - free_hru = free_hru - (NEW.hru - COALESCE(OLD.hru, 0)), - free_sru = free_sru - (NEW.sru - COALESCE(OLD.sru, 0)) - WHERE - -- (SELECT state from node_contract where id = NEW.contract_id) != 'Deleted' AND - resources_cache.node_id = ( - SELECT node_id FROM node_contract WHERE node_contract.id = NEW.contract_id - ); - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error reflecting contract_resources changes %', SQLERRM; - END; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_contract_resources - AFTER INSERT OR UPDATE ON contract_resources FOR EACH ROW - EXECUTE PROCEDURE reflect_contract_resources_changes(); - -/* - Node contract trigger - - Insert new contract > increment resources_cache node_contracts_count - - Update contract state to 'Deleted' > decrement used an increment free fields on resources_cache -*/ -CREATE OR REPLACE FUNCTION reflect_node_contract_changes() RETURNS TRIGGER AS -$$ -BEGIN - IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN - BEGIN - -- lock cache row to prevent concurrent updates - PERFORM 1 - FROM resources_cache - WHERE node_id = NEW.node_id - FOR UPDATE; - - UPDATE resources_cache - SET - used_cru = resources_cache.used_cru - contract_resources.cru, - used_mru = resources_cache.used_mru - contract_resources.mru, - used_sru = resources_cache.used_sru - contract_resources.sru, - used_hru = resources_cache.used_hru - contract_resources.hru, - free_mru = resources_cache.free_mru + contract_resources.mru, - free_sru = resources_cache.free_sru + contract_resources.sru, - free_hru = resources_cache.free_hru + contract_resources.hru, - node_contracts_count = COALESCE(ncc.count, 0) - FROM contract_resources - LEFT JOIN - (SELECT node_id, COUNT(contract_id) as count - FROM node_contract - WHERE state IN ('Created', 'GracePeriod') - GROUP BY node_id) AS ncc - ON ncc.node_id = NEW.node_id - WHERE - contract_resources.contract_id = NEW.id - AND resources_cache.node_id = NEW.node_id; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'failed reflecting node_contract updates %', SQLERRM; - END; - - ELSIF (TG_OP = 'INSERT') THEN - BEGIN - UPDATE resources_cache - SET node_contracts_count = ( - SELECT COALESCE(COUNT(contract_id), 0) - FROM node_contract - WHERE node_id = NEW.node_id - AND state IN ('Created', 'GracePeriod') - ) - WHERE resources_cache.node_id = NEW.node_id; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'failed calc node_contracts_count %', SQLERRM; - END; - END IF; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_node_contract - AFTER INSERT OR UPDATE OF state ON node_contract FOR EACH ROW - EXECUTE PROCEDURE reflect_node_contract_changes(); - -/* - Gpu trigger - - Insert new node_gpu > increase the gpu_num in resources cache - - Delete node_gpu > decrease the gpu_num in resources cache -*/ -CREATE OR REPLACE FUNCTION reflect_node_gpu_count_change() RETURNS TRIGGER AS -$$ -BEGIN - BEGIN - UPDATE resources_cache - SET node_gpu_count = gpu.count, gpus = gpu.gpus - FROM ( - SELECT COUNT(*) AS count, - json_agg( - json_build_object( - 'id', id, - 'vendor', vendor, - 'device', device, - 'vram', vram, - 'contract', contract - ) - ) AS gpus - FROM node_gpu - WHERE node_twin_id = COALESCE(NEW.node_twin_id, OLD.node_twin_id) - ) AS gpu - WHERE resources_cache.node_id = ( - SELECT node_id from node where node.twin_id = COALESCE(NEW.node_twin_id, OLD.node_twin_id) - ); - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error updating resources_cache gpu fields %', SQLERRM; - END; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_node_gpu_count - AFTER INSERT OR DELETE OR UPDATE ON node_gpu FOR EACH ROW - EXECUTE PROCEDURE reflect_node_gpu_count_change(); - -/* - Rent contract trigger - - Insert new rent contract > Update resources_cache renter/rent_contract_id - - Update (state to 'Deleted') > nullify resources_cache renter/rent_contract_id -*/ - -CREATE OR REPLACE FUNCTION reflect_rent_contract_changes() RETURNS TRIGGER AS -$$ -BEGIN - IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN - BEGIN - UPDATE resources_cache - SET renter = NULL, - rent_contract_id = NULL - WHERE - resources_cache.node_id = NEW.node_id; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error removing resources_cache rent fields %', SQLERRM; - END; - ELSIF (TG_OP = 'INSERT') THEN - BEGIN - UPDATE resources_cache - SET renter = NEW.twin_id, - rent_contract_id = NEW.contract_id - WHERE - resources_cache.node_id = NEW.node_id; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error reflecting rent_contract changes %', SQLERRM; - END; - END IF; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_rent_contract - AFTER INSERT OR UPDATE OF state ON rent_contract FOR EACH ROW - EXECUTE PROCEDURE reflect_rent_contract_changes(); - -/* - Dmi trigger - - Insert new record/Update > update resources_cache -*/ -CREATE OR REPLACE FUNCTION reflect_dmi_changes() RETURNS TRIGGER AS -$$ -BEGIN - BEGIN - UPDATE resources_cache - SET bios = NEW.bios, - baseboard = NEW.baseboard, - processor = NEW.processor, - memory = NEW.memory - WHERE resources_cache.node_id = ( - SELECT node_id from node where node.twin_id = NEW.node_twin_id - ); - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error updating resources_cache dmi fields %', SQLERRM; - END; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_dmi - AFTER INSERT OR UPDATE ON dmi FOR EACH ROW - EXECUTE PROCEDURE reflect_dmi_changes(); - - -/* - speed trigger - - Insert new record/Update > update resources_cache -*/ -CREATE OR REPLACE FUNCTION reflect_speed_changes() RETURNS TRIGGER AS -$$ -BEGIN - BEGIN - UPDATE resources_cache - SET upload_speed = NEW.upload, - download_speed = NEW.download, - udp_download_ipv4 = NEW.udp_download_ipv4, - udp_upload_ipv4 = NEW.udp_upload_ipv4, - tcp_download_ipv6 = NEW.tcp_download_ipv6, - tcp_upload_ipv6 = NEW.tcp_upload_ipv6, - udp_download_ipv6 = NEW.udp_download_ipv6, - udp_upload_ipv6 = NEW.udp_upload_ipv6 - WHERE resources_cache.node_id = ( - SELECT node_id from node where node.twin_id = NEW.node_twin_id - ); - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error updating resources_cache speed fields %', SQLERRM; - END; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_speed - AFTER INSERT OR UPDATE ON speed FOR EACH ROW - EXECUTE PROCEDURE reflect_speed_changes(); - -/* - cpu_benchmark trigger - - Insert new record/Update > update resources_cache -*/ -CREATE OR REPLACE FUNCTION reflect_cpu_benchmark_changes() RETURNS TRIGGER AS -$$ -BEGIN - BEGIN - UPDATE resources_cache - SET single_threaded_cpu = NEW.single_threaded, - multi_threaded_cpu = NEW.multi_threaded, - threads_cpu = NEW.threads, - workloads_cpu = NEW.workloads - WHERE resources_cache.node_id = ( - SELECT node_id from node where node.twin_id = NEW.node_twin_id - ); - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error updating resources_cache cpu_benchmark fields %', SQLERRM; - END; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_cpu_benchmark - AFTER INSERT OR UPDATE ON cpu_benchmark FOR EACH ROW - EXECUTE PROCEDURE reflect_cpu_benchmark_changes(); - -/* - Public ips trigger - - Insert new ip > increment free/total ips + re-aggregate ips object - - Deleted > decrement total, decrement free ips (if it was used) + re-aggregate ips object - - Update > increment/decrement free ips based on usage + re-aggregate ips object - - - reserve ip > free_ips decrease - - unreserve ip > free_ips increase - - insert new ip (expected be free) > free_ips increase - - remove reserved ip > free_ips does not change - - remove free ip > free_ips decrease -*/ -CREATE OR REPLACE FUNCTION reflect_public_ip_changes() RETURNS TRIGGER AS -$$ -BEGIN - - BEGIN - UPDATE public_ips_cache - SET free_ips = free_ips + ( - CASE - -- handles insertion/update by freeing ip - WHEN TG_OP = 'INSERT' AND NEW.contract_id = 0 OR - TG_OP = 'UPDATE' AND NEW.contract_id = 0 AND OLD.contract_id != 0 - THEN 1 - -- handles deletion/update by reserving ip - WHEN TG_OP = 'DELETE' AND OLD.contract_id = 0 OR - TG_OP = 'UPDATE' AND OLD.contract_id = 0 AND NEW.contract_id != 0 - THEN -1 - -- handles delete reserved ips - ELSE 0 - END - ), - - total_ips = total_ips + ( - CASE - WHEN TG_OP = 'INSERT' - THEN 1 - WHEn TG_OP = 'DELETE' - THEN -1 - ELSE 0 - END - ), - - ips = ( - SELECT jsonb_agg( - jsonb_build_object( - 'id', - public_ip.id, - 'ip', - public_ip.ip, - 'contract_id', - public_ip.contract_id, - 'gateway', - public_ip.gateway - ) - ) - -- old/new farm_id are the same - from public_ip where farm_id = COALESCE(NEW.farm_id, OLD.farm_id) - ) - WHERE - public_ips_cache.farm_id = ( - SELECT farm_id FROM farm WHERE farm.id = COALESCE(NEW.farm_id, OLD.farm_id) - ); - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error reflect public_ips changes %s', SQLERRM; - END; - -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_public_ip - AFTER INSERT OR DELETE OR UPDATE OF contract_id ON public_ip FOR EACH ROW - EXECUTE PROCEDURE reflect_public_ip_changes(); - - -CREATE OR REPLACE FUNCTION reflect_farm_changes() RETURNS TRIGGER AS -$$ -BEGIN - IF TG_OP = 'INSERT' THEN - BEGIN - INSERT INTO public_ips_cache VALUES( - NEW.farm_id, - 0, - 0, - '[]' - ); - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error inserting public_ips_cache record %s', SQLERRM; - END; - - ELSIF (TG_OP = 'DELETE') THEN - BEGIN - DELETE FROM public_ips_cache WHERE public_ips_cache.farm_id = OLD.farm_id; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error deleting public_ips_cache record %s', SQLERRM; - END; - END IF; - -RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER tg_farm - AFTER INSERT OR DELETE ON farm FOR EACH ROW - EXECUTE PROCEDURE reflect_farm_changes(); - -COMMIT;