diff --git a/VERSION.in b/VERSION.in index e71519696..b48f32260 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -1.16 +1.17 diff --git a/cuebot/src/main/java/com/imageworks/spcue/MaintenanceTask.java b/cuebot/src/main/java/com/imageworks/spcue/MaintenanceTask.java index 90be9ca29..2163d9659 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/MaintenanceTask.java +++ b/cuebot/src/main/java/com/imageworks/spcue/MaintenanceTask.java @@ -44,5 +44,10 @@ public enum MaintenanceTask { /** * Lock the stale checkpoint task. */ - LOCK_STALE_CHECKPOINT + LOCK_STALE_CHECKPOINT, + + /** + * Lock the subscription recalculation task. + */ + LOCK_SUBSCRIPTION_RECALCULATION } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/MaintenanceDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/MaintenanceDao.java index 3ae7c71b2..98542c6c6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/MaintenanceDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/MaintenanceDao.java @@ -57,4 +57,10 @@ public interface MaintenanceDao { */ void unlockTask(MaintenanceTask task); + /** + * Recalculates subscription core usage values by calling the recalculate_subs() database + * function. This fixes subscription accountability issues that can occur at large scale. + */ + void recalculateSubscriptions(); + } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/MaintenanceDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/MaintenanceDaoJdbc.java index 50ca6bfba..049b61841 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/MaintenanceDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/MaintenanceDaoJdbc.java @@ -55,4 +55,11 @@ public void unlockTask(MaintenanceTask task) { getJdbcTemplate().update("UPDATE task_lock SET int_lock = 0 WHERE str_name=?", task.toString()); } + + private static final String RECALCULATE_SUBS = "SELECT recalculate_subs()"; + + @Override + public void recalculateSubscriptions() { + getJdbcTemplate().execute(RECALCULATE_SUBS); + } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java index fff3d1a41..67e53d3a8 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java @@ -224,6 +224,25 @@ public void updateTaskValues() { } } + /** + * Recalculates subscription core usage values to fix accountability issues that can occur at + * large scale. This calls the recalculate_subs() database function that was added in PR #1380. + */ + public void recalculateSubscriptions() { + if (!maintenanceDao.lockTask(MaintenanceTask.LOCK_SUBSCRIPTION_RECALCULATION)) { + return; + } + try { + logger.info("running subscription recalculation"); + maintenanceDao.recalculateSubscriptions(); + logger.info("subscription recalculation completed"); + } catch (Exception e) { + logger.warn("failed to recalculate subscriptions: " + e); + } finally { + maintenanceDao.unlockTask(MaintenanceTask.LOCK_SUBSCRIPTION_RECALCULATION); + } + } + public FrameDao getFrameDao() { return frameDao; } diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V35__Add_subscription_recalculation_task_lock.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V35__Add_subscription_recalculation_task_lock.sql new file mode 100644 index 000000000..3a2d5a4af --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V35__Add_subscription_recalculation_task_lock.sql @@ -0,0 +1,6 @@ +-- Add task_lock entry for subscription recalculation maintenance task +-- This task periodically recalculates subscription core usage values +-- to fix accountability issues that can occur at large scale. + +INSERT INTO task_lock (pk_task_lock, str_name, int_lock, int_timeout) +VALUES ('00000000-0000-0000-0000-000000000006', 'LOCK_SUBSCRIPTION_RECALCULATION', 0, 7200); diff --git a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql index 7b189174c..bd52252f2 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql @@ -82,3 +82,5 @@ Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('0000 Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000003','LOCK_ORPHANED_PROC_CHECK',0,30); Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000005','LOCK_TASK_UPDATE',1240618998852,3600); + +Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000006','LOCK_SUBSCRIPTION_RECALCULATION',0,7200); diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 65e65d315..9a805ee2a 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -591,6 +591,19 @@ + + + + + + + + + + + + + false @@ -607,6 +620,7 @@ + diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 1d450d317..f4b6796dc 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -224,6 +224,11 @@ history.archive_jobs_cutoff_hours=72 # Delete down hosts automatically. maintenance.auto_delete_down_hosts=false +# Interval in milliseconds for subscription recalculation task. +# Default is 2 hours (7200000 ms). This task fixes subscription +# accountability issues that can occur at large scale. +maintenance.subscription_recalculation_interval_ms=7200000 + # Set hostname/IP of the smtp host. Will be used for mailing smtp_host=smtp diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 956332681..ae59a3662 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -97,3 +97,6 @@ dispatcher.memory.mem_gpu_reserved_max = 104857600 # Loki log.loki.url = http://localhost/loki/api + +# Interval in milliseconds for subscription recalculation task. +maintenance.subscription_recalculation_interval_ms=7200000