[cuebot] Add scheduled subscription recalculation task

VERSION.in

@@ -1 +1 @@
-1.15
+1.16

cuebot/src/main/java/com/imageworks/spcue/MaintenanceTask.java

@@ -44,5 +44,10 @@ public enum MaintenanceTask {
     /**
      * Lock the stale checkpoint task.
      */
-    LOCK_STALE_CHECKPOINT
+    LOCK_STALE_CHECKPOINT,
+
+    /**
+     * Lock the subscription recalculation task.
+     */
+    LOCK_SUBSCRIPTION_RECALCULATION
 }

cuebot/src/main/java/com/imageworks/spcue/dao/MaintenanceDao.java

@@ -57,4 +57,10 @@ public interface MaintenanceDao {
      */
     void unlockTask(MaintenanceTask task);
 
+    /**
+     * Recalculates subscription core usage values by calling the recalculate_subs() database
+     * function. This fixes subscription accountability issues that can occur at large scale.
+     */
+    void recalculateSubscriptions();
+
 }

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/MaintenanceDaoJdbc.java

@@ -55,4 +55,11 @@ public void unlockTask(MaintenanceTask task) {
         getJdbcTemplate().update("UPDATE task_lock SET int_lock = 0 WHERE str_name=?",
                 task.toString());
     }
+
+    private static final String RECALCULATE_SUBS = "SELECT recalculate_subs()";
+
+    @Override
+    public void recalculateSubscriptions() {
+        getJdbcTemplate().execute(RECALCULATE_SUBS);
+    }
 }

cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java

@@ -224,6 +224,25 @@ public void updateTaskValues() {
         }
     }
 
+    /**
+     * Recalculates subscription core usage values to fix accountability issues that can occur at
+     * large scale. This calls the recalculate_subs() database function that was added in PR #1380.
+     */
+    public void recalculateSubscriptions() {
+        if (!maintenanceDao.lockTask(MaintenanceTask.LOCK_SUBSCRIPTION_RECALCULATION)) {
+            return;
+        }
+        try {
+            logger.info("running subscription recalculation");
+            maintenanceDao.recalculateSubscriptions();
+            logger.info("subscription recalculation completed");
+        } catch (Exception e) {
+            logger.warn("failed to recalculate subscriptions: " + e);
+        } finally {
+            maintenanceDao.unlockTask(MaintenanceTask.LOCK_SUBSCRIPTION_RECALCULATION);
+        }
+    }
+
     public FrameDao getFrameDao() {
         return frameDao;
     }

cuebot/src/main/resources/conf/ddl/postgres/migrations/V35__Add_subscription_recalculation_task_lock.sql

@@ -0,0 +1,6 @@
+-- Add task_lock entry for subscription recalculation maintenance task
+-- This task periodically recalculates subscription core usage values
+-- to fix accountability issues that can occur at large scale.
+
+INSERT INTO task_lock (pk_task_lock, str_name, int_lock, int_timeout)
+VALUES ('00000000-0000-0000-0000-000000000006', 'LOCK_SUBSCRIPTION_RECALCULATION', 0, 7200);

cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql

@@ -82,3 +82,5 @@ Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('0000
 Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000003','LOCK_ORPHANED_PROC_CHECK',0,30);
 
 Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000005','LOCK_TASK_UPDATE',1240618998852,3600);
+
+Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000006','LOCK_SUBSCRIPTION_RECALCULATION',0,7200);

cuebot/src/main/resources/conf/spring/applicationContext-service.xml

@@ -591,6 +591,19 @@
     <property name="repeatInterval" value="60000" />
   </bean>
 
+  <bean id="subscriptionRecalculation" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
+    <property name="targetObject" ref="maintenanceManager" />
+    <property name="targetMethod" value="recalculateSubscriptions" />
+  </bean>
+
+  <bean id="subscriptionRecalculationTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerFactoryBean">
+    <property name="jobDetail" ref="subscriptionRecalculation" />
+    <!-- 10 minute start delay to let the cue get up and running -->
+    <property name="startDelay" value="600000" />
+    <!-- repeat every 2 hours by default (configurable via maintenance.subscription_recalculation_interval_ms) -->
+    <property name="repeatInterval" value="${maintenance.subscription_recalculation_interval_ms}" />
+  </bean>
+
   <bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean" destroy-method="destroy">
    <property name="waitForJobsToCompleteOnShutdown"><value>false</value></property>
     <property name="triggers">
@@ -607,6 +620,7 @@
         <ref bean="killQueueSchedule" />
         <ref bean="updateShowsStatusTrigger" />
         <ref bean="collectPrometheusMetricsTrigger" />
+        <ref bean="subscriptionRecalculationTrigger" />
       </list>
     </property>
   </bean>

cuebot/src/main/resources/opencue.properties

@@ -224,6 +224,11 @@ history.archive_jobs_cutoff_hours=72
 # Delete down hosts automatically.
 maintenance.auto_delete_down_hosts=false
 
+# Interval in milliseconds for subscription recalculation task.
+# Default is 2 hours (7200000 ms). This task fixes subscription
+# accountability issues that can occur at large scale.
+maintenance.subscription_recalculation_interval_ms=7200000
+
 # Set hostname/IP of the smtp host. Will be used for mailing
 smtp_host=smtp