diff --git a/api/cmd/midas-api/main.go b/api/cmd/midas-api/main.go index b3e8a760..0b835d22 100644 --- a/api/cmd/midas-api/main.go +++ b/api/cmd/midas-api/main.go @@ -21,7 +21,7 @@ func main() { } } -func run(ctx context.Context, cfg *config.ApiConfig, l *logger.Logger) (err error) { +func run(ctx context.Context, cfg *config.ApiConfig, l logger.Logger) (err error) { h, err := handler.NewApi(ctx, cfg, l) if err != nil { return err diff --git a/api/cmd/midas-dcs-loader/main.go b/api/cmd/midas-dcs-loader/main.go index fa7c9eba..d918adcd 100644 --- a/api/cmd/midas-dcs-loader/main.go +++ b/api/cmd/midas-dcs-loader/main.go @@ -21,7 +21,7 @@ func main() { } } -func run(ctx context.Context, cfg *config.DcsLoaderConfig, l *logger.Logger) (err error) { +func run(ctx context.Context, cfg *config.DcsLoaderConfig, l logger.Logger) (err error) { h, err := handler.NewDcsLoader(ctx, cfg, l) if err != nil { return err diff --git a/api/cmd/midas-task/main.go b/api/cmd/midas-task/main.go index 3c0ad44d..403f8fe3 100644 --- a/api/cmd/midas-task/main.go +++ b/api/cmd/midas-task/main.go @@ -9,11 +9,11 @@ import ( "github.com/USACE/instrumentation-api/api/v4/internal/cloud" "github.com/USACE/instrumentation-api/api/v4/internal/config" + "github.com/USACE/instrumentation-api/api/v4/internal/eval" "github.com/USACE/instrumentation-api/api/v4/internal/logger" "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" "github.com/USACE/instrumentation-api/api/v4/internal/service" "github.com/USACE/instrumentation-api/api/v4/internal/worker" - "github.com/google/uuid" "github.com/riverqueue/river" ) @@ -28,7 +28,9 @@ func main() { } } -func run(ctx context.Context, cfg *config.TaskConfig, l *logger.Logger) (err error) { +var riverTimeout = 15 * time.Second + +func run(ctx context.Context, cfg *config.TaskConfig, l logger.Logger) (err error) { dbpool, err := service.NewDBPool(ctx, cfg.DBConfig) if err != nil { return err @@ -42,83 +44,91 @@ func run(ctx context.Context, cfg *config.TaskConfig, l *logger.Logger) (err err if err != nil { return err } - defer func() { - err = errors.Join(err, taskServices.Shutdown(ctx)) - }() + defer func() { err = errors.Join(err, taskServices.Shutdown(ctx)) }() - workers := river.NewWorkers() - periodicJobs := make([]*river.PeriodicJob, 0) - - alertEventWorker := worker.NewAlertEventWorker(dbservice, taskServices) - river.AddWorker(workers, alertEventWorker) - - emailEventWorker := worker.NewEamilEventWorker(dbservice, taskServices) - river.AddWorker(workers, emailEventWorker) - - // TODO - // fetchThinglogixScheduleWorker := worker.NewFetchThinglogixScheduleWorker(dbservice, cfg) - // river.AddWorker(workers, fetchThinglogixScheduleWorker) - // - // fetchThinglogixEventWorker := worker.NewFetchThinglogixEventWorker(dbservice, cfg) - // river.AddWorker(workers, fetchThinglogixEventWorker) - - // V1 batches submittals every 15 minutes, V2 runs dynamic scheduling per-submittal. - // We need to benchmark the performance of V2 before enabling it by default. It allows - // for higher accuracy scheduling of alerts, but may be more resource intensive. - // - // Note that V2 will also require reqorking how measurement submittals are handled. - // While Evaluation submittals are always done manually, some measurement submittals - // are automated via telemetry, while others are done manually. Instead of the user manually - // assigning a submittal to a measurement upload, we simple check if the measurement exists for - // the given instrument/timeseries/interval. Technically, if we wanted the submittals to behave - // sumilarly, we would need to complete and create a new submittal for each measurement upload, - // which would increase database round trips. Maybe instead, we can make a clearer - // differentation between measurement submittals and evaluation submittals, where we create the - // check for existing measurements and create the next submittal during the scheduled alert check job. - // - if cfg.FeatureFlags.DynamicAlertSchedulerDisabled { - river.AddWorker(workers, worker.NewAlertScheduleWorkerV1(dbservice)) - periodicJobs = append(periodicJobs, worker.AlertScheduleV1JobOptions) - } else { - river.AddWorker(workers, worker.NewAlertScheduleWorkerV2(dbservice)) + // NewEnv loads custom Measurement type from protobuf + baseEnv, err := eval.NewEnv() + if err != nil { + return fmt.Errorf("failed to create program baseEnv: %w", err) } - - tlgxWorker, err := worker.NewFetchThinglogixScheduleWorker(ctx, dbservice, &cfg.ThinglogixConfig, l) + // set up lru program cache + cache, err := eval.NewProgramCache(1024) if err != nil { - l.Error(ctx, "failed to initialize thinglogix; skipping worker", "error", err) - } else { - river.AddWorker(workers, tlgxWorker) - periodicJobs = append(periodicJobs, worker.FetchThinglogixJobOptions) + return fmt.Errorf("failed to create program cache: %w", err) } - rivercfg := &river.Config{ - ID: "midas-task__" + uuid.New().String() + "__" + time.Now().Format("2006_01_02T15_04_05_000000"), - Logger: l.Slogger(), + // Need references for in-process bus used for batching + alertEventW := worker.NewAlertEventWorker(dbservice) + exprTsComputeFullEventW := worker.NewEvaluationTimeseriesComputeFullEventWorker(dbservice, baseEnv, cache) + exprTsComputePartialEventW := worker.NewEvaluationTimeseriesComputePartialEventWorker(dbservice, baseEnv, cache) + + q, err := pgqueue.NewWorkerClient(ctx, dbpool, &pgqueue.Options{ + ClientName: "midas-task", + Logger: l.Slogger(), + Schema: cfg.RiverQueueSchema, + DefaultMaxAttempts: 1, Queues: map[string]river.QueueConfig{ river.QueueDefault: {MaxWorkers: cfg.MaxWorkers}, + // worker.HighPriorityQueue: {MaxWorkers: 100}, }, - Schema: cfg.RiverQueueSchema, - Workers: workers, - PeriodicJobs: periodicJobs, - ErrorHandler: pgqueue.NewErrorHandler(l), - MaxAttempts: 1, - } - pgq, err := pgqueue.New(ctx, dbpool, rivercfg) + // Register all River workers + RegisterWorkers: func(ws *river.Workers) error { + // Workers that publish into the in-process bus will be injected after creation + // Used for tasks with high throughput + river.AddWorker(ws, alertEventW) + river.AddWorker(ws, exprTsComputePartialEventW) + + // Other event workers + river.AddWorker(ws, worker.NewEmailEventWorker(dbservice, taskServices)) + + // Periodic workers + river.AddWorker(ws, worker.NewAlertScheduleWorkerV1(dbservice)) + if tlgx, e := worker.NewFetchThinglogixScheduleWorker(ctx, dbservice, &cfg.ThinglogixConfig, l); e != nil { + l.Error(ctx, "failed to initialize thinglogix; skipping worker", "error", e) + } else { + river.AddWorker(ws, tlgx) + } + return nil + }, + Periodic: func(p *pgqueue.Periodics) error { + worker.RegisterAlertScheduleV1(p) + worker.RegisterFetchThinglogixPeriodic(p) + return nil + }, + }) if err != nil { return err } - dbservice.PGQueue = pgq + // TODO: This isn't a great pattern, if we forget to inject q into dbservice.PGQueue we will panic at runtime for certain tasks. + // Inject q into workers and injected database + dbservice.PGQueue = q + alertEventW.PGQueue = q + exprTsComputeFullEventW.PGQueue = q + exprTsComputePartialEventW.PGQueue = q + + // Start River workers defer func() { - err = errors.Join(err, pgq.Stop(ctx)) + err = errors.Join(err, q.Stop(ctx, riverTimeout)) }() - l.Info(ctx, "Starting worker pool (background)...") - if err := pgq.Start(ctx); err != nil { + if err := q.Start(ctx); err != nil { return fmt.Errorf("error starting riverqueue client: %w", err) } - l.Info(ctx, "Starting ListenPoolProcess for AlertEventWorker") - return alertEventWorker.ListenPoolProcess(ctx, cfg, l) + // Build in-process routes and run the aggregator (blocks until ctx done or error) + routes := pgqueue.Routes{} + if alertEventW != nil { + k, r := alertEventW.Route() + routes[k] = r + } + if exprTsComputePartialEventW != nil { + k, r := exprTsComputePartialEventW.Route() + routes[k] = r + } + + l.Info(ctx, "Starting in-process batch dispatcher...") + bw := pgqueue.NewBatchWorker(cfg, l, q.WorkerBus.Sub, routes) + return bw.Start(ctx) } diff --git a/api/cmd/midas-telemetry/main.go b/api/cmd/midas-telemetry/main.go index 30b3c95d..f03d476a 100644 --- a/api/cmd/midas-telemetry/main.go +++ b/api/cmd/midas-telemetry/main.go @@ -20,7 +20,7 @@ func main() { } } -func run(ctx context.Context, cfg *config.TelemetryConfig, l *logger.Logger) error { +func run(ctx context.Context, cfg *config.TelemetryConfig, l logger.Logger) error { h, err := handler.NewTelemetry(ctx, cfg, l) if err != nil { return err diff --git a/api/go.mod b/api/go.mod index ad9b99de..be456548 100644 --- a/api/go.mod +++ b/api/go.mod @@ -16,31 +16,41 @@ require ( github.com/danielgtaylor/huma/v2 v2.32.0 github.com/gofrs/uuid v4.4.0+incompatible github.com/golang-jwt/jwt/v5 v5.2.3 + github.com/google/cel-go v0.26.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 - github.com/jackc/pgx/v5 v5.7.4 + github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/jackc/pgx/v5 v5.7.5 github.com/labstack/echo/v4 v4.13.3 - github.com/riverqueue/river v0.22.0 - github.com/riverqueue/river/riverdriver/riverpgxv5 v0.22.0 - github.com/riverqueue/river/rivertype v0.22.0 - github.com/stretchr/testify v1.10.0 + github.com/riverqueue/river v0.24.0 + github.com/riverqueue/river/riverdriver/riverpgxv5 v0.24.0 + github.com/riverqueue/river/rivertype v0.24.0 + github.com/robfig/cron/v3 v3.0.1 + github.com/stretchr/testify v1.11.0 github.com/tidwall/btree v1.7.0 github.com/twpayne/go-geom v1.6.1 github.com/twpayne/pgx-geom v1.0.0 github.com/xeipuuv/gojsonschema v1.2.0 + github.com/xnacly/go-iso8601-duration v1.1.0 github.com/xuri/excelize/v2 v2.9.0 gocloud.dev v0.43.0 - golang.org/x/crypto v0.40.0 - golang.org/x/term v0.33.0 - golang.org/x/text v0.27.0 + golang.org/x/crypto v0.41.0 + golang.org/x/term v0.34.0 + golang.org/x/text v0.28.0 + google.golang.org/protobuf v1.36.6 ) require ( + cel.dev/expr v0.24.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/aws/aws-sdk-go v1.55.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.70 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.84 // indirect +) + +require ( github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect @@ -72,8 +82,9 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.4 // indirect - github.com/riverqueue/river/riverdriver v0.22.0 // indirect - github.com/riverqueue/river/rivershared v0.22.0 // indirect + github.com/riverqueue/river/riverdriver v0.24.0 // indirect + github.com/riverqueue/river/rivershared v0.24.0 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -91,15 +102,16 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect go.opentelemetry.io/otel/trace v1.37.0 // indirect go.uber.org/goleak v1.3.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/image v0.27.0 // indirect golang.org/x/net v0.42.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sys v0.35.0 // indirect golang.org/x/time v0.12.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.242.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250715232539-7130f93afb79 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250715232539-7130f93afb79 // indirect google.golang.org/grpc v1.73.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/api/go.sum b/api/go.sum index e8feaeb0..0ee8e9ae 100644 --- a/api/go.sum +++ b/api/go.sum @@ -26,8 +26,6 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/USACE/instrumentation-api/api v0.0.0-20250128053738-f5dcec98f2b4 h1:+V8qMbEmvpCUTBalr9TfejD6cPWugHLWKkU8E57WTM8= -github.com/USACE/instrumentation-api/api v0.0.0-20250128053738-f5dcec98f2b4/go.mod h1:j5bSeEjURqIXv6W1/RZjbcvabAcuZwmkQFsBXjnu3Wc= github.com/USACE/instrumentation-api/api v0.0.0-20251007013530-2901caab5e33 h1:jCkSI3UqOvV2VMEaxcPBoIaj+R5VCkQ0W5EjCMKf4NM= github.com/USACE/instrumentation-api/api v0.0.0-20251007013530-2901caab5e33/go.mod h1:j5bSeEjURqIXv6W1/RZjbcvabAcuZwmkQFsBXjnu3Wc= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -35,6 +33,8 @@ github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZ github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/aws/aws-lambda-go v1.48.0 h1:1aZUYsrJu0yo5fC4z+Rba1KhNImXcJcvHu763BxoyIo= @@ -129,6 +129,8 @@ github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVI github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -151,6 +153,8 @@ github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81 github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -160,8 +164,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.4 h1:9wKznZrhWa2QiHL+NjTSPP6yjl3451BX3imWDnokYlg= -github.com/jackc/pgx/v5 v5.7.4/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -202,18 +206,16 @@ github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7 github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/riverqueue/river v0.22.0 h1:PO4Ula2RqViQqNs6xjze7yFV6Zq4T3Ffv092+f4S8xQ= -github.com/riverqueue/river v0.22.0/go.mod h1:IRoWoK4RGCiPuVJUV4EWcCl9d/TMQYkk0EEYV/Wgq+U= -github.com/riverqueue/river/riverdriver v0.22.0 h1:i7OSFkUi6x4UKvttdFOIg7NYLYaBOFLJZvkZ0+JWS/8= -github.com/riverqueue/river/riverdriver v0.22.0/go.mod h1:oNdjJCeAJhN/UiZGLNL+guNqWaxMFuSD4lr5x/v/was= -github.com/riverqueue/river/riverdriver/riverdatabasesql v0.22.0 h1:+no3gToOK9SmWg0pDPKfOGSCsrxqqaFdD8K1NQndRbY= -github.com/riverqueue/river/riverdriver/riverdatabasesql v0.22.0/go.mod h1:mygiHa1dnlKRjxT1//wIvfT2fMTbfXKm37NcsxoyBoQ= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.22.0 h1:2TWbVL73gipJ2/4JNCQbifaNj+BCC/Zxpp30o1D8RTg= -github.com/riverqueue/river/riverdriver/riverpgxv5 v0.22.0/go.mod h1:TZY/BG8w/nDxkraAEvvgyVupIz0b4+PQVUW0kIiy1fc= -github.com/riverqueue/river/rivershared v0.22.0 h1:hLPHr98d6OEfmUJ4KpIXgoy2tbQ14htWILcRBHJF11U= -github.com/riverqueue/river/rivershared v0.22.0/go.mod h1:BK+hvhECfdDLWNDH3xiGI95m2YoPfVtECZLT+my8XM8= -github.com/riverqueue/river/rivertype v0.22.0 h1:rSRhbd5uV/BaFTPxReCxuYTAzx+/riBZJlZdREADvO4= -github.com/riverqueue/river/rivertype v0.22.0/go.mod h1:lmdl3vLNDfchDWbYdW2uAocIuwIN+ZaXqAukdSCFqWs= +github.com/riverqueue/river v0.24.0 h1:CesL6vymWgz0d+zNwtnSGRWaB+E8Dax+o9cxD7sUmKc= +github.com/riverqueue/river v0.24.0/go.mod h1:UZ3AxU5t6WtyqNssaea/AkRS8h/kJ+E9ImSB3xyb3ns= +github.com/riverqueue/river/riverdriver v0.24.0 h1:HqGgGkls11u+YKDA7cKOdYKlQwRNJyHuGa3UtOvpdT0= +github.com/riverqueue/river/riverdriver v0.24.0/go.mod h1:dEew9DDIKenNvzpm8Edw8+PkqP3c0zl1fKjiQTq2n/w= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.24.0 h1:yV37OIbRrhRwIiGeRT7P4D3szhAemu87BgCf8gTCoU4= +github.com/riverqueue/river/riverdriver/riverpgxv5 v0.24.0/go.mod h1:QfznySVKC4ljx53syd/bA/LRSsydAyuD3Q9/EbSniKA= +github.com/riverqueue/river/rivershared v0.24.0 h1:KysokksW75pug2a5RTOc6WESOupWmsylVc6VWvAx+4Y= +github.com/riverqueue/river/rivershared v0.24.0/go.mod h1:UIBfSdai0oWFlwFcoqG4DZX83iA/fLWTEBGrj7Oe1ho= +github.com/riverqueue/river/rivertype v0.24.0 h1:xrQZm/h6U8TBPyTsQPYD5leOapuoBAcdz30bdBwTqOg= +github.com/riverqueue/river/rivertype v0.24.0/go.mod h1:lmdl3vLNDfchDWbYdW2uAocIuwIN+ZaXqAukdSCFqWs= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= @@ -222,11 +224,14 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= +github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -253,6 +258,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xnacly/go-iso8601-duration v1.1.0 h1:5FQeoPSs0lxzCGp82TLR8F6Rm3N6idawnNBf+fsJSZs= +github.com/xnacly/go-iso8601-duration v1.1.0/go.mod h1:S1+1mC/X7HsHlc0DZNB+ljkZoopmYjzJnEXTHkq/nKE= github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE= @@ -290,8 +297,10 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.27.0 h1:C8gA4oWU/tKkdCfYT6T2u4faJu3MeNS5O8UPWlPF61w= golang.org/x/image v0.27.0/go.mod h1:xbdrClrAUway1MUTEZDq9mz/UpRwYAkFFNUslZtcB+g= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -331,16 +340,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -348,8 +357,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -379,6 +388,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/api/internal/cloud/api.go b/api/internal/cloud/api.go index e55215e3..780521d5 100644 --- a/api/internal/cloud/api.go +++ b/api/internal/cloud/api.go @@ -18,10 +18,6 @@ type ApiServices struct { MediaBucket *blob.Bucket // ReportTaskPub publishes report jobs to a queue that are sent to an async PDF report renderer ReportTaskPub *pubsub.Topic - // AlertEventBatcherPub publishes alert event messages to an in-memory queue - AlertEventBatcherPub *pubsub.Topic - // AlertEventBatcherSub subscribes to the AlertEventBatcherPub topic to batch messages for PGQueue - AlertEventBatcherSub *pubsub.Subscription } // NewApiServices creates a new ApiServices instance @@ -36,22 +32,9 @@ func NewApiServices(ctx context.Context, cfg *config.ApiConfig) (*ApiServices, e return nil, err } - mq := "mem://task-batch-send" - alertEventBatcherPub, err := pubsub.OpenTopic(ctx, mq) - if err != nil { - return nil, err - } - - alertEventBatcherSub, err := pubsub.OpenSubscription(ctx, mq) - if err != nil { - return nil, err - } - return &ApiServices{ - MediaBucket: mediaBucket, - ReportTaskPub: reportTaskPub, - AlertEventBatcherPub: alertEventBatcherPub, - AlertEventBatcherSub: alertEventBatcherSub, + MediaBucket: mediaBucket, + ReportTaskPub: reportTaskPub, }, nil } @@ -64,11 +47,5 @@ func (s *ApiServices) Shutdown(ctx context.Context) error { if a := s.ReportTaskPub; a != nil { errs = append(errs, a.Shutdown(ctx)) } - if a := s.AlertEventBatcherPub; a != nil { - errs = append(errs, a.Shutdown(ctx)) - } - if a := s.AlertEventBatcherSub; a != nil { - errs = append(errs, a.Shutdown(ctx)) - } return errors.Join(errs...) } diff --git a/api/internal/cloud/dcsloader.go b/api/internal/cloud/dcsloader.go index 0d2c9428..1338f65a 100644 --- a/api/internal/cloud/dcsloader.go +++ b/api/internal/cloud/dcsloader.go @@ -55,7 +55,7 @@ func (s *DcsLoaderServices) Shutdown(ctx context.Context) error { } // messageToS3Event converts a pubsub message to an S3 event -func MessageToS3Event(ctx context.Context, msg *pubsub.Message, l *logger.Logger) (events.S3Event, error) { +func MessageToS3Event(ctx context.Context, msg *pubsub.Message, l logger.Logger) (events.S3Event, error) { var evt events.S3Event var sqsMessage sqstypes.Message diff --git a/api/internal/db/batch.go b/api/internal/db/batch.go index 43af0d4b..94dc027c 100644 --- a/api/internal/db/batch.go +++ b/api/internal/db/batch.go @@ -657,6 +657,118 @@ func (b *EvaluationInstrumentCreateBatchBatchResults) Close() error { return b.br.Close() } +const expressionGetBatch = `-- name: ExpressionGetBatch :batchone +select id, instrument_id, name, expression, mode, created_at, created_by, updated_at, updated_by, variables, target_timeseries_id, opts +from v_expression +where id = $1 +` + +type ExpressionGetBatchBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +func (q *Queries) ExpressionGetBatch(ctx context.Context, id []uuid.UUID) *ExpressionGetBatchBatchResults { + batch := &pgx.Batch{} + for _, a := range id { + vals := []interface{}{ + a, + } + batch.Queue(expressionGetBatch, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &ExpressionGetBatchBatchResults{br, len(id), false} +} + +func (b *ExpressionGetBatchBatchResults) QueryRow(f func(int, VExpression, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + var i VExpression + if b.closed { + if f != nil { + f(t, i, ErrBatchAlreadyClosed) + } + continue + } + row := b.br.QueryRow() + err := row.Scan( + &i.ID, + &i.InstrumentID, + &i.Name, + &i.Expression, + &i.Mode, + &i.CreatedAt, + &i.CreatedBy, + &i.UpdatedAt, + &i.UpdatedBy, + &i.Variables, + &i.TargetTimeseriesID, + &i.Opts, + ) + if f != nil { + f(t, i, err) + } + } +} + +func (b *ExpressionGetBatchBatchResults) Close() error { + b.closed = true + return b.br.Close() +} + +const expressionTimeseriesVariableCreateOrUpdateBatch = `-- name: ExpressionTimeseriesVariableCreateOrUpdateBatch :batchexec +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values ($1, $2, $3) +on conflict (expression_id, timeseries_id) do update set variable_name = excluded.variable_name +` + +type ExpressionTimeseriesVariableCreateOrUpdateBatchBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type ExpressionTimeseriesVariableCreateOrUpdateBatchParams struct { + ExpressionID uuid.UUID `json:"expression_id"` + TimeseriesID uuid.UUID `json:"timeseries_id"` + VariableName string `json:"variable_name"` +} + +func (q *Queries) ExpressionTimeseriesVariableCreateOrUpdateBatch(ctx context.Context, arg []ExpressionTimeseriesVariableCreateOrUpdateBatchParams) *ExpressionTimeseriesVariableCreateOrUpdateBatchBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []interface{}{ + a.ExpressionID, + a.TimeseriesID, + a.VariableName, + } + batch.Queue(expressionTimeseriesVariableCreateOrUpdateBatch, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &ExpressionTimeseriesVariableCreateOrUpdateBatchBatchResults{br, len(arg), false} +} + +func (b *ExpressionTimeseriesVariableCreateOrUpdateBatchBatchResults) Exec(f func(int, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + if b.closed { + if f != nil { + f(t, ErrBatchAlreadyClosed) + } + continue + } + _, err := b.br.Exec() + if f != nil { + f(t, err) + } + } +} + +func (b *ExpressionTimeseriesVariableCreateOrUpdateBatchBatchResults) Close() error { + b.closed = true + return b.br.Close() +} + const inclOptsCreateBatch = `-- name: InclOptsCreateBatch :batchexec insert into incl_opts (instrument_id, num_segments, bottom_elevation_timeseries_id, initial_time) values ($1, $2, $3, $4) @@ -3126,90 +3238,86 @@ func (b *TimeseriesMeasurementCreateOrUpdateBatchBatchResults) Close() error { return b.br.Close() } -const timeseriesMeasurementCreateOrUpdateBatchShouldAlertCheck = `-- name: TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheck :batchone -insert into timeseries_measurement (timeseries_id, time, value) values ($1, $2, $3) -on conflict on constraint timeseries_unique_time do update set value = excluded.value -returning (select true = any(select true from alert_config_timeseries where timeseries_id = $1)) +const timeseriesMeasurementDeleteBatch = `-- name: TimeseriesMeasurementDeleteBatch :batchexec +delete from timeseries_measurement where timeseries_id=$1 and time=$2 ` -type TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckBatchResults struct { +type TimeseriesMeasurementDeleteBatchBatchResults struct { br pgx.BatchResults tot int closed bool } -type TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckParams struct { +type TimeseriesMeasurementDeleteBatchParams struct { TimeseriesID uuid.UUID `json:"timeseries_id"` Time time.Time `json:"time"` - Value float64 `json:"value"` } -func (q *Queries) TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheck(ctx context.Context, arg []TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckParams) *TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckBatchResults { +func (q *Queries) TimeseriesMeasurementDeleteBatch(ctx context.Context, arg []TimeseriesMeasurementDeleteBatchParams) *TimeseriesMeasurementDeleteBatchBatchResults { batch := &pgx.Batch{} for _, a := range arg { vals := []interface{}{ a.TimeseriesID, a.Time, - a.Value, } - batch.Queue(timeseriesMeasurementCreateOrUpdateBatchShouldAlertCheck, vals...) + batch.Queue(timeseriesMeasurementDeleteBatch, vals...) } br := q.db.SendBatch(ctx, batch) - return &TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckBatchResults{br, len(arg), false} + return &TimeseriesMeasurementDeleteBatchBatchResults{br, len(arg), false} } -func (b *TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckBatchResults) QueryRow(f func(int, interface{}, error)) { +func (b *TimeseriesMeasurementDeleteBatchBatchResults) Exec(f func(int, error)) { defer b.br.Close() for t := 0; t < b.tot; t++ { - var exists interface{} if b.closed { if f != nil { - f(t, exists, ErrBatchAlreadyClosed) + f(t, ErrBatchAlreadyClosed) } continue } - row := b.br.QueryRow() - err := row.Scan(&exists) + _, err := b.br.Exec() if f != nil { - f(t, exists, err) + f(t, err) } } } -func (b *TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckBatchResults) Close() error { +func (b *TimeseriesMeasurementDeleteBatchBatchResults) Close() error { b.closed = true return b.br.Close() } -const timeseriesMeasurementDeleteBatch = `-- name: TimeseriesMeasurementDeleteBatch :batchexec -delete from timeseries_measurement where timeseries_id=$1 and time=$2 +const timeseriesMeasurementDeleteRangeBatch = `-- name: TimeseriesMeasurementDeleteRangeBatch :batchexec +delete from timeseries_measurement where timeseries_id = $1 and time > $2 and time < $3 ` -type TimeseriesMeasurementDeleteBatchBatchResults struct { +type TimeseriesMeasurementDeleteRangeBatchBatchResults struct { br pgx.BatchResults tot int closed bool } -type TimeseriesMeasurementDeleteBatchParams struct { +type TimeseriesMeasurementDeleteRangeBatchParams struct { TimeseriesID uuid.UUID `json:"timeseries_id"` - Time time.Time `json:"time"` + After time.Time `json:"after"` + Before time.Time `json:"before"` } -func (q *Queries) TimeseriesMeasurementDeleteBatch(ctx context.Context, arg []TimeseriesMeasurementDeleteBatchParams) *TimeseriesMeasurementDeleteBatchBatchResults { +func (q *Queries) TimeseriesMeasurementDeleteRangeBatch(ctx context.Context, arg []TimeseriesMeasurementDeleteRangeBatchParams) *TimeseriesMeasurementDeleteRangeBatchBatchResults { batch := &pgx.Batch{} for _, a := range arg { vals := []interface{}{ a.TimeseriesID, - a.Time, + a.After, + a.Before, } - batch.Queue(timeseriesMeasurementDeleteBatch, vals...) + batch.Queue(timeseriesMeasurementDeleteRangeBatch, vals...) } br := q.db.SendBatch(ctx, batch) - return &TimeseriesMeasurementDeleteBatchBatchResults{br, len(arg), false} + return &TimeseriesMeasurementDeleteRangeBatchBatchResults{br, len(arg), false} } -func (b *TimeseriesMeasurementDeleteBatchBatchResults) Exec(f func(int, error)) { +func (b *TimeseriesMeasurementDeleteRangeBatchBatchResults) Exec(f func(int, error)) { defer b.br.Close() for t := 0; t < b.tot; t++ { if b.closed { @@ -3225,58 +3333,229 @@ func (b *TimeseriesMeasurementDeleteBatchBatchResults) Exec(f func(int, error)) } } -func (b *TimeseriesMeasurementDeleteBatchBatchResults) Close() error { +func (b *TimeseriesMeasurementDeleteRangeBatchBatchResults) Close() error { b.closed = true return b.br.Close() } -const timeseriesMeasurementDeleteRangeBatch = `-- name: TimeseriesMeasurementDeleteRangeBatch :batchexec -delete from timeseries_measurement where timeseries_id = $1 and time > $2 and time < $3 +const timeseriesMeasurementListBatchEdgesForTimeseriesIDs = `-- name: TimeseriesMeasurementListBatchEdgesForTimeseriesIDs :batchmany +select + ts.id as timeseries_id, + b.time as before_time, + b.value as before_value, + coalesce(bn.masked, false) as before_masked, + coalesce(bn.validated, false) as before_validated, + a.time as after_time, + a.value as after_value, + coalesce(an.masked, false) as after_masked, + coalesce(an.validated, false) as after_validated +from timeseries ts +left join lateral ( + select m.time, m.value + from timeseries_measurement m + where m.timeseries_id = ts.id + and m.time < $1::timestamptz + order by m.time desc + limit 1 +) b on true +left join lateral ( + select n.masked, n.validated + from timeseries_notes n + where n.timeseries_id = ts.id + and n.time = b.time + limit 1 +) bn on true +left join lateral ( + select m.time, m.value + from timeseries_measurement m + where m.timeseries_id = ts.id + and m.time >= $2::timestamptz + order by m.time asc + limit 1 +) a on true +left join lateral ( + select n.masked, n.validated + from timeseries_notes n + where n.timeseries_id = ts.id + and n.time = a.time + limit 1 +) an on true +where ts.id = any($3::uuid[]) ` -type TimeseriesMeasurementDeleteRangeBatchBatchResults struct { +type TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults struct { br pgx.BatchResults tot int closed bool } -type TimeseriesMeasurementDeleteRangeBatchParams struct { +type TimeseriesMeasurementListBatchEdgesForTimeseriesIDsParams struct { + StartsAt time.Time `json:"starts_at"` + EndsBefore time.Time `json:"ends_before"` + TimeseriesIDs []uuid.UUID `json:"timeseries_ids"` +} + +type TimeseriesMeasurementListBatchEdgesForTimeseriesIDsRow struct { + TimeseriesID uuid.UUID `json:"timeseries_id"` + BeforeTime time.Time `json:"before_time"` + BeforeValue float64 `json:"before_value"` + BeforeMasked bool `json:"before_masked"` + BeforeValidated bool `json:"before_validated"` + AfterTime time.Time `json:"after_time"` + AfterValue float64 `json:"after_value"` + AfterMasked bool `json:"after_masked"` + AfterValidated bool `json:"after_validated"` +} + +func (q *Queries) TimeseriesMeasurementListBatchEdgesForTimeseriesIDs(ctx context.Context, arg []TimeseriesMeasurementListBatchEdgesForTimeseriesIDsParams) *TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []interface{}{ + a.StartsAt, + a.EndsBefore, + a.TimeseriesIDs, + } + batch.Queue(timeseriesMeasurementListBatchEdgesForTimeseriesIDs, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults{br, len(arg), false} +} + +func (b *TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults) Query(f func(int, []TimeseriesMeasurementListBatchEdgesForTimeseriesIDsRow, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + items := []TimeseriesMeasurementListBatchEdgesForTimeseriesIDsRow{} + if b.closed { + if f != nil { + f(t, items, ErrBatchAlreadyClosed) + } + continue + } + err := func() error { + rows, err := b.br.Query() + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var i TimeseriesMeasurementListBatchEdgesForTimeseriesIDsRow + if err := rows.Scan( + &i.TimeseriesID, + &i.BeforeTime, + &i.BeforeValue, + &i.BeforeMasked, + &i.BeforeValidated, + &i.AfterTime, + &i.AfterValue, + &i.AfterMasked, + &i.AfterValidated, + ); err != nil { + return err + } + items = append(items, i) + } + return rows.Err() + }() + if f != nil { + f(t, items, err) + } + } +} + +func (b *TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults) Close() error { + b.closed = true + return b.br.Close() +} + +const timeseriesMeasurementListBatchForTimeseriesIDs = `-- name: TimeseriesMeasurementListBatchForTimeseriesIDs :batchmany +select + m.timeseries_id, + m.time, + m.value, + coalesce(m.masked, false) as masked, + coalesce(m.validated, false) as validated +from v_timeseries_measurement m +where m.timeseries_id = any($1::uuid[]) +and m.time >= $2::timestamptz +and m.time < $3::timestamptz +order by m.time asc +limit $4::int +` + +type TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type TimeseriesMeasurementListBatchForTimeseriesIDsParams struct { + TimeseriesIDs []uuid.UUID `json:"timeseries_ids"` + StartsAt time.Time `json:"starts_at"` + EndsBefore time.Time `json:"ends_before"` + PageSize int32 `json:"page_size"` +} + +type TimeseriesMeasurementListBatchForTimeseriesIDsRow struct { TimeseriesID uuid.UUID `json:"timeseries_id"` - After time.Time `json:"after"` - Before time.Time `json:"before"` + Time time.Time `json:"time"` + Value float64 `json:"value"` + Masked bool `json:"masked"` + Validated bool `json:"validated"` } -func (q *Queries) TimeseriesMeasurementDeleteRangeBatch(ctx context.Context, arg []TimeseriesMeasurementDeleteRangeBatchParams) *TimeseriesMeasurementDeleteRangeBatchBatchResults { +func (q *Queries) TimeseriesMeasurementListBatchForTimeseriesIDs(ctx context.Context, arg []TimeseriesMeasurementListBatchForTimeseriesIDsParams) *TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults { batch := &pgx.Batch{} for _, a := range arg { vals := []interface{}{ - a.TimeseriesID, - a.After, - a.Before, + a.TimeseriesIDs, + a.StartsAt, + a.EndsBefore, + a.PageSize, } - batch.Queue(timeseriesMeasurementDeleteRangeBatch, vals...) + batch.Queue(timeseriesMeasurementListBatchForTimeseriesIDs, vals...) } br := q.db.SendBatch(ctx, batch) - return &TimeseriesMeasurementDeleteRangeBatchBatchResults{br, len(arg), false} + return &TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults{br, len(arg), false} } -func (b *TimeseriesMeasurementDeleteRangeBatchBatchResults) Exec(f func(int, error)) { +func (b *TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults) Query(f func(int, []TimeseriesMeasurementListBatchForTimeseriesIDsRow, error)) { defer b.br.Close() for t := 0; t < b.tot; t++ { + items := []TimeseriesMeasurementListBatchForTimeseriesIDsRow{} if b.closed { if f != nil { - f(t, ErrBatchAlreadyClosed) + f(t, items, ErrBatchAlreadyClosed) } continue } - _, err := b.br.Exec() + err := func() error { + rows, err := b.br.Query() + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var i TimeseriesMeasurementListBatchForTimeseriesIDsRow + if err := rows.Scan( + &i.TimeseriesID, + &i.Time, + &i.Value, + &i.Masked, + &i.Validated, + ); err != nil { + return err + } + items = append(items, i) + } + return rows.Err() + }() if f != nil { - f(t, err) + f(t, items, err) } } } -func (b *TimeseriesMeasurementDeleteRangeBatchBatchResults) Close() error { +func (b *TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults) Close() error { b.closed = true return b.br.Close() } diff --git a/api/internal/db/expression.sql_gen.go b/api/internal/db/expression.sql_gen.go new file mode 100644 index 00000000..863b3090 --- /dev/null +++ b/api/internal/db/expression.sql_gen.go @@ -0,0 +1,335 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: expression.sql + +package db + +import ( + "context" + + "github.com/google/uuid" +) + +const expressionCreate = `-- name: ExpressionCreate :one +insert into expression (instrument_id, name, expression, mode, created_by) values ($1, $2, $3, $4, $5) +returning id +` + +type ExpressionCreateParams struct { + InstrumentID uuid.UUID `json:"instrument_id"` + Name string `json:"name"` + Expression string `json:"expression"` + Mode ExpressionMode `json:"mode"` + CreatedBy uuid.UUID `json:"created_by"` +} + +func (q *Queries) ExpressionCreate(ctx context.Context, arg ExpressionCreateParams) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, expressionCreate, + arg.InstrumentID, + arg.Name, + arg.Expression, + arg.Mode, + arg.CreatedBy, + ) + var id uuid.UUID + err := row.Scan(&id) + return id, err +} + +const expressionDelete = `-- name: ExpressionDelete :exec +delete from expression +where id = $1 +` + +func (q *Queries) ExpressionDelete(ctx context.Context, id uuid.UUID) error { + _, err := q.db.Exec(ctx, expressionDelete, id) + return err +} + +const expressionDownstreamListForTimeseries = `-- name: ExpressionDownstreamListForTimeseries :many +with recursive expr_deps as ( + -- Anchor term: get direct downstreams + select + e.id as expression_id, + tgt.timeseries_id as target_timeseries_id, + var.timeseries_id as variable_timeseries_id, + 1 as depth + from expression_timeseries_variable var + inner join expression e on e.id = var.expression_id + inner join expression_timeseries_target tgt on tgt.expression_id = e.id + where var.timeseries_id = any($1::uuid[]) + + union all + + -- Recursive term: get downstreams of downstreams + select + e.id as expression_id, + tgt.timeseries_id as target_timeseries_id, + var.timeseries_id as variable_timeseries_id, + ed.depth + 1 as depth + from expr_deps ed + join expression_timeseries_variable var on var.timeseries_id = ed.target_timeseries_id + join expression e on e.id = var.expression_id + join expression_timeseries_target tgt on tgt.expression_id = e.id + where ed.depth < 10 -- for safety, max depth should be below this +) +select + expression_id, + target_timeseries_id, + array_agg(variable_timeseries_id)::uuid[] as variable_timeseries_ids, + min(depth)::int as min_depth, + max(depth)::int as max_depth +from expr_deps +group by expression_id, target_timeseries_id +` + +type ExpressionDownstreamListForTimeseriesRow struct { + ExpressionID uuid.UUID `json:"expression_id"` + TargetTimeseriesID uuid.UUID `json:"target_timeseries_id"` + VariableTimeseriesIds []uuid.UUID `json:"variable_timeseries_ids"` + MinDepth int32 `json:"min_depth"` + MaxDepth int32 `json:"max_depth"` +} + +func (q *Queries) ExpressionDownstreamListForTimeseries(ctx context.Context, depTimeseriesIds []uuid.UUID) ([]ExpressionDownstreamListForTimeseriesRow, error) { + rows, err := q.db.Query(ctx, expressionDownstreamListForTimeseries, depTimeseriesIds) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ExpressionDownstreamListForTimeseriesRow{} + for rows.Next() { + var i ExpressionDownstreamListForTimeseriesRow + if err := rows.Scan( + &i.ExpressionID, + &i.TargetTimeseriesID, + &i.VariableTimeseriesIds, + &i.MinDepth, + &i.MaxDepth, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const expressionGet = `-- name: ExpressionGet :one +select id, instrument_id, name, expression, mode, created_at, created_by, updated_at, updated_by, variables, target_timeseries_id, opts +from v_expression +where id = $1 +` + +func (q *Queries) ExpressionGet(ctx context.Context, id uuid.UUID) (VExpression, error) { + row := q.db.QueryRow(ctx, expressionGet, id) + var i VExpression + err := row.Scan( + &i.ID, + &i.InstrumentID, + &i.Name, + &i.Expression, + &i.Mode, + &i.CreatedAt, + &i.CreatedBy, + &i.UpdatedAt, + &i.UpdatedBy, + &i.Variables, + &i.TargetTimeseriesID, + &i.Opts, + ) + return i, err +} + +const expressionGetWouldCreateCycleOrExceedDepth = `-- name: ExpressionGetWouldCreateCycleOrExceedDepth :one +with recursive dep_graph as ( + -- Anchor: start from the target timeseries of the expression being checked + select + tgt.timeseries_id as start_timeseries_id, + tgt.timeseries_id as current_timeseries_id, + tgt.expression_id as current_expression_id, + 0 as depth + from expression_timeseries_target tgt + where tgt.expression_id = $1::uuid + + union all + + -- Recursive: follow dependencies from the current expression + select + dep_graph.start_timeseries_id, + var.timeseries_id as current_timeseries_id, + var.expression_id as current_expression_id, + dep_graph.depth + 1 as depth + from dep_graph + join expression_timeseries_variable var + on var.expression_id = dep_graph.current_expression_id + join expression_timeseries_target tgt + on tgt.expression_id = var.expression_id + where dep_graph.depth < 5 +) +select + exists ( + select 1 + from dep_graph + where start_timeseries_id = current_timeseries_id + and depth > 0 -- exclude the anchor node itself + ) as would_create_cycle, + max(depth) > 5 as would_exceed_max_depth +from dep_graph +` + +type ExpressionGetWouldCreateCycleOrExceedDepthRow struct { + WouldCreateCycle bool `json:"would_create_cycle"` + WouldExceedMaxDepth bool `json:"would_exceed_max_depth"` +} + +func (q *Queries) ExpressionGetWouldCreateCycleOrExceedDepth(ctx context.Context, expressionID uuid.UUID) (ExpressionGetWouldCreateCycleOrExceedDepthRow, error) { + row := q.db.QueryRow(ctx, expressionGetWouldCreateCycleOrExceedDepth, expressionID) + var i ExpressionGetWouldCreateCycleOrExceedDepthRow + err := row.Scan(&i.WouldCreateCycle, &i.WouldExceedMaxDepth) + return i, err +} + +const expressionListForInstrument = `-- name: ExpressionListForInstrument :many +select id, instrument_id, name, expression, mode, created_at, created_by, updated_at, updated_by, variables, target_timeseries_id, opts +from v_expression +where instrument_id = $1 +order by name asc +` + +func (q *Queries) ExpressionListForInstrument(ctx context.Context, instrumentID uuid.UUID) ([]VExpression, error) { + rows, err := q.db.Query(ctx, expressionListForInstrument, instrumentID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []VExpression{} + for rows.Next() { + var i VExpression + if err := rows.Scan( + &i.ID, + &i.InstrumentID, + &i.Name, + &i.Expression, + &i.Mode, + &i.CreatedAt, + &i.CreatedBy, + &i.UpdatedAt, + &i.UpdatedBy, + &i.Variables, + &i.TargetTimeseriesID, + &i.Opts, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const expressionTimeseriesTargetCreate = `-- name: ExpressionTimeseriesTargetCreate :exec +insert into expression_timeseries_target (expression_id, timeseries_id) values ($1, $2) +` + +type ExpressionTimeseriesTargetCreateParams struct { + ExpressionID uuid.UUID `json:"expression_id"` + TimeseriesID uuid.UUID `json:"timeseries_id"` +} + +func (q *Queries) ExpressionTimeseriesTargetCreate(ctx context.Context, arg ExpressionTimeseriesTargetCreateParams) error { + _, err := q.db.Exec(ctx, expressionTimeseriesTargetCreate, arg.ExpressionID, arg.TimeseriesID) + return err +} + +const expressionTimeseriesVariableDeleteAllForExpression = `-- name: ExpressionTimeseriesVariableDeleteAllForExpression :exec +delete from expression_timeseries_variable +where expression_id = $1 +` + +func (q *Queries) ExpressionTimeseriesVariableDeleteAllForExpression(ctx context.Context, expressionID uuid.UUID) error { + _, err := q.db.Exec(ctx, expressionTimeseriesVariableDeleteAllForExpression, expressionID) + return err +} + +const expressionUpdate = `-- name: ExpressionUpdate :one +update expression set + name = $2, + expression = $3, + mode = $4, + updated_at = now(), + updated_by = $5 +where id = $1 +returning id +` + +type ExpressionUpdateParams struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + Expression string `json:"expression"` + Mode ExpressionMode `json:"mode"` + UpdatedBy *uuid.UUID `json:"updated_by"` +} + +func (q *Queries) ExpressionUpdate(ctx context.Context, arg ExpressionUpdateParams) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, expressionUpdate, + arg.ID, + arg.Name, + arg.Expression, + arg.Mode, + arg.UpdatedBy, + ) + var id uuid.UUID + err := row.Scan(&id) + return id, err +} + +const expressionWindowOptsCreate = `-- name: ExpressionWindowOptsCreate :exec +insert into expression_window_opts (expression_id, duration, tz_name, duration_offset, timestamping_policy) +values ($1, $2, $3, $4, $5) +` + +type ExpressionWindowOptsCreateParams struct { + ExpressionID uuid.UUID `json:"expression_id"` + Duration string `json:"duration"` + TzName string `json:"tz_name"` + DurationOffset string `json:"duration_offset"` + TimestampingPolicy ExpressionWindowTimestampingPolicy `json:"timestamping_policy"` +} + +func (q *Queries) ExpressionWindowOptsCreate(ctx context.Context, arg ExpressionWindowOptsCreateParams) error { + _, err := q.db.Exec(ctx, expressionWindowOptsCreate, + arg.ExpressionID, + arg.Duration, + arg.TzName, + arg.DurationOffset, + arg.TimestampingPolicy, + ) + return err +} + +const expressionWindowOptsDelete = `-- name: ExpressionWindowOptsDelete :exec +delete from expression_window_opts where expression_id = $1 +` + +func (q *Queries) ExpressionWindowOptsDelete(ctx context.Context, expressionID uuid.UUID) error { + _, err := q.db.Exec(ctx, expressionWindowOptsDelete, expressionID) + return err +} + +const timeseriesDeleteTargetForExpression = `-- name: TimeseriesDeleteTargetForExpression :exec +delete from timeseries t +using expression_timeseries_target e +where t.id = e.timeseries_id +and e.expression_id = $1 +` + +func (q *Queries) TimeseriesDeleteTargetForExpression(ctx context.Context, expressionID uuid.UUID) error { + _, err := q.db.Exec(ctx, timeseriesDeleteTargetForExpression, expressionID) + return err +} diff --git a/api/internal/db/measurement.manual.go b/api/internal/db/measurement.manual.go index b5b67073..134f74e6 100644 --- a/api/internal/db/measurement.manual.go +++ b/api/internal/db/measurement.manual.go @@ -194,3 +194,106 @@ func LTTB[T MeasurementGetter](data []T, threshold int) []T { return sampled } + +const timeseriesMeasurementCreateOrUpdateBatchShouldCheck = ` +with upserted as ( + insert into timeseries_measurement as ts (timeseries_id, time, value) + select + u.timeseries_id, + u.time, + u.value + from unnest( + $1::uuid[], + $2::timestamptz[], + $3::double precision[] + ) as u(timeseries_id, time, value) + on conflict on constraint timeseries_unique_time do update set value = excluded.value + where ts.value is distinct from excluded.value + returning timeseries_id +) +select + u.timeseries_id, + exists ( + select 1 from alert_config_timeseries ac + where ac.timeseries_id = u.timeseries_id + ) as should_check_alert, + exists ( + select 1 from expression_timeseries_variable ev + where ev.timeseries_id = u.timeseries_id + ) as should_check_expression +from (select distinct timeseries_id from upserted) u +` + +type TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams struct { + TimeseriesIDs []uuid.UUID `json:"timeseries_ids"` + Times []time.Time `json:"times"` + Values []float64 `json:"values"` +} + +type TimeseriesMeasurementCreateOrUpdateBatchShouldCheckBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type TimeseriesMeasurementCreateOrUpdateBatchShouldCheckRow struct { + TimeseriesID uuid.UUID `db:"timeseries_id"` + ShouldCheckAlert bool `db:"should_check_alert"` + ShouldCheckExpression bool `db:"should_check_expression"` +} + +// This query must be manual because sqlc has a bug where unnest is not supported with multiple arrays. +// https://github.com/sqlc-dev/sqlc/issues/3507 +func (q *Queries) TimeseriesMeasurementCreateOrUpdateBatchShouldCheck(ctx context.Context, arg []TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams) *TimeseriesMeasurementCreateOrUpdateBatchShouldCheckBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []any{ + a.TimeseriesIDs, + a.Times, + a.Values, + } + batch.Queue(timeseriesMeasurementCreateOrUpdateBatchShouldCheck, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &TimeseriesMeasurementCreateOrUpdateBatchShouldCheckBatchResults{br, len(arg), false} +} + +func (b *TimeseriesMeasurementCreateOrUpdateBatchShouldCheckBatchResults) Query(f func(int, []TimeseriesMeasurementCreateOrUpdateBatchShouldCheckRow, error)) { + defer b.br.Close() + for t := range b.tot { + items := []TimeseriesMeasurementCreateOrUpdateBatchShouldCheckRow{} + if b.closed { + if f != nil { + f(t, items, ErrBatchAlreadyClosed) + } + continue + } + err := func() error { + rows, err := b.br.Query() + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var i TimeseriesMeasurementCreateOrUpdateBatchShouldCheckRow + if err := rows.Scan( + &i.TimeseriesID, + &i.ShouldCheckAlert, + &i.ShouldCheckExpression, + ); err != nil { + return err + } + items = append(items, i) + } + return rows.Err() + }() + if f != nil { + f(t, items, err) + } + } +} + +func (b *TimeseriesMeasurementCreateOrUpdateBatchShouldCheckBatchResults) Close() error { + b.closed = true + return b.br.Close() +} diff --git a/api/internal/db/measurement.sql_gen.go b/api/internal/db/measurement.sql_gen.go index 498dfea2..8301cecf 100644 --- a/api/internal/db/measurement.sql_gen.go +++ b/api/internal/db/measurement.sql_gen.go @@ -58,6 +58,15 @@ func (q *Queries) TimeseriesMeasurementDelete(ctx context.Context, arg Timeserie return err } +const timeseriesMeasurementDeleteAllForTimeseries = `-- name: TimeseriesMeasurementDeleteAllForTimeseries :exec +delete from timeseries_measurement where timeseries_id = $1 +` + +func (q *Queries) TimeseriesMeasurementDeleteAllForTimeseries(ctx context.Context, timeseriesID uuid.UUID) error { + _, err := q.db.Exec(ctx, timeseriesMeasurementDeleteAllForTimeseries, timeseriesID) + return err +} + const timeseriesMeasurementDeleteRange = `-- name: TimeseriesMeasurementDeleteRange :exec delete from timeseries_measurement where timeseries_id = $1 and time > $2 and time < $3 ` diff --git a/api/internal/db/models.go b/api/internal/db/models.go index 997a8271..d7b2ceeb 100644 --- a/api/internal/db/models.go +++ b/api/internal/db/models.go @@ -15,6 +15,91 @@ import ( "github.com/twpayne/go-geom" ) +type ExpressionMode string + +const ( + ExpressionModePoint ExpressionMode = "point" + ExpressionModeWindow ExpressionMode = "window" +) + +func (e *ExpressionMode) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = ExpressionMode(s) + case string: + *e = ExpressionMode(s) + default: + return fmt.Errorf("unsupported scan type for ExpressionMode: %T", src) + } + return nil +} + +type NullExpressionMode struct { + ExpressionMode ExpressionMode `json:"expression_mode"` + Valid bool `json:"valid"` // Valid is true if ExpressionMode is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullExpressionMode) Scan(value interface{}) error { + if value == nil { + ns.ExpressionMode, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.ExpressionMode.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullExpressionMode) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.ExpressionMode), nil +} + +type ExpressionWindowTimestampingPolicy string + +const ( + ExpressionWindowTimestampingPolicyStart ExpressionWindowTimestampingPolicy = "start" + ExpressionWindowTimestampingPolicyCenter ExpressionWindowTimestampingPolicy = "center" + ExpressionWindowTimestampingPolicyEnd ExpressionWindowTimestampingPolicy = "end" +) + +func (e *ExpressionWindowTimestampingPolicy) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = ExpressionWindowTimestampingPolicy(s) + case string: + *e = ExpressionWindowTimestampingPolicy(s) + default: + return fmt.Errorf("unsupported scan type for ExpressionWindowTimestampingPolicy: %T", src) + } + return nil +} + +type NullExpressionWindowTimestampingPolicy struct { + ExpressionWindowTimestampingPolicy ExpressionWindowTimestampingPolicy `json:"expression_window_timestamping_policy"` + Valid bool `json:"valid"` // Valid is true if ExpressionWindowTimestampingPolicy is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullExpressionWindowTimestampingPolicy) Scan(value interface{}) error { + if value == nil { + ns.ExpressionWindowTimestampingPolicy, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.ExpressionWindowTimestampingPolicy.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullExpressionWindowTimestampingPolicy) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.ExpressionWindowTimestampingPolicy), nil +} + type JobStatus string const ( @@ -242,10 +327,11 @@ func (ns NullTelemetryPreviewType) Value() (driver.Value, error) { type TimeseriesType string const ( - TimeseriesTypeStandard TimeseriesType = "standard" - TimeseriesTypeConstant TimeseriesType = "constant" - TimeseriesTypeComputed TimeseriesType = "computed" - TimeseriesTypeCwms TimeseriesType = "cwms" + TimeseriesTypeStandard TimeseriesType = "standard" + TimeseriesTypeConstant TimeseriesType = "constant" + TimeseriesTypeComputed TimeseriesType = "computed" + TimeseriesTypeCwms TimeseriesType = "cwms" + TimeseriesTypeExpression TimeseriesType = "expression" ) func (e *TimeseriesType) Scan(src interface{}) error { @@ -643,6 +729,37 @@ type EvaluationInstrument struct { InstrumentID *uuid.UUID `json:"instrument_id"` } +type Expression struct { + ID uuid.UUID `json:"id"` + InstrumentID uuid.UUID `json:"instrument_id"` + Name string `json:"name"` + Expression string `json:"expression"` + Mode ExpressionMode `json:"mode"` + CreatedAt time.Time `json:"created_at"` + CreatedBy uuid.UUID `json:"created_by"` + UpdatedAt *time.Time `json:"updated_at"` + UpdatedBy *uuid.UUID `json:"updated_by"` +} + +type ExpressionTimeseriesTarget struct { + ExpressionID uuid.UUID `json:"expression_id"` + TimeseriesID uuid.UUID `json:"timeseries_id"` +} + +type ExpressionTimeseriesVariable struct { + ExpressionID uuid.UUID `json:"expression_id"` + TimeseriesID uuid.UUID `json:"timeseries_id"` + VariableName string `json:"variable_name"` +} + +type ExpressionWindowOpts struct { + ExpressionID uuid.UUID `json:"expression_id"` + Duration string `json:"duration"` + TzName string `json:"tz_name"` + DurationOffset string `json:"duration_offset"` + TimestampingPolicy ExpressionWindowTimestampingPolicy `json:"timestamping_policy"` +} + type Heartbeat struct { Time time.Time `json:"time"` } @@ -1392,6 +1509,21 @@ type VEvaluation struct { Instruments []InstrumentIDName `json:"instruments"` } +type VExpression struct { + ID uuid.UUID `json:"id"` + InstrumentID uuid.UUID `json:"instrument_id"` + Name string `json:"name"` + Expression string `json:"expression"` + Mode ExpressionMode `json:"mode"` + CreatedAt time.Time `json:"created_at"` + CreatedBy uuid.UUID `json:"created_by"` + UpdatedAt *time.Time `json:"updated_at"` + UpdatedBy *uuid.UUID `json:"updated_by"` + Variables []VExpressionTimeseriesVariable `json:"variables"` + TargetTimeseriesID uuid.UUID `json:"target_timeseries_id"` + Opts json.RawMessage `json:"opts"` +} + type VInclMeasurement struct { InstrumentID uuid.UUID `json:"instrument_id"` Time time.Time `json:"time"` diff --git a/api/internal/db/overrides.go b/api/internal/db/overrides.go index 6f1b457a..d1dbfbaf 100644 --- a/api/internal/db/overrides.go +++ b/api/internal/db/overrides.go @@ -204,3 +204,8 @@ func (pc *VPlotConfiguration) WithOverrides(ov ReportConfigGlobalOverrides) { pc.ShowNonvalidated = ov.ShowNonvalidated.Value } } + +type VExpressionTimeseriesVariable struct { + VariableName string `json:"variable_name"` + TimeseriesID uuid.UUID `json:"timeseries_id"` +} diff --git a/api/internal/db/querier.go b/api/internal/db/querier.go index b6697586..719dbc2c 100644 --- a/api/internal/db/querier.go +++ b/api/internal/db/querier.go @@ -113,6 +113,19 @@ type Querier interface { EvaluationListForProject(ctx context.Context, projectID uuid.UUID) ([]VEvaluation, error) EvaluationListForProjectAlertConfig(ctx context.Context, arg EvaluationListForProjectAlertConfigParams) ([]VEvaluation, error) EvaluationUpdate(ctx context.Context, arg EvaluationUpdateParams) error + ExpressionCreate(ctx context.Context, arg ExpressionCreateParams) (uuid.UUID, error) + ExpressionDelete(ctx context.Context, id uuid.UUID) error + ExpressionDownstreamListForTimeseries(ctx context.Context, depTimeseriesIds []uuid.UUID) ([]ExpressionDownstreamListForTimeseriesRow, error) + ExpressionGet(ctx context.Context, id uuid.UUID) (VExpression, error) + ExpressionGetBatch(ctx context.Context, id []uuid.UUID) *ExpressionGetBatchBatchResults + ExpressionGetWouldCreateCycleOrExceedDepth(ctx context.Context, expressionID uuid.UUID) (ExpressionGetWouldCreateCycleOrExceedDepthRow, error) + ExpressionListForInstrument(ctx context.Context, instrumentID uuid.UUID) ([]VExpression, error) + ExpressionTimeseriesTargetCreate(ctx context.Context, arg ExpressionTimeseriesTargetCreateParams) error + ExpressionTimeseriesVariableCreateOrUpdateBatch(ctx context.Context, arg []ExpressionTimeseriesVariableCreateOrUpdateBatchParams) *ExpressionTimeseriesVariableCreateOrUpdateBatchBatchResults + ExpressionTimeseriesVariableDeleteAllForExpression(ctx context.Context, expressionID uuid.UUID) error + ExpressionUpdate(ctx context.Context, arg ExpressionUpdateParams) (uuid.UUID, error) + ExpressionWindowOptsCreate(ctx context.Context, arg ExpressionWindowOptsCreateParams) error + ExpressionWindowOptsDelete(ctx context.Context, expressionID uuid.UUID) error HeartbeatCreate(ctx context.Context, argTime time.Time) (time.Time, error) HeartbeatGetLatest(ctx context.Context) (time.Time, error) HeartbeatList(ctx context.Context, resultLimit int32) ([]time.Time, error) @@ -327,6 +340,7 @@ type Querier interface { TimeseriesCwmsList(ctx context.Context, instrumentID uuid.UUID) ([]VTimeseriesCwms, error) TimeseriesCwmsUpdate(ctx context.Context, arg TimeseriesCwmsUpdateParams) error TimeseriesDelete(ctx context.Context, id uuid.UUID) error + TimeseriesDeleteTargetForExpression(ctx context.Context, expressionID uuid.UUID) error TimeseriesGet(ctx context.Context, id uuid.UUID) (VTimeseries, error) TimeseriesGetAllBelongToProject(ctx context.Context, arg TimeseriesGetAllBelongToProjectParams) (bool, error) TimeseriesGetExistsStored(ctx context.Context, id uuid.UUID) (bool, error) @@ -340,12 +354,14 @@ type Querier interface { TimeseriesMeasurementCreateBatch(ctx context.Context, arg []TimeseriesMeasurementCreateBatchParams) *TimeseriesMeasurementCreateBatchBatchResults TimeseriesMeasurementCreateOrUpdate(ctx context.Context, arg TimeseriesMeasurementCreateOrUpdateParams) error TimeseriesMeasurementCreateOrUpdateBatch(ctx context.Context, arg []TimeseriesMeasurementCreateOrUpdateBatchParams) *TimeseriesMeasurementCreateOrUpdateBatchBatchResults - TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheck(ctx context.Context, arg []TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckParams) *TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheckBatchResults TimeseriesMeasurementDelete(ctx context.Context, arg TimeseriesMeasurementDeleteParams) error + TimeseriesMeasurementDeleteAllForTimeseries(ctx context.Context, timeseriesID uuid.UUID) error TimeseriesMeasurementDeleteBatch(ctx context.Context, arg []TimeseriesMeasurementDeleteBatchParams) *TimeseriesMeasurementDeleteBatchBatchResults TimeseriesMeasurementDeleteRange(ctx context.Context, arg TimeseriesMeasurementDeleteRangeParams) error TimeseriesMeasurementDeleteRangeBatch(ctx context.Context, arg []TimeseriesMeasurementDeleteRangeBatchParams) *TimeseriesMeasurementDeleteRangeBatchBatchResults TimeseriesMeasurementGetMostRecent(ctx context.Context, timeseriesID uuid.UUID) (TimeseriesMeasurement, error) + TimeseriesMeasurementListBatchEdgesForTimeseriesIDs(ctx context.Context, arg []TimeseriesMeasurementListBatchEdgesForTimeseriesIDsParams) *TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults + TimeseriesMeasurementListBatchForTimeseriesIDs(ctx context.Context, arg []TimeseriesMeasurementListBatchForTimeseriesIDsParams) *TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults TimeseriesMeasurementNoteRangeList(ctx context.Context, arg TimeseriesMeasurementNoteRangeListParams) ([]TimeseriesMeasurementNoteRangeListRow, error) TimeseriesNoteCreate(ctx context.Context, arg TimeseriesNoteCreateParams) error TimeseriesNoteCreateBatch(ctx context.Context, arg []TimeseriesNoteCreateBatchParams) *TimeseriesNoteCreateBatchBatchResults diff --git a/api/internal/dto/expression.go b/api/internal/dto/expression.go new file mode 100644 index 00000000..c03980b5 --- /dev/null +++ b/api/internal/dto/expression.go @@ -0,0 +1,51 @@ +package dto + +import ( + "encoding/json" + "strings" + + "github.com/google/uuid" +) + +type ExpressionDTO struct { + Name string `json:"name"` + Expression string `json:"expression"` + Mode string `json:"mode" enum:"point,window"` + Variables []ExpressionTimeseriesVariableDTO `json:"variables"` + RawOpts json.RawMessage `json:"opts" required:"false"` + Opts struct { + ExpressionWindowOptsDTO + } `json:"-"` +} + +type ExpressionTimeseriesVariableDTO struct { + VariableName string `json:"variable_name"` + TimeseriesID uuid.UUID `json:"timeseries_id"` +} + +type ExpressionWindowOptsDTO struct { + Duration string `json:"duration"` + TzName string `json:"tz_name" default:"UTC"` + DurationOffset string `json:"duration_offset" default:"PT0"` + TimestampingPolicy string `json:"timestamping_policy" enum:"start,center,end" default:"start"` +} + +// ExpressionDTO implements the UnmarshalJSON interface +// This unpacks the contents of JSON-encoded RawOpts into typed Opts based on the enum value of Mode +func (d *ExpressionDTO) UnmarshalJSON(b []byte) error { + // alias so we don't call this recursively + type alias ExpressionDTO + tmp := &struct{ *alias }{alias: (*alias)(d)} + if err := json.Unmarshal(b, &tmp); err != nil { + return err + } + *d = ExpressionDTO(*tmp.alias) + var err error + switch strings.ToLower(d.Mode) { + case "window": + err = json.Unmarshal(d.RawOpts, &d.Opts.ExpressionWindowOptsDTO) + default: + return nil + } + return err +} diff --git a/api/internal/dto/worker_expression_ts_compute_event.go b/api/internal/dto/worker_expression_ts_compute_event.go new file mode 100644 index 00000000..2646b0c5 --- /dev/null +++ b/api/internal/dto/worker_expression_ts_compute_event.go @@ -0,0 +1,27 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type WorkerExpressionTimeseriesComputeFullEventArgs struct { + TargetTimeseriesID uuid.UUID + ExpressionID uuid.UUID + Expression ExpressionDTO +} + +func (WorkerExpressionTimeseriesComputeFullEventArgs) Kind() string { + return "ExpressionTimeseriesComputeFullEvent" +} + +type WorkerExpressionTimeseriesComputePartialEventArgs struct { + WorkerExpressionTimeseriesComputeFullEventArgs + MinTime time.Time + MaxTime time.Time +} + +func (WorkerExpressionTimeseriesComputePartialEventArgs) Kind() string { + return "ExpressionTimeseriesComputePartialEvent" +} diff --git a/api/internal/eval/bind.go b/api/internal/eval/bind.go new file mode 100644 index 00000000..57ee319c --- /dev/null +++ b/api/internal/eval/bind.go @@ -0,0 +1,39 @@ +package eval + +import ( + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" +) + +// bindConstDouble f() -> double +func bindConstDouble(name string, f func() float64) cel.EnvOption { + return cel.Function(name, + cel.Overload(name+"_const", + []*cel.Type{}, + cel.DoubleType, + cel.FunctionBinding(func(args ...ref.Val) ref.Val { + return types.Double(f()) + }), + ), + ) +} + +// bindUnaryMeas: f(Measurement) -> Measurement +// Convenience function for unary math functions that accept and return Measurement type +func (lib *mathLib) bindUnaryMeas(name string, f func(float64) float64) cel.EnvOption { + return cel.Function(name, + cel.Overload(name+"_meas", + []*cel.Type{CELMeasurementType}, + CELMeasurementType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + m, err := convertToMeasurement(v) + if err != nil { + return types.NewErr("%s expected Measurement, got: %w", name, err) + } + m.Value = f(m.Value) + return lib.reg.NativeToValue(m) + }), + ), + ) +} diff --git a/api/internal/eval/cache.go b/api/internal/eval/cache.go new file mode 100644 index 00000000..b2f41682 --- /dev/null +++ b/api/internal/eval/cache.go @@ -0,0 +1,43 @@ +package eval + +import ( + "github.com/google/uuid" + lru "github.com/hashicorp/golang-lru/v2" +) + +// ProgramCacheEntry holds a compiled program and its checksum. +type ProgramCacheEntry struct { + Compiled *compiled + Checksum string +} + +// ProgramCache is a concurrent cache for compiled expressions. +type ProgramCache struct { + lru *lru.TwoQueueCache[uuid.UUID, *ProgramCacheEntry] +} + +func NewProgramCache(size int) (*ProgramCache, error) { + cache, err := lru.New2Q[uuid.UUID, *ProgramCacheEntry](size) + if err != nil { + return nil, err + } + return &ProgramCache{cache}, nil +} + +func (c *ProgramCache) Get(id uuid.UUID) (*ProgramCacheEntry, bool) { + val, ok := c.lru.Get(id) + if !ok { + return nil, false + } + return val, true +} + +func (c *ProgramCache) Set(id uuid.UUID, entry *ProgramCacheEntry) { + c.lru.Add(id, entry) +} + +func WithProgramCache(cache *ProgramCache) EvalSessionOption { + return func(es *EvalSession) { + es.programCache = cache + } +} diff --git a/api/internal/eval/cache_test.go b/api/internal/eval/cache_test.go new file mode 100644 index 00000000..c88b3ee5 --- /dev/null +++ b/api/internal/eval/cache_test.go @@ -0,0 +1,21 @@ +package eval + +import ( + "testing" + + "github.com/google/uuid" +) + +func TestProgramCache(t *testing.T) { + cache, err := NewProgramCache(2) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + id := uuid.New() + entry := &ProgramCacheEntry{Checksum: "abc"} + cache.Set(id, entry) + got, ok := cache.Get(id) + if !ok || got.Checksum != "abc" { + t.Errorf("cache get/set failed") + } +} diff --git a/api/internal/eval/compile.go b/api/internal/eval/compile.go new file mode 100644 index 00000000..0ddf4456 --- /dev/null +++ b/api/internal/eval/compile.go @@ -0,0 +1,63 @@ +package eval + +import ( + "errors" + "slices" + + "github.com/google/cel-go/cel" +) + +type Mode string + +const ( + ModePoint Mode = "point" + ModeWindow Mode = "window" +) + +type compiled struct { + Prog cel.Program + Vars []string +} + +func (e *Env) CompileExpression( + eCtx *evalContext, + exprSrc string, + vars []string, + extraOpts ...cel.EnvOption, +) (*compiled, error) { + opts := []cel.EnvOption{ + NewContextMathLib(eCtx, e.reg), + } + + for _, v := range vars { + opts = append(opts, cel.Variable(v, cel.ObjectType("eval.Measurement"))) + } + + opts = append(opts, extraOpts...) + + env, err := e.env.Extend(opts...) + if err != nil { + return nil, err + } + + ast, iss := env.Compile(exprSrc) + if iss != nil && iss.Err() != nil { + return nil, iss.Err() + } + + // only accept Measurement or list return types + ot := ast.OutputType() + if !ot.IsEquivalentType(CELMeasurementType) && !ot.IsEquivalentType(cel.ListType(CELMeasurementType)) { + return nil, errors.New("CEL expression expected Measurement or list result, got: " + ot.DeclaredTypeName()) + } + + prog, err := env.Program(ast, cel.InterruptCheckFrequency(1000)) + if err != nil { + return nil, err + } + + return &compiled{ + Prog: prog, + Vars: slices.Clone(vars), + }, nil +} diff --git a/api/internal/eval/completion.go b/api/internal/eval/completion.go new file mode 100644 index 00000000..555d1dd0 --- /dev/null +++ b/api/internal/eval/completion.go @@ -0,0 +1,619 @@ +package eval + +import ( + "fmt" + "math" + "strings" + "time" +) + +// https://codemirror.net/docs/ref/#autocomplete +type Completion struct { + // The label to show in the completion picker. + Label string `json:"label"` + // Optional override for the completion's visible label. + DisplayLabel string `json:"display_label"` + // Optional short piece of information to show after the label. + Detail string `json:"detail,omitempty"` + // Info subsection Signature + Signature string `json:"signature,omitempty"` + // Info subsection Definition + Definition string `json:"definition,omitempty"` + // Info subsection Example + Example string `json:"example,omitempty"` + // How to apply the completion. + Apply string `json:"apply"` + // The type of the completion. + // options: + // class, constant, enum, function, interface, keyword, + // method, namespace, property, text, type, variable + Type string `json:"type"` + // Characters that trigger commit of the completion. + CommitCharacters []string `json:"commit_characters,omitempty"` + // Adjusts ranking (-99 to 99). + Boost int `json:"boost"` + // Section name for grouping completions. + Section string `json:"section"` +} + +func c( + label, display, apply, typ, section string, boost int, + signature, definition, example string, +) Completion { + return Completion{ + Label: label, + DisplayLabel: display, + Apply: apply, + Type: typ, + Section: section, + Boost: boost, + Signature: signature, + Definition: definition, + Example: example, + } +} + +var completions = []Completion{ + // Measurement Functions + c("@abs", "abs(x)", "abs(x)", "function", "Measurement Functions", 50, + "abs(x Measurement) -> Measurement", + "Absolute value of a measurement’s value.", + "abs("+fmtM(-1)+") // "+fmtM(1), + ), + c("@round", "round(x)", "round(x)", "function", "Measurement Functions", 50, + "round(x Measurement) -> Measurement", + "Rounds the measurement’s value to the nearest integer.", + "round("+fmtM(0.5)+") // "+fmtM(1), + ), + c("@floor", "floor(x)", "floor(x)", "function", "Measurement Functions", 50, + "floor(x Measurement) -> Measurement", + "Rounds the measurement’s value down to the nearest integer.", + "floor("+fmtM(0.5)+") // "+fmtM(0), + ), + c("@ceil", "ceil(x)", "ceil(x)", "function", "Measurement Functions", 50, + "ceil(x Measurement) -> Measurement", + "Rounds the measurement’s value up to the nearest integer.", + "ceil("+fmtM(0.5)+") // "+fmtM(1), + ), + c("@clamp", "clamp(m, lo, hi)", "clamp(m, lo, hi)", "function", "Measurement Functions", 50, + "clamp(m Measurement, lo double, hi double) -> Measurement", + "Clamps a measurement’s value between lo and hi.", + "clamp("+fmtM(101)+", 0, 100) // "+fmtM(100), + ), + c("@pow", "pow(x, y)", "pow(x, y)", "function", "Measurement Functions", 50, + "pow(x Measurement, y double) -> Measurement", + "Raises a measurement’s value to the power y.", + "pow("+fmtM(2)+", 2) // "+fmtM(4), + ), + c("@sqrt", "sqrt(x)", "sqrt(x)", "function", "Measurement Functions", 50, + "sqrt(x Measurement) -> Measurement", + "Square root of a measurement’s value.", + "sqrt("+fmtM(4)+") // "+fmtM(2), + ), + c("@log", "log(x)", "log(x)", "function", "Measurement Functions", 50, + "log(x Measurement) -> Measurement", + "Natural logarithm of a measurement’s value.", + "log("+fmtM(2)+") // "+fmtM(0.69), + ), + c("@log10", "log10(x)", "log10(x)", "function", "Measurement Functions", 50, + "log10(x Measurement) -> Measurement", + "Base-10 logarithm of a measurement’s value.", + "log10("+fmtM(2)+") // "+fmtM(0.30), + ), + c("@exp", "exp(x)", "exp(x)", "function", "Measurement Functions", 50, + "exp(x Measurement) -> Measurement", + "Exponential of a measurement’s value.", + "exp("+fmtM(2)+") // "+fmtM(7.38), + ), + c("@sin", "sin(x)", "sin(x)", "function", "Measurement Functions", 50, + "sin(x Measurement) -> Measurement", + "Sine of a measurement’s value (radians).", + "sin("+fmtM(2)+") // "+fmtM(0.90), + ), + c("@cos", "cos(x)", "cos(x)", "function", "Measurement Functions", 50, + "cos(x Measurement) -> Measurement", + "Cosine of a measurement’s value (radians).", + "cos("+fmtM(2)+") // "+fmtM(-0.41), + ), + c("@tan", "tan(x)", "tan(x)", "function", "Measurement Functions", 50, + "tan(x Measurement) -> Measurement", + "Tangent of a measurement’s value (radians).", + "tan("+fmtM(2)+") // "+fmtM(-2.18), + ), + c("@isnan", "isnan(x)", "isnan(x)", "function", "Measurement Functions", 50, + "isnan(x Measurement) -> bool", + "Checks whether a measurement’s value is NaN.", + "isnan("+fmtM(2)+") // false", + ), + c("@coalesce", "coalesce(a, b)", "coalesce(a, b)", "function", "Measurement Functions", 50, + "coalesce(a Measurement, b Measurement) -> Measurement", + "Returns b if a’s value is NaN, otherwise a.", + "coalesce("+fmtM(math.NaN())+", "+fmtM(2)+") // "+fmtM(2), + ), + c("@min", "min(mm)", "min(mm)", "function", "Measurement Functions", 50, + "min(mm list) -> Measurement", + "Minimum value of a list of measurements.", + "min("+fmtMm(1, 2)+") // "+fmtM(1), + ), + c("@max", "max(mm)", "max(mm)", "function", "Measurement Functions", 50, + "max(mm list) -> Measurement", + "Maximum value of a list of measurements.", + "max("+fmtMm(2, 1)+") // "+fmtM(2), + ), + c("@mean", "mean(mm)", "mean(mm)", "function", "Measurement Functions", 50, + "mean(mm list) -> Measurement", + "Arithmetic mean of a list of measurements. Timestamping follows the expression’s policy.", + "mean("+fmtMm(1, 2)+") // "+fmtM(1.5)+" (timestamp policy “start”)", + ), + c("@stddev", "stddev(mm)", "stddev(mm)", "function", "Measurement Functions", 50, + "stddev(mm list) -> Measurement", + "Sample standard deviation of a list of measurements. Timestamping follows the expression’s policy.", + "stddev("+fmtMm(1, 2)+") // "+fmtM(0.5)+" (timestamp policy “start”)", + ), + c("@moving_avg", "moving_avg(mm, win)", "moving_avg(mm, win)", "function", "Measurement Functions", 50, + "moving_avg(mm list, win int) -> list", + "Simple moving average over a sliding window of size win.", + "moving_avg(measurements, 5)", + ), + c("@exp_moving_avg", "exp_moving_avg(mm, alpha)", "exp_moving_avg(mm, alpha)", "function", "Measurement Functions", 50, + "exp_moving_avg(mm list, alpha double) -> list", + "Exponential moving average with smoothing factor alpha in (0, 1].", + "exp_moving_avg(measurements, 0.5)", + ), + c("@delta", "delta(mm)", "delta(mm)", "function", "Measurement Functions", 50, + "delta(mm list) -> list", + "Differences between consecutive measurement values.", + "delta(measurements)", + ), + c("@cumsum", "cumsum(mm)", "cumsum(mm)", "function", "Measurement Functions", 50, + "cumsum(mm list) -> list", + "Cumulative sum of measurement values.", + "cumsum(measurements)", + ), + + // Custom Types + c("@Measurement", "Measurement", "Measurement", "type", "Custom Types", 50, + "message Measurement { time: google.protobuf.Timestamp, value: double, masked: bool, validated: bool }", + "Custom protobuf type representing a single measurement sample.", + `Measurement{time: timestamp("2023-08-26T12:39:00-07:00"), value: 42.0, masked: false, validated: true}`, + ), + + // Mathematical Constants + c("@NAN", "NAN", "NAN", "constant", "Mathematical Constants", 50, + "NAN -> double", + "IEEE-754 Not-a-Number.", + "NAN", + ), + c("@INF", "INF", "INF", "constant", "Mathematical Constants", 50, + "INF -> double", + "Positive infinity per IEEE-754.", + "INF", + ), + c("@NEGATIVE_INF", "NEGATIVE_INF", "NEGATIVE_INF", "constant", "Mathematical Constants", 50, + "NEGATIVE_INF -> double", + "Negative infinity per IEEE-754.", + "NEGATIVE_INF", + ), + c("@PI", "PI", "PI", "constant", "Mathematical Constants", 50, + "PI -> double", + "The constant π.", + "PI", + ), + c("@E", "E", "E", "constant", "Mathematical Constants", 50, + "E -> double", + "Euler’s number.", + "E", + ), + c("@PHI", "PHI", "PHI", "constant", "Mathematical Constants", 50, + "PHI -> double", + "Golden ratio (≈ 1.618).", + "PHI", + ), + c("@TAU", "TAU", "TAU", "constant", "Mathematical Constants", 50, + "TAU -> double", + "Circle constant τ = 2π.", + "TAU", + ), + c("@SQRT_2", "SQRT_2", "SQRT_2", "constant", "Mathematical Constants", 50, + "SQRT_2 -> double", + "Square root of 2.", + "SQRT_2", + ), + c("@SQRT_PI", "SQRT_PI", "SQRT_PI", "constant", "Mathematical Constants", 50, + "SQRT_PI -> double", + "Square root of π.", + "SQRT_PI", + ), + c("@SQRT_E", "SQRT_E", "SQRT_E", "constant", "Mathematical Constants", 50, + "SQRT_E -> double", + "Square root of e.", + "SQRT_E", + ), + c("@SQRT_PHI", "SQRT_PHI", "SQRT_PHI", "constant", "Mathematical Constants", 50, + "SQRT_PHI -> double", + "Square root of the golden ratio.", + "SQRT_PHI", + ), + c("@LN_2", "LN_2", "LN_2", "constant", "Mathematical Constants", 50, + "LN_2 -> double", + "Natural logarithm of 2.", + "LN_2", + ), + c("@LN_10", "LN_10", "LN_10", "constant", "Mathematical Constants", 50, + "LN_10 -> double", + "Natural logarithm of 10.", + "LN_10", + ), + c("@GRAVITY", "GRAVITY", "GRAVITY", "constant", "Mathematical Constants", 50, + "GRAVITY -> double", + "Standard gravity (9.80665 m/s²).", + "GRAVITY", + ), + + // CEL Types + c("@int", "int", "int", "type", "CEL Types", 10, + "int", + "64-bit signed integer type.", + "int", + ), + c("@uint", "uint", "uint", "type", "CEL Types", 10, + "uint", + "64-bit unsigned integer type.", + "uint", + ), + c("@double", "double", "double", "type", "CEL Types", 10, + "double", + "64-bit IEEE floating-point type.", + "double", + ), + c("@bool", "bool", "bool", "type", "CEL Types", 10, + "bool", + "Boolean type.", + "bool", + ), + c("@string", "string", "string", "type", "CEL Types", 10, + "string", + "Unicode string type.", + "string", + ), + c("@bytes", "bytes", "bytes", "type", "CEL Types", 10, + "bytes", + "Byte sequence type.", + "bytes", + ), + c("@list", "list", "list", "type", "CEL Types", 10, + "list", + "Ordered collection type.", + "list", + ), + c("@map_type", "map", "map", "type", "CEL Types", 10, + "map", + "Associative array with int/uint/bool/string keys.", + "map", + ), + c("@null_type", "null_type", "null_type", "type", "CEL Types", 10, + "null_type", + "The null value type.", + "null_type", + ), + c("@type", "type", "type", "type", "CEL Types", 10, + "type", + "Values representing CEL types.", + "type", + ), + + // CEL Macros (methods & helpers) + c("@all", "e.all(x,p)", "e.all(x,p)", "method", "CEL Macros", 10, + "e.all(x, p) -> bool", + "True if predicate p holds for all elements/keys.", + fmtMmMethod(0, 1)+".all(m, m.value > 0) // false", + ), + c("@exists", "e.exists(x,p)", "e.exists(x,p)", "method", "CEL Macros", 10, + "e.exists(x, p) -> bool", + "True if predicate p holds for any element/key.", + fmtMmMethod(0, 101)+".exists(m, m.value > 100) // true", + ), + c("@exists_one", "e.exists_one(x,p)", "e.exists_one(x,p)", "method", "CEL Macros", 10, + "e.exists_one(x, p) -> bool", + "True if exactly one element satisfies p.", + fmtMmMethod(42, 42)+".exists_one(m, m.value == 42) // false", + ), + c("@map", "e.map(x,t)", "e.map(x,t)", "method", "CEL Macros", 10, + "e.map(x, t) -> list", + "Transforms each element using t.", + fmtMmMethod(5, 6)+".map(m, Measurement{time: m.time, value: m.value * m.value})", + ), + c("@map_cond", "e.map(x,p,t)", "e.map(x,p,t)", "method", "CEL Macros", 10, + "e.map(x, p, t) -> list", + "Filters elements by p, then transforms with t.", + fmtMmMethod(1, 2)+".map(m, m.value > 1, Measurement{time: m.time, value: m.value * m.value})", + ), + c("@filter", "e.filter(x,p)", "e.filter(x,p)", "method", "CEL Macros", 10, + "e.filter(x, p) -> list", + "Filters a list/map by predicate p.", + fmtMmMethod(1, 2)+".filter(m, m.value % 2 > 0)", + ), + + // Map Functions + c("@map.size", "map.size()", "map.size()", "method", "Map Functions", 10, + "map.size() -> int", + "Number of entries in the map.", + "{'hello': 'world'}.size() // 1", + ), + c("@size_map", "size(map)", "size(map)", "function", "Map Functions", 10, + "size(map) -> int", + "Number of entries in the map.", + "size({1: true, 2: false}) // 2", + ), + + // Bytes Functions + c("@bytes.size", "bytes.size()", "bytes.size()", "method", "Bytes Functions", 10, + "bytes.size() -> int", + "Number of bytes in the sequence.", + "b'hello'.size() // 5", + ), + c("@size_bytes", "size(bytes)", "size(bytes)", "function", "Bytes Functions", 10, + "size(bytes) -> int", + "Number of bytes in the sequence.", + "size(b'world!') // 6; size(b'\\xF0\\x9F\\xA4\\xAA') // 4", + ), + + // String Functions + c("@string.contains", "string.contains(string)", "string.contains(string)", "method", "String Functions", 10, + "string.contains(sub string) -> bool", + "Whether the string contains the given substring.", + `"hello world".contains("world") // true`, + ), + c("@string.endsWith", "string.endsWith(string)", "string.endsWith(string)", "method", "String Functions", 10, + "string.endsWith(suffix string) -> bool", + "Whether the string ends with suffix.", + `"foobar".endsWith("bar") // true`, + ), + c("@matches", "matches(string, string)", "matches(string, string)", "function", "String Functions", 10, + "matches(value string, re string) -> bool", + "RE2 regular expression match.", + `matches("foobar", "foo.*") // true`, + ), + c("@string.matches", "string.matches(string)", "string.matches(string)", "method", "String Functions", 10, + "string.matches(re string) -> bool", + "RE2 regular expression match.", + `"foobar".matches("foo.*") // true`, + ), + c("@string.startsWith", "string.startsWith(string)", "string.startsWith(string)", "method", "String Functions", 10, + "string.startsWith(prefix string) -> bool", + "Whether the string starts with prefix.", + `"foobar".startsWith("foo") // true`, + ), + c("@string.size", "string.size()", "string.size()", "method", "String Functions", 10, + "string.size() -> int", + "Length measured in Unicode code points.", + `"hello".size() // 5`, + ), + c("@size_string", "size(string)", "size(string)", "function", "String Functions", 10, + "size(string) -> int", + "Length measured in Unicode code points.", + `size("world!") // 6; "fiance\u0301".size() // 7; size(string(b'\xF0\x9F\xA4\xAA')) // 1`, + ), + + // Date/Time Functions + c("@timestamp", "timestamp(string)", "timestamp(string)", "function", "Date/Time Functions", 10, + "timestamp(text string) -> google.protobuf.Timestamp", + "Parses an RFC3339 timestamp string.", + `timestamp("2023-08-26T12:39:00-07:00")`, + ), + c("@duration", "duration(string)", "duration(string)", "function", "Date/Time Functions", 10, + "duration(text string) -> google.protobuf.Duration", + `Parses a duration string with units: h, m, s, ms, us, ns.`, + `duration("1m")`, + ), + c("@timestamp.getDate", "google.protobuf.Timestamp.getDate()", "google.protobuf.Timestamp.getDate()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getDate([tz string]) -> int", + "Day of month (1–31). Defaults to UTC; pass an IANA timezone to localize.", + `t(timestamp("2023-12-25T00:00:00Z")).getDate() // 25; ...getDate("America/Los_Angeles") // 24`, + ), + c("@timestamp.getDayOfMonth", "google.protobuf.Timestamp.getDayOfMonth()", "google.protobuf.Timestamp.getDayOfMonth()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getDayOfMonth([tz string]) -> int", + "Zero-based day of month (0–30). Defaults to UTC; pass a timezone to localize.", + `t(...).getDayOfMonth() // 24`, + ), + c("@timestamp.getDayOfWeek", "google.protobuf.Timestamp.getDayOfWeek()", "google.protobuf.Timestamp.getDayOfWeek()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getDayOfWeek([tz string]) -> int", + "Zero-based day of week (0 = Sunday).", + `t(...).getDayOfWeek() // 1 (Monday)`, + ), + c("@timestamp.getDayOfYear", "google.protobuf.Timestamp.getDayOfYear()", "google.protobuf.Timestamp.getDayOfYear()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getDayOfYear([tz string]) -> int", + "Zero-based day of year.", + `t(...).getDayOfYear() // 358`, + ), + c("@timestamp.getFullYear", "google.protobuf.Timestamp.getFullYear()", "google.protobuf.Timestamp.getFullYear()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getFullYear([tz string]) -> int", + "Four-digit year extracted from a timestamp.", + `t(...).getFullYear() // 2023`, + ), + c("@timestamp.getHours", "google.protobuf.Timestamp.getHours()", "google.protobuf.Timestamp.getHours()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getHours([tz string]) -> int; google.protobuf.Duration.getHours() -> int", + "Hour component for timestamps (optionally localized) or duration converted to hours.", + `t(...).getHours() // 12; duration("3h").getHours() // 3`, + ), + c("@timestamp.getMilliseconds", "google.protobuf.Timestamp.getMilliseconds()", "google.protobuf.Timestamp.getMilliseconds()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getMilliseconds([tz string]) -> int; google.protobuf.Duration.getMilliseconds() -> int", + "Millisecond component for timestamps (optionally localized) or duration milliseconds.", + `t(...500Z).getMilliseconds() // 500; duration("1.234s").getMilliseconds() // 234`, + ), + c("@timestamp.getMinutes", "google.protobuf.Timestamp.getMinutes()", "google.protobuf.Timestamp.getMinutes()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getMinutes([tz string]) -> int; google.protobuf.Duration.getMinutes() -> int", + "Minute component for timestamps (optionally localized) or duration minutes.", + `t(...30:00Z).getMinutes() // 30; duration("1h30m").getMinutes() // 90`, + ), + c("@timestamp.getMonth", "google.protobuf.Timestamp.getMonth()", "google.protobuf.Timestamp.getMonth()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getMonth([tz string]) -> int", + "Zero-based month (0 = January).", + `t(...).getMonth() // 11 (December)`, + ), + c("@timestamp.getSeconds", "google.protobuf.Timestamp.getSeconds()", "google.protobuf.Timestamp.getSeconds()", "method", "Date/Time Functions", 10, + "google.protobuf.Timestamp.getSeconds([tz string]) -> int; google.protobuf.Duration.getSeconds() -> int", + "Second component for timestamps (optionally localized) or duration seconds.", + `t(...30Z).getSeconds() // 30; duration("1m30s").getSeconds() // 90`, + ), + + // CEL Keywords + c("@true", "true", "true", "keyword", "CEL Keywords", 10, + "true", + "Boolean literal true.", + "true", + ), + c("@false", "false", "false", "keyword", "CEL Keywords", 10, + "false", + "Boolean literal false.", + "false", + ), + c("@null", "null", "null", "keyword", "CEL Keywords", 10, + "null", + "The null literal.", + "null", + ), + c("@in", "in", "in", "keyword", "CEL Keywords", 10, + "in", + "Membership operator for lists/maps.", + `x in [1, 2, 3]`, + ), + c("@has", "has(\n e.f\n)", "has(\n e.f\n)", "keyword", "CEL Keywords", 10, + "has(e.f)", + "Field presence test.", + "has(measurement.value)", + ), + + // Reserved Keywords (Do Not Use) + c("@as", "as", "as", "keyword", "Reserved Keywords (Do Not Use)", -50, + "as", + reservedKeywordInfo, + "as", + ), + c("@break", "break", "break", "keyword", "Reserved Keywords (Do Not Use)", -50, + "break", + reservedKeywordInfo, + "break", + ), + c("@const", "const", "const", "keyword", "Reserved Keywords (Do Not Use)", -50, + "const", + reservedKeywordInfo, + "const", + ), + c("@continue", "continue", "continue", "keyword", "Reserved Keywords (Do Not Use)", -50, + "continue", + reservedKeywordInfo, + "continue", + ), + c("@else", "else", "else", "keyword", "Reserved Keywords (Do Not Use)", -50, + "else", + reservedKeywordInfo, + "else", + ), + c("@for", "for", "for", "keyword", "Reserved Keywords (Do Not Use)", -50, + "for", + reservedKeywordInfo, + "for", + ), + c("@function", "function", "function", "keyword", "Reserved Keywords (Do Not Use)", -50, + "function", + reservedKeywordInfo, + "function", + ), + c("@if", "if", "if", "keyword", "Reserved Keywords (Do Not Use)", -50, + "if", + reservedKeywordInfo, + "if", + ), + c("@import", "import", "import", "keyword", "Reserved Keywords (Do Not Use)", -50, + "import", + reservedKeywordInfo, + "import", + ), + c("@let", "let", "let", "keyword", "Reserved Keywords (Do Not Use)", -50, + "let", + reservedKeywordInfo, + "let", + ), + c("@loop", "loop", "loop", "keyword", "Reserved Keywords (Do Not Use)", -50, + "loop", + reservedKeywordInfo, + "loop", + ), + c("@package", "package", "package", "keyword", "Reserved Keywords (Do Not Use)", -50, + "package", + reservedKeywordInfo, + "package", + ), + c("@namespace", "namespace", "namespace", "keyword", "Reserved Keywords (Do Not Use)", -50, + "namespace", + reservedKeywordInfo, + "namespace", + ), + c("@return", "return", "return", "keyword", "Reserved Keywords (Do Not Use)", -50, + "return", + reservedKeywordInfo, + "return", + ), + c("@var", "var", "var", "keyword", "Reserved Keywords (Do Not Use)", -50, + "var", + reservedKeywordInfo, + "var", + ), + c("@void", "void", "void", "keyword", "Reserved Keywords (Do Not Use)", -50, + "void", + reservedKeywordInfo, + "void", + ), + c("@while", "while", "while", "keyword", "Reserved Keywords (Do Not Use)", -50, + "while", + reservedKeywordInfo, + "while", + ), +} + +// GetCompletionOptions gets a slice of completion options structs +// based on the CodeMirror 6 Completions interface +func GetCompletionOptions() []Completion { + return completions +} + +func fmtM(m float64) string { + t := time.Date(2025, 01, 01, 00, 00, 00, 00, time.UTC) + mStr := fmt.Sprintf(`Measurement{time: timestamp("%s"), value: %.2f}`, t.Format(time.RFC3339), m) + return mStr +} + +func fmtMm(mm ...float64) string { + const prefix = "[" + mmStr := make([]string, len(mm)) + t := time.Date(2025, 01, 01, 00, 00, 00, 00, time.UTC) + for i, m := range mm { + mmStr[i] = fmt.Sprintf(` + Measurement{ + time: timestamp("%s"), + value: %.2f + }`, t.Format(time.RFC3339), m) + t = t.Add(24 * time.Hour) + } + const suffix = "\n]" + return prefix + strings.Join(mmStr, ",") + suffix +} + +func fmtMmMethod(mm ...float64) string { + const prefix = "Example:\n[" + mmStr := make([]string, len(mm)) + t := time.Date(2025, 01, 01, 00, 00, 00, 00, time.UTC) + for i, m := range mm { + mmStr[i] = fmt.Sprintf(` + Measurement{ + time: timestamp("%s"), + value: %.2f + }`, t.Format(time.RFC3339), m) + t = t.Add(24 * time.Hour) + } + const suffix = "\n]" + return prefix + strings.Join(mmStr, ",") + suffix +} + +const reservedKeywordInfo = "Reserved keyword (do not use)" diff --git a/api/internal/eval/completion_test.go b/api/internal/eval/completion_test.go new file mode 100644 index 00000000..64bc2eb8 --- /dev/null +++ b/api/internal/eval/completion_test.go @@ -0,0 +1,26 @@ +package eval + +import ( + "testing" +) + +func TestFmtM(t *testing.T) { + s := fmtM(42) + if s == "" { + t.Error("expected non-empty string") + } +} + +func TestFmtMm(t *testing.T) { + s := fmtMm(1, 2) + if s == "" { + t.Error("expected non-empty string") + } +} + +func TestFmtMmMethod(t *testing.T) { + s := fmtMmMethod(1, 2) + if s == "" { + t.Error("expected non-empty string") + } +} diff --git a/api/internal/eval/context.go b/api/internal/eval/context.go new file mode 100644 index 00000000..e46400ad --- /dev/null +++ b/api/internal/eval/context.go @@ -0,0 +1,32 @@ +package eval + +import "time" + +// TimestampingPolicy controls how aggregate results get a timestamp when +// there is no single “winner” (mean, stddev, etc). +type TimestampingPolicy string + +const ( + PolicyStart TimestampingPolicy = "start" + PolicyCenter TimestampingPolicy = "center" + PolicyEnd TimestampingPolicy = "end" +) + +// evalContext is provided per compiled expression to the library so that +// dynamic functions can shape outputs correctly for point vs window mode. +type evalContext struct { + // Used only in window mode when there is no unique winning element + // (mean/stddev). If there are multiple winners (ties in min/max), + // the first match’s timestamp is used instead of Policy*. + TimestampingPolicy TimestampingPolicy + PolicyWindowStart time.Time + PolicyWindowEnd time.Time +} + +func Context() *evalContext { + return &evalContext{ + TimestampingPolicy: PolicyStart, + PolicyWindowStart: defaultStartsAt, + PolicyWindowEnd: defaultEndsBefore.Add(1), + } +} diff --git a/api/internal/eval/context_test.go b/api/internal/eval/context_test.go new file mode 100644 index 00000000..0f393f1d --- /dev/null +++ b/api/internal/eval/context_test.go @@ -0,0 +1,18 @@ +package eval + +import ( + "testing" +) + +func TestEvalContextDefaults(t *testing.T) { + ctx := Context() + if ctx.TimestampingPolicy != PolicyStart { + t.Errorf("expected PolicyStart, got %v", ctx.TimestampingPolicy) + } + if ctx.PolicyWindowStart != defaultStartsAt { + t.Errorf("unexpected PolicyWindowStart") + } + if !ctx.PolicyWindowEnd.After(defaultEndsBefore) { + t.Errorf("unexpected PolicyWindowEnd") + } +} diff --git a/api/internal/eval/conv.go b/api/internal/eval/conv.go new file mode 100644 index 00000000..7460a644 --- /dev/null +++ b/api/internal/eval/conv.go @@ -0,0 +1,88 @@ +package eval + +import ( + "errors" + "fmt" + "math" + + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/common/types/traits" +) + +func convertToMeasurement(v ref.Val) (*Measurement, error) { + value := v.Value() + m, ok := value.(*Measurement) + if !ok { + return nil, errors.New("failed type assertion") + } + return m, nil +} + +func celNumberToFloat(v ref.Val) float64 { + switch t := v.(type) { + case types.Double: + return float64(t) + case types.Int: + return float64(t) + case types.Uint: + return float64(t) + default: + cv := t.ConvertToType(types.DoubleType) + if types.IsError(cv) { + return math.NaN() + } + return float64(cv.(types.Double)) + } +} + +func celMeasListToFloat64(v ref.Val) ([]float64, error) { + lst, ok := v.(traits.Lister) + if !ok { + return nil, errors.New("expected list") + } + capacity := 128 + if sizer, ok := v.(traits.Sizer); ok { + if sz, ok := sizer.Size().(types.Int); ok { + capacity = int(sz) + } + } + out := make([]float64, 0, capacity) + it := lst.Iterator() + idx := 0 + for it.HasNext() == types.True { + elem := it.Next() + m, err := convertToMeasurement(elem) + if err != nil { + return nil, fmt.Errorf("element at index %d is not of type Measurement: %w", idx, err) + } + out = append(out, m.Value) + idx++ + } + return out, nil +} + +// celMeasListToMeasGoSlice converts a CEL list to a go []*Measurement +func celMeasListToMeasGoSlice(v ref.Val) ([]*Measurement, error) { + lst, ok := v.(traits.Lister) + if !ok { + return nil, errors.New("expected list") + } + capacity := 128 + if sizer, ok := v.(traits.Sizer); ok { + if sz, ok := sizer.Size().(types.Int); ok { + capacity = int(sz) + } + } + it := lst.Iterator() + out := make([]*Measurement, 0, capacity) + for it.HasNext() == types.True { + elem := it.Next() + meas, err := convertToMeasurement(elem) + if err != nil { + return nil, errors.New("element is not of type Measurement") + } + out = append(out, meas) + } + return out, nil +} diff --git a/api/internal/eval/dag.go b/api/internal/eval/dag.go new file mode 100644 index 00000000..058c0298 --- /dev/null +++ b/api/internal/eval/dag.go @@ -0,0 +1,57 @@ +package eval + +import ( + "github.com/USACE/instrumentation-api/api/v4/internal/db" + "github.com/google/uuid" +) + +// ExpressionNode represents a node in the dependency graph. +type ExpressionNode struct { + ExpressionID uuid.UUID + TargetTimeseriesID uuid.UUID + Dependencies []uuid.UUID + Dependents []uuid.UUID +} + +// ExpressionDAG manages the dependency graph and provides query/sort utilities. +type ExpressionDAG struct { + nodes map[uuid.UUID]*ExpressionNode + tsToExprID map[uuid.UUID][]uuid.UUID +} + +// NewExpressionDAG builds the DAG from recursive query results. +func NewExpressionDAG(rows []db.ExpressionDownstreamListForTimeseriesRow) *ExpressionDAG { + dag := &ExpressionDAG{ + nodes: make(map[uuid.UUID]*ExpressionNode), + tsToExprID: make(map[uuid.UUID][]uuid.UUID), + } + // Build nodes and dependency edges + for _, row := range rows { + node, ok := dag.nodes[row.ExpressionID] + if !ok { + node = &ExpressionNode{ + ExpressionID: row.ExpressionID, + TargetTimeseriesID: row.TargetTimeseriesID, + } + dag.nodes[row.ExpressionID] = node + } + node.Dependencies = row.VariableTimeseriesIds + dag.tsToExprID[row.TargetTimeseriesID] = append(dag.tsToExprID[row.TargetTimeseriesID], row.ExpressionID) + } + // Build reverse edges (dependents) + for _, node := range dag.nodes { + for _, depTsID := range node.Dependencies { + for _, depExprID := range dag.tsToExprID[depTsID] { + if depNode, ok := dag.nodes[depExprID]; ok { + depNode.Dependents = append(depNode.Dependents, node.ExpressionID) + } + } + } + } + return dag +} + +// directExpressionsForTimeseries returns expressions that directly depend on the given timeseries. +func (dag *ExpressionDAG) DirectExpressionsForTimeseries(tsID uuid.UUID) []uuid.UUID { + return dag.tsToExprID[tsID] +} diff --git a/api/internal/eval/dag_test.go b/api/internal/eval/dag_test.go new file mode 100644 index 00000000..28722f0e --- /dev/null +++ b/api/internal/eval/dag_test.go @@ -0,0 +1,26 @@ +package eval + +import ( + "testing" + + "github.com/USACE/instrumentation-api/api/v4/internal/db" + "github.com/google/uuid" +) + +func TestNewExpressionDAG(t *testing.T) { + id1 := uuid.New() + id2 := uuid.New() + ts1 := uuid.New() + ts2 := uuid.New() + rows := []db.ExpressionDownstreamListForTimeseriesRow{ + {ExpressionID: id1, TargetTimeseriesID: ts1, VariableTimeseriesIds: []uuid.UUID{ts2}}, + {ExpressionID: id2, TargetTimeseriesID: ts2, VariableTimeseriesIds: []uuid.UUID{}}, + } + dag := NewExpressionDAG(rows) + if dag.nodes[id1].TargetTimeseriesID != ts1 { + t.Errorf("unexpected TargetTimeseriesID") + } + if len(dag.DirectExpressionsForTimeseries(ts1)) != 1 { + t.Errorf("unexpected direct expressions") + } +} diff --git a/api/internal/eval/env.go b/api/internal/eval/env.go new file mode 100644 index 00000000..2e5b82e0 --- /dev/null +++ b/api/internal/eval/env.go @@ -0,0 +1,47 @@ +package eval + +import ( + "fmt" + reflect "reflect" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/containers" + "github.com/google/cel-go/common/types" +) + +var ( + measProtoMsg = &Measurement{} + measType = reflect.TypeFor[*Measurement]() + CELMeasurementType = cel.ObjectType("eval.Measurement") +) + +type Env struct { + env *cel.Env + reg *types.Registry +} + +func NewEnv() (*Env, error) { + reg, err := types.NewRegistry(measProtoMsg) + if err != nil { + return nil, fmt.Errorf("types.NewRegistry(measProtoMsg): failed to create new registry: %w", err) + } + + e, err := cel.NewEnv( + cel.CustomTypeProvider(reg), + cel.CustomTypeAdapter(reg), + cel.Types(measProtoMsg), + NewMathLib(reg), + NewTelemetryLib(reg), + ) + if err != nil { + return nil, fmt.Errorf("error creating new Env: %w", err) + } + + co, err := e.Container.Extend(containers.Abbrevs("eval.Measurement")) + if err != nil { + return nil, fmt.Errorf("e.Container.Extend: failed to extend default container: %w", err) + } + e.Container = co + + return &Env{env: e, reg: reg}, nil +} diff --git a/api/internal/eval/eval.go b/api/internal/eval/eval.go new file mode 100644 index 00000000..043aa7bb --- /dev/null +++ b/api/internal/eval/eval.go @@ -0,0 +1,622 @@ +package eval + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "regexp" + "time" + + "github.com/USACE/instrumentation-api/api/v4/internal/db" + "github.com/USACE/instrumentation-api/api/v4/internal/dto" + "github.com/USACE/instrumentation-api/api/v4/internal/logger" + "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/common/types/traits" + "github.com/google/uuid" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + defaultPageSize = 200_000 +) + +var ( + reSpace = regexp.MustCompile(`\s+`) +) + +var ( + // If no date range is specified, capture the full dependency + // timeseries when creating new expression. + defaultStartsAt = time.Date(0001, 1, 1, 0, 0, 0, 0, time.UTC) + defaultEndsBefore = time.Date(9999, 12, 31, 23, 59, 59, 0, time.UTC) +) + +type EvalSession struct { + // q embeds a EvalSessionQuerier for running database queries + q EvalSessionQuerier + + // enq embeds a EvalSessionEnqueuer for enqueuing dependant expression compute jobs + enq EvalSessionEnqueuer + + // logger embeds a logger.Logger interface for strutured logging. + logger logger.Logger + + // pageSize is the number of records limited in each chunk when querying the database. + pageSize []int32 + + // args are required arguments for creating an EvalSession. + // Multiple args can be passed for batch processing, which may be more efficient for + // updates to shorter time ranges that run frequently. + args []EvalSessionParams + + // vars are the variables for each expression, as a slice to correspond + // with the number of args for batching. + vars [][]string + + // ts2Var keeps track of timeseries variable mappings for + // lookup during expression preprocessing. + ts2Var []map[uuid.UUID]string + + // tsIDs is a convenience array passed to the batchmany query + // based on timeseries variables for each arg. + tsIDs [][]uuid.UUID + + // evalContext is used for keeping track of window start and end + // and is accessed by expressions at runtime. + // It is used to determine the timestamping policy for window functions + // which may have ambiguous times (start, middle, or end). + evalContext []*evalContext + + // compiledExpr is an array of compiled expressions for each batch. + compiledExpr []*compiled + + // remember holds the last known value of a given point for point-mode + // LOCF (Last Observation Carried Forward) expression evaluation. + remember []map[string]*Measurement + + // windowBuffer and nextWindowBuffer are used for keeping + // track of quantized windows across different chunks. + windowBuffer [][]*Measurement + nextWindowBuffer [][]*Measurement + + // checkAlertTsIDs keeps track of which alert configs should + // be checked based on their assigned timeseries. + checkAlertTsIDs []map[uuid.UUID]struct{} + + // checkExpressionTsIDs keeps track of which expressions need + // to be evalauted which may depend on the output of expressions in this batch. + checkExpressionTsIDs []map[uuid.UUID]struct{} + + // programRegistry is a custom registry for the program to correctly reference + // and resolve the custom "eval.Measurement" protobuf message type + programRegistry *types.Registry + + // programCache is an optional in-memory two-queue lru (Least Recently Used) cache of compiled expressions. + programCache *ProgramCache +} + +type EvalSessionParams struct { + // TargetTimeseriesID represents the timeseries which resulting + // measurements of the expression will be written. + // Required. + TargetTimeseriesID uuid.UUID + + // ExpressionID is the ID of the expression to be evaluated. + // Required. + ExpressionID uuid.UUID + + // Expression is the DTO expression configuration. + // Required. + Expression dto.ExpressionDTO + + // StartsAt is the (inclusive) start time which measurements for the expression will be evaluated. + // Optional. Defaults to 0001-01-01T00:00:00Z. + StartsAt time.Time + + // EndsBefore is the (exclusive) end time which measurements for the expression will be evaluated. + // Optional. Defaults to 9999-01-01T00:00:00Z. + EndsBefore time.Time + + // PageSize defines the number of records the database will query PER CHUNK (the length of EvalSessionParams). + // The database will continue to query chunks until the length of queried records + // is 0 or less than the PageSize. + // Optional. Defaults to 20_000. + PageSize int32 +} + +// EvalSessionOption defines the type signature for variadic optional parameters. +type EvalSessionOption func(*EvalSession) + +type EvalSessionQuerier interface { + TimeseriesMeasurementListBatchForTimeseriesIDs(ctx context.Context, arg []db.TimeseriesMeasurementListBatchForTimeseriesIDsParams) *db.TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults + TimeseriesMeasurementCreateOrUpdateBatchShouldCheck(ctx context.Context, arg []db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams) *db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckBatchResults + ExpressionDownstreamListForTimeseries(ctx context.Context, depTimeseriesIds []uuid.UUID) ([]db.ExpressionDownstreamListForTimeseriesRow, error) + TimeseriesMeasurementListBatchEdgesForTimeseriesIDs(ctx context.Context, arg []db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsParams) *db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults + ExpressionGetBatch(ctx context.Context, id []uuid.UUID) *db.ExpressionGetBatchBatchResults +} + +type EvalSessionEnqueuer interface { + InsertMany(ctx context.Context, params []pgqueue.InsertManyParams) ([]int64, error) +} + +// NewEvalSession compiles a batch of expressions and prepares them for evaluation (session.Run). +func NewEvalSession(q EvalSessionQuerier, enq EvalSessionEnqueuer, l logger.Logger, baseEnv *Env, args []EvalSessionParams, options ...EvalSessionOption) (*EvalSession, error) { + n := len(args) + vars := make([][]string, n) + ts2Var := make([]map[uuid.UUID]string, n) + tsIDs := make([][]uuid.UUID, n) + evalCtx := make([]*evalContext, n) + compiledExpr := make([]*compiled, n) + remember := make([]map[string]*Measurement, n) + windowBuffer := make([][]*Measurement, n) + alertCheckTsIDs := make([]map[uuid.UUID]struct{}, n) + nextWindowBuffer := make([][]*Measurement, n) + pageSize := make([]int32, n) + + es := &EvalSession{ + q: q, + enq: enq, + logger: l, + args: args, + vars: vars, + ts2Var: ts2Var, + tsIDs: tsIDs, + evalContext: evalCtx, + compiledExpr: compiledExpr, + remember: remember, + windowBuffer: windowBuffer, + checkAlertTsIDs: alertCheckTsIDs, + nextWindowBuffer: nextWindowBuffer, + pageSize: pageSize, + } + + for _, optFn := range options { + optFn(es) + } + + for i, arg := range args { + // Set default values for time and page size. + if arg.StartsAt.IsZero() { + arg.StartsAt = defaultStartsAt + } + if arg.EndsBefore.IsZero() { + arg.EndsBefore = defaultEndsBefore + } + if arg.PageSize == 0 { + arg.PageSize = defaultPageSize + } + + // Validate required fields. + if arg.TargetTimeseriesID == uuid.Nil { + return nil, fmt.Errorf("EvalSessionParams[%d]: TargetTimeseriesID is required", i) + } + if arg.ExpressionID == uuid.Nil { + return nil, fmt.Errorf("EvalSessionParams[%d]: ExpressionID is required", i) + } + if arg.Expression.Expression == "" { + return nil, fmt.Errorf("EvalSessionParams[%d]: Expression.Expression is required", i) + } + + // Store possibly updated arg back in es.args. + es.args[i] = arg + + vars[i] = make([]string, len(arg.Expression.Variables)) + ts2Var[i] = make(map[uuid.UUID]string) + tsIDs[i] = make([]uuid.UUID, len(arg.Expression.Variables)) + for idx, v := range arg.Expression.Variables { + vars[i][idx] = v.VariableName + ts2Var[i][v.TimeseriesID] = v.VariableName + tsIDs[i][idx] = v.TimeseriesID + } + eCtx := Context() + if db.ExpressionMode(arg.Expression.Mode) == db.ExpressionModeWindow { + eCtx.TimestampingPolicy = TimestampingPolicy(arg.Expression.Opts.TimestampingPolicy) + } + + exprStr := reSpace.ReplaceAllString(arg.Expression.Expression, " ") + + checksumBytes := sha256.Sum256([]byte(exprStr)) + checksum := hex.EncodeToString(checksumBytes[:]) + + var ce *compiled + if es.programCache != nil && arg.ExpressionID != uuid.Nil { + entry, ok := es.programCache.Get(arg.ExpressionID) + needsCompile := true + if ok && entry.Checksum == checksum { + ce = entry.Compiled + needsCompile = false + } + if needsCompile { + newCe, err := baseEnv.CompileExpression(eCtx, exprStr, vars[i]) + if err != nil { + return nil, err + } + es.programCache.Set(arg.ExpressionID, &ProgramCacheEntry{Compiled: newCe, Checksum: checksum}) + ce = newCe + } + } else { + var err error + ce, err = baseEnv.CompileExpression(eCtx, exprStr, vars[i]) + if err != nil { + return nil, err + } + } + + evalCtx[i] = eCtx + compiledExpr[i] = ce + remember[i] = make(map[string]*Measurement) + windowBuffer[i] = []*Measurement{} + alertCheckTsIDs[i] = make(map[uuid.UUID]struct{}) + nextWindowBuffer[i] = []*Measurement{} + if arg.PageSize == 0 { + arg.PageSize = defaultPageSize + } + pageSize[i] = arg.PageSize + } + + return es, nil +} + +// Run is a top-level API for running an evaluation session. +func (sess *EvalSession) Run(ctx context.Context) error { + n := len(sess.args) + chunkStart := make([]time.Time, n) + for i := range sess.args { + chunkStart[i] = sess.args[i].StartsAt + } + + err := sess.seedEdgeMeasurements(ctx) + if err != nil { + return fmt.Errorf("seedEdgeMeasurements failed: %w", err) + } + + for { + params := make([]db.TimeseriesMeasurementListBatchForTimeseriesIDsParams, n) + for i := range sess.args { + params[i] = db.TimeseriesMeasurementListBatchForTimeseriesIDsParams{ + TimeseriesIDs: sess.tsIDs[i], + StartsAt: chunkStart[i], + EndsBefore: sess.args[i].EndsBefore, + PageSize: sess.pageSize[i], + } + } + + batch := sess.q.TimeseriesMeasurementListBatchForTimeseriesIDs(ctx, params) + batchResults := make([][]db.TimeseriesMeasurementListBatchForTimeseriesIDsRow, n) + var batchErr error + + batch.Query(func(idx int, mm []db.TimeseriesMeasurementListBatchForTimeseriesIDsRow, e error) { + if e != nil { + if errors.Is(e, sql.ErrNoRows) { + batchResults[idx] = nil + return + } + batchErr = fmt.Errorf("TimeseriesMeasurementListBatchForTimeseriesIDs SQL query error: %w", e) + return + } + batchResults[idx] = mm + }) + if batchErr != nil { + return batchErr + } + + upsertParams := make([]db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams, len(batchResults)) + + allDone := true + for i := range batchResults { + mm := batchResults[i] + if len(mm) == 0 && len(sess.windowBuffer[i]) == 0 { + continue + } + allDone = false + + switch db.ExpressionMode(sess.args[i].Expression.Mode) { + case db.ExpressionModePoint: + upsertParams[i], chunkStart[i] = sess.processPointModeBatch(ctx, i, mm) + + case db.ExpressionModeWindow: + upsertParams[i], chunkStart[i] = sess.processWindowModeBatch(ctx, i, mm) + sess.windowBuffer[i] = sess.nextWindowBuffer[i] + + default: + return errors.New("expression mode " + sess.args[i].Expression.Mode + " not supported") + } + } + + if len(upsertParams) > 0 { + bq := sess.q.TimeseriesMeasurementCreateOrUpdateBatchShouldCheck(ctx, upsertParams) + bq.Query(func(i int, rr []db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckRow, e error) { + if e != nil { + batchErr = e + return + } + for _, r := range rr { + if r.ShouldCheckAlert { + sess.checkAlertTsIDs[i][r.TimeseriesID] = struct{}{} + } + if r.ShouldCheckExpression { + sess.checkExpressionTsIDs[i][r.TimeseriesID] = struct{}{} + } + } + }) + if batchErr != nil { + return batchErr + } + } + + if allDone { + break + } + } + + if err := sess.batchEnqueueAfterInsertJobs(ctx); err != nil { + return err + } + return nil +} + +// dbRowToMeasurement converts a db row to a *Measurement. +func dbRowToMeasurement(row db.TimeseriesMeasurementListBatchForTimeseriesIDsRow) *Measurement { + return &Measurement{ + Time: timestamppb.New(row.Time), + Value: row.Value, + Masked: &row.Masked, + Validated: &row.Validated, + } +} + +// measurementsFromRefVal attempts to convert a ref.Val (result type returned from cel-go) to a *Measurement or []*Measurement. +func measurementsFromRefVal(res ref.Val, tsID uuid.UUID) (db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams, error) { + var out db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams + if mIface, err := res.ConvertToNative(measType); err == nil { + if m, ok := mIface.(*Measurement); ok && m.Time != nil { + out.TimeseriesIDs = append(out.TimeseriesIDs, tsID) + out.Times = append(out.Times, m.Time.AsTime()) + out.Values = append(out.Values, m.Value) + return out, nil + } + } + + if lister, ok := res.(traits.Lister); ok { + it := lister.Iterator() + for it.HasNext() == types.True { + elem := it.Next() + mIface, err := elem.ConvertToNative(measType) + if err != nil { + continue + } + if m, ok := mIface.(*Measurement); ok && m.Time != nil { + out.TimeseriesIDs = append(out.TimeseriesIDs, tsID) + out.Times = append(out.Times, m.Time.AsTime()) + out.Values = append(out.Values, m.Value) + } + } + if len(out.TimeseriesIDs) > 0 { + return out, nil + } + } + + return out, errors.New("unsupported result type for upload") +} + +// batchEnqueueAfterInsertJobs queues alert check event jobs for the target timeseries and timeframe. +func (sess *EvalSession) batchEnqueueAfterInsertJobs(ctx context.Context) error { + + affectedSet := make(map[uuid.UUID]struct{}) + for _, tsIDSet := range sess.checkExpressionTsIDs { + for tsID := range tsIDSet { + affectedSet[tsID] = struct{}{} + } + } + affectedDeps := make([]uuid.UUID, 0, len(affectedSet)) + for tsID := range affectedSet { + affectedDeps = append(affectedDeps, tsID) + } + + downstream, err := sess.q.ExpressionDownstreamListForTimeseries(ctx, affectedDeps) + if err != nil { + return err + } + + dag := NewExpressionDAG(downstream) + + // Build a lookup for time ranges by target timeseries (from args) + timeRangeByTs := make(map[uuid.UUID]struct { + MinTime time.Time + MaxTime time.Time + }) + for batchIdx, tsIDSet := range sess.checkExpressionTsIDs { + for tsID := range tsIDSet { + timeRangeByTs[tsID] = struct { + MinTime time.Time + MaxTime time.Time + }{ + MinTime: sess.args[batchIdx].StartsAt, + MaxTime: sess.args[batchIdx].EndsBefore, + } + } + } + + // Gather direct children timeseries for expressions that depend on any of the recently updated timeseries. + directChildren := map[uuid.UUID]struct{}{} + for tsID := range affectedSet { + for _, exprID := range dag.DirectExpressionsForTimeseries(tsID) { + directChildren[exprID] = struct{}{} + } + } + + // Check if exprID depends (directly or transitively) on any other direct child. + // Implement depth-first-search to traverse child nodes to determine if a direct + // child should be enqueued. If the child node depends on another child of the current node, + // skip enqueuing for that node, as it will be enqueued by the other child it depends on. + dependsOnOtherChild := func(exprID uuid.UUID) bool { + visited := map[uuid.UUID]struct{}{} + stack := []uuid.UUID{exprID} + + for len(stack) > 0 { + // Pop from stack + eid := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + if _, ok := visited[eid]; ok { + continue + } + visited[eid] = struct{}{} + + n := dag.nodes[eid] + for _, depTsID := range n.Dependencies { + for otherExprID := range directChildren { + if otherExprID != exprID && depTsID == dag.nodes[otherExprID].TargetTimeseriesID { + return true + } + } + // Push transitive dependencies onto stack + for _, childExprID := range dag.DirectExpressionsForTimeseries(depTsID) { + stack = append(stack, childExprID) + } + } + } + return false + } + + type key struct { + TimeseriesID uuid.UUID + MinTime time.Time + MaxTime time.Time + } + visitedKeys := make(map[key]struct{}) + jobs := []pgqueue.InsertManyParams{} + + exprGetBatchParams := make([]uuid.UUID, len(directChildren)) + i := 0 + for exprID := range directChildren { + exprGetBatchParams[i] = exprID + i++ + } + xx := make([]db.VExpression, len(exprGetBatchParams)) + var batchErr error + sess.q.ExpressionGetBatch(ctx, exprGetBatchParams).QueryRow(func(i int, r db.VExpression, err error) { + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + sess.logger.Warn( + ctx, + "expression could not be found potentially due to a race condition if it was recently deleted", + "err", err, + ) + return + } + batchErr = errors.Join(batchErr, err) + return + } + xx[i] = r + }) + if batchErr != nil { + return err + } + xLookup := make(map[uuid.UUID]dto.ExpressionDTO) + for _, x := range xx { + vars := make([]dto.ExpressionTimeseriesVariableDTO, len(x.Variables)) + for i, v := range x.Variables { + vars[i] = dto.ExpressionTimeseriesVariableDTO(v) + } + // pgqueue (river) encodes args as JSON, so we can use RawOpts + xLookup[x.ID] = dto.ExpressionDTO{ + Name: x.Name, + Expression: x.Expression, + Variables: vars, + Mode: string(x.Mode), + RawOpts: x.Opts, + } + } + + for exprID := range directChildren { + node := dag.nodes[exprID] + tr, ok := timeRangeByTs[node.TargetTimeseriesID] + if !ok { + continue + } + k := key{ + TimeseriesID: node.TargetTimeseriesID, + MinTime: tr.MinTime, + MaxTime: tr.MaxTime, + } + if _, seen := visitedKeys[k]; seen { + continue + } + if dependsOnOtherChild(exprID) { + continue + } + visitedKeys[k] = struct{}{} + + x, exists := xLookup[exprID] + if !exists { + sess.logger.Warn( + ctx, + "unable to get dto.ExpressionDTO by id from lookup table; skipping...", + "expression_id", exprID, + ) + continue + } + + jobArgs := dto.WorkerExpressionTimeseriesComputePartialEventArgs{ + MinTime: k.MinTime, + MaxTime: k.MaxTime, + } + jobArgs.TargetTimeseriesID = node.TargetTimeseriesID + jobArgs.ExpressionID = exprID + jobArgs.Expression = x + + jobs = append(jobs, pgqueue.InsertManyParams{ + Args: jobArgs, + InsertOpts: &pgqueue.InsertOpts{ + UniqueOpts: pgqueue.UniqueOpts{ + ByArgs: true, + }, + }, + }) + } + + uniqueAlerts := make(map[key]struct{}) + for batchIdx, checkAlertTsIDs := range sess.checkAlertTsIDs { + for tsID := range checkAlertTsIDs { + k := key{ + TimeseriesID: tsID, + MinTime: sess.args[batchIdx].StartsAt, + MaxTime: sess.args[batchIdx].EndsBefore, + } + uniqueAlerts[k] = struct{}{} + } + } + for k := range uniqueAlerts { + jobs = append(jobs, pgqueue.InsertManyParams{ + Args: dto.WorkerAlertEventArgs{ + TimeseriesToCheck: []dto.TimeseriesToCheck{{ + TimeseriesID: k.TimeseriesID, + MinTime: k.MinTime, + MaxTime: k.MaxTime, + }}, + }, + InsertOpts: &pgqueue.InsertOpts{ + UniqueOpts: pgqueue.UniqueOpts{ + ByArgs: true, + }, + }, + }) + } + + if len(jobs) == 0 { + return nil + } + + _, err = sess.enq.InsertMany(ctx, jobs) + if err != nil { + return fmt.Errorf("job InsertMany failed after expression timeseries compute: %w", err) + } + sess.logger.Debug(ctx, "jobs queued after expression timeseries compute", "count", len(jobs)) + return nil +} diff --git a/api/internal/eval/eval_point.go b/api/internal/eval/eval_point.go new file mode 100644 index 00000000..24c3cafb --- /dev/null +++ b/api/internal/eval/eval_point.go @@ -0,0 +1,106 @@ +package eval + +import ( + "context" + "database/sql" + "errors" + "math" + "time" + + "github.com/USACE/instrumentation-api/api/v4/internal/db" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// processPointMode evaluates an expression for a chunk of db rows in point mode. It assumes the records are in ascending order. +// Point mode runs the given expression for each point, using Last-Observation-Carried-Forward to fill in mis-matched dates +func (sess *EvalSession) processPointModeBatch( + ctx context.Context, + batchIdx int, + mm []db.TimeseriesMeasurementListBatchForTimeseriesIDsRow, +) (db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams, time.Time) { + var upsertParams db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams + var lastTime time.Time + for _, m := range mm { + varName := sess.ts2Var[batchIdx][m.TimeseriesID] + if varName == "" { + sess.logger.Warn(ctx, "variable name is empty; this should never happen", "mode", sess.args[batchIdx].Expression.Mode) + continue + } + sess.remember[batchIdx][varName] = dbRowToMeasurement(m) + evalArgs := make(map[string]any) + for _, v := range sess.vars[batchIdx] { + if meas, ok := sess.remember[batchIdx][v]; ok { + evalArgs[v] = meas + } else { + evalArgs[v] = &Measurement{Time: timestamppb.New(m.Time), Value: math.NaN()} + } + } + res, details, err := sess.compiledExpr[batchIdx].Prog.ContextEval(ctx, evalArgs) + if err != nil { + if res == nil { + sess.logger.Error(ctx, "critical error evaluating expression", "time", m.Time, "error", err) + return upsertParams, lastTime + } + sess.logger.Error(ctx, "runtime error evaluating expression", "time", m.Time, "error", err) + continue + } + sess.logger.Debug(ctx, "expression evaluated", "time", m.Time, "result", res, "details", details) + results, err := measurementsFromRefVal(res, sess.args[batchIdx].TargetTimeseriesID) + if err != nil { + sess.logger.Error(ctx, "failed to process result for upload", "error", err) + continue + } + upsertParams.TimeseriesIDs = append(upsertParams.TimeseriesIDs, results.TimeseriesIDs...) + upsertParams.Times = append(upsertParams.Times, results.Times...) + upsertParams.Values = append(upsertParams.Values, results.Values...) + lastTime = m.Time + } + return upsertParams, lastTime +} + +// seedEdgeMeasurements fetches edge measurements for all batches (needed for point mode only). +// Note, only Before* row results are implemented. After* rows are not needed for LOCF. +// After* rows could be used if interpolation is added in the future. +func (sess *EvalSession) seedEdgeMeasurements(ctx context.Context) error { + n := len(sess.args) + params := make([]db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsParams, n) + for i := range sess.args { + params[i] = db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsParams{ + TimeseriesIDs: sess.tsIDs[i], + StartsAt: sess.args[i].StartsAt, + EndsBefore: sess.args[i].EndsBefore, + } + } + + var batchErr error + batchEdges := make([][]db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsRow, n) + + bq := sess.q.TimeseriesMeasurementListBatchEdgesForTimeseriesIDs(ctx, params) + bq.Query(func(idx int, rows []db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsRow, err error) { + if err != nil && !errors.Is(err, sql.ErrNoRows) { + batchErr = err + return + } + batchEdges[idx] = rows + }) + if batchErr != nil { + return batchErr + } + + // Seed the remember cache for LOCF processing. + for batchIdx, batch := range batchEdges { + for _, edge := range batch { + variable, exists := sess.ts2Var[batchIdx][edge.TimeseriesID] + if !exists { + continue + } + sess.remember[batchIdx][variable] = &Measurement{ + Time: timestamppb.New(edge.BeforeTime), + Value: edge.BeforeValue, + Masked: &edge.BeforeMasked, + Validated: &edge.BeforeValidated, + } + } + } + return nil +} diff --git a/api/internal/eval/eval_test.go b/api/internal/eval/eval_test.go new file mode 100644 index 00000000..c86c482c --- /dev/null +++ b/api/internal/eval/eval_test.go @@ -0,0 +1,88 @@ +package eval + +import ( + "context" + "testing" + "time" + + "github.com/USACE/instrumentation-api/api/v4/internal/config" + "github.com/USACE/instrumentation-api/api/v4/internal/db" + "github.com/USACE/instrumentation-api/api/v4/internal/dto" + "github.com/USACE/instrumentation-api/api/v4/internal/logger" + "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" + "github.com/google/uuid" +) + +type mockEvalSession struct{} + +func (*mockEvalSession) TimeseriesMeasurementListBatchForTimeseriesIDs(ctx context.Context, arg []db.TimeseriesMeasurementListBatchForTimeseriesIDsParams) *db.TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults { + return &db.TimeseriesMeasurementListBatchForTimeseriesIDsBatchResults{} +} + +func (*mockEvalSession) TimeseriesMeasurementCreateOrUpdateBatchShouldCheck(ctx context.Context, arg []db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams) *db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckBatchResults { + return &db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckBatchResults{} +} + +func (*mockEvalSession) ExpressionDownstreamListForTimeseries(ctx context.Context, depTimeseriesIds []uuid.UUID) ([]db.ExpressionDownstreamListForTimeseriesRow, error) { + return make([]db.ExpressionDownstreamListForTimeseriesRow, 0), nil +} + +func (*mockEvalSession) TimeseriesMeasurementListBatchEdgesForTimeseriesIDs(ctx context.Context, arg []db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsParams) *db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults { + return &db.TimeseriesMeasurementListBatchEdgesForTimeseriesIDsBatchResults{} +} + +func (*mockEvalSession) ExpressionGetBatch(ctx context.Context, id []uuid.UUID) *db.ExpressionGetBatchBatchResults { + return &db.ExpressionGetBatchBatchResults{} +} + +func (*mockEvalSession) InsertMany(ctx context.Context, params []pgqueue.InsertManyParams) ([]int64, error) { + return make([]int64, 0), nil +} + +func TestNewEvalSession_Valid(t *testing.T) { + env, err := NewEnv() + if err != nil { + t.Fatal(err) + } + + args := []EvalSessionParams{ + { + TargetTimeseriesID: uuid.New(), + ExpressionID: uuid.New(), + Expression: dto.ExpressionDTO{ + Expression: "abs(x)", + Variables: []dto.ExpressionTimeseriesVariableDTO{ + {VariableName: "x", TimeseriesID: uuid.New()}, + }, + Mode: "point", + }, + StartsAt: time.Now(), + EndsBefore: time.Now().Add(time.Hour), + PageSize: 10, + }, + } + mock := &mockEvalSession{} + sess, err := NewEvalSession(mock, mock, logger.NewLogger(&config.LoggerConfig{}), env, args) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if sess == nil { + t.Fatal("expected non-nil EvalSession") + } +} + +func TestNewEvalSession_InvalidArgs(t *testing.T) { + env, _ := NewEnv() + args := []EvalSessionParams{ + { + TargetTimeseriesID: uuid.Nil, + ExpressionID: uuid.New(), + Expression: dto.ExpressionDTO{Expression: "abs(x)"}, + }, + } + mock := &mockEvalSession{} + _, err := NewEvalSession(mock, mock, logger.NewLogger(&config.LoggerConfig{}), env, args) + if err == nil { + t.Error("expected error for missing TargetTimeseriesID") + } +} diff --git a/api/internal/eval/eval_window.go b/api/internal/eval/eval_window.go new file mode 100644 index 00000000..f1cc24ec --- /dev/null +++ b/api/internal/eval/eval_window.go @@ -0,0 +1,139 @@ +package eval + +import ( + "context" + "time" + + "github.com/USACE/instrumentation-api/api/v4/internal/db" + "github.com/USACE/instrumentation-api/api/v4/internal/dto" + iso8601duration "github.com/xnacly/go-iso8601-duration" +) + +type window struct { + start time.Time + end time.Time +} + +// processWindowModeBatch evaluates an expression for a chunk of db rows in window mode. +// Window mode aggregates []*Measurement data into fixed windows, quantized based on a user-specified duration. +// An internal windowBuffer is set to handle the case where a chunk of database rows does not align with a "window". +func (sess *EvalSession) processWindowModeBatch( + ctx context.Context, + batchIdx int, + mm []db.TimeseriesMeasurementListBatchForTimeseriesIDsRow, +) (db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams, time.Time) { + var upsertParams db.TimeseriesMeasurementCreateOrUpdateBatchShouldCheckParams + allMeasurements := append(sess.windowBuffer[batchIdx], make([]*Measurement, 0, len(mm))...) + for _, m := range mm { + varName := sess.ts2Var[batchIdx][m.TimeseriesID] + if varName == "" { + sess.logger.Warn(ctx, "variable name is empty; this should never happen", "mode", sess.args[batchIdx].Expression.Mode) + continue + } + allMeasurements = append(allMeasurements, dbRowToMeasurement(m)) + } + if len(allMeasurements) == 0 { + return upsertParams, time.Time{} + } + chunkEnd := allMeasurements[len(allMeasurements)-1].Time.AsTime() + windows, err := sess.quantizeWindows(ctx, allMeasurements, sess.args[batchIdx].Expression.Opts.ExpressionWindowOptsDTO) + if err != nil { + sess.logger.Error(ctx, "failed to quantize windows", "error", err) + return upsertParams, chunkEnd + } + loc, err := time.LoadLocation(sess.args[batchIdx].Expression.Opts.TzName) + if err != nil { + sess.logger.Warn(ctx, "invalid tz_name option passed, defaulting to UTC", "tz_name", sess.args[batchIdx].Expression.Opts.ExpressionWindowOptsDTO.TzName) + loc = time.UTC + } + var nextBuffer []*Measurement + for _, w := range windows { + if w.end.After(chunkEnd) { + nextBuffer = measurementsInWindow(allMeasurements, w, loc) + continue + } + evalArgs := make(map[string]any) + for _, varName := range sess.vars[batchIdx] { + ms := measurementsInWindow(allMeasurements, w, loc) + evalArgs[varName] = ms + } + sess.evalContext[batchIdx].PolicyWindowStart = w.start + sess.evalContext[batchIdx].PolicyWindowEnd = w.end + + res, details, err := sess.compiledExpr[batchIdx].Prog.ContextEval(ctx, evalArgs) + if err != nil { + if res == nil { + sess.logger.Error(ctx, "critical error evaluating window expression", "window", w, "error", err) + return upsertParams, chunkEnd + } + sess.logger.Error(ctx, "runtime error evaluating window expression", "window", w, "error", err) + continue + } + sess.logger.Debug(ctx, "window expression evaluated", "window", w, "result", res, "details", details) + results, err := measurementsFromRefVal(res, sess.args[batchIdx].TargetTimeseriesID) + if err != nil { + sess.logger.Error(ctx, "failed to process result for upload", "error", err) + continue + } + upsertParams.TimeseriesIDs = append(upsertParams.TimeseriesIDs, results.TimeseriesIDs...) + upsertParams.Times = append(upsertParams.Times, results.Times...) + upsertParams.Values = append(upsertParams.Values, results.Values...) + } + sess.nextWindowBuffer[batchIdx] = nextBuffer + return upsertParams, chunkEnd +} + +// quantizeWindows quantizes windows based on user options +func (sess *EvalSession) quantizeWindows(ctx context.Context, measurements []*Measurement, opts dto.ExpressionWindowOptsDTO) ([]window, error) { + dur, err := parseISODuration(opts.Duration) + if err != nil { + return nil, err + } + offset, err := parseISODuration(opts.DurationOffset) + if err != nil { + return nil, err + } + loc, err := time.LoadLocation(opts.TzName) + if err != nil { + sess.logger.Warn(ctx, "invalid tz_name option passed, defaulting to UTC", "tz_name", opts.TzName) + loc = time.UTC + } + var minTime, maxTime time.Time + for i, m := range measurements { + t := m.Time.AsTime().In(loc) + if i == 0 || t.Before(minTime) { + minTime = t + } + if i == 0 || t.After(maxTime) { + maxTime = t + } + } + var windows []window + windowStart := minTime.Truncate(dur).Add(offset) + for ws := windowStart; ws.Before(maxTime); ws = ws.Add(dur) { + we := ws.Add(dur) + windows = append(windows, window{start: ws, end: we}) + } + return windows, nil +} + +// measurementsInWindow buffers measurements for incomplete windows across chunks +func measurementsInWindow(measurements []*Measurement, w window, loc *time.Location) []*Measurement { + var out []*Measurement + for _, m := range measurements { + t := m.Time.AsTime().In(loc) + if !t.Before(w.start) && t.Before(w.end) { + out = append(out, m) + } + } + return out +} + +// parseISODuration parses an ISO8601 duration to time.Duration +func parseISODuration(s string) (time.Duration, error) { + dur, err := iso8601duration.From(s) + if err != nil { + return 0, err + } + return dur.Duration(), nil +} diff --git a/api/internal/eval/libmath.go b/api/internal/eval/libmath.go new file mode 100644 index 00000000..d8fd5bd5 --- /dev/null +++ b/api/internal/eval/libmath.go @@ -0,0 +1,420 @@ +package eval + +import ( + "math" + "time" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/common/types/traits" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" +) + +func NewMathLib(reg *types.Registry) cel.EnvOption { + return cel.Lib(mathLib{reg}) +} + +type mathLib struct { + reg *types.Registry +} + +func (mathLib) ProgramOptions() []cel.ProgramOption { + return []cel.ProgramOption{} +} + +// MathLib returns a collection of CEL EnvOptions that define mathematical +// functions and constants +func (lib mathLib) CompileOptions() []cel.EnvOption { + // Functions + // abs(x Measurement) -> Measurement + absFn := lib.bindUnaryMeas("abs", math.Abs) + + // round(x Measurement) -> Measurement + roundFn := lib.bindUnaryMeas("round", math.Round) + + // floor(x Measurement) -> Measurement + floorFn := lib.bindUnaryMeas("floor", math.Floor) + + // ceil(x Measurement) -> Measurement + ceilFn := lib.bindUnaryMeas("ceil", math.Ceil) + + // clamp(m Measurement, lo number, hi number) -> Measurement + clampFn := cel.Function("clamp", + cel.Overload("clamp_meas_number_number", + []*cel.Type{CELMeasurementType, cel.DoubleType, cel.DoubleType}, + CELMeasurementType, + cel.FunctionBinding(func(args ...ref.Val) ref.Val { + if len(args) != 3 { + return types.NewErr("clamp expects 3 args") + } + m, err := convertToMeasurement(args[0]) + if err != nil { + return types.NewErr("clamp expects Measurement as arg1: %v", err) + } + x := m.Value + lo := celNumberToFloat(args[1]) + hi := celNumberToFloat(args[2]) + if lo > hi { + lo, hi = hi, lo + } + var outVal float64 + switch { + case x < lo: + outVal = lo + case x > hi: + outVal = hi + default: + outVal = x + } + out := &Measurement{ + Time: m.Time, + Value: outVal, + Masked: m.Masked, + Validated: m.Validated, + } + return lib.reg.NativeToValue(out) + }), + ), + ) + + // pow(x Measurement, y number) -> Measurement + powFn := cel.Function("pow", + cel.Overload("pow_meas_number", + []*cel.Type{CELMeasurementType, cel.DoubleType}, + CELMeasurementType, + cel.BinaryBinding(func(a, b ref.Val) ref.Val { + m, err := convertToMeasurement(a) + if err != nil { + return types.NewErr("pow expects Measurement as arg1: %v", err) + } + m.Value = math.Pow(m.Value, celNumberToFloat(b)) + return lib.reg.NativeToValue(m) + }), + ), + ) + + // sqrt(x Measurement) -> Measurement + sqrtFn := lib.bindUnaryMeas("sqrt", math.Sqrt) + + // log(x Measurement) -> Measurement + logFn := lib.bindUnaryMeas("log", math.Log) + + // log10(x Measurement) -> Measurement + log10Fn := lib.bindUnaryMeas("log10", math.Log10) + + // exp(x Measurement) -> Measurement + expFn := lib.bindUnaryMeas("exp", math.Exp) + + // sin(x Measurement) -> Measurement + sinFn := lib.bindUnaryMeas("sin", math.Sin) + + // cos(x Measurement) -> Measurement + cosFn := lib.bindUnaryMeas("cos", math.Cos) + + // tan(x Measurement) -> Measurement + tanFn := lib.bindUnaryMeas("tan", math.Tan) + + // Helper functions for dealing with NaN values + + // isnan(x Measurement) -> bool + isnanFn := cel.Function("isnan", + cel.Overload("isnan_meas", + []*cel.Type{CELMeasurementType}, + cel.BoolType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + m, err := convertToMeasurement(v) + if err != nil { + return types.NewErr("isnan expects Measurement as arg: %v", err) + } + return types.Bool(math.IsNaN(m.Value)) + }), + ), + ) + + // coalesce(a Measurement, b Measurement) -> Measurement + // If first value is NaN, return second + coalesceFn := cel.Function("coalesce", + cel.Overload("coalesce_meas_meas", + []*cel.Type{CELMeasurementType, CELMeasurementType}, + CELMeasurementType, + cel.BinaryBinding(func(a, b ref.Val) ref.Val { + ma, err := convertToMeasurement(a) + if err != nil { + return types.NewErr("isnan expects Measurement as arg: %v", err) + } + if math.IsNaN(ma.Value) { + mb, err := convertToMeasurement(a) + if err != nil { + return types.NewErr("isnan expects Measurement as arg: %v", err) + } + return lib.reg.NativeToValue(mb) + } + return lib.reg.NativeToValue(ma) + }), + ), + ) + + // Constants + // NAN -> double + nanConst := bindConstDouble("NAN", func() float64 { + return math.NaN() + }) + // INF -> double + infConst := bindConstDouble("INF", func() float64 { + return math.Inf(1) + }) + // NEGATIVE_INF -> double + negativeInfConst := bindConstDouble("NEGATIVE_INF", func() float64 { + return math.Inf(-1) + }) + // PI -> double + piConst := bindConstDouble("PI", func() float64 { + return math.Pi + }) + // E -> double + eConst := bindConstDouble("E", func() float64 { + return math.E + }) + // PHI -> double + phiConst := bindConstDouble("PHI", func() float64 { + return (1 + math.Sqrt(5)) / 2 + }) + // TAU -> double + tauConst := bindConstDouble("TAU", func() float64 { + return 2 * math.Pi + }) + // SQRT_2 -> double + sqrt2Const := bindConstDouble("SQRT_2", func() float64 { + return math.Sqrt2 + }) + // SQRT_PI -> double + sqrtPiConst := bindConstDouble("SQRT_PI", func() float64 { + return math.SqrtPi + }) + // SQRT_E -> double + sqrtEConst := bindConstDouble("SQRT_E", func() float64 { + return math.SqrtE + }) + // SQRT_PHI -> double + sqrtPhiConst := bindConstDouble("SQRT_PHI", func() float64 { + return math.SqrtPhi + }) + // LN_2 -> double + ln2Const := bindConstDouble("LN_2", func() float64 { + return math.Ln2 + }) + // LN_10 -> double + ln10Const := bindConstDouble("LN_10", func() float64 { + return math.Ln10 + }) + // GRAVITY -> double + gravityConst := bindConstDouble("GRAVITY", func() float64 { + return 9.80665 + }) + + return []cel.EnvOption{ + // Functions + absFn, roundFn, floorFn, ceilFn, clampFn, + powFn, sqrtFn, logFn, log10Fn, expFn, + sinFn, cosFn, tanFn, + isnanFn, coalesceFn, + + // Constants + nanConst, infConst, negativeInfConst, + piConst, eConst, phiConst, tauConst, + sqrt2Const, sqrtPiConst, sqrtEConst, sqrtPhiConst, + ln2Const, ln10Const, gravityConst, + } +} + +func celMeasListMinByValue(v ref.Val) (best ref.Val, ok bool) { + lst, ok2 := v.(traits.Lister) + if !ok2 { + return nil, false + } + it := lst.Iterator() + if it.HasNext() != types.True { + return nil, false + } + bRaw := it.Next() + b, err := convertToMeasurement(bRaw) + if err != nil { + return nil, false + } + bv := b.Value + bestRaw := bRaw + for it.HasNext() == types.True { + curRaw := it.Next() + cur, err := convertToMeasurement(curRaw) + if err != nil { + continue + } + cv := cur.Value + if cv < bv || (math.IsNaN(bv) && !math.IsNaN(cv)) { + bestRaw, b, bv = curRaw, cur, cv + } + } + return bestRaw, true +} + +func celMeasListMaxByValue(v ref.Val) (best ref.Val, ok bool) { + lst, ok2 := v.(traits.Lister) + if !ok2 { + return nil, false + } + it := lst.Iterator() + if it.HasNext() != types.True { + return nil, false + } + bRaw := it.Next() + b, err := convertToMeasurement(bRaw) + if err != nil { + return nil, false + } + bv := b.Value + bestRaw := bRaw + for it.HasNext() == types.True { + curRaw := it.Next() + cur, err := convertToMeasurement(curRaw) + if err != nil { + continue + } + cv := cur.Value + if cv > bv || (!math.IsNaN(cv) && math.IsNaN(bv)) { + bestRaw, b, bv = curRaw, cur, cv + } + } + return bestRaw, true +} + +func NewContextMathLib(ctx *evalContext, reg *types.Registry) cel.EnvOption { + return cel.Lib(contextMathLib{ctx, reg}) +} + +type contextMathLib struct { + ctx *evalContext + reg *types.Registry +} + +func (contextMathLib) ProgramOptions() []cel.ProgramOption { + return []cel.ProgramOption{} +} + +func (lib contextMathLib) CompileOptions() []cel.EnvOption { + // Functions to aggregate over list + + // emitWithPolicy is a helper function that assigns a value val to + // a winner if the timestamp measurement exists, falling back to + // the timestamp_policy set in the window_opts + emitWithPolicy := func(val float64, winner ref.Val) ref.Val { + if winner != nil { + m, err := convertToMeasurement(winner) + if err != nil { + return types.NewErr("emitWithPolicy: winner is not Measurement: %v", err) + } + m.Value = val + return lib.reg.NativeToValue(m) + } + var t time.Time + switch lib.ctx.TimestampingPolicy { + case PolicyCenter: + t = lib.ctx.PolicyWindowStart.Add(lib.ctx.PolicyWindowEnd.Sub(lib.ctx.PolicyWindowStart) / 2) + case PolicyEnd: + t = lib.ctx.PolicyWindowEnd + default: + t = lib.ctx.PolicyWindowStart + } + out := &Measurement{ + Time: timestamppb.New(t), + Value: val, + Masked: nil, + Validated: nil, + } + return lib.reg.NativeToValue(out) + } + + // min(mm list) -> Measurement + minFn := cel.Function("min", + cel.Overload("min_list_meas", + []*cel.Type{cel.ListType(CELMeasurementType)}, + CELMeasurementType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + best, ok := celMeasListMinByValue(v) + if !ok { + return emitWithPolicy(math.NaN(), nil) + } + return best + }), + ), + ) + + // max(mm list) -> Measurement + maxFn := cel.Function("max", + cel.Overload("max_list_meas", + []*cel.Type{cel.ListType(CELMeasurementType)}, + CELMeasurementType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + best, ok := celMeasListMaxByValue(v) + if !ok { + return emitWithPolicy(math.NaN(), nil) + } + return best + }), + ), + ) + + // mean(mm list) -> Measurement + meanFn := cel.Function("mean", + cel.Overload("mean_list_meas", + []*cel.Type{cel.ListType(CELMeasurementType)}, + CELMeasurementType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + xs, err := celMeasListToFloat64(v) + if err != nil { + return types.NewErr("mean: %v", err) + } + if len(xs) == 0 { + return emitWithPolicy(math.NaN(), nil) + } + sum := 0.0 + for _, x := range xs { + sum += x + } + return emitWithPolicy(sum/float64(len(xs)), nil) + }), + ), + ) + + // stddev(mm list) -> Measurement + stddevFn := cel.Function("stddev", + cel.Overload("stddev_list_meas", + []*cel.Type{cel.ListType(CELMeasurementType)}, + CELMeasurementType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + xs, err := celMeasListToFloat64(v) + if err != nil { + return types.NewErr("stddev: %v", err) + } + if len(xs) == 0 { + return emitWithPolicy(math.NaN(), nil) + } + m := 0.0 + for _, x := range xs { + m += x + } + m /= float64(len(xs)) + var ss float64 + for _, x := range xs { + d := x - m + ss += d * d + } + return emitWithPolicy(math.Sqrt(ss/float64(len(xs))), nil) + }), + ), + ) + + return []cel.EnvOption{ + minFn, maxFn, meanFn, stddevFn, + } +} diff --git a/api/internal/eval/libmath_test.go b/api/internal/eval/libmath_test.go new file mode 100644 index 00000000..a2f7f5c7 --- /dev/null +++ b/api/internal/eval/libmath_test.go @@ -0,0 +1,25 @@ +package eval + +import ( + "testing" +) + +func TestCumSum(t *testing.T) { + out, err := cumSum([]float64{1, 2, 3}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(out) != 3 || out[2] != 6 { + t.Errorf("unexpected output: %v", out) + } +} + +func TestDelta(t *testing.T) { + out, err := delta([]float64{1, 3, 6}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(out) != 2 || out[0] != 2 || out[1] != 3 { + t.Errorf("unexpected output: %v", out) + } +} diff --git a/api/internal/eval/libtelemetry.go b/api/internal/eval/libtelemetry.go new file mode 100644 index 00000000..a82300cb --- /dev/null +++ b/api/internal/eval/libtelemetry.go @@ -0,0 +1,223 @@ +package eval + +import ( + "errors" + "fmt" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" +) + +func NewTelemetryLib(reg *types.Registry) cel.EnvOption { + return cel.Lib(telemetryLib{reg}) +} + +type telemetryLib struct { + reg *types.Registry +} + +func (telemetryLib) ProgramOptions() []cel.ProgramOption { + return []cel.ProgramOption{} +} + +// TelemetryLib returns a collection of CEL EnvOptions useful for working with remote sensing telemetry data +func (l telemetryLib) CompileOptions() []cel.EnvOption { + // Functions + // moving_avg(mm list, win int) -> list + movingAvgFn := cel.Function("moving_avg", + cel.Overload("moving_avg_list_meas_number", + []*cel.Type{cel.ListType(CELMeasurementType), cel.IntType}, + cel.ListType(CELMeasurementType), + cel.FunctionBinding(func(args ...ref.Val) ref.Val { + mm, err := celMeasListToMeasGoSlice(args[0]) + if err != nil { + return types.NewErr("moving_avg: %v", err) + } + win := int(celNumberToFloat(args[1])) + outVals, kerr := sma(mm, win) + if kerr != nil { + return types.NewErr("moving_avg: %v", kerr) + } + return types.NewDynamicList(l.reg, outVals) + }), + ), + ) + + // exp_moving_avg(mm list, alpha float) -> list + expMovingAvgFn := cel.Function("exp_moving_avg", + cel.Overload("exp_moving_avg_list_meas_number", + []*cel.Type{cel.ListType(CELMeasurementType), cel.DoubleType}, + cel.ListType(CELMeasurementType), + cel.FunctionBinding(func(args ...ref.Val) ref.Val { + mm, err := celMeasListToMeasGoSlice(args[0]) + if err != nil { + return types.NewErr("exp_moving_avg: %v", err) + } + alpha := celNumberToFloat(args[1]) + outVals, kerr := ema(mm, alpha) + if kerr != nil { + return types.NewErr("exp_moving_avg: %v", kerr) + } + return types.NewDynamicList(l.reg, outVals) + }), + ), + ) + + // delta(mm list) -> list + deltaFn := cel.Function("delta", + cel.Overload("delta_list_meas", + []*cel.Type{cel.ListType(CELMeasurementType)}, + cel.ListType(CELMeasurementType), + cel.UnaryBinding(func(arg ref.Val) ref.Val { + mm, err := celMeasListToMeasGoSlice(arg) + if err != nil { + return types.NewErr("delta: %v", err) + } + values := make([]float64, len(mm)) + for i, m := range mm { + values[i] = m.Value + } + outVals, kerr := delta(values) + if kerr != nil { + return types.NewErr("delta: %v", kerr) + } + out := make([]*Measurement, len(outVals)) + for i := range outVals { + // Align to the later sample + m := mm[i+1] + out[i] = &Measurement{ + Time: m.Time, + Value: outVals[i], + Masked: m.Masked, + Validated: m.Validated, + } + } + return types.NewDynamicList(l.reg, out) + }), + ), + ) + + // cumsum(mm list) -> list + cumsumFn := cel.Function("cumsum", + cel.Overload("cumsum_list_meas", + []*cel.Type{cel.ListType(CELMeasurementType)}, + cel.ListType(CELMeasurementType), + cel.UnaryBinding(func(arg ref.Val) ref.Val { + mm, err := celMeasListToMeasGoSlice(arg) + if err != nil { + return types.NewErr("cumsum: %v", err) + } + values := make([]float64, len(mm)) + for i, m := range mm { + values[i] = m.Value + } + outVals, kerr := cumSum(values) + if kerr != nil { + return types.NewErr("cumsum: %v", kerr) + } + out := make([]*Measurement, len(outVals)) + for i := range outVals { + m := mm[i] + out[i] = &Measurement{ + Time: m.Time, + Value: outVals[i], + Masked: m.Masked, + Validated: m.Validated, + } + } + return types.NewDynamicList(l.reg, out) + }), + ), + ) + + return []cel.EnvOption{ + movingAvgFn, expMovingAvgFn, deltaFn, cumsumFn, + } +} + +// sma returns a slice of *Measurement representing the Simple Moving Average. +func sma(measurements []*Measurement, window int) ([]*Measurement, error) { + if window <= 0 || window > len(measurements) { + window = len(measurements) + } + out := make([]*Measurement, 0, len(measurements)-window+1) + for i := 0; i <= len(measurements)-window; i++ { + sum := 0.0 + for j := i; j < i+window; j++ { + sum += measurements[j].Value + } + avg := sum / float64(window) + lastPos := i + window - 1 + if lastPos < 0 { + // this should never happen + return nil, fmt.Errorf("invalid array length %d for window of %d", len(measurements), window) + } + last := measurements[lastPos] + out = append(out, &Measurement{ + Time: last.Time, + Value: avg, + Masked: last.Masked, + Validated: last.Validated, + }) + } + return out, nil +} + +// ema returns a slice of *Measurement representing the Exponential Moving Average. +func ema(measurements []*Measurement, alpha float64) ([]*Measurement, error) { + if len(measurements) == 0 { + return make([]*Measurement, 0), nil + } + if alpha <= 0 || alpha > 1 { + return nil, errors.New("invalid alpha") + } + out := make([]*Measurement, len(measurements)) + ema := measurements[0].Value + first := measurements[0] + out[0] = &Measurement{ + Time: first.Time, + Value: ema, + Masked: first.Masked, + Validated: first.Validated, + } + for i := 1; i < len(measurements); i++ { + ema = alpha*measurements[i].Value + (1-alpha)*ema + cur := measurements[i] + out[i] = &Measurement{ + Time: cur.Time, + Value: ema, + Masked: cur.Masked, + Validated: cur.Validated, + } + } + return out, nil +} + +// delta returns the difference between consecutive values in a slice. +func delta(xs []float64) ([]float64, error) { + if len(xs) < 2 { + return nil, errors.New("need at least 2 values") + } + out := make([]float64, len(xs)-1) + prev := xs[0] + for i := 1; i < len(xs); i++ { + out[i-1] = xs[i] - prev + prev = xs[i] + } + return out, nil +} + +// cumSum returns the cumulative sum of a slice of float64. +func cumSum(xs []float64) ([]float64, error) { + if len(xs) == 0 { + return nil, errors.New("empty list") + } + out := make([]float64, len(xs)) + sum := 0.0 + for i, v := range xs { + sum += v + out[i] = sum + } + return out, nil +} diff --git a/api/internal/eval/libtelemetry_test.go b/api/internal/eval/libtelemetry_test.go new file mode 100644 index 00000000..d6644d12 --- /dev/null +++ b/api/internal/eval/libtelemetry_test.go @@ -0,0 +1,31 @@ +package eval + +import ( + "testing" +) + +func TestSMA(t *testing.T) { + m1 := &Measurement{Value: 1} + m2 := &Measurement{Value: 2} + m3 := &Measurement{Value: 3} + out, err := sma([]*Measurement{m1, m2, m3}, 2) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(out) != 2 || out[0].Value != 1.5 || out[1].Value != 2.5 { + t.Errorf("unexpected output: %v", out) + } +} + +func TestEMA(t *testing.T) { + m1 := &Measurement{Value: 1} + m2 := &Measurement{Value: 2} + m3 := &Measurement{Value: 3} + out, err := ema([]*Measurement{m1, m2, m3}, 0.5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(out) != 3 || out[2].Value <= 0 { + t.Errorf("unexpected output: %v", out) + } +} diff --git a/api/internal/eval/measurement.pb.go b/api/internal/eval/measurement.pb.go new file mode 100644 index 00000000..c908fee7 --- /dev/null +++ b/api/internal/eval/measurement.pb.go @@ -0,0 +1,156 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.9 +// protoc v3.21.12 +// source: api/internal/eval/measurement.proto + +package eval + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Measurement struct { + state protoimpl.MessageState `protogen:"open.v1"` + Time *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=time,proto3" json:"time,omitempty"` + Value float64 `protobuf:"fixed64,2,opt,name=value,proto3" json:"value,omitempty"` + Masked *bool `protobuf:"varint,3,opt,name=masked,proto3,oneof" json:"masked,omitempty"` + Validated *bool `protobuf:"varint,4,opt,name=validated,proto3,oneof" json:"validated,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Measurement) Reset() { + *x = Measurement{} + mi := &file_api_internal_eval_measurement_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Measurement) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Measurement) ProtoMessage() {} + +func (x *Measurement) ProtoReflect() protoreflect.Message { + mi := &file_api_internal_eval_measurement_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Measurement.ProtoReflect.Descriptor instead. +func (*Measurement) Descriptor() ([]byte, []int) { + return file_api_internal_eval_measurement_proto_rawDescGZIP(), []int{0} +} + +func (x *Measurement) GetTime() *timestamppb.Timestamp { + if x != nil { + return x.Time + } + return nil +} + +func (x *Measurement) GetValue() float64 { + if x != nil { + return x.Value + } + return 0 +} + +func (x *Measurement) GetMasked() bool { + if x != nil && x.Masked != nil { + return *x.Masked + } + return false +} + +func (x *Measurement) GetValidated() bool { + if x != nil && x.Validated != nil { + return *x.Validated + } + return false +} + +var File_api_internal_eval_measurement_proto protoreflect.FileDescriptor + +const file_api_internal_eval_measurement_proto_rawDesc = "" + + "\n" + + "#api/internal/eval/measurement.proto\x12\x04eval\x1a\x1fgoogle/protobuf/timestamp.proto\"\xac\x01\n" + + "\vMeasurement\x12.\n" + + "\x04time\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x04time\x12\x14\n" + + "\x05value\x18\x02 \x01(\x01R\x05value\x12\x1b\n" + + "\x06masked\x18\x03 \x01(\bH\x00R\x06masked\x88\x01\x01\x12!\n" + + "\tvalidated\x18\x04 \x01(\bH\x01R\tvalidated\x88\x01\x01B\t\n" + + "\a_maskedB\f\n" + + "\n" + + "_validatedB=Z;github.com/USACE/instrumentation-api/api/internal/eval;evalb\x06proto3" + +var ( + file_api_internal_eval_measurement_proto_rawDescOnce sync.Once + file_api_internal_eval_measurement_proto_rawDescData []byte +) + +func file_api_internal_eval_measurement_proto_rawDescGZIP() []byte { + file_api_internal_eval_measurement_proto_rawDescOnce.Do(func() { + file_api_internal_eval_measurement_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_api_internal_eval_measurement_proto_rawDesc), len(file_api_internal_eval_measurement_proto_rawDesc))) + }) + return file_api_internal_eval_measurement_proto_rawDescData +} + +var file_api_internal_eval_measurement_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_api_internal_eval_measurement_proto_goTypes = []any{ + (*Measurement)(nil), // 0: eval.Measurement + (*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp +} +var file_api_internal_eval_measurement_proto_depIdxs = []int32{ + 1, // 0: eval.Measurement.time:type_name -> google.protobuf.Timestamp + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_api_internal_eval_measurement_proto_init() } +func file_api_internal_eval_measurement_proto_init() { + if File_api_internal_eval_measurement_proto != nil { + return + } + file_api_internal_eval_measurement_proto_msgTypes[0].OneofWrappers = []any{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_internal_eval_measurement_proto_rawDesc), len(file_api_internal_eval_measurement_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_api_internal_eval_measurement_proto_goTypes, + DependencyIndexes: file_api_internal_eval_measurement_proto_depIdxs, + MessageInfos: file_api_internal_eval_measurement_proto_msgTypes, + }.Build() + File_api_internal_eval_measurement_proto = out.File + file_api_internal_eval_measurement_proto_goTypes = nil + file_api_internal_eval_measurement_proto_depIdxs = nil +} diff --git a/api/internal/eval/measurement.proto b/api/internal/eval/measurement.proto new file mode 100644 index 00000000..bf801770 --- /dev/null +++ b/api/internal/eval/measurement.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package eval; + +option go_package = "github.com/USACE/instrumentation-api/api/internal/eval;eval"; + +import "google/protobuf/timestamp.proto"; + +message Measurement { + google.protobuf.Timestamp time = 1; + double value = 2; + optional bool masked = 3; + optional bool validated = 4; +} diff --git a/api/internal/handler/api.go b/api/internal/handler/api.go index 94db907e..5c700aeb 100644 --- a/api/internal/handler/api.go +++ b/api/internal/handler/api.go @@ -12,15 +12,14 @@ import ( "github.com/USACE/instrumentation-api/api/v4/internal/cloud" "github.com/USACE/instrumentation-api/api/v4/internal/config" "github.com/USACE/instrumentation-api/api/v4/internal/dto" + "github.com/USACE/instrumentation-api/api/v4/internal/eval" "github.com/USACE/instrumentation-api/api/v4/internal/logger" "github.com/USACE/instrumentation-api/api/v4/internal/middleware" "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" "github.com/USACE/instrumentation-api/api/v4/internal/service" "github.com/danielgtaylor/huma/v2" "github.com/danielgtaylor/huma/v2/adapters/humaecho" - "github.com/google/uuid" "github.com/labstack/echo/v4" - "github.com/riverqueue/river" ) // TODO: make this a build tag @@ -50,23 +49,25 @@ type ApiHandler struct { CloudServices *cloud.ApiServices HTTPClient *http.Client Config *config.ApiConfig - Logger *logger.Logger + Logger logger.Logger router *Router + // baseEnv is used for validating expression compilation + baseEnv *eval.Env apiMiddlewares } -func NewApi(ctx context.Context, cfg *config.ApiConfig, l *logger.Logger) (*ApiHandler, error) { +func NewApi(ctx context.Context, cfg *config.ApiConfig, l logger.Logger) (*ApiHandler, error) { dbpool, err := service.NewDBPool(ctx, cfg.DBConfig) if err != nil { return nil, err } l.Info(ctx, "Database connection established") - pgq, err := pgqueue.New(ctx, dbpool, &river.Config{ - ID: "midas-api__" + uuid.New().String() + "__" + time.Now().Format("2006_01_02T15_04_05_000000"), - Logger: l.Slogger(), - Schema: cfg.RiverQueueSchema, - ErrorHandler: pgqueue.NewErrorHandler(l), + pgq, err := pgqueue.NewInsertOnlyClient(ctx, dbpool, &pgqueue.Options{ + ClientName: "midas-api", + Logger: l.Slogger(), + Schema: cfg.RiverQueueSchema, + DefaultMaxAttempts: 1, }) if err != nil { return nil, err @@ -83,12 +84,18 @@ func NewApi(ctx context.Context, cfg *config.ApiConfig, l *logger.Logger) (*ApiH httpClient := NewHttpClient() + baseEnv, err := eval.NewEnv() + if err != nil { + return nil, err + } + return &ApiHandler{ DBService: dbService, CloudServices: cloudServices, HTTPClient: httpClient, Config: cfg, Logger: l, + baseEnv: baseEnv, }, nil } @@ -271,7 +278,7 @@ func NewApiRouter(ctx context.Context, h *ApiHandler) *Router { return &Router{r} } -func (h *ApiHandler) startAlertEventBatcher(ctx context.Context) <-chan error { +func (h *ApiHandler) startBatchInserter(ctx context.Context) <-chan error { decode := func(b []byte) (dto.WorkerAlertEventArgs, error) { var ev dto.WorkerAlertEventArgs return ev, json.Unmarshal(b, &ev) @@ -288,13 +295,13 @@ func (h *ApiHandler) startAlertEventBatcher(ctx context.Context) <-chan error { return nil } - b := pgqueue.NewBatcher( - &pgqueue.BatcherConfig{ + b := pgqueue.NewBatchInserter( + &pgqueue.BatchInserterConfig{ BatchSize: h.Config.AlertEventBatchSize, BatchTimeout: h.Config.AlertEventBatchTimeout, FlushWorkers: h.Config.AlertEventFlushWorkers, }, - h.CloudServices.AlertEventBatcherSub, + h.DBService.PGQueue.InserterBus.Sub, decode, insertFn, ) @@ -304,7 +311,7 @@ func (h *ApiHandler) startAlertEventBatcher(ctx context.Context) <-chan error { func (h *ApiHandler) Run(ctx context.Context) error { h.Logger.Info(ctx, "Starting MIDAS API service...") - batchErrCh := h.startAlertEventBatcher(ctx) + batchErrCh := h.startBatchInserter(ctx) router := NewApiRouter(ctx, h) srv := &http.Server{ diff --git a/api/internal/handler/datalogger_telemetry.go b/api/internal/handler/datalogger_telemetry.go index 971c6565..6aba80af 100644 --- a/api/internal/handler/datalogger_telemetry.go +++ b/api/internal/handler/datalogger_telemetry.go @@ -210,7 +210,7 @@ func (h *ApiHandler) RegisterDataloggerTelemetry(api huma.API) { ) } - if err := h.DBService.TimeseriesMeasurementCreateOrUpdateBatch(ctx, mcs, h.CloudServices.AlertEventBatcherPub); err != nil { + if err := h.DBService.TimeseriesMeasurementCreateOrUpdateBatch(ctx, mcs); err != nil { em = append(em, fmt.Sprintf("%d: %s", http.StatusInternalServerError, err.Error())) return nil, httperr.InternalServerError(err) } diff --git a/api/internal/handler/dcsloader.go b/api/internal/handler/dcsloader.go index 79009dc5..9085516e 100644 --- a/api/internal/handler/dcsloader.go +++ b/api/internal/handler/dcsloader.go @@ -18,10 +18,10 @@ type DcsLoaderHandler struct { Config *config.DcsLoaderConfig DcsLoaderService *service.DcsLoaderService CloudServices *cloud.DcsLoaderServices - Logger *logger.Logger + Logger logger.Logger } -func NewDcsLoader(ctx context.Context, cfg *config.DcsLoaderConfig, l *logger.Logger) (*DcsLoaderHandler, error) { +func NewDcsLoader(ctx context.Context, cfg *config.DcsLoaderConfig, l logger.Logger) (*DcsLoaderHandler, error) { cloudServices, err := cloud.NewDcsLoaderServices(ctx, cfg) if err != nil { return nil, err diff --git a/api/internal/handler/expression.go b/api/internal/handler/expression.go new file mode 100644 index 00000000..f5debbf9 --- /dev/null +++ b/api/internal/handler/expression.go @@ -0,0 +1,261 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "reflect" + + "github.com/USACE/instrumentation-api/api/v4/internal/ctxkey" + "github.com/USACE/instrumentation-api/api/v4/internal/db" + "github.com/USACE/instrumentation-api/api/v4/internal/dto" + "github.com/USACE/instrumentation-api/api/v4/internal/eval" + "github.com/USACE/instrumentation-api/api/v4/internal/httperr" + "github.com/USACE/instrumentation-api/api/v4/internal/service" + "github.com/danielgtaylor/huma/v2" +) + +var expressionTags = []string{"Expressions"} + +type ExpressionIDParam struct { + ExpressionID UUID `path:"expression_id"` +} + +func (h *ApiHandler) RegisterExpression(api huma.API) { + registry := api.OpenAPI().Components.Schemas + + expressionOptsDTOSchema := &huma.Schema{ + OneOf: []*huma.Schema{ + registry.Schema(reflect.TypeFor[dto.ExpressionWindowOptsDTO](), true, ""), + }, + Nullable: true, + } + expressionDTOSchema := registry.SchemaFromRef(registry.Schema(reflect.TypeFor[dto.ExpressionDTO](), true, "").Ref) + expressionDTOSchema.Properties["opts"] = expressionOptsDTOSchema + expressionDTOSchema.PrecomputeMessages() + + expressionRequestBody := &huma.RequestBody{ + Required: true, + Content: map[string]*huma.MediaType{"application/json": { + Schema: &huma.Schema{ + Type: huma.TypeObject, + Items: expressionOptsDTOSchema, + }, + }}, + } + + huma.Register(api, huma.Operation{ + Middlewares: h.Public, + OperationID: "expression-list-for-instrument", + Method: http.MethodGet, + Path: "/projects/{project_id}/instruments/{instrument_id}/expressions", + Description: "lists expressions for an instrument", + Tags: expressionTags, + }, func(ctx context.Context, input *struct { + ProjectIDParam + InstrumentIDParam + }) (*Response[[]db.VExpression], error) { + aa, err := h.DBService.ExpressionListForInstrument(ctx, input.InstrumentID.UUID) + if err != nil { + return nil, httperr.InternalServerError(err) + } + return NewResponse(aa), nil + }) + + huma.Register(api, huma.Operation{ + Middlewares: h.Public, + OperationID: "expression-get-for-id", + Method: http.MethodGet, + Path: "/projects/{project_id}/instruments/{instrument_id}/expressions/{expression_id}", + Description: "gets an expression", + Tags: expressionTags, + }, func(ctx context.Context, input *struct { + ProjectIDParam + InstrumentIDParam + ExpressionIDParam + }) (*Response[db.VExpression], error) { + a, err := h.DBService.ExpressionGet(ctx, input.ExpressionID.UUID) + if err != nil { + return nil, httperr.InternalServerError(err) + } + return NewResponse(a), nil + }) + + huma.Register(api, huma.Operation{ + Middlewares: h.ProjectAdmin, + OperationID: "expression-create-validate", + Method: http.MethodPost, + Path: "/projects/{project_id}/instruments/{instrument_id}/expressions/validations", + RequestBody: expressionRequestBody, + Description: "validates an expression without creating it", + Tags: expressionTags, + }, func(ctx context.Context, input *struct { + ProjectIDParam + InstrumentIDParam + Body dto.ExpressionDTO + }) (*Response[service.ExpressionValidation], error) { + p, ok := ctx.Value(ctxkey.Profile).(db.VProfile) + if !ok { + return nil, httperr.Unauthorized(errNoProfileAttached) + } + xNew, a, err := h.DBService.ExpressionCreateValidation(ctx, input.InstrumentID.UUID, p.ID, input.Body) + if err != nil { + var uerr service.NestedUserError + if errors.As(err, &uerr) { + return nil, httperr.BadRequest(uerr) + } + return nil, httperr.InternalServerError(err) + } + + _, err = eval.NewEvalSession( + h.DBService.Queries, + h.DBService.PGQueue, + h.DBService.Logger, + h.baseEnv, + []eval.EvalSessionParams{{ + TargetTimeseriesID: xNew.TargetTimeseriesID, + ExpressionID: xNew.ID, + // we don't need to fill all of this out since we're just validating + // that the expression compiles and returns one of the correct types + Expression: dto.ExpressionDTO{}, + }}, + ) + if err != nil { + return nil, httperr.InternalServerError(err) + } + + return NewResponse(a), nil + }) + + huma.Register(api, huma.Operation{ + Middlewares: h.ProjectAdmin, + OperationID: "expression-update-validate", + Method: http.MethodPut, + Path: "/projects/{project_id}/instruments/{instrument_id}/expressions/{expression_id}/validations", + RequestBody: expressionRequestBody, + Description: "validates an expression without creating it", + Tags: expressionTags, + }, func(ctx context.Context, input *struct { + ProjectIDParam + InstrumentIDParam + ExpressionIDParam + Body dto.ExpressionDTO + }) (*Response[service.ExpressionValidation], error) { + p, ok := ctx.Value(ctxkey.Profile).(db.VProfile) + if !ok { + return nil, httperr.Unauthorized(errNoProfileAttached) + } + xUpdated, a, err := h.DBService.ExpressionUpdateValidation(ctx, input.InstrumentID.UUID, p.ID, input.Body) + if err != nil { + var uerr service.NestedUserError + if errors.As(err, &uerr) { + return nil, httperr.BadRequest(err) + } + return nil, httperr.InternalServerError(err) + } + _, err = eval.NewEvalSession( + h.DBService.Queries, + h.DBService.PGQueue, + h.DBService.Logger, + h.baseEnv, + []eval.EvalSessionParams{{ + TargetTimeseriesID: xUpdated.TargetTimeseriesID, + ExpressionID: xUpdated.ID, + // we don't need to fill all of this out since we're just validating + // that the expression compiles and returns one of the correct types + Expression: dto.ExpressionDTO{}, + }}, + ) + if err != nil { + return nil, httperr.InternalServerError(err) + } + return NewResponse(a), nil + }) + + huma.Register(api, huma.Operation{ + Middlewares: h.ProjectAdmin, + OperationID: "expression-create", + Method: http.MethodPost, + Path: "/projects/{project_id}/instruments/{instrument_id}/expressions", + RequestBody: expressionRequestBody, + Description: "creates an expression and starts an expression timeseries compute job", + Tags: expressionTags, + }, func(ctx context.Context, input *struct { + ProjectIDParam + InstrumentIDParam + Body dto.ExpressionDTO + }) (*Response[db.VExpression], error) { + p, ok := ctx.Value(ctxkey.Profile).(db.VProfile) + if !ok { + return nil, httperr.Unauthorized(errNoProfileAttached) + } + a, err := h.DBService.ExpressionCreate(ctx, input.InstrumentID.UUID, p.ID, input.Body) + if err != nil { + var uerr service.NestedUserError + if errors.As(err, &uerr) { + return nil, httperr.BadRequest(err) + } + return nil, httperr.InternalServerError(err) + } + return NewResponse(a), nil + }) + + huma.Register(api, huma.Operation{ + Middlewares: h.ProjectAdmin, + OperationID: "expression-update", + Method: http.MethodPut, + Path: "/projects/{project_id}/instruments/{instrument_id}/expressions/{expression_id}", + RequestBody: expressionRequestBody, + Description: "updates an expression and starts an expression timeseries compute job", + Tags: expressionTags, + }, func(ctx context.Context, input *struct { + ProjectIDParam + InstrumentIDParam + ExpressionIDParam + Body dto.ExpressionDTO + }) (*Response[db.VExpression], error) { + p, ok := ctx.Value(ctxkey.Profile).(db.VProfile) + if !ok { + return nil, httperr.Unauthorized(errNoProfileAttached) + } + a, err := h.DBService.ExpressionUpdate(ctx, input.ExpressionID.UUID, p.ID, input.Body) + if err != nil { + var uerr service.NestedUserError + if errors.As(err, &uerr) { + return nil, httperr.BadRequest(err) + } + return nil, httperr.InternalServerError(err) + } + return NewResponse(a), nil + }) + + huma.Register(api, huma.Operation{ + Middlewares: h.ProjectAdmin, + OperationID: "expression-delete", + Method: http.MethodDelete, + Path: "/projects/{project_id}/instruments/{instrument_id}/expressions/{expression_id}", + Description: "deletes an expression and its corresponding timeseries/measurements", + Tags: expressionTags, + }, func(ctx context.Context, input *struct { + ProjectIDParam + InstrumentIDParam + ExpressionIDParam + }) (*Response[db.VExpression], error) { + if err := h.DBService.ExpressionDelete(ctx, input.ExpressionID.UUID); err != nil { + return nil, httperr.InternalServerError(err) + } + return nil, nil + }) + + huma.Register(api, huma.Operation{ + Middlewares: h.Public, + OperationID: "expression-get-autocomplete-options", + Method: http.MethodGet, + Path: "/domains/expressions/completions", + Description: "lists autocomplete options for expressions", + Tags: expressionTags, + }, func(ctx context.Context, input *struct{}) (*Response[[]eval.Completion], error) { + completions := eval.GetCompletionOptions() + return NewResponse(completions), nil + }) +} diff --git a/api/internal/handler/instrument_seis.go b/api/internal/handler/instrument_seis.go index bc3edf5a..f8fd51fd 100644 --- a/api/internal/handler/instrument_seis.go +++ b/api/internal/handler/instrument_seis.go @@ -97,7 +97,7 @@ func (h *ApiHandler) RegisterInstrumentSeis(api huma.API) { for idx := range cc { tsIDs[idx] = cc[idx].TimeseriesID } - if err := h.DBService.SeisTimeseriesMeasurementCreateOrUpdateBatch(ctx, cc, h.CloudServices.AlertEventBatcherPub); err != nil { + if err := h.DBService.SeisTimeseriesMeasurementCreateOrUpdateBatch(ctx, cc); err != nil { return nil, httperr.InternalServerError(err) } return NewResponse(tsIDs), nil diff --git a/api/internal/handler/measurement.go b/api/internal/handler/measurement.go index d2cfe262..38017a2a 100644 --- a/api/internal/handler/measurement.go +++ b/api/internal/handler/measurement.go @@ -52,7 +52,7 @@ func (h *ApiHandler) RegisterMeasurement(api huma.API) { return nil, httperr.BadRequest(errors.New("one or more timeseries do not belong to an instrument in this project")) } - if err := h.DBService.TimeseriesMeasurementCreateOrUpdateBatch(ctx, cc, h.CloudServices.AlertEventBatcherPub); err != nil { + if err := h.DBService.TimeseriesMeasurementCreateOrUpdateBatch(ctx, cc); err != nil { return nil, httperr.InternalServerError(err) } @@ -76,7 +76,7 @@ func (h *ApiHandler) RegisterMeasurement(api huma.API) { for idx := range cc { tsIDs[idx] = cc[idx].TimeseriesID } - if err := h.DBService.TimeseriesMeasurementCreateOrUpdateBatch(ctx, cc, h.CloudServices.AlertEventBatcherPub); err != nil { + if err := h.DBService.TimeseriesMeasurementCreateOrUpdateBatch(ctx, cc); err != nil { return nil, httperr.InternalServerError(err) } @@ -102,7 +102,7 @@ func (h *ApiHandler) RegisterMeasurement(api huma.API) { twParam.SetWindow(input.After, input.Before, input.After, input.Before) tw = &twParam cc := input.Body - if err := h.DBService.TimeseriesMeasurementUpdateBatch(ctx, cc, tw, h.CloudServices.AlertEventBatcherPub); err != nil { + if err := h.DBService.TimeseriesMeasurementUpdateBatch(ctx, cc, tw); err != nil { return nil, httperr.InternalServerError(err) } diff --git a/api/internal/handler/report_config.go b/api/internal/handler/report_config.go index 644cc965..34bf76ab 100644 --- a/api/internal/handler/report_config.go +++ b/api/internal/handler/report_config.go @@ -316,6 +316,7 @@ func (h *ApiHandler) runLambdaRIE(_ context.Context, body []byte) error { if err != nil { return err } + h.Logger.Info(ctx, "mock lambda invocation request", "body", string(b)) go func() { r := bytes.NewReader(b) diff --git a/api/internal/handler/telemetry.go b/api/internal/handler/telemetry.go index cabd9239..693ada87 100644 --- a/api/internal/handler/telemetry.go +++ b/api/internal/handler/telemetry.go @@ -15,11 +15,11 @@ import ( type TelemetryHandler struct { Client *http.Client Config *config.TelemetryConfig - Logger *logger.Logger + Logger logger.Logger telemetryMiddlewares } -func NewTelemetry(ctx context.Context, cfg *config.TelemetryConfig, l *logger.Logger) (*TelemetryHandler, error) { +func NewTelemetry(ctx context.Context, cfg *config.TelemetryConfig, l logger.Logger) (*TelemetryHandler, error) { h := &TelemetryHandler{ Client: NewHttpClient(), Config: cfg, diff --git a/api/internal/logger/logger.go b/api/internal/logger/logger.go index 619690fd..b2026d1d 100644 --- a/api/internal/logger/logger.go +++ b/api/internal/logger/logger.go @@ -9,46 +9,55 @@ import ( ) // Logger wraps slog.Logger to provide simplified logging methods. -type Logger struct { - logger *slog.Logger +type logger struct { + *slog.Logger } -func (l *Logger) Slogger() *slog.Logger { - return l.logger +type Logger interface { + Error(ctx context.Context, msg string, args ...any) + Info(ctx context.Context, msg string, args ...any) + Debug(ctx context.Context, msg string, args ...any) + Warn(ctx context.Context, msg string, args ...any) + LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) + Slogger() *slog.Logger +} + +func (l *logger) Slogger() *slog.Logger { + return l.Logger } // NewLogger creates a new Logger instance with a JSON configuration and sets it as the default logger. -func NewLogger(cfg *config.LoggerConfig) *Logger { +func NewLogger(cfg *config.LoggerConfig) Logger { var level slog.Level if err := level.UnmarshalText([]byte(cfg.LogLevel)); err != nil { - panic(err) + level = slog.LevelWarn } h := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level}) l := slog.New(h) slog.SetDefault(l) - return &Logger{logger: l} + return &logger{l} } // Error logs an error message. -func (l *Logger) Error(ctx context.Context, msg string, args ...any) { - l.logger.ErrorContext(ctx, msg, args...) +func (l *logger) Error(ctx context.Context, msg string, args ...any) { + l.Logger.ErrorContext(ctx, msg, args...) } // Info logs an informational message. -func (l *Logger) Info(ctx context.Context, msg string, args ...any) { - l.logger.InfoContext(ctx, msg, args...) +func (l *logger) Info(ctx context.Context, msg string, args ...any) { + l.Logger.InfoContext(ctx, msg, args...) } // Debug logs a debug message. -func (l *Logger) Debug(ctx context.Context, msg string, args ...any) { - l.logger.DebugContext(ctx, msg, args...) +func (l *logger) Debug(ctx context.Context, msg string, args ...any) { + l.Logger.DebugContext(ctx, msg, args...) } // Warn logs a warning message. -func (l *Logger) Warn(ctx context.Context, msg string, args ...any) { - l.logger.WarnContext(ctx, msg, args...) +func (l *logger) Warn(ctx context.Context, msg string, args ...any) { + l.Logger.WarnContext(ctx, msg, args...) } -func (l *Logger) LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { - l.logger.LogAttrs(ctx, level, msg, attrs...) +func (l *logger) LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { + l.Logger.LogAttrs(ctx, level, msg, attrs...) } diff --git a/api/internal/middleware/logger.go b/api/internal/middleware/logger.go index 45baaa44..413ba828 100644 --- a/api/internal/middleware/logger.go +++ b/api/internal/middleware/logger.go @@ -10,7 +10,7 @@ import ( echomw "github.com/labstack/echo/v4/middleware" ) -func (m *echoMw) NewRequestLoggerMiddleware(l *logger.Logger) func(echo.HandlerFunc) echo.HandlerFunc { +func (m *echoMw) NewRequestLoggerMiddleware(l logger.Logger) func(echo.HandlerFunc) echo.HandlerFunc { return echomw.RequestLoggerWithConfig(echomw.RequestLoggerConfig{ LogStatus: true, LogURI: true, diff --git a/api/internal/middleware/middleware.go b/api/internal/middleware/middleware.go index 92207d3d..190622d9 100644 --- a/api/internal/middleware/middleware.go +++ b/api/internal/middleware/middleware.go @@ -33,7 +33,7 @@ type EchoMiddleware interface { GZIP(next echo.HandlerFunc) echo.HandlerFunc Recover(next echo.HandlerFunc) echo.HandlerFunc RequestID(next echo.HandlerFunc) echo.HandlerFunc - NewRequestLoggerMiddleware(l *logger.Logger) func(echo.HandlerFunc) echo.HandlerFunc + NewRequestLoggerMiddleware(l logger.Logger) func(echo.HandlerFunc) echo.HandlerFunc } type echoMw struct { @@ -56,12 +56,12 @@ type apiMw struct { *echoMw Config *config.ServerConfig DBService *service.DBService - Logger *logger.Logger + Logger logger.Logger } var _ ApiMiddleware = (*apiMw)(nil) -func NewApiMiddleware(cfg *config.ServerConfig, db *service.DBService, l *logger.Logger) *apiMw { +func NewApiMiddleware(cfg *config.ServerConfig, db *service.DBService, l logger.Logger) *apiMw { e := NewEchoMiddeware(cfg) return &apiMw{e, cfg, db, l} } @@ -72,12 +72,12 @@ type TelemetryMiddleware interface { type telemetryMw struct { *echoMw - Logger *logger.Logger + Logger logger.Logger } var _ TelemetryMiddleware = (*telemetryMw)(nil) -func NewTelemetryMiddleware(cfg *config.ServerConfig, l *logger.Logger) *telemetryMw { +func NewTelemetryMiddleware(cfg *config.ServerConfig, l logger.Logger) *telemetryMw { e := NewEchoMiddeware(cfg) return &telemetryMw{e, l} } diff --git a/api/internal/pgqueue/batch_inserter.go b/api/internal/pgqueue/batch_inserter.go new file mode 100644 index 00000000..d26db42f --- /dev/null +++ b/api/internal/pgqueue/batch_inserter.go @@ -0,0 +1,146 @@ +package pgqueue + +import ( + "context" + "slices" + "sync" + "time" + + "gocloud.dev/pubsub" +) + +var flushOnErrorGracePeriod = 15 * time.Second + +type BatchInserterConfig struct { + BatchSize int + BatchTimeout time.Duration + FlushWorkers int +} + +type envelope[T any] struct { + msg *pubsub.Message + data T +} + +type BatchInserter[T any] struct { + cfg *BatchInserterConfig + sub *pubsub.Subscription + decode func([]byte) (T, error) + insertFn func(context.Context, []T) error + + sem chan struct{} + ErrCh chan error + wg sync.WaitGroup +} + +func NewBatchInserter[T any](cfg *BatchInserterConfig, sub *pubsub.Subscription, decode func([]byte) (T, error), insertFn func(context.Context, []T) error) *BatchInserter[T] { + return &BatchInserter[T]{ + cfg: cfg, + sub: sub, + decode: decode, + insertFn: insertFn, + sem: make(chan struct{}, cfg.FlushWorkers), + ErrCh: make(chan error, 1), + } +} + +func (b *BatchInserter[T]) Run(ctx context.Context) <-chan error { + msgCh := make(chan *pubsub.Message, b.cfg.BatchSize*2) + + b.wg.Add(1) + go func() { + defer b.wg.Done() + defer close(msgCh) + for { + m, err := b.sub.Receive(ctx) + if err != nil { + return + } + msgCh <- m + } + }() + + b.wg.Add(1) + go func() { + defer b.wg.Done() + + timer := time.NewTimer(b.cfg.BatchTimeout) + defer timer.Stop() + + var batch []envelope[T] + + flushAndReset := func(flushCtx context.Context) { + if len(batch) == 0 { + return + } + b.flush(flushCtx, batch) + batch = nil + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(b.cfg.BatchTimeout) + } + + for { + select { + case <-ctx.Done(): + graceCtx, cancel := context.WithTimeout(context.Background(), flushOnErrorGracePeriod) + flushAndReset(graceCtx) + cancel() + return + + case m, ok := <-msgCh: + if !ok { + graceCtx, cancel := context.WithTimeout(context.Background(), flushOnErrorGracePeriod) + flushAndReset(graceCtx) + cancel() + return + } + + evt, err := b.decode(m.Body) + if err != nil { + m.Ack() + continue + } + batch = append(batch, envelope[T]{msg: m, data: evt}) + + if len(batch) >= b.cfg.BatchSize { + flushAndReset(ctx) + } + + case <-timer.C: + flushAndReset(ctx) + } + } + }() + + return b.ErrCh +} + +func (b *BatchInserter[T]) flush(ctx context.Context, batch []envelope[T]) { + snap := slices.Clone(batch) + items := make([]T, len(snap)) + for i, it := range snap { + items[i] = it.data + } + + b.sem <- struct{}{} + b.wg.Add(1) + go func() { + defer b.wg.Done() + defer func() { <-b.sem }() + if err := b.insertFn(ctx, items); err != nil { + select { + case b.ErrCh <- err: + default: + } + return + } + for _, it := range snap { + it.msg.Ack() + } + }() +} diff --git a/api/internal/pgqueue/batch_inserter_test.go b/api/internal/pgqueue/batch_inserter_test.go new file mode 100644 index 00000000..47a49833 --- /dev/null +++ b/api/internal/pgqueue/batch_inserter_test.go @@ -0,0 +1,151 @@ +package pgqueue + +import ( + "context" + "encoding/json" + "errors" + "slices" + "sync" + "testing" + "time" + + "gocloud.dev/pubsub" + "gocloud.dev/pubsub/mempubsub" +) + +func decodeInt(b []byte) (int, error) { + var v int + if err := json.Unmarshal(b, &v); err != nil { + return 0, err + } + return v, nil +} + +func TestBatcher_FlushOnSize(t *testing.T) { + t.Parallel() + + topic := mempubsub.NewTopic() + sub := mempubsub.NewSubscription(topic, 2*time.Second) + defer topic.Shutdown(context.Background()) + defer sub.Shutdown(context.Background()) + + var mu sync.Mutex + var got [][]int + insertFn := func(_ context.Context, xs []int) error { + mu.Lock() + defer mu.Unlock() + cp := slices.Clone(xs) + got = append(got, cp) + return nil + } + + cfg := &BatchInserterConfig{BatchSize: 3, BatchTimeout: time.Hour, FlushWorkers: 2} + b := NewBatchInserter(cfg, sub, decodeInt, insertFn) + + ctx := t.Context() + errCh := b.Run(ctx) + + // Send 3 messages to trigger size flush. + for _, n := range []int{1, 2, 3} { + body, _ := json.Marshal(n) + if err := topic.Send(ctx, &pubsub.Message{Body: body}); err != nil { + t.Fatalf("send: %v", err) + } + } + + // Wait a beat for the worker to flush. + time.Sleep(150 * time.Millisecond) + + select { + case err := <-errCh: + t.Fatalf("unexpected error: %v", err) + default: + } + + mu.Lock() + defer mu.Unlock() + if len(got) != 1 { + t.Fatalf("expected 1 flush, got %d", len(got)) + } + if len(got[0]) != 3 { + t.Fatalf("expected batch of 3, got %d", len(got[0])) + } +} + +func TestBatcher_FlushOnTimeout(t *testing.T) { + t.Parallel() + + topic := mempubsub.NewTopic() + sub := mempubsub.NewSubscription(topic, 2*time.Second) + defer topic.Shutdown(context.Background()) + defer sub.Shutdown(context.Background()) + + var mu sync.Mutex + var got [][]int + insertFn := func(_ context.Context, xs []int) error { + mu.Lock() + defer mu.Unlock() + got = append(got, slices.Clone(xs)) + return nil + } + + cfg := &BatchInserterConfig{BatchSize: 10, BatchTimeout: 50 * time.Millisecond, FlushWorkers: 1} + b := NewBatchInserter(cfg, sub, decodeInt, insertFn) + + ctx := t.Context() + errCh := b.Run(ctx) + + // Send fewer than BatchSize and rely on timeout. + for _, n := range []int{7, 8} { + body, _ := json.Marshal(n) + _ = topic.Send(ctx, &pubsub.Message{Body: body}) + } + + time.Sleep(150 * time.Millisecond) + + select { + case err := <-errCh: + t.Fatalf("unexpected error: %v", err) + default: + } + + mu.Lock() + defer mu.Unlock() + if len(got) == 0 { + t.Fatalf("expected at least 1 flush on timeout, got 0") + } + if want := 2; len(got[0]) != want { + t.Fatalf("expected first batch size %d, got %d", want, len(got[0])) + } +} + +func TestBatcher_ErrorsPropagate_NoAck(t *testing.T) { + t.Parallel() + + topic := mempubsub.NewTopic() + sub := mempubsub.NewSubscription(topic, 50*time.Millisecond) + defer topic.Shutdown(context.Background()) + defer sub.Shutdown(context.Background()) + + insertErr := errors.New("boom") + insertFn := func(_ context.Context, _ []int) error { return insertErr } + + cfg := &BatchInserterConfig{BatchSize: 1, BatchTimeout: time.Hour, FlushWorkers: 1} + b := NewBatchInserter(cfg, sub, decodeInt, insertFn) + + ctx := t.Context() + errCh := b.Run(ctx) + + // Send a message to trigger failure. + body, _ := json.Marshal(42) + _ = topic.Send(ctx, &pubsub.Message{Body: body}) + + select { + case err := <-errCh: + if !errors.Is(err, insertErr) { + t.Fatalf("got %v, want %v", err, insertErr) + } + case <-time.After(250 * time.Millisecond): + t.Fatal("timeout waiting for error") + } +} diff --git a/api/internal/pgqueue/batch_worker.go b/api/internal/pgqueue/batch_worker.go new file mode 100644 index 00000000..77e8e074 --- /dev/null +++ b/api/internal/pgqueue/batch_worker.go @@ -0,0 +1,141 @@ +package pgqueue + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + + "github.com/USACE/instrumentation-api/api/v4/internal/config" + "github.com/USACE/instrumentation-api/api/v4/internal/logger" + "gocloud.dev/pubsub" +) + +type Envelope struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` +} + +type BatchWorker struct { + Config *config.TaskConfig + Logger logger.Logger + Sub *pubsub.Subscription + Routes Routes +} + +func NewBatchWorker(cfg *config.TaskConfig, l logger.Logger, sub *pubsub.Subscription, routes Routes) *BatchWorker { + return &BatchWorker{ + Config: cfg, + Logger: l, + Sub: sub, + Routes: routes, + } +} + +func (bw *BatchWorker) Start(ctx context.Context) error { + if bw.Sub == nil { + return fmt.Errorf("pgqueue: task bus not initialized") + } + + sem := make(chan struct{}, bw.Config.AggMaxHandlers) + errCh := make(chan error, 1) + var wg sync.WaitGroup + defer wg.Wait() + +recvLoop: + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-errCh: + return err + default: + } + + batchCtx, cancel := context.WithTimeout(ctx, bw.Config.AggBatchTimeout) + + type bucket struct { + msgs []*pubsub.Message + vals []any + } + buckets := make(map[string]*bucket, 4) + + collectLoop: + for range bw.Config.AggBatchSize { + msg, err := bw.Sub.Receive(batchCtx) + if err != nil { + switch { + case errors.Is(err, context.DeadlineExceeded): + break collectLoop // flush current buckets + case errors.Is(err, context.Canceled): + cancel() + break recvLoop + default: + cancel() + return fmt.Errorf("receive: %w", err) + } + } + + var env Envelope + if err := json.Unmarshal(msg.Body, &env); err != nil { + bw.Logger.Warn(ctx, "bad envelope", "err", err) + msg.Ack() + continue + } + rt, ok := bw.Routes[env.Type] + if !ok { + bw.Logger.Warn(ctx, "unknown message type", "type", env.Type) + msg.Ack() + continue + } + val, err := rt.Decode(env.Payload) + if err != nil { + bw.Logger.Warn(ctx, "decode error", "type", env.Type, "err", err) + msg.Ack() + continue + } + + b, ok := buckets[env.Type] + if !ok { + b = &bucket{msgs: make([]*pubsub.Message, 0, 16), vals: make([]any, 0, 16)} + buckets[env.Type] = b + } + b.msgs = append(b.msgs, msg) + b.vals = append(b.vals, val) + } + cancel() + + if len(buckets) == 0 { + continue + } + + for typ, b := range buckets { + sem <- struct{}{} + wg.Add(1) + + go func(typ string, b *bucket) { + defer func() { <-sem; wg.Done() }() + + gctx, cancel := context.WithTimeout(context.Background(), flushOnErrorGracePeriod) + defer cancel() + + bw.Logger.Info(ctx, "processing batch", "type", typ, "count", len(b.vals)) + + if err := bw.Routes[typ].Handle(gctx, b.vals); err != nil { + bw.Logger.Error(ctx, "handler failed", "type", typ, "err", err) + select { + case errCh <- fmt.Errorf("batch %s failed: %w", typ, err): + default: + } + return + } + + for _, m := range b.msgs { + m.Ack() + } + }(typ, b) + } + } + return <-errCh +} diff --git a/api/internal/pgqueue/batch_worker_test.go b/api/internal/pgqueue/batch_worker_test.go new file mode 100644 index 00000000..ca3bae9f --- /dev/null +++ b/api/internal/pgqueue/batch_worker_test.go @@ -0,0 +1,93 @@ +package pgqueue + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/USACE/instrumentation-api/api/v4/internal/config" + "github.com/USACE/instrumentation-api/api/v4/internal/logger" +) + +type testLogger logger.Logger + +func TestListenPoolProcess_BatchesByType(t *testing.T) { + t.Parallel() + + q := &PGQueue{WorkerBus: newMemBus()} + + decodeStr := func(b []byte) (any, error) { + var s string + return s, json.Unmarshal(b, &s) + } + decodeNum := func(b []byte) (any, error) { + var n int + return n, json.Unmarshal(b, &n) + } + + var mu sync.Mutex + counts := map[string]int{} + collect := func(kind string) HandleFunc { + return func(_ context.Context, vals []any) error { + mu.Lock() + defer mu.Unlock() + counts[kind] += len(vals) + return nil + } + } + + routes := Routes{ + "str": {Decode: decodeStr, Handle: collect("str")}, + "num": {Decode: decodeNum, Handle: collect("num")}, + } + + cfg := &config.TaskConfig{ + AggMaxHandlers: 3, + AggBatchTimeout: 50 * time.Millisecond, + AggBatchSize: 10, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + errCh := make(chan error, 1) + bw := NewBatchWorker(cfg, logger.NewLogger(&config.LoggerConfig{LogLevel: "WARN"}), q.WorkerBus.Sub, routes) + go func() { + if err := bw.Start(ctx); err != nil && err != context.Canceled { + errCh <- err + } + }() + + // Publish mixed envelopes. + send := func(tp string, payload any) { + b, _ := json.Marshal(payload) + env, _ := json.Marshal(Envelope{Type: tp, Payload: b}) + _ = q.BatchWorkerSend(ctx, env) + } + send("str", "a") + send("str", "b") + send("num", 1) + send("str", "c") + send("num", 2) + + // Give the loop time to receive and flush. + time.Sleep(200 * time.Millisecond) + cancel() + + select { + case err := <-errCh: + t.Fatalf("unexpected error: %v", err) + default: + } + + mu.Lock() + defer mu.Unlock() + if counts["str"] != 3 { + t.Fatalf("want 3 'str' handled, got %d", counts["str"]) + } + if counts["num"] != 2 { + t.Fatalf("want 2 'num' handled, got %d", counts["num"]) + } +} diff --git a/api/internal/pgqueue/batcher.go b/api/internal/pgqueue/batcher.go deleted file mode 100644 index 9d09db5c..00000000 --- a/api/internal/pgqueue/batcher.go +++ /dev/null @@ -1,118 +0,0 @@ -package pgqueue - -import ( - "context" - "slices" - "time" - - "gocloud.dev/pubsub" -) - -type BatcherConfig struct { - BatchSize int - BatchTimeout time.Duration - FlushWorkers int -} - -// A generic batcher over any event type T. -type Batcher[T any] struct { - cfg *BatcherConfig - sub *pubsub.Subscription - decode func([]byte) (T, error) - insertFn func(context.Context, []T) error - - sem chan struct{} - ErrCh chan error -} - -// NewBatcher constructs a new Batcher[T]. -func NewBatcher[T any](cfg *BatcherConfig, sub *pubsub.Subscription, decode func([]byte) (T, error), insertFn func(context.Context, []T) error) *Batcher[T] { - return &Batcher[T]{ - cfg: cfg, - sub: sub, - decode: decode, - insertFn: insertFn, - sem: make(chan struct{}, cfg.FlushWorkers), - ErrCh: make(chan error, 1), - } -} - -// Run starts reading from the subscription, buffering up to batchSize or batchTimeout, -// and calling insertFn in non-blocking worker goroutines. Returns ErrCh for the first error. -func (b *Batcher[T]) Run(ctx context.Context) <-chan error { - msgCh := make(chan *pubsub.Message, b.cfg.BatchSize*2) - go func() { - defer close(msgCh) - for { - m, err := b.sub.Receive(ctx) - if err != nil { - return - } - msgCh <- m - } - }() - - go func() { - timer := time.NewTimer(b.cfg.BatchTimeout) - defer timer.Stop() - - var batch []T - - for { - select { - case <-ctx.Done(): - if len(batch) > 0 { - b.flush(ctx, batch) - } - return - - case m, ok := <-msgCh: - if !ok { - if len(batch) > 0 { - b.flush(ctx, batch) - } - return - } - - evt, err := b.decode(m.Body) - m.Ack() - if err == nil { - batch = append(batch, evt) - } - - if len(batch) >= b.cfg.BatchSize { - b.flush(ctx, batch) - batch = nil - if !timer.Stop() { - <-timer.C - } - timer.Reset(b.cfg.BatchTimeout) - } - - case <-timer.C: - if len(batch) > 0 { - b.flush(ctx, batch) - batch = nil - } - timer.Reset(b.cfg.BatchTimeout) - } - } - }() - - return b.ErrCh -} - -// flush snapshots and launches a worker to call insertFn. -func (b *Batcher[T]) flush(ctx context.Context, batch []T) { - snap := slices.Clone(batch) - b.sem <- struct{}{} - go func() { - defer func() { <-b.sem }() - if err := b.insertFn(ctx, snap); err != nil { - select { - case b.ErrCh <- err: - default: - } - } - }() -} diff --git a/api/internal/pgqueue/batcher_test.go b/api/internal/pgqueue/batcher_test.go deleted file mode 100644 index ffcdc357..00000000 --- a/api/internal/pgqueue/batcher_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package pgqueue_test - -import ( - "context" - "slices" - "testing" - "time" - - "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" - "github.com/stretchr/testify/assert" - "gocloud.dev/pubsub" - _ "gocloud.dev/pubsub/mempubsub" -) - -func TestBatcher_FlushOnBatchSize(t *testing.T) { - ctx := t.Context() - - // set up an in-memory topic+subscription - url := "mem://flush-by-size" - topic, err := pubsub.OpenTopic(ctx, url) - assert.NoError(t, err) - defer topic.Shutdown(ctx) - - sub, err := pubsub.OpenSubscription(ctx, url) - assert.NoError(t, err) - defer sub.Shutdown(ctx) - - // batcher config: size=2, timeout=long, one worker - cfg := &pgqueue.BatcherConfig{BatchSize: 2, BatchTimeout: time.Hour, FlushWorkers: 1} - decode := func(b []byte) (string, error) { return string(b), nil } - - // collect flushed batches here - got := make(chan []string, 1) - insertFn := func(_ context.Context, batch []string) error { - got <- slices.Clone(batch) - return nil - } - - b := pgqueue.NewBatcher(cfg, sub, decode, insertFn) - errCh := b.Run(ctx) - - // publish two messages → should trigger a flush immediately - assert.NoError(t, topic.Send(ctx, &pubsub.Message{Body: []byte("A")})) - assert.NoError(t, topic.Send(ctx, &pubsub.Message{Body: []byte("B")})) - - select { - case batch := <-got: - assert.Contains(t, batch, "A", "B") - case <-time.After(time.Second): - t.Fatal("timeout waiting for flush by size") - } - - // ensure no error signaled - select { - case e := <-errCh: - t.Fatalf("unexpected error from batcher: %v", e) - default: - } -} - -func TestBatcher_FlushOnTimeout(t *testing.T) { - ctx := t.Context() - - url := "mem://flush-on-timeout" - topic, err := pubsub.OpenTopic(ctx, url) - assert.NoError(t, err) - defer topic.Shutdown(ctx) - - sub, err := pubsub.OpenSubscription(ctx, url) - assert.NoError(t, err) - defer sub.Shutdown(ctx) - - // batcher config: size large, timeout short - cfg := &pgqueue.BatcherConfig{BatchSize: 10, BatchTimeout: 100 * time.Millisecond, FlushWorkers: 1} - decode := func(b []byte) (string, error) { return string(b), nil } - - got := make(chan []string, 1) - insertFn := func(_ context.Context, batch []string) error { - got <- slices.Clone(batch) - return nil - } - - b := pgqueue.NewBatcher(cfg, sub, decode, insertFn) - errCh := b.Run(ctx) - - // publish a single message - assert.NoError(t, topic.Send(ctx, &pubsub.Message{Body: []byte("X")})) - - // wait a bit longer than the timeout for the flush - select { - case batch := <-got: - assert.Equal(t, []string{"X"}, batch) - case <-time.After(time.Second): - t.Fatal("timeout waiting for flush by timeout") - } - - // ensure no error signaled - select { - case e := <-errCh: - t.Fatalf("unexpected error from batcher: %v", e) - default: - } -} diff --git a/api/internal/pgqueue/bus.go b/api/internal/pgqueue/bus.go new file mode 100644 index 00000000..a7c0ca4d --- /dev/null +++ b/api/internal/pgqueue/bus.go @@ -0,0 +1,29 @@ +package pgqueue + +import ( + "context" + "time" + + "gocloud.dev/pubsub" + "gocloud.dev/pubsub/mempubsub" +) + +type memBus struct { + Topic *pubsub.Topic + Sub *pubsub.Subscription +} + +func newMemBus() *memBus { + t := mempubsub.NewTopic() + // ack deadline: messages re-deliver if not Ack()'d by then + s := mempubsub.NewSubscription(t, 1*time.Minute) + return &memBus{Topic: t, Sub: s} +} + +func (q *PGQueue) BatchInserterSend(ctx context.Context, body []byte) error { + return q.InserterBus.Topic.Send(ctx, &pubsub.Message{Body: body}) +} + +func (q *PGQueue) BatchWorkerSend(ctx context.Context, body []byte) error { + return q.WorkerBus.Topic.Send(ctx, &pubsub.Message{Body: body}) +} diff --git a/api/internal/pgqueue/bus_test.go b/api/internal/pgqueue/bus_test.go new file mode 100644 index 00000000..ba996097 --- /dev/null +++ b/api/internal/pgqueue/bus_test.go @@ -0,0 +1,37 @@ +package pgqueue + +import ( + "context" + "testing" + "time" + + "gocloud.dev/pubsub" +) + +func TestMemBus_SendReceive(t *testing.T) { + t.Parallel() + + bus := newMemBus() + defer func() { + _ = bus.Topic.Shutdown(context.Background()) + _ = bus.Sub.Shutdown(context.Background()) + }() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + want := []byte("hello") + if err := bus.Topic.Send(ctx, &pubsub.Message{Body: want}); err != nil { + t.Fatalf("send: %v", err) + } + + gotMsg, err := bus.Sub.Receive(ctx) + if err != nil { + t.Fatalf("receive: %v", err) + } + defer gotMsg.Ack() + + if string(gotMsg.Body) != string(want) { + t.Fatalf("got %q, want %q", string(gotMsg.Body), string(want)) + } +} diff --git a/api/internal/pgqueue/error.go b/api/internal/pgqueue/error.go index 013a0326..0d366dba 100644 --- a/api/internal/pgqueue/error.go +++ b/api/internal/pgqueue/error.go @@ -10,10 +10,10 @@ import ( ) type ErrorHandler struct { - logger *logger.Logger + logger logger.Logger } -func NewErrorHandler(l *logger.Logger) *ErrorHandler { +func NewErrorHandler(l logger.Logger) *ErrorHandler { return &ErrorHandler{l} } diff --git a/api/internal/pgqueue/periodics.go b/api/internal/pgqueue/periodics.go new file mode 100644 index 00000000..bbb346aa --- /dev/null +++ b/api/internal/pgqueue/periodics.go @@ -0,0 +1,34 @@ +package pgqueue + +import ( + "time" + + "github.com/riverqueue/river" + "github.com/robfig/cron/v3" +) + +type Periodics struct { + jobs []*river.PeriodicJob +} + +func (p *Periodics) Jobs() []*river.PeriodicJob { + return p.jobs +} + +func (p *Periodics) Add(sched river.PeriodicSchedule, ctor river.PeriodicJobConstructor, opts *river.PeriodicJobOpts) *river.PeriodicJob { + j := river.NewPeriodicJob(sched, ctor, opts) + p.jobs = append(p.jobs, j) + return j +} + +func (p *Periodics) Every(d time.Duration, ctor river.PeriodicJobConstructor, opts *river.PeriodicJobOpts) *river.PeriodicJob { + return p.Add(river.PeriodicInterval(d), ctor, opts) +} + +func (p *Periodics) Cron(spec string, ctor river.PeriodicJobConstructor, opts *river.PeriodicJobOpts) (*river.PeriodicJob, error) { + s, err := cron.ParseStandard(spec) + if err != nil { + return nil, err + } + return p.Add(s, ctor, opts), nil +} diff --git a/api/internal/pgqueue/periodics_test.go b/api/internal/pgqueue/periodics_test.go new file mode 100644 index 00000000..208f8546 --- /dev/null +++ b/api/internal/pgqueue/periodics_test.go @@ -0,0 +1,45 @@ +package pgqueue + +import ( + "testing" + "time" + + "github.com/riverqueue/river" +) + +type dummyArgs struct{} + +func (dummyArgs) Kind() string { return "test.dummy" } + +func TestPeriodicsEvery(t *testing.T) { + p := &Periodics{} + p.Every(15*time.Minute, func() (river.JobArgs, *river.InsertOpts) { + return dummyArgs{}, nil + }, &river.PeriodicJobOpts{RunOnStart: true}) + + if got := len(p.Jobs()); got != 1 { + t.Fatalf("expected 1 job, got %d", got) + } +} + +func TestPeriodicsCron(t *testing.T) { + p := &Periodics{} + _, err := p.Cron("0 * * * *", func() (river.JobArgs, *river.InsertOpts) { + return dummyArgs{}, nil + }, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(p.Jobs()) != 1 { + t.Fatalf("expected 1 job, got %d", len(p.Jobs())) + } +} + +func TestPeriodicsCron_Invalid(t *testing.T) { + p := &Periodics{} + if _, err := p.Cron("not a cron", func() (river.JobArgs, *river.InsertOpts) { + return dummyArgs{}, nil + }, nil); err == nil { + t.Fatal("expected cron parse error, got nil") + } +} diff --git a/api/internal/pgqueue/pgqueue.go b/api/internal/pgqueue/pgqueue.go index 8ae1e4c9..34cc6ec4 100644 --- a/api/internal/pgqueue/pgqueue.go +++ b/api/internal/pgqueue/pgqueue.go @@ -4,25 +4,26 @@ import ( "context" "errors" "fmt" + "log/slog" "time" "github.com/USACE/instrumentation-api/api/v4/internal/db" + "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "github.com/riverqueue/river" "github.com/riverqueue/river/riverdriver/riverpgxv5" ) +// PGQueue is used for both insert-only clients and worker processes. +// Insert-only callers never see River types via the public API. type PGQueue struct { client *river.Client[pgx.Tx] -} -func New(ctx context.Context, dbpool *pgxpool.Pool, config *river.Config) (*PGQueue, error) { - riverClient, err := river.NewClient(riverpgxv5.New(dbpool), config) - if err != nil { - return nil, fmt.Errorf("error creating riverqueue client: %w", err) - } - return &PGQueue{riverClient}, nil + // internal in-memory buses + InserterBus *memBus // used by insert-only client Batcher + WorkerBus *memBus // used by worker client ListenPoolProcess } type PGQueueClient interface { @@ -39,8 +40,12 @@ type JobSnoozer interface { JobSnooze(duration time.Duration) error } +type JobArgs interface { + Kind() string +} + type InsertManyParams struct { - Args interface{ Kind() string } + Args JobArgs InsertOpts *InsertOpts } @@ -56,86 +61,255 @@ type UniqueOpts struct { type JobInserter interface { InsertManyTx(ctx context.Context, tx db.DBTX, params []InsertManyParams) ([]int64, error) + InsertManyFast(ctx context.Context, params []InsertManyParams) error } type JobCanceller interface { JobCancelTx(ctx context.Context, tx db.DBTX, jobID int64) error } +const defaultMaxWorkers = 100 + +type Options struct { + // Common + ClientName string + Logger *slog.Logger + Schema string + DefaultMaxAttempts int + + // Worker-only + Queues map[string]river.QueueConfig + StopTimeout time.Duration + + // Register periodic jobs for worker. Ignored for insert-only clients. + Periodic func(*Periodics) error + + // Avoid exposing River in insert-only signatures. + RegisterWorkers func(*river.Workers) error +} + +func convertTx(tx db.DBTX) (pgx.Tx, error) { + if t, ok := tx.(pgx.Tx); ok && t != nil { + return t, nil + } + return nil, fmt.Errorf("pgqueue: expected pgx.Tx-compatible tx, got %T", tx) +} + +// withConfig maps Options to river.Config with defaults, periodic jobs are passed in. +func (o *Options) withConfig(workers *river.Workers, periodicJobs []*river.PeriodicJob) *river.Config { + idPrefix := o.ClientName + if idPrefix != "" { + idPrefix += "__" + } + cfg := &river.Config{ + ID: idPrefix + uuid.New().String() + "__" + time.Now().Format("2006_01_02T15_04_05_000000"), + Logger: o.Logger, + Schema: o.Schema, + MaxAttempts: o.DefaultMaxAttempts, + Workers: workers, + PeriodicJobs: periodicJobs, + } + if workers != nil && len(o.Queues) == 0 { + cfg.Queues = map[string]river.QueueConfig{ + river.QueueDefault: {MaxWorkers: defaultMaxWorkers}, + } + } else { + cfg.Queues = o.Queues + } + return cfg.WithDefaults() +} + +// NewInsertOnlyClient builds an insert-only PGQueue Client. +func NewInsertOnlyClient(ctx context.Context, pool *pgxpool.Pool, opt *Options) (*PGQueue, error) { + var o Options + if opt != nil { + o = *opt + } + + // no workers or periodic jobs, insert-only client + cfg := o.withConfig(nil, nil) + + rc, err := river.NewClient(riverpgxv5.New(pool), cfg) + if err != nil { + return nil, fmt.Errorf("pgqueue: create client: %w", err) + } + + return &PGQueue{ + client: rc, + InserterBus: newMemBus(), + }, nil +} + +// NewWorkerClient builds a worker client that registers workers and periodic jobs. +func NewWorkerClient(ctx context.Context, pool *pgxpool.Pool, opt *Options) (*PGQueue, error) { + if opt == nil || opt.RegisterWorkers == nil { + return nil, fmt.Errorf("pgqueue: worker requires Options.RegisterWorkers") + } + ws := river.NewWorkers() + if err := opt.RegisterWorkers(ws); err != nil { + return nil, fmt.Errorf("pgqueue: register workers: %w", err) + } + + var pjobs []*river.PeriodicJob + if opt.Periodic != nil { + pb := &Periodics{} + if err := opt.Periodic(pb); err != nil { + return nil, fmt.Errorf("pgqueue: build periodics: %w", err) + } + pjobs = pb.Jobs() + } + + cfg := opt.withConfig(ws, pjobs) + + rc, err := river.NewClient(riverpgxv5.New(pool), cfg) + if err != nil { + return nil, fmt.Errorf("pgqueue: create worker client: %w", err) + } + return &PGQueue{ + client: rc, + WorkerBus: newMemBus(), + }, nil +} + func (q *PGQueue) Start(ctx context.Context) error { if err := q.client.Start(ctx); err != nil { - return fmt.Errorf("error starting riverqueue client: %w", err) + return fmt.Errorf("pgqueue: start: %w", err) } return nil } -func (q *PGQueue) Stop(ctx context.Context) error { +func (q *PGQueue) Stop(ctx context.Context, stopTimeout time.Duration) error { + if stopTimeout > 0 { + ctx2, cancel := context.WithTimeout(ctx, stopTimeout) + defer cancel() + if err := q.client.Stop(ctx2); err != nil { + return fmt.Errorf("pgqueue: stop: %w", err) + } + return nil + } if err := q.client.Stop(ctx); err != nil { - return fmt.Errorf("error stopping riverqueue client: %w", err) + return fmt.Errorf("pgqueue: stop: %w", err) } return nil } -func (q *PGQueue) InsertManyTx(ctx context.Context, tx db.DBTX, params []InsertManyParams) ([]int64, error) { - pgxTx, ok := tx.(pgx.Tx) - if !ok { - return nil, errors.New("failed to assert db.DBTX as pgx.Tx") +func (q *PGQueue) Insert(ctx context.Context, args JobArgs, opts InsertOpts) (int64, error) { + o := ConvertInsertOpts(opts) + r, err := q.client.Insert(ctx, args, o) + if err != nil { + return 0, fmt.Errorf("pgqueue: insert: %w", err) } + if r == nil || r.Job == nil { + return 0, errors.New("no job was enqueued") + } + return r.Job.ID, nil +} - riverParams := ConvertInsertManyParamsToRiver(params) +func (q *PGQueue) InsertTx(ctx context.Context, tx db.DBTX, args JobArgs, opts InsertOpts) (int64, error) { + pgxTx, err := convertTx(tx) + if err != nil { + return 0, err + } + o := ConvertInsertOpts(opts) + r, err := q.client.InsertTx(ctx, pgxTx, args, o) + if err != nil { + return 0, fmt.Errorf("pgqueue: insert tx: %w", err) + } + if r == nil || r.Job == nil { + return 0, errors.New("no job was enqueued") + } + return r.Job.ID, nil +} - jobResults, err := q.client.InsertManyTx(ctx, pgxTx, riverParams) +func (q *PGQueue) InsertMany(ctx context.Context, params []InsertManyParams) ([]int64, error) { + if len(params) == 0 { + return nil, nil + } + rparams := ConvertInsertManyParams(params) + res, err := q.client.InsertMany(ctx, rparams) if err != nil { - return nil, fmt.Errorf("error inserting jobs: %w", err) + return nil, fmt.Errorf("pgqueue: insert many: %w", err) } - jobIDs := make([]int64, len(jobResults)) - for idx := range jobResults { - jobIDs[idx] = jobResults[idx].Job.ID + ids := make([]int64, 0) + for _, r := range res { + if r == nil { + continue + } + if j := r.Job; j != nil { + ids = append(ids, j.ID) + } } - return jobIDs, nil + return ids, nil } -func (q *PGQueue) JobCancelTx(ctx context.Context, tx db.DBTX, jobID int64) error { - pgxTx, ok := tx.(pgx.Tx) - if !ok { - return errors.New("failed to assert db.DBTX as pgx.Tx") +func (q *PGQueue) InsertManyTx(ctx context.Context, tx db.DBTX, params []InsertManyParams) ([]int64, error) { + if len(params) == 0 { + return nil, nil } - _, err := q.client.JobCancelTx(ctx, pgxTx, jobID) + pgxTx, err := convertTx(tx) if err != nil { - return fmt.Errorf("error cancelling job: %w", err) + return nil, err } - return nil + rparams := ConvertInsertManyParams(params) + res, err := q.client.InsertManyTx(ctx, pgxTx, rparams) + if err != nil { + return nil, fmt.Errorf("pgqueue: insert many tx: %w", err) + } + ids := make([]int64, 0) + for _, r := range res { + if r == nil { + continue + } + if j := r.Job; j != nil { + ids = append(ids, j.ID) + } + } + return ids, nil } func (q *PGQueue) InsertManyFast(ctx context.Context, params []InsertManyParams) error { if len(params) == 0 { return nil } - rparams := ConvertInsertManyParamsToRiver(params) + rparams := ConvertInsertManyParams(params) if _, err := q.client.InsertManyFast(ctx, rparams); err != nil { - return fmt.Errorf("insert fast many: %w", err) + return fmt.Errorf("pgqueue: insert many fast: %w", err) } return nil } -func ConvertInsertManyParamsToRiver(params []InsertManyParams) []river.InsertManyParams { - riverParams := make([]river.InsertManyParams, len(params)) - for idx, p := range params { - riverParams[idx] = river.InsertManyParams{ - Args: p.Args, - } +func (q *PGQueue) JobCancelTx(ctx context.Context, tx db.DBTX, jobID int64) error { + pgxTx, err := convertTx(tx) + if err != nil { + return err + } + if _, err := q.client.JobCancelTx(ctx, pgxTx, jobID); err != nil { + return fmt.Errorf("pgqueue: cancel job: %w", err) + } + return nil +} + +func ConvertInsertManyParams(params []InsertManyParams) []river.InsertManyParams { + out := make([]river.InsertManyParams, len(params)) + for i, p := range params { + out[i] = river.InsertManyParams{Args: p.Args} if p.InsertOpts != nil { - riverParams[idx].InsertOpts = &river.InsertOpts{} - if p.InsertOpts.ScheduledAt != nil { - riverParams[idx].InsertOpts.ScheduledAt = *p.InsertOpts.ScheduledAt - } - if p.InsertOpts.UniqueOpts.ByArgs { - riverParams[idx].InsertOpts.UniqueOpts.ByArgs = true - } - if p.InsertOpts.UniqueOpts.ByPeriod != nil { - riverParams[idx].InsertOpts.UniqueOpts.ByPeriod = *p.InsertOpts.UniqueOpts.ByPeriod - } + out[i].InsertOpts = ConvertInsertOpts(*p.InsertOpts) } } - return riverParams + return out +} + +func ConvertInsertOpts(opts InsertOpts) *river.InsertOpts { + o := river.InsertOpts{} + if opts.ScheduledAt != nil { + o.ScheduledAt = *opts.ScheduledAt + } + if opts.UniqueOpts.ByArgs { + o.UniqueOpts.ByArgs = true + } + if opts.UniqueOpts.ByPeriod != nil { + o.UniqueOpts.ByPeriod = *opts.UniqueOpts.ByPeriod + } + return &o } diff --git a/api/internal/pgqueue/pgqueue_test.go b/api/internal/pgqueue/pgqueue_test.go index d301f80d..068685a0 100644 --- a/api/internal/pgqueue/pgqueue_test.go +++ b/api/internal/pgqueue/pgqueue_test.go @@ -1,52 +1,86 @@ -package pgqueue_test +package pgqueue import ( "testing" "time" - "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" - "github.com/stretchr/testify/assert" - _ "gocloud.dev/pubsub/mempubsub" + "github.com/riverqueue/river" ) -// A dummy type implementing Kind() so we can use it in InsertManyParams. -type dummyArg string +type argsA struct{} -func (d dummyArg) Kind() string { return string(d) } +func (argsA) Kind() string { return "kind.a" } -func TestConvertInsertManyParamsToRiver(t *testing.T) { - now := time.Now().Truncate(time.Second) - period := 5 * time.Minute +func TestConvertInsertManyParams(t *testing.T) { + when := time.Now().Add(5 * time.Minute) + period := 15 * time.Minute - params := []pgqueue.InsertManyParams{ + in := []InsertManyParams{ { - Args: dummyArg("foo"), - InsertOpts: &pgqueue.InsertOpts{ - ScheduledAt: &now, - UniqueOpts: pgqueue.UniqueOpts{ + Args: argsA{}, + InsertOpts: &InsertOpts{ + ScheduledAt: &when, + UniqueOpts: UniqueOpts{ ByArgs: true, ByPeriod: &period, }, }, }, { - Args: dummyArg("bar"), + Args: argsA{}, + InsertOpts: nil, }, } - riverParams := pgqueue.ConvertInsertManyParamsToRiver(params) - assert.Len(t, riverParams, 2) + got := ConvertInsertManyParams(in) + if len(got) != 2 { + t.Fatalf("expected 2 params, got %d", len(got)) + } + + // First item: options mapped + if got[0].Args.(argsA).Kind() != "kind.a" { + t.Fatalf("unexpected args kind") + } + if got[0].InsertOpts == nil { + t.Fatalf("InsertOpts was nil") + } + if !got[0].InsertOpts.ScheduledAt.Equal(when) { + t.Fatalf("scheduled mismatch") + } + if !got[0].InsertOpts.UniqueOpts.ByArgs { + t.Fatalf("ByArgs not set") + } + if got[0].InsertOpts.UniqueOpts.ByPeriod != period { + t.Fatalf("ByPeriod mismatch") + } + + // Second item: options empty + if got[1].InsertOpts != nil { + t.Fatalf("expected nil opts, got %#v", got[1].InsertOpts) + } +} - // first record - p0 := riverParams[0] - assert.Equal(t, "foo", p0.Args.Kind()) - assert.NotNil(t, p0.InsertOpts) - assert.True(t, p0.InsertOpts.ScheduledAt.Equal(now)) - assert.True(t, p0.InsertOpts.UniqueOpts.ByArgs) - assert.Equal(t, period, p0.InsertOpts.UniqueOpts.ByPeriod) +func TestOptions_withConfig_Basics(t *testing.T) { + o := &Options{ + ClientName: "api", + Schema: "midas", + DefaultMaxAttempts: 7, + Queues: map[string]river.QueueConfig{ + river.QueueDefault: {MaxWorkers: 5}, + }, + } + cfg := o.withConfig(nil, nil) - // second record - p1 := riverParams[1] - assert.Equal(t, "bar", p1.Args.Kind()) - assert.Nil(t, p1.InsertOpts) + if cfg.Schema != "midas" { + t.Fatalf("schema: got %q", cfg.Schema) + } + if cfg.MaxAttempts != 7 { + t.Fatalf("max attempts: got %d", cfg.MaxAttempts) + } + if cfg.Queues[river.QueueDefault].MaxWorkers != 5 { + t.Fatalf("max workers: got %d", cfg.Queues[river.QueueDefault].MaxWorkers) + } + if cfg.ID == "" { + t.Fatal("expected non-empty client ID") + } } diff --git a/api/internal/pgqueue/route.go b/api/internal/pgqueue/route.go new file mode 100644 index 00000000..511f8243 --- /dev/null +++ b/api/internal/pgqueue/route.go @@ -0,0 +1,48 @@ +package pgqueue + +import ( + "context" + "encoding/json" + "fmt" +) + +type DecodeFunc func([]byte) (any, error) +type HandleFunc func(context.Context, []any) error + +type Route struct { + Decode DecodeFunc + Handle HandleFunc +} +type Routes map[string]Route + +// TypedRoute builds a Route for a concrete payload type T. +// You provide a decoder to T, and a typed batch handler ([]T). +func TypedRoute[T any](decode func([]byte) (T, error), handle func(context.Context, []T) error) Route { + d := func(b []byte) (any, error) { + v, err := decode(b) + if err != nil { + return nil, err + } + return v, nil + } + h := func(ctx context.Context, vals []any) error { + items := make([]T, 0, len(vals)) + for i, v := range vals { + tv, ok := v.(T) + if !ok { + return fmt.Errorf("pgqueue: typed route: item %d has type %T (want %T)", i, v, *new(T)) + } + items = append(items, tv) + } + return handle(ctx, items) + } + return Route{Decode: d, Handle: h} +} + +// JSONRoute is a convenience over TypedRoute that uses json.Unmarshal into T. +func JSONRoute[T any](handle func(context.Context, []T) error) Route { + return TypedRoute(func(b []byte) (T, error) { + var t T + return t, json.Unmarshal(b, &t) + }, handle) +} diff --git a/api/internal/service/alert_check_event.go b/api/internal/service/alert_check_event.go index 7ae9219b..e72d166e 100644 --- a/api/internal/service/alert_check_event.go +++ b/api/internal/service/alert_check_event.go @@ -110,28 +110,28 @@ func (s *DBService) DoAlertCheckEvent(ctx context.Context, messages []dto.Worker mopts, ok := r.Opts.(map[string]any) if !ok { // invalid opts - s.logger.Warn(ctx, "alter config opts type assertion failed in doAlertAfterRequestChecks; skipping") + s.Logger.Warn(ctx, "alter config opts type assertion failed in doAlertAfterRequestChecks; skipping") continue } bopts, err := json.Marshal(mopts) if err != nil { - s.logger.Warn(ctx, "failed to call json.Marshal on r.Opts (map[string]any); skipping") + s.Logger.Warn(ctx, "failed to call json.Marshal on r.Opts (map[string]any); skipping") continue } switch r.AlertTypeID { case db.ThresholdAlertTypeID: if err := json.Unmarshal(bopts, &a.Opts.ThresholdOpts); err != nil { - s.logger.Warn(ctx, "failed to unmarshal ThresholdOpts; skipping", "error", err) + s.Logger.Warn(ctx, "failed to unmarshal ThresholdOpts; skipping", "error", err) continue } case db.RateOfChangeAlertTypeID: if err := json.Unmarshal(bopts, &a.Opts.ChangeOpts); err != nil { - s.logger.Warn(ctx, "failed to unmarshal ChangeOpts; skipping", "error", err) + s.Logger.Warn(ctx, "failed to unmarshal ChangeOpts; skipping", "error", err) continue } default: - s.logger.Warn(ctx, "unsupported alert type; skipping", "alert_type_id", r.AlertTypeID) + s.Logger.Warn(ctx, "unsupported alert type; skipping", "alert_type_id", r.AlertTypeID) continue } aa = append(aa, a) @@ -195,7 +195,7 @@ func (s *DBService) doAlertEventChecks(ctx context.Context, acc []AlertConfigEve case db.RateOfChangeAlertTypeID: statusID, vv = doAlertCheckChanges(ac.Opts.ChangeOpts, ac.Measurements, ac.LastMeasurement, ac.NextMeasurement) default: - s.logger.Error(ctx, "alert type not supported", "alert_type", ac.AlertType) + s.Logger.Error(ctx, "alert type not supported", "alert_type", ac.AlertType) continue } ac.Violations = vv @@ -208,9 +208,9 @@ func (s *DBService) doAlertEventChecks(ctx context.Context, acc []AlertConfigEve emailAlertType = "Warning" case GreenAlertStatusID: case uuid.Nil: - s.logger.Debug(ctx, "no severity level set in doAlertCheckThresholds or doAlertCheckChanges; skipping...") + s.Logger.Debug(ctx, "no severity level set in doAlertCheckThresholds or doAlertCheckChanges; skipping...") default: - s.logger.Error(ctx, "unknown AlertStatusID; skipping...", "alert_status_id", statusID) + s.Logger.Error(ctx, "unknown AlertStatusID; skipping...", "alert_status_id", statusID) continue } @@ -218,7 +218,7 @@ func (s *DBService) doAlertEventChecks(ctx context.Context, acc []AlertConfigEve if emailAlertType != "" { ec, err = ac.EmailContent(emailAlertType) if err != nil { - s.logger.Error(ctx, "formatting alert config to template", "error", err) + s.Logger.Error(ctx, "formatting alert config to template", "error", err) } } @@ -282,11 +282,11 @@ func (s *DBService) doAlertEventChecks(ctx context.Context, acc []AlertConfigEve if _, exists := uniqueIDSet[emailArgs[idx].UniqueID]; !exists { uniqueIDSet[emailArgs[idx].UniqueID] = struct{}{} } else { - s.logger.Debug(ctx, "unique alert_config_id already exists in batch", "alert_config_id", emailArgs[idx].UniqueID) + s.Logger.Debug(ctx, "unique alert_config_id already exists in batch", "alert_config_id", emailArgs[idx].UniqueID) continue } if len(emailArgs[idx].Bcc) == 0 { - s.logger.Warn(ctx, "no bcc recipients for alert config", "alert_config_id", emailArgs[idx].UniqueID) + s.Logger.Warn(ctx, "no bcc recipients for alert config", "alert_config_id", emailArgs[idx].UniqueID) continue } pgqParams = append(pgqParams, pgqueue.InsertManyParams{Args: emailArgs[idx]}) diff --git a/api/internal/service/alert_check_schedule_v1.go b/api/internal/service/alert_check_schedule_v1.go index a8cf679f..592f0701 100644 --- a/api/internal/service/alert_check_schedule_v1.go +++ b/api/internal/service/alert_check_schedule_v1.go @@ -54,7 +54,7 @@ type alertChecker interface { } func (s *DBService) DoAlertCheckScheduleV1(ctx context.Context) error { - s.logger.Info(ctx, "starting DoAlertCheckScheduleV1") + s.Logger.Info(ctx, "starting DoAlertCheckScheduleV1") subs, err := s.Queries.SubmittalListUnverifiedMissing(ctx) if err != nil { return err @@ -64,7 +64,7 @@ func (s *DBService) DoAlertCheckScheduleV1(ctx context.Context) error { return err } if len(acs) == 0 { - s.logger.Info(ctx, "no alert configs to check") + s.Logger.Info(ctx, "no alert configs to check") return nil } @@ -87,26 +87,27 @@ func (s *DBService) DoAlertCheckScheduleV1(ctx context.Context) error { defer s.TxDo(ctx, tx.Rollback) qtx := s.WithTx(tx) - p1, err := handleChecks(ctx, qtx, evaluationChecks, s.logger) + p1, err := handleChecks(ctx, qtx, evaluationChecks, s.Logger) if err != nil { return err } - p2, err := handleChecks(ctx, qtx, measurementChecks, s.logger) + p2, err := handleChecks(ctx, qtx, measurementChecks, s.Logger) if err != nil { return err } - params := append(p1, p2...) + params := make([]pgqueue.InsertManyParams, 0) + params = append(p1, p2...) if len(params) > 0 { - s.logger.Debug(ctx, "batch insert alert check params", "params", fmt.Sprintf("%+v", params)) + s.Logger.Debug(ctx, "batch insert alert check params", "params", fmt.Sprintf("%+v", params)) if _, err = s.PGQueue.InsertManyTx(ctx, tx, params); err != nil { return err } } - s.logger.Info(ctx, "alert checks completed sucessfully", "email_jobs_queued_deduped", len(params)) + s.Logger.Info(ctx, "alert checks completed sucessfully", "email_jobs_queued_deduped", len(params)) return tx.Commit(ctx) } @@ -115,7 +116,7 @@ func (s *DBService) getEvaluationChecks(ctx context.Context, subMap submittalMap alertConfigEvaluationChecks := make([]*AlertConfigEvaluationCheck, 0) ecs, err := s.Queries.SubmittalListIncompleteEvaluation(ctx) if err != nil { - s.logger.Error( + s.Logger.Error( ctx, "SubmittalListIncompleteEvaluation", "error", err, ) @@ -157,7 +158,7 @@ func (s *DBService) getMeasurementChecks(ctx context.Context, subMap submittalMa alertConfigMeasurementChecks := make([]*AlertConfigMeasurementCheck, 0) mcs, err := s.Queries.SubmittalListIncompleteMeasurement(ctx) if err != nil { - s.logger.Error( + s.Logger.Error( ctx, "SubmittalListIncompleteMeasurement", "error", err, ) @@ -197,7 +198,7 @@ func (s *DBService) getMeasurementChecks(ctx context.Context, subMap submittalMa return alertConfigMeasurementChecks } -func updateAlertConfigChecks[T alertChecker, PT alertConfigChecker[T]](ctx context.Context, q *db.Queries, accs []PT, l *logger.Logger) error { +func updateAlertConfigChecks[T alertChecker, PT alertConfigChecker[T]](ctx context.Context, q *db.Queries, accs []PT, l logger.Logger) error { updateLastRemindedBatchParams := make([]db.AlertConfigScheduleUpdateLastRemindedAtBatchParams, len(accs)) updateCompletionWarningParams := make([]db.SubmittalUpdateCompletionDateOrWarningSentBatchParams, 0) createFromAlertConfigDateParams := make([]db.SubmittalCreateNextFromExistingAlertConfigDateBatchParams, 0) @@ -306,7 +307,7 @@ func updateAlertConfigChecks[T alertChecker, PT alertConfigChecker[T]](ctx conte // for measurements, the next submittal is created the first time this function runs after the due date // // No "Yellow" Status Submittals should be passed to this function as it implies the submittal has been completed -func handleChecks[T alertChecker, PT alertConfigChecker[T]](ctx context.Context, q *db.Queries, accs []PT, l *logger.Logger) ([]pgqueue.InsertManyParams, error) { +func handleChecks[T alertChecker, PT alertConfigChecker[T]](ctx context.Context, q *db.Queries, accs []PT, l logger.Logger) ([]pgqueue.InsertManyParams, error) { defer util.Timer()() alertCreateBatchParams := make([]db.AlertCreateBatchParams, 0) diff --git a/api/internal/service/alert_check_schedule_v2.go b/api/internal/service/alert_check_schedule_v2.go index 4ab8f936..69d98fa5 100644 --- a/api/internal/service/alert_check_schedule_v2.go +++ b/api/internal/service/alert_check_schedule_v2.go @@ -41,7 +41,7 @@ func (s *DBService) DoAlertCheckScheduleV2(ctx context.Context, msg dto.WorkerAl return schedulingError("WarnAfter") } if !sub.WarningsEnabled { - s.logger.Warn(ctx, "warnings are not enabled for this alert config; either the alert config settings have changed or there is an error and this job should not have been scheduled", "submittal", sub) + s.Logger.Warn(ctx, "warnings are not enabled for this alert config; either the alert config settings have changed or there is an error and this job should not have been scheduled", "submittal", sub) return nil } diff --git a/api/internal/service/db.go b/api/internal/service/db.go index 7c5669c4..0be71c2c 100644 --- a/api/internal/service/db.go +++ b/api/internal/service/db.go @@ -15,7 +15,7 @@ import ( ) type DBService struct { - logger *logger.Logger + Logger logger.Logger db *pgxpool.Pool PGQueue *pgqueue.PGQueue FeatureFlags *config.FeatureFlags @@ -24,11 +24,11 @@ type DBService struct { type DBServiceOpts struct { Config *config.DBConfig - Logger *logger.Logger + Logger logger.Logger } -func NewDBServiceFromPool(dbpool *pgxpool.Pool, logger *logger.Logger, flags *config.FeatureFlags) *DBService { - return &DBService{logger: logger, db: dbpool, Queries: gen.New(dbpool), FeatureFlags: flags} +func NewDBServiceFromPool(dbpool *pgxpool.Pool, logger logger.Logger, flags *config.FeatureFlags) *DBService { + return &DBService{Logger: logger, db: dbpool, Queries: gen.New(dbpool), FeatureFlags: flags} } func NewDBPool(ctx context.Context, cfg config.DBConfig) (*pgxpool.Pool, error) { @@ -52,7 +52,7 @@ func NewDBPool(ctx context.Context, cfg config.DBConfig) (*pgxpool.Pool, error) func (s *DBService) TxDo(ctx context.Context, rollback func(ctx context.Context) error) { err := rollback(ctx) if err != nil && !errors.Is(err, pgx.ErrTxClosed) { - s.logger.Error(ctx, "unable to rollback transaction", "error", err) + s.Logger.Error(ctx, "unable to rollback transaction", "error", err) } } @@ -93,3 +93,5 @@ func batchQueryCollect[T any](rrr [][]T, err *error) func(int, []T, error) { rrr[i] = rr } } + +type NestedUserError error diff --git a/api/internal/service/dcsloader.go b/api/internal/service/dcsloader.go index 766bbb66..3cb5231c 100644 --- a/api/internal/service/dcsloader.go +++ b/api/internal/service/dcsloader.go @@ -27,10 +27,10 @@ import ( type DcsLoaderService struct { apiClient *http.Client cfg *config.DcsLoaderConfig - logger *logger.Logger + logger logger.Logger } -func NewDcsLoaderService(apiClient *http.Client, cfg *config.DcsLoaderConfig, logger *logger.Logger) *DcsLoaderService { +func NewDcsLoaderService(apiClient *http.Client, cfg *config.DcsLoaderConfig, logger logger.Logger) *DcsLoaderService { return &DcsLoaderService{apiClient, cfg, logger} } diff --git a/api/internal/service/expression.go b/api/internal/service/expression.go new file mode 100644 index 00000000..8beae350 --- /dev/null +++ b/api/internal/service/expression.go @@ -0,0 +1,303 @@ +package service + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/USACE/instrumentation-api/api/v4/internal/db" + "github.com/USACE/instrumentation-api/api/v4/internal/dto" + "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" + "github.com/google/uuid" +) + +func expressionCreate(ctx context.Context, qtx *db.Queries, instrumentID, profileID uuid.UUID, x dto.ExpressionDTO) (db.VExpression, error) { + var a db.VExpression + mode := db.ExpressionMode(x.Mode) + idNew, err := qtx.ExpressionCreate(ctx, db.ExpressionCreateParams{ + InstrumentID: instrumentID, + Name: x.Name, + Expression: x.Expression, + Mode: mode, + CreatedBy: profileID, + }) + if err != nil { + return a, err + } + + switch mode { + case db.ExpressionModeWindow: + qtx.ExpressionWindowOptsCreate(ctx, db.ExpressionWindowOptsCreateParams{ + ExpressionID: idNew, + Duration: x.Opts.Duration, + TzName: x.Opts.TzName, + DurationOffset: x.Opts.DurationOffset, + TimestampingPolicy: db.ExpressionWindowTimestampingPolicy(x.Opts.TimestampingPolicy), + }) + default: + } + + params := make([]db.ExpressionTimeseriesVariableCreateOrUpdateBatchParams, len(x.Variables)) + for idx, v := range x.Variables { + params[idx] = db.ExpressionTimeseriesVariableCreateOrUpdateBatchParams{ + ExpressionID: idNew, + TimeseriesID: v.TimeseriesID, + VariableName: v.VariableName, + } + } + + qtx.ExpressionTimeseriesVariableCreateOrUpdateBatch(ctx, params).Exec(batchExecErr(&err)) + if err != nil { + return a, err + } + + tsNew, err := qtx.TimeseriesCreate(ctx, db.TimeseriesCreateParams{ + InstrumentID: instrumentID, + Name: x.Name, + ParameterID: uuid.Nil, + UnitID: uuid.Nil, + Type: db.TimeseriesTypeExpression, + }) + if err != nil { + return a, err + } + + if err := qtx.ExpressionTimeseriesTargetCreate(ctx, db.ExpressionTimeseriesTargetCreateParams{ + ExpressionID: idNew, + TimeseriesID: tsNew.ID, + }); err != nil { + return a, err + } + + checks, err := qtx.ExpressionGetWouldCreateCycleOrExceedDepth(ctx, idNew) + if err != nil { + return a, fmt.Errorf("query ExpressionGetWouldCreateCycleOrExceedDepth: %w", err) + } + if checks.WouldCreateCycle { + return a, NestedUserError(errors.New("import cycles not allowed")) + } + if checks.WouldExceedMaxDepth { + return a, NestedUserError(errors.New("maximum depth of 5 exceeded")) + } + + a, err = qtx.ExpressionGet(ctx, idNew) + if err != nil { + return a, err + } + return a, nil +} + +func (s *DBService) ExpressionCreate(ctx context.Context, instrumentID, profileID uuid.UUID, x dto.ExpressionDTO) (db.VExpression, error) { + var a db.VExpression + tx, err := s.db.Begin(ctx) + if err != nil { + return a, err + } + defer s.TxDo(ctx, tx.Rollback) + qtx := s.WithTx(tx) + + a, err = expressionCreate(ctx, qtx, instrumentID, profileID, x) + if err != nil { + return a, err + } + + arg := dto.WorkerExpressionTimeseriesComputeFullEventArgs{ + TargetTimeseriesID: a.TargetTimeseriesID, + ExpressionID: a.ID, + Expression: x, + } + s.Logger.Debug(ctx, "enqueue evaluate timeseries event (create)", "event_payload", fmt.Sprintf("%+v", arg)) + jobID, err := s.PGQueue.InsertTx(ctx, tx, arg, pgqueue.InsertOpts{}) + if err != nil { + return a, err + } + s.Logger.Debug(ctx, "enqueued job", "job_id", jobID) + + return a, tx.Commit(ctx) +} + +func expressionUpdate(ctx context.Context, qtx *db.Queries, expressionID, profileID uuid.UUID, x dto.ExpressionDTO) (db.VExpression, error) { + var a db.VExpression + mode := db.ExpressionMode(x.Mode) + _, err := qtx.ExpressionUpdate(ctx, db.ExpressionUpdateParams{ + ID: expressionID, + Name: x.Name, + Expression: x.Expression, + Mode: mode, + UpdatedBy: &profileID, + }) + if err != nil { + return a, err + } + + if err := qtx.ExpressionWindowOptsDelete(ctx, expressionID); err != nil { + return a, err + } + + switch mode { + case db.ExpressionModeWindow: + qtx.ExpressionWindowOptsCreate(ctx, db.ExpressionWindowOptsCreateParams{ + ExpressionID: expressionID, + Duration: x.Opts.Duration, + TzName: x.Opts.TzName, + DurationOffset: x.Opts.DurationOffset, + TimestampingPolicy: db.ExpressionWindowTimestampingPolicy(x.Opts.TimestampingPolicy), + }) + default: + } + + if err := qtx.ExpressionTimeseriesVariableDeleteAllForExpression(ctx, expressionID); err != nil { + return a, err + } + + params := make([]db.ExpressionTimeseriesVariableCreateOrUpdateBatchParams, len(x.Variables)) + for idx, v := range x.Variables { + params[idx] = db.ExpressionTimeseriesVariableCreateOrUpdateBatchParams{ + ExpressionID: expressionID, + TimeseriesID: v.TimeseriesID, + VariableName: v.VariableName, + } + } + + qtx.ExpressionTimeseriesVariableCreateOrUpdateBatch(ctx, params).Exec(batchExecErr(&err)) + if err != nil { + return a, err + } + + checks, err := qtx.ExpressionGetWouldCreateCycleOrExceedDepth(ctx, expressionID) + if err != nil { + return a, fmt.Errorf("query ExpressionGetWouldCreateCycleOrExceedDepth: %w", err) + } + if checks.WouldCreateCycle { + return a, NestedUserError(errors.New("import cycles not allowed")) + } + if checks.WouldExceedMaxDepth { + return a, NestedUserError(errors.New("maximum depth of 5 exceeded")) + } + + a, err = qtx.ExpressionGet(ctx, expressionID) + if err != nil { + return a, err + } + return a, nil +} + +func (s *DBService) ExpressionUpdate(ctx context.Context, expressionID, profileID uuid.UUID, x dto.ExpressionDTO) (db.VExpression, error) { + var a db.VExpression + tx, err := s.db.Begin(ctx) + if err != nil { + return a, err + } + defer s.TxDo(ctx, tx.Rollback) + qtx := s.WithTx(tx) + + a, err = expressionUpdate(ctx, qtx, expressionID, profileID, x) + if err != nil { + return a, err + } + + arg := dto.WorkerExpressionTimeseriesComputeFullEventArgs{ + TargetTimeseriesID: a.TargetTimeseriesID, + ExpressionID: expressionID, + Expression: x, + } + + s.Logger.Debug(ctx, "enqueue evaluate timeseries event (update)", "event_payload", fmt.Sprintf("%+v", arg)) + jobID, err := s.PGQueue.InsertTx(ctx, tx, arg, pgqueue.InsertOpts{}) + if err != nil { + return a, err + } + + if err := qtx.TimeseriesMeasurementDeleteAllForTimeseries(ctx, a.TargetTimeseriesID); err != nil && !errors.Is(err, sql.ErrNoRows) { + return a, err + } + s.Logger.Debug(ctx, "enqueued job", "job_id", jobID) + + return a, tx.Commit(ctx) +} + +func (s *DBService) ExpressionDelete(ctx context.Context, expressionID uuid.UUID) error { + tx, err := s.db.Begin(ctx) + if err != nil { + return err + } + defer s.TxDo(ctx, tx.Rollback) + qtx := s.WithTx(tx) + + if err := qtx.TimeseriesDeleteTargetForExpression(ctx, expressionID); err != nil { + return err + } + if err := qtx.ExpressionDelete(ctx, expressionID); err != nil { + return err + } + + return nil +} + +type ExpressionValidation struct { + Dependencies []db.ExpressionDownstreamListForTimeseriesRow `json:"dependencies"` +} + +func (s *DBService) ExpressionCreateValidation(ctx context.Context, instrumentID, profileID uuid.UUID, x dto.ExpressionDTO) (db.VExpression, ExpressionValidation, error) { + var xNew db.VExpression + var v ExpressionValidation + tx, err := s.db.Begin(ctx) + if err != nil { + return xNew, v, err + } + qtx := s.WithTx(tx) + + xNew, err = expressionCreate(ctx, qtx, instrumentID, profileID, x) + if err != nil { + return xNew, v, err + } + + tsIDs := make([]uuid.UUID, len(x.Variables)) + for i, v := range x.Variables { + tsIDs[i] = v.TimeseriesID + } + graph, err := s.Queries.ExpressionDownstreamListForTimeseries(ctx, tsIDs) + if err != nil { + return xNew, v, err + } + v.Dependencies = graph + + if err := tx.Rollback(ctx); err != nil { + return xNew, v, err + } + + return xNew, v, nil +} + +func (s *DBService) ExpressionUpdateValidation(ctx context.Context, expressionID, profileID uuid.UUID, x dto.ExpressionDTO) (db.VExpression, ExpressionValidation, error) { + var xUpdated db.VExpression + var v ExpressionValidation + tx, err := s.db.Begin(ctx) + if err != nil { + return xUpdated, v, err + } + qtx := s.WithTx(tx) + + xUpdated, err = expressionUpdate(ctx, qtx, expressionID, profileID, x) + if err != nil { + return xUpdated, v, err + } + + tsIDs := make([]uuid.UUID, len(x.Variables)+1) + for i, v := range x.Variables { + tsIDs[i] = v.TimeseriesID + } + tsIDs[len(x.Variables)] = xUpdated.TargetTimeseriesID + graph, err := s.Queries.ExpressionDownstreamListForTimeseries(ctx, tsIDs) + if err != nil { + return xUpdated, v, err + } + v.Dependencies = graph + + if err := tx.Rollback(ctx); err != nil { + return xUpdated, v, err + } + + return xUpdated, v, nil +} diff --git a/api/internal/service/measurement.go b/api/internal/service/measurement.go index af7ef6de..c4648dc7 100644 --- a/api/internal/service/measurement.go +++ b/api/internal/service/measurement.go @@ -12,7 +12,6 @@ import ( "github.com/USACE/instrumentation-api/api/v4/internal/httperr" "github.com/USACE/instrumentation-api/api/v4/internal/util" "github.com/google/uuid" - "gocloud.dev/pubsub" ) type TimeseriesMeasurementInfo struct { @@ -31,7 +30,7 @@ func (s *DBService) TimeseriesMeasurementListForRange(ctx context.Context, arg d // TimeseriesMeasurementCreateOrUpdateBatch creates many timeseries from an array of timeseries // If a timeseries measurement already exists for a given timeseries_id and time, the value is updated -func (s *DBService) TimeseriesMeasurementCreateOrUpdateBatch(ctx context.Context, mc []dto.MeasurementCollectionDTO, pub *pubsub.Topic) error { +func (s *DBService) TimeseriesMeasurementCreateOrUpdateBatch(ctx context.Context, mc []dto.MeasurementCollectionDTO) error { tx, err := s.db.Begin(ctx) if err != nil { return err @@ -51,17 +50,17 @@ func (s *DBService) TimeseriesMeasurementCreateOrUpdateBatch(ctx context.Context if len(args.TimeseriesToCheck) != 0 { b, err := json.Marshal(args) if err != nil { - s.logger.Error(ctx, "incorrect message format", "error", err) + s.Logger.Error(ctx, "incorrect message format", "error", err) } - if err := pub.Send(ctx, &pubsub.Message{Body: b}); err != nil { - s.logger.Error(ctx, "failed to queue job", "error", err) + if err := s.PGQueue.BatchInserterSend(ctx, b); err != nil { + s.Logger.Error(ctx, "failed to queue job", "error", err) } } return nil } -func (s *DBService) SeisTimeseriesMeasurementCreateOrUpdateBatch(ctx context.Context, mc []dto.SeisMeasurementCollectionDTO, pub *pubsub.Topic) error { +func (s *DBService) SeisTimeseriesMeasurementCreateOrUpdateBatch(ctx context.Context, mc []dto.SeisMeasurementCollectionDTO) error { tx, err := s.db.Begin(ctx) if err != nil { return err @@ -81,17 +80,17 @@ func (s *DBService) SeisTimeseriesMeasurementCreateOrUpdateBatch(ctx context.Con if len(args.TimeseriesToCheck) != 0 { b, err := json.Marshal(args) if err != nil { - s.logger.Error(ctx, "incorrect message format", "error", err) + s.Logger.Error(ctx, "incorrect message format", "error", err) } - if err := pub.Send(ctx, &pubsub.Message{Body: b}); err != nil { - s.logger.Error(ctx, "failed to queue job", "error", err) + if err := s.PGQueue.BatchInserterSend(ctx, b); err != nil { + s.Logger.Error(ctx, "failed to queue job", "error", err) } } return nil } -func (s *DBService) TimeseriesMeasurementUpdateBatch(ctx context.Context, mc []dto.MeasurementCollectionDTO, tw *util.TimeWindow, pub *pubsub.Topic) error { +func (s *DBService) TimeseriesMeasurementUpdateBatch(ctx context.Context, mc []dto.MeasurementCollectionDTO, tw *util.TimeWindow) error { tx, err := s.db.Begin(ctx) if err != nil { return err @@ -116,10 +115,10 @@ func (s *DBService) TimeseriesMeasurementUpdateBatch(ctx context.Context, mc []d if len(args.TimeseriesToCheck) != 0 { b, err := json.Marshal(args) if err != nil { - s.logger.Error(ctx, "incorrect message format", "error", err) + s.Logger.Error(ctx, "incorrect message format", "error", err) } - if err := pub.Send(ctx, &pubsub.Message{Body: b}); err != nil { - s.logger.Error(ctx, "failed to queue job", "error", err) + if err := s.PGQueue.BatchInserterSend(ctx, b); err != nil { + s.Logger.Error(ctx, "failed to queue job", "error", err) } } diff --git a/api/internal/service/survey123.go b/api/internal/service/survey123.go index 05901f3e..b64fa3eb 100644 --- a/api/internal/service/survey123.go +++ b/api/internal/service/survey123.go @@ -154,7 +154,7 @@ func (s *DBService) Survey123MeasurementCreateOrUpdateBatch(ctx context.Context, em := make([]string, 0) defer func() { if err := s.createOrUpdateSurvey123PayloadError(ctx, survey123ID, em); err != nil { - s.logger.Error(ctx, "unable to write Survey123 error payload to database", "error", err) + s.Logger.Error(ctx, "unable to write Survey123 error payload to database", "error", err) } }() diff --git a/api/internal/worker/alert_event.go b/api/internal/worker/alert_event.go index b0304041..51b7c686 100644 --- a/api/internal/worker/alert_event.go +++ b/api/internal/worker/alert_event.go @@ -3,112 +3,43 @@ package worker import ( "context" "encoding/json" - "errors" "fmt" - "github.com/USACE/instrumentation-api/api/v4/internal/cloud" - "github.com/USACE/instrumentation-api/api/v4/internal/config" "github.com/USACE/instrumentation-api/api/v4/internal/dto" - "github.com/USACE/instrumentation-api/api/v4/internal/logger" + "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" "github.com/USACE/instrumentation-api/api/v4/internal/service" "github.com/riverqueue/river" - "gocloud.dev/pubsub" ) type AlertEventWorker struct { river.WorkerDefaults[dto.WorkerAlertEventArgs] - DBService *service.DBService - TaskServices *cloud.TaskServices + DBService *service.DBService + PGQueue *pgqueue.PGQueue // internal task bus/publisher } -func NewAlertEventWorker(dbservice *service.DBService, taskServices *cloud.TaskServices) *AlertEventWorker { - return &AlertEventWorker{DBService: dbservice, TaskServices: taskServices} +func NewAlertEventWorker(dbservice *service.DBService) *AlertEventWorker { + return &AlertEventWorker{DBService: dbservice} } +// Work publishes the args into pgqueue's internal task bus as an envelope. +// The "Type" uses the JobArgs.Kind() so routing can be consistent. func (w *AlertEventWorker) Work(ctx context.Context, job *river.Job[dto.WorkerAlertEventArgs]) error { - b, err := json.Marshal(job.Args) + if w.PGQueue == nil { + return fmt.Errorf("ComputeTsWorker.Work: internal bus Q is nil (expected *pgqueue.PGQueue)") + } + env, err := json.Marshal(pgqueue.Envelope{ + Type: job.Args.Kind(), + Payload: job.EncodedArgs, + }) if err != nil { - return err + return fmt.Errorf("marshal env: %w", err) } - return w.TaskServices.TaskPub.Send(ctx, &pubsub.Message{Body: b}) + return w.PGQueue.BatchWorkerSend(ctx, env) } -func (w *AlertEventWorker) ListenPoolProcess(ctx context.Context, cfg *config.TaskConfig, l *logger.Logger) error { - sem := make(chan struct{}, cfg.AggMaxHandlers) - errCh := make(chan error, 1) // capture the first critical error - -recvLoop: - for { - select { - case <-ctx.Done(): - return ctx.Err() - case err := <-errCh: - return err - default: - } - - batchCtx, cancel := context.WithTimeout(ctx, cfg.AggBatchTimeout) - - batchArgs := make([]dto.WorkerAlertEventArgs, 0) - msgs := make([]*pubsub.Message, 0) - - collectLoop: - for len(batchArgs) < cfg.AggBatchSize { - msg, err := w.TaskServices.TaskSub.Receive(batchCtx) - if err != nil { - switch { - case errors.Is(err, context.DeadlineExceeded): - // timeout, flush whatever we have - break collectLoop - case errors.Is(err, context.Canceled): - // parent ctx done, exit - cancel() - break recvLoop - default: - // critical receive error, exit and bubble up - cancel() - return fmt.Errorf("failed to receive message: %w", err) - } - } - - var args dto.WorkerAlertEventArgs - if err := json.Unmarshal(msg.Body, &args); err != nil { - msg.Ack() - continue collectLoop - } - - batchArgs = append(batchArgs, args) - msgs = append(msgs, msg) - } - cancel() - - if len(batchArgs) == 0 { - // nothing to do; loop again for new batches or cancellation - continue recvLoop - } - - sem <- struct{}{} - go func(msgs []*pubsub.Message, args []dto.WorkerAlertEventArgs) { - defer func() { <-sem }() - - l.Info(ctx, "Processing batch of alert event messages", "count", len(args)) - if err := w.DBService.DoAlertCheckEvent(ctx, args); err != nil { - select { - case errCh <- fmt.Errorf("batch processing failed: %w", err): - default: - } - } - - // ack every message once we’ve attempted processing - for _, m := range msgs { - m.Ack() - } - }(msgs, batchArgs) - } - - // wait for in-flight handlers to finish - for range cfg.AggMaxHandlers { - sem <- struct{}{} - } - return nil +// Route exposes a decode/handle pair so the caller can register it with pgqueue.NewBatchWorker +func (w *AlertEventWorker) Route() (string, pgqueue.Route) { + kind := (dto.WorkerAlertEventArgs{}).Kind() + route := pgqueue.JSONRoute(w.DBService.DoAlertCheckEvent) + return kind, route } diff --git a/api/internal/worker/alert_schedule_v1.go b/api/internal/worker/alert_schedule_v1.go index 99981f41..4d82f095 100644 --- a/api/internal/worker/alert_schedule_v1.go +++ b/api/internal/worker/alert_schedule_v1.go @@ -5,25 +5,23 @@ import ( "time" "github.com/USACE/instrumentation-api/api/v4/internal/dto" + "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" "github.com/USACE/instrumentation-api/api/v4/internal/service" "github.com/riverqueue/river" ) var alertScheduleV1Interval = 15 * time.Minute -var AlertScheduleV1JobOptions = river.NewPeriodicJob( - river.PeriodicInterval(alertScheduleV1Interval), - func() (river.JobArgs, *river.InsertOpts) { +// RegisterAlertScheduleV1 adds the periodic schedule to the worker's Periodics builder. +func RegisterAlertScheduleV1(p *pgqueue.Periodics) { + p.Every(alertScheduleV1Interval, func() (river.JobArgs, *river.InsertOpts) { return dto.WorkerAlertCheckScheduleV1Args{}, &river.InsertOpts{ - UniqueOpts: river.UniqueOpts{ - ByPeriod: alertScheduleV1Interval, - }, + UniqueOpts: river.UniqueOpts{ByPeriod: alertScheduleV1Interval}, } - }, - &river.PeriodicJobOpts{RunOnStart: true}, -) + }, &river.PeriodicJobOpts{RunOnStart: true}) +} -// AlertScheduleWorkerV1 runs every 15 minutes and batch processes submittals +// AlertScheduleWorkerV1 runs every 15 minutes and batch processes submittals. type AlertScheduleWorkerV1 struct { river.WorkerDefaults[dto.WorkerAlertCheckScheduleV1Args] DBService *service.DBService diff --git a/api/internal/worker/compute_ts_event.go b/api/internal/worker/compute_ts_event.go new file mode 100644 index 00000000..bceb8264 --- /dev/null +++ b/api/internal/worker/compute_ts_event.go @@ -0,0 +1,107 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/USACE/instrumentation-api/api/v4/internal/dto" + "github.com/USACE/instrumentation-api/api/v4/internal/eval" + "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" + "github.com/USACE/instrumentation-api/api/v4/internal/service" + "github.com/riverqueue/river" +) + +// large, less frequent jobs (when a user creates or updates an expression) +type EvaluationTimeseriesComputeFullEventWorker struct { + river.WorkerDefaults[dto.WorkerExpressionTimeseriesComputeFullEventArgs] + DBService *service.DBService + PGQueue *pgqueue.PGQueue + baseEnv *eval.Env + programCache *eval.ProgramCache +} + +func NewEvaluationTimeseriesComputeFullEventWorker(dbservice *service.DBService, baseEnv *eval.Env, cache *eval.ProgramCache) *EvaluationTimeseriesComputeFullEventWorker { + return &EvaluationTimeseriesComputeFullEventWorker{DBService: dbservice, baseEnv: baseEnv, programCache: cache} +} + +func (w *EvaluationTimeseriesComputeFullEventWorker) Work(ctx context.Context, job *river.Job[dto.WorkerExpressionTimeseriesComputeFullEventArgs]) error { + if w.PGQueue == nil { + return fmt.Errorf("ComputeTsWorker.Work: internal bus PGQueue is nil (expected *pgqueue.PGQueue)") + } + opts := []eval.EvalSessionParams{{ + TargetTimeseriesID: job.Args.TargetTimeseriesID, + ExpressionID: job.Args.ExpressionID, + Expression: job.Args.Expression, + }} + sess, err := eval.NewEvalSession( + w.DBService.Queries, + w.DBService.PGQueue, + w.DBService.Logger, + w.baseEnv, + opts, + eval.WithProgramCache(w.programCache), + ) + if err != nil { + return err + } + return sess.Run(ctx) +} + +// small, frequent jobs (when measurements are uploaded to a source timeseries) +type EvaluationTimeseriesComputePartialEventWorker struct { + river.WorkerDefaults[dto.WorkerExpressionTimeseriesComputePartialEventArgs] + DBService *service.DBService + PGQueue *pgqueue.PGQueue + baseEnv *eval.Env + programCache *eval.ProgramCache +} + +func NewEvaluationTimeseriesComputePartialEventWorker(dbservice *service.DBService, baseEnv *eval.Env, cache *eval.ProgramCache) *EvaluationTimeseriesComputePartialEventWorker { + return &EvaluationTimeseriesComputePartialEventWorker{DBService: dbservice, baseEnv: baseEnv, programCache: cache} +} + +func (w *EvaluationTimeseriesComputePartialEventWorker) Work(ctx context.Context, job *river.Job[dto.WorkerExpressionTimeseriesComputePartialEventArgs]) error { + if w.PGQueue == nil { + return fmt.Errorf("ComputeTsWorker.Work: internal bus PGQueue is nil (expected *pgqueue.PGQueue)") + } + env, err := json.Marshal(pgqueue.Envelope{ + Type: job.Args.Kind(), + Payload: job.EncodedArgs, + }) + if err != nil { + return fmt.Errorf("marshal env: %w", err) + } + return w.PGQueue.BatchWorkerSend(ctx, env) +} + +// Route exposes a decode/handle pair so the caller can register it with +// q.ListenPoolProcess(ctx, cfg, logger, routes). +func (w *EvaluationTimeseriesComputePartialEventWorker) Route() (string, pgqueue.Route) { + kind := (dto.WorkerExpressionTimeseriesComputePartialEventArgs{}).Kind() + route := pgqueue.JSONRoute(func(ctx context.Context, args []dto.WorkerExpressionTimeseriesComputePartialEventArgs) error { + opts := make([]eval.EvalSessionParams, len(args)) + for i, arg := range args { + opts[i] = eval.EvalSessionParams{ + TargetTimeseriesID: arg.TargetTimeseriesID, + ExpressionID: arg.ExpressionID, + Expression: arg.Expression, + StartsAt: arg.MinTime, + EndsBefore: arg.MaxTime.Add(1), + } + } + sess, err := eval.NewEvalSession( + w.DBService.Queries, + w.DBService.PGQueue, + w.DBService.Logger, + w.baseEnv, + opts, + eval.WithProgramCache(w.programCache), + ) + if err != nil { + return err + } + return sess.Run(ctx) + }) + return kind, route +} diff --git a/api/internal/worker/email_schedule.go b/api/internal/worker/email_schedule.go index 661fcefc..04ca9e75 100644 --- a/api/internal/worker/email_schedule.go +++ b/api/internal/worker/email_schedule.go @@ -15,7 +15,7 @@ type EmailEventWorker struct { TaskServices *cloud.TaskServices } -func NewEamilEventWorker(dbservice *service.DBService, taskServices *cloud.TaskServices) *EmailEventWorker { +func NewEmailEventWorker(dbservice *service.DBService, taskServices *cloud.TaskServices) *EmailEventWorker { return &EmailEventWorker{DBService: dbservice, TaskServices: taskServices} } diff --git a/api/internal/worker/fetch_thinglogix.go b/api/internal/worker/fetch_thinglogix.go index e4858589..29cfd8a8 100644 --- a/api/internal/worker/fetch_thinglogix.go +++ b/api/internal/worker/fetch_thinglogix.go @@ -11,46 +11,36 @@ import ( "github.com/USACE/instrumentation-api/api/v4/internal/db" "github.com/USACE/instrumentation-api/api/v4/internal/dto" "github.com/USACE/instrumentation-api/api/v4/internal/logger" + "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" "github.com/USACE/instrumentation-api/api/v4/internal/service" "github.com/riverqueue/river" ) var fetchThinglogixInterval = 5 * time.Minute -var FetchThinglogixJobOptions = river.NewPeriodicJob( - river.PeriodicInterval(fetchThinglogixInterval), - func() (river.JobArgs, *river.InsertOpts) { +// RegisterFetchThinglogixPeriodic adds the periodic schedule to the worker's Periodics builder. +func RegisterFetchThinglogixPeriodic(p *pgqueue.Periodics) { + p.Every(fetchThinglogixInterval, func() (river.JobArgs, *river.InsertOpts) { return dto.WorkerThinglogixScheduledFetchArgs{}, &river.InsertOpts{ - UniqueOpts: river.UniqueOpts{ - ByPeriod: fetchThinglogixInterval, - }, + UniqueOpts: river.UniqueOpts{ByPeriod: fetchThinglogixInterval}, } - }, - &river.PeriodicJobOpts{RunOnStart: true}, -) + }, &river.PeriodicJobOpts{RunOnStart: true}) +} -// FetchThinglogixScheduleWorker pulls all of the available devices from the Thinglogix API every N minutes. -// -// The most recent successful response payload is stored in the MIDAS database. This can be used to create -// timeseries and associated mappings for any instrument with an assigned thinglogix_device_id. The inital -// readings can be created by the client from this inital payload. Subsequent payloads will be parsed. -// -// Note: for state of health status fields are hard-coded. If this needs to be more extensible, we can -// have some mapping mechanism to extrapolate numerical data from a matching string. +// FetchThinglogixScheduleWorker pulls all available devices from the Thinglogix API. type FetchThinglogixScheduleWorker struct { river.WorkerDefaults[dto.WorkerThinglogixScheduledFetchArgs] DBService *service.DBService ThinglogixService *service.ThinglogixService Config *config.ThinglogixConfig - Logger *logger.Logger + Logger logger.Logger } -func NewFetchThinglogixScheduleWorker(ctx context.Context, dbservice *service.DBService, cfg *config.ThinglogixConfig, l *logger.Logger) (*FetchThinglogixScheduleWorker, error) { +func NewFetchThinglogixScheduleWorker(ctx context.Context, dbservice *service.DBService, cfg *config.ThinglogixConfig, l logger.Logger) (*FetchThinglogixScheduleWorker, error) { tlgx, err := service.NewThinglogixService(ctx, cfg) if err != nil { return nil, fmt.Errorf("failed to initialize thinglogix service authentication on initialization: %w", err) } - return &FetchThinglogixScheduleWorker{DBService: dbservice, ThinglogixService: tlgx, Config: cfg, Logger: l}, nil } @@ -68,7 +58,6 @@ func (w *FetchThinglogixScheduleWorker) Work(ctx context.Context, job *river.Job payload, err := w.ThinglogixService.FetchDevices(ctx) if err != nil { - // Force worker to re-authentication on subsequent tries w.ThinglogixService = nil return fmt.Errorf("fetchDevices(ctx, w.Config.ThinglogixConfig, creds) failed: %w", err) } diff --git a/api/internal/worker/worker.go b/api/internal/worker/worker.go index 2b0bdeab..0969c81d 100644 --- a/api/internal/worker/worker.go +++ b/api/internal/worker/worker.go @@ -9,8 +9,11 @@ import ( "github.com/USACE/instrumentation-api/api/v4/internal/pgqueue" "github.com/jackc/pgx/v5" "github.com/riverqueue/river" + "github.com/riverqueue/river/rivertype" ) +const HighPriorityQueue = "high_priority" + type workerClient struct{} var _ pgqueue.PGQueueWorker = (*workerClient)(nil) @@ -20,45 +23,57 @@ func (wo workerClient) JobSnooze(duration time.Duration) error { } func (wo workerClient) InsertManyTx(ctx context.Context, tx db.DBTX, params []pgqueue.InsertManyParams) ([]int64, error) { - // Not sure if this is needed, we have the river.Client available as a field in DBService, - // but using it directly may cause issues with context cancellations. - // https://github.com/riverqueue/river/blob/27975e3a993e14045d3d1c98e48c04dd3cd646cf/client.go#L945 - client := river.ClientFromContext[pgx.Tx](ctx) - riverParams := pgqueue.ConvertInsertManyParamsToRiver(params) + client := clientFromCtx(ctx) - pgxTx, ok := tx.(pgx.Tx) - if !ok { - return nil, errors.New("failed to assert db.DBTX as pgx.Tx") + pgxTx, err := resolvePgxTx(tx) + if err != nil { + return nil, err } + rparams := pgqueue.ConvertInsertManyParams(params) - jobResults, err := client.InsertManyTx(ctx, pgxTx, riverParams) + results, err := client.InsertManyTx(ctx, pgxTx, rparams) if err != nil { return nil, err } - jobIDs := make([]int64, len(jobResults)) - for idx := range jobResults { - jobIDs[idx] = jobResults[idx].Job.ID - } - - return jobIDs, nil + return extractIDs(results), nil } func (wo workerClient) InsertManyFast(ctx context.Context, params []pgqueue.InsertManyParams) error { - client := river.ClientFromContext[pgx.Tx](ctx) - riverParams := pgqueue.ConvertInsertManyParamsToRiver(params) - _, err := client.InsertManyFast(ctx, riverParams) + client := clientFromCtx(ctx) + rparams := pgqueue.ConvertInsertManyParams(params) + _, err := client.InsertManyFast(ctx, rparams) + return err +} + +func (wo workerClient) JobCancelTx(ctx context.Context, tx db.DBTX, jobID int64) error { + client := clientFromCtx(ctx) + + pgxTx, err := resolvePgxTx(tx) if err != nil { return err } - return nil + _, err = client.JobCancelTx(ctx, pgxTx, jobID) + return err } -func (wo workerClient) JobCancelTx(ctx context.Context, tx db.DBTX, jobID int64) error { - client := river.ClientFromContext[pgx.Tx](ctx) - pgxTx, ok := tx.(pgx.Tx) - if !ok { - return errors.New("failed to assert db.DBTX as pgx.Tx") +func clientFromCtx(ctx context.Context) *river.Client[pgx.Tx] { + return river.ClientFromContext[pgx.Tx](ctx) +} + +func resolvePgxTx(tx db.DBTX) (pgx.Tx, error) { + if t, ok := tx.(pgx.Tx); ok && t != nil { + return t, nil } - _, err := client.JobCancelTx(ctx, pgxTx, jobID) - return err + return nil, errors.New("failed to assert db.DBTX as pgx.Tx") +} + +func extractIDs(results []*rivertype.JobInsertResult) []int64 { + if len(results) == 0 { + return nil + } + ids := make([]int64, len(results)) + for i := range results { + ids[i] = results[i].Job.ID + } + return ids } diff --git a/api/migrations/repeat/0190__views_expression.sql b/api/migrations/repeat/0190__views_expression.sql new file mode 100644 index 00000000..f9409189 --- /dev/null +++ b/api/migrations/repeat/0190__views_expression.sql @@ -0,0 +1,29 @@ +create or replace view v_expression as ( + with expression_opts as ( + select expression_id, row_to_json(o1) as opts + from expression_window_opts o1 + ) + select + x.id, + x.instrument_id, + x.name, + x.expression, + x.mode, + x.created_at, + x.created_by, + x.updated_at, + x.updated_by, + deps.variables, + ett.timeseries_id as target_timeseries_id, + o.opts + from expression x + inner join expression_timeseries_target ett on ett.expression_id = x.id + left join lateral ( + select coalesce(json_agg(json_build_object( + 'timeseries_id', etv.timeseries_id, + 'variable_name', etv.variable_name + ) order by etv.variable_name), '[]'::json) variables from expression_timeseries_variable etv + where etv.expression_id = x.id + ) deps on true + left join expression_opts o on o.expression_id = x.id +); diff --git a/api/migrations/schema/V1.52.00__expression.sql b/api/migrations/schema/V1.52.00__expression.sql new file mode 100644 index 00000000..74541580 --- /dev/null +++ b/api/migrations/schema/V1.52.00__expression.sql @@ -0,0 +1,59 @@ +alter type timeseries_type add value 'expression'; + + +create type expression_mode as enum ( + 'point', + 'window' +); + + +create table expression ( + id uuid primary key default uuid_generate_v4(), + instrument_id uuid not null references instrument(id), + name text not null, + expression text not null, + mode expression_mode not null, + created_at timestamptz not null default now(), + created_by uuid not null references profile(id), + updated_at timestamptz, + updated_by uuid references profile(id), + check (expression != ''), + constraint expression_name_instrument_id_key unique(name, instrument_id) +); + + +create table expression_timeseries_variable ( + expression_id uuid not null references expression(id) on delete cascade, + timeseries_id uuid not null references timeseries(id) on delete cascade, + variable_name text not null, + constraint expression_timeseries_variable_expression_id_variable_name_key unique(expression_id, variable_name), + constraint expression_timeseries_variable_expression_id_timeseries_id_key unique(expression_id, timeseries_id), + check (variable_name != '') +); + + +create table expression_timeseries_target ( + expression_id uuid unique not null references expression(id) on delete cascade, + timeseries_id uuid unique not null references timeseries(id) on delete cascade +); + + +create type expression_window_timestamping_policy as enum ( + 'start', + 'center', + 'end' +); + + +create table expression_window_opts ( + expression_id uuid not null references expression(id) on delete cascade, + duration interval not null, + tz_name text not null default 'UTC', + duration_offset interval not null default 'PT0', + timestamping_policy expression_window_timestamping_policy not null default 'start', + check (duration != 'PT0'::interval) +); + + +create index if not exists idx_expression_timeseries_variable_timeseries_id on expression_timeseries_variable(timeseries_id); +create index if not exists idx_alert_config_timeseries_timeseries_id on alert_config_timeseries(timeseries_id); diff --git a/api/migrations/seed/V1.52.01__seed_expression.sql b/api/migrations/seed/V1.52.01__seed_expression.sql new file mode 100644 index 00000000..891e10b8 --- /dev/null +++ b/api/migrations/seed/V1.52.01__seed_expression.sql @@ -0,0 +1,81 @@ +-- create new computed timeseries for point-mode expressions +insert into timeseries (id, slug, name, instrument_id, parameter_id, unit_id, type) values +('a3f8c6e2-7b4e-4c9a-8e3d-2b6f8e9d2a1b', 'point-ts-c', 'point ts c', 'a7540f69-c41e-43b3-b655-6e44097edb7e', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', 'expression'), +('b2e7d1c3-5f6a-4e8b-9c2d-3a7e8f9d1b2c', 'point-ts-d', 'point ts d', 'a7540f69-c41e-43b3-b655-6e44097edb7e', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', 'expression'), +('c1d6e2b3-8f7a-4c9e-8d2b-1a6e7f8d2c3b', 'point-ts-e', 'point ts e', 'a7540f69-c41e-43b3-b655-6e44097edb7e', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', 'expression'), +('d4e5f6a7-1b2c-4d3e-9f8a-2c1b3d4e5f6a', 'point-ts-f', 'point ts f', 'a7540f69-c41e-43b3-b655-6e44097edb7e', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', 'expression'); + +-- create point-mode expressions +insert into expression (id, instrument_id, name, expression, mode, created_by) +values +('e1c11111-1111-4111-8111-111111111111', 'a7540f69-c41e-43b3-b655-6e44097edb7e', 'c = a + b', 'a + b', 'point', '57329df6-9f7a-4dad-9383-4633b452efab'), +('e2c22222-2222-4222-8222-222222222222', 'a7540f69-c41e-43b3-b655-6e44097edb7e', 'd = pow(c, 2) + 1', 'pow(c, 2) + 1', 'point', '57329df6-9f7a-4dad-9383-4633b452efab'), +('e3c33333-3333-4333-8333-333333333333', 'a7540f69-c41e-43b3-b655-6e44097edb7e', 'e = stddev(d) + a', 'stddev(d) + a', 'point', '57329df6-9f7a-4dad-9383-4633b452efab'), +('e4c44444-4444-4444-8444-444444444444', 'a7540f69-c41e-43b3-b655-6e44097edb7e', 'f = sin(c) + e', 'sin(c) + e', 'point', '57329df6-9f7a-4dad-9383-4633b452efab'); + +-- link expressions to their new computed timeseries targets +insert into expression_timeseries_target (expression_id, timeseries_id) values +('e1c11111-1111-4111-8111-111111111111', 'a3f8c6e2-7b4e-4c9a-8e3d-2b6f8e9d2a1b'), -- c +('e2c22222-2222-4222-8222-222222222222', 'b2e7d1c3-5f6a-4e8b-9c2d-3a7e8f9d1b2c'), -- d +('e3c33333-3333-4333-8333-333333333333', 'c1d6e2b3-8f7a-4c9e-8d2b-1a6e7f8d2c3b'), -- e +('e4c44444-4444-4444-8444-444444444444', 'd4e5f6a7-1b2c-4d3e-9f8a-2c1b3d4e5f6a'); -- f + +-- expression variables (dependencies) +-- c = a + b +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values +('e1c11111-1111-4111-8111-111111111111', '3c4a0e1d-03a1-4d2b-9b6f-4521b52f491d', 'a'), -- uploader timeseries 3 +('e1c11111-1111-4111-8111-111111111111', '4d5b281f-14b8-42d7-bb1e-9c6118da813f', 'b'); -- uploader timeseries 4 + +-- d = c + 1 +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values +('e2c22222-2222-4222-8222-222222222222', 'a3f8c6e2-7b4e-4c9a-8e3d-2b6f8e9d2a1b', 'c'); -- computed c + +-- e = d + a +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values +('e3c33333-3333-4333-8333-333333333333', 'b2e7d1c3-5f6a-4e8b-9c2d-3a7e8f9d1b2c', 'd'), -- computed d +('e3c33333-3333-4333-8333-333333333333', '3c4a0e1d-03a1-4d2b-9b6f-4521b52f491d', 'a'); -- uploader timeseries 3 + +-- f = c + e +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values +('e4c44444-4444-4444-8444-444444444444', 'a3f8c6e2-7b4e-4c9a-8e3d-2b6f8e9d2a1b', 'c'), -- computed c +('e4c44444-4444-4444-8444-444444444444', 'c1d6e2b3-8f7a-4c9e-8d2b-1a6e7f8d2c3b', 'e'); -- computed e + +-- create new computed timeseries for window-mode expressions +insert into timeseries (id, slug, name, instrument_id, parameter_id, unit_id, type) values +('e7f8a9b0-1c2d-4e3f-8a9b-0c1d2e3f4a5b', 'window-ts-w1', 'window ts w1', 'a7540f69-c41e-43b3-b655-6e44097edb7e', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', 'expression'), +('f8a9b0c1-2d3e-4f5a-9b0c-1d2e3f4a5b6c', 'window-ts-w2', 'window ts w2', 'a7540f69-c41e-43b3-b655-6e44097edb7e', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', 'expression'), +('a9b0c1d2-3e4f-5a6b-8c1d-2e3f4a5b6c7d', 'window-ts-w3', 'window ts w3', 'a7540f69-c41e-43b3-b655-6e44097edb7e', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', 'expression'); + +-- create window-mode expressions +insert into expression (id, instrument_id, name, expression, mode, created_by) +values +('e5a11111-1111-4111-8111-111111111111', 'a7540f69-c41e-43b3-b655-6e44097edb7e', 'w1 = mean(a)', 'mean(a)', 'window', '57329df6-9f7a-4dad-9383-4633b452efab'), +('e6a22222-2222-4222-8222-222222222222', 'a7540f69-c41e-43b3-b655-6e44097edb7e', 'w2 = w1.map(w, Measurement{time: w.time, value: w.value * 2})', 'w1.map(w, Measurement{time: w.time, value: w.value * 2})', 'window', '57329df6-9f7a-4dad-9383-4633b452efab'), +('e7a33333-3333-4333-8333-333333333333', 'a7540f69-c41e-43b3-b655-6e44097edb7e', 'w3 = mean(w2) + min(a)', 'mean(w2) + min(a)', 'window', '57329df6-9f7a-4dad-9383-4633b452efab'); + +-- link expressions to their new computed timeseries targets +insert into expression_timeseries_target (expression_id, timeseries_id) values +('e5a11111-1111-4111-8111-111111111111', 'e7f8a9b0-1c2d-4e3f-8a9b-0c1d2e3f4a5b'), -- w1 +('e6a22222-2222-4222-8222-222222222222', 'f8a9b0c1-2d3e-4f5a-9b0c-1d2e3f4a5b6c'), -- w2 +('e7a33333-3333-4333-8333-333333333333', 'a9b0c1d2-3e4f-5a6b-8c1d-2e3f4a5b6c7d'); -- w3 + +-- expression variables (dependencies) +-- w1 = mean(a, b) +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values +('e5a11111-1111-4111-8111-111111111111', '3c4a0e1d-03a1-4d2b-9b6f-4521b52f491d', 'a'), -- uploader timeseries 3 +('e5a11111-1111-4111-8111-111111111111', '4d5b281f-14b8-42d7-bb1e-9c6118da813f', 'b'); -- uploader timeseries 4 + +-- w2 = w1 * 2 +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values +('e6a22222-2222-4222-8222-222222222222', 'e7f8a9b0-1c2d-4e3f-8a9b-0c1d2e3f4a5b', 'w1'); -- computed w1 + +-- w3 = mean(w2, a) +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values +('e7a33333-3333-4333-8333-333333333333', 'f8a9b0c1-2d3e-4f5a-9b0c-1d2e3f4a5b6c', 'w2'), -- computed w2 +('e7a33333-3333-4333-8333-333333333333', '3c4a0e1d-03a1-4d2b-9b6f-4521b52f491d', 'a'); -- uploader timeseries 3 + +-- window options for each expression (1 day window, utc) +insert into expression_window_opts (expression_id, duration, tz_name, duration_offset, timestamping_policy) values +('e5a11111-1111-4111-8111-111111111111', '1 day', 'UTC', 'PT0', 'start'), +('e6a22222-2222-4222-8222-222222222222', '1 week', 'UTC', 'PT0', 'center'), +('e7a33333-3333-4333-8333-333333333333', '1 month', 'UTC', 'PT0', 'end'); diff --git a/api/queries/expression.sql b/api/queries/expression.sql new file mode 100644 index 00000000..3a39e7b7 --- /dev/null +++ b/api/queries/expression.sql @@ -0,0 +1,202 @@ +-- name: ExpressionListForInstrument :many +select * +from v_expression +where instrument_id = $1 +order by name asc; + + +-- name: ExpressionGet :one +select * +from v_expression +where id = $1; + + +-- name: ExpressionGetBatch :batchone +select * +from v_expression +where id = $1; + + +-- name: ExpressionCreate :one +insert into expression (instrument_id, name, expression, mode, created_by) values ($1, $2, $3, $4, $5) +returning id; + + +-- name: ExpressionTimeseriesTargetCreate :exec +insert into expression_timeseries_target (expression_id, timeseries_id) values ($1, $2); + + +-- name: ExpressionUpdate :one +update expression set + name = $2, + expression = $3, + mode = $4, + updated_at = now(), + updated_by = $5 +where id = $1 +returning id; + + +-- name: ExpressionDelete :exec +delete from expression +where id = $1; + + +-- name: TimeseriesDeleteTargetForExpression :exec +delete from timeseries t +using expression_timeseries_target e +where t.id = e.timeseries_id +and e.expression_id = $1; + + +-- name: ExpressionTimeseriesVariableCreateOrUpdateBatch :batchexec +insert into expression_timeseries_variable (expression_id, timeseries_id, variable_name) values ($1, $2, $3) +on conflict (expression_id, timeseries_id) do update set variable_name = excluded.variable_name; + + +-- name: ExpressionTimeseriesVariableDeleteAllForExpression :exec +delete from expression_timeseries_variable +where expression_id = $1; + + +-- name: ExpressionWindowOptsCreate :exec +insert into expression_window_opts (expression_id, duration, tz_name, duration_offset, timestamping_policy) +values ($1, $2, $3, $4, $5); + + +-- name: ExpressionWindowOptsDelete :exec +delete from expression_window_opts where expression_id = $1; + + +-- name: TimeseriesMeasurementListBatchForTimeseriesIDs :batchmany +select + m.timeseries_id, + m.time, + m.value, + coalesce(m.masked, false) as masked, + coalesce(m.validated, false) as validated +from v_timeseries_measurement m +where m.timeseries_id = any(sqlc.arg(timeseries_ids)::uuid[]) +and m.time >= sqlc.arg(starts_at)::timestamptz +and m.time < sqlc.arg(ends_before)::timestamptz +order by m.time asc +limit sqlc.arg(page_size)::int; + + +-- name: TimeseriesMeasurementListBatchEdgesForTimeseriesIDs :batchmany +select + ts.id as timeseries_id, + b.time as before_time, + b.value as before_value, + coalesce(bn.masked, false) as before_masked, + coalesce(bn.validated, false) as before_validated, + a.time as after_time, + a.value as after_value, + coalesce(an.masked, false) as after_masked, + coalesce(an.validated, false) as after_validated +from timeseries ts +left join lateral ( + select m.time, m.value + from timeseries_measurement m + where m.timeseries_id = ts.id + and m.time < sqlc.arg(starts_at)::timestamptz + order by m.time desc + limit 1 +) b on true +left join lateral ( + select n.masked, n.validated + from timeseries_notes n + where n.timeseries_id = ts.id + and n.time = b.time + limit 1 +) bn on true +left join lateral ( + select m.time, m.value + from timeseries_measurement m + where m.timeseries_id = ts.id + and m.time >= sqlc.arg(ends_before)::timestamptz + order by m.time asc + limit 1 +) a on true +left join lateral ( + select n.masked, n.validated + from timeseries_notes n + where n.timeseries_id = ts.id + and n.time = a.time + limit 1 +) an on true +where ts.id = any(sqlc.arg(timeseries_ids)::uuid[]); + + +-- name: ExpressionDownstreamListForTimeseries :many +with recursive expr_deps as ( + -- Anchor term: get direct downstreams + select + e.id as expression_id, + tgt.timeseries_id as target_timeseries_id, + var.timeseries_id as variable_timeseries_id, + 1 as depth + from expression_timeseries_variable var + inner join expression e on e.id = var.expression_id + inner join expression_timeseries_target tgt on tgt.expression_id = e.id + where var.timeseries_id = any(sqlc.arg(dep_timeseries_ids)::uuid[]) + + union all + + -- Recursive term: get downstreams of downstreams + select + e.id as expression_id, + tgt.timeseries_id as target_timeseries_id, + var.timeseries_id as variable_timeseries_id, + ed.depth + 1 as depth + from expr_deps ed + join expression_timeseries_variable var on var.timeseries_id = ed.target_timeseries_id + join expression e on e.id = var.expression_id + join expression_timeseries_target tgt on tgt.expression_id = e.id + where ed.depth < 10 -- for safety, max depth should be below this +) +select + expression_id, + target_timeseries_id, + array_agg(variable_timeseries_id)::uuid[] as variable_timeseries_ids, + min(depth)::int as min_depth, + max(depth)::int as max_depth +from expr_deps +group by expression_id, target_timeseries_id; + + +-- name: ExpressionGetWouldCreateCycleOrExceedDepth :one +with recursive dep_graph as ( + -- Anchor: start from the target timeseries of the expression being checked + select + tgt.timeseries_id as start_timeseries_id, + tgt.timeseries_id as current_timeseries_id, + tgt.expression_id as current_expression_id, + 0 as depth + from expression_timeseries_target tgt + where tgt.expression_id = sqlc.arg(expression_id)::uuid + + union all + + -- Recursive: follow dependencies from the current expression + select + dep_graph.start_timeseries_id, + var.timeseries_id as current_timeseries_id, + var.expression_id as current_expression_id, + dep_graph.depth + 1 as depth + from dep_graph + join expression_timeseries_variable var + on var.expression_id = dep_graph.current_expression_id + join expression_timeseries_target tgt + on tgt.expression_id = var.expression_id + where dep_graph.depth < 5 +) +select + exists ( + select 1 + from dep_graph + where start_timeseries_id = current_timeseries_id + and depth > 0 -- exclude the anchor node itself + ) as would_create_cycle, + max(depth) > 5 as would_exceed_max_depth +from dep_graph; diff --git a/api/queries/measurement.sql b/api/queries/measurement.sql index 7b7c0f5a..e76db8a4 100644 --- a/api/queries/measurement.sql +++ b/api/queries/measurement.sql @@ -26,12 +26,6 @@ insert into timeseries_measurement (timeseries_id, time, value) values ($1, $2, on conflict on constraint timeseries_unique_time do update set value = excluded.value; --- name: TimeseriesMeasurementCreateOrUpdateBatchShouldAlertCheck :batchone -insert into timeseries_measurement (timeseries_id, time, value) values ($1, $2, $3) -on conflict on constraint timeseries_unique_time do update set value = excluded.value -returning (select true = any(select true from alert_config_timeseries where timeseries_id = $1)); - - -- name: TimeseriesNoteCreate :exec insert into timeseries_notes (timeseries_id, time, masked, validated, annotation) values ($1, $2, $3, $4, $5) on conflict on constraint notes_unique_time do nothing; @@ -97,6 +91,10 @@ delete from timeseries_measurement where timeseries_id = sqlc.arg(timeseries_id) delete from timeseries_measurement where timeseries_id = sqlc.arg(timeseries_id) and time > sqlc.arg(after) and time < sqlc.arg(before); +-- name: TimeseriesMeasurementDeleteAllForTimeseries :exec +delete from timeseries_measurement where timeseries_id = $1; + + -- name: TimeseriesNoteDelete :exec delete from timeseries_notes where timeseries_id=$1 and time=$2; diff --git a/compose.sh b/compose.sh index 3da4fc6e..187df97f 100755 --- a/compose.sh +++ b/compose.sh @@ -19,10 +19,10 @@ mkdocs() { ALL_SERVICES=("midas-api" "midas-sql" "midas-telemetry" "midas-task" "midas-dcs-loader" "midas-report" "midas-sl-client") if [ "$1" = "gen" ]; then - DOCKER_BUILDKIT=1 $COMPOSECMD run --rm sqlc + DOCKER_BUILDKIT=1 $COMPOSECMD run --build --rm gen elif [ "$1" = "migrate" ]; then - DOCKER_BUILDKIT=1 $COMPOSECMD run --rm migrate + DOCKER_BUILDKIT=1 $COMPOSECMD run --build --rm migrate elif [ "$1" = "watch" ]; then DOCKER_BUILDKIT=1 $COMPOSECMD up --watch diff --git a/docker-compose.yaml b/docker-compose.yaml index a2c54164..57a6f0b1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -210,10 +210,12 @@ services: target: /usr/app/src restart: unless-stopped - sqlc: + gen: build: dockerfile_inline: | - FROM debian:bookworm-slim + FROM golang:bookworm + RUN apt update && apt install -y protobuf-compiler + RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest COPY --from=sqlc/sqlc:1.29.0 /workspace/sqlc /usr/local/bin/sqlc entrypoint: - /bin/sh @@ -224,8 +226,13 @@ services: sqlc vet -f sqlc.vet.yaml echo "[sqlc] running generate..." sqlc generate -f sqlc.generate.yaml - echo "[sqlc] done - exit 0" + echo "[sqlc] done" + echo "[protoc] running protoc" + protoc \ + --go_out=. \ + --go_opt=paths=source_relative \ + api/internal/eval/measurement.proto + exit 0 volumes: - .:/src working_dir: /src diff --git a/go.work.sum b/go.work.sum index 9b836424..d0876e88 100644 --- a/go.work.sum +++ b/go.work.sum @@ -20,6 +20,7 @@ cloud.google.com/go/trace v1.11.5/go.mod h1:TwblCcqNInriu5/qzaeYEIH7wzUcchSdeY2l cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= contrib.go.opencensus.io/exporter/aws v0.0.0-20230502192102-15967c811cec/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/stackdriver v0.13.14/go.mod h1:5pSSGY0Bhuk7waTHuDf4aQ8D2DrhgETRo9fy6k3Xlzc= +contrib.go.opencensus.io/integrations/ocsql v0.1.7 h1:G3k7C0/W44zcqkpRSFyjU9f6HZkbwIrL//qqnlqWZ60= contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= @@ -65,7 +66,6 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/XSAM/otelsql v0.39.0/go.mod h1:uMOXLUX+wkuAuP0AR3B45NXX7E9lJS2mERa8gqdU8R0= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.19.5/go.mod h1:VNM08cHlOsIbSHRqb6D/M2L4kKXfJv3A2/f0GNbOQSc= github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.87/go.mod h1:ZeQC4gVarhdcWeM1c90DyBLaBCNhEeAbKUXwVI/byvw= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.44.0/go.mod h1:mWB0GE1bqcVSvpW7OtFA0sKuHk52+IqtnsYU2jUfYAs= @@ -241,7 +241,7 @@ golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeId golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -255,7 +255,7 @@ golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ= google.golang.org/genproto/googleapis/bytestream v0.0.0-20250313205543-e70fdf4c4cb4/go.mod h1:WkJpQl6Ujj3ElX4qZaNm5t6cT95ffI4K+HKQ0+1NyMw= google.golang.org/genproto/googleapis/bytestream v0.0.0-20250603155806-513f23925822/go.mod h1:h6yxum/C2qRb4txaZRLDHK8RyS0H/o2oEDeKY4onY/Y= diff --git a/sqlc.generate.yaml b/sqlc.generate.yaml index 4739e67f..f8bd695f 100644 --- a/sqlc.generate.yaml +++ b/sqlc.generate.yaml @@ -131,6 +131,14 @@ sql: type: InstrumentIDName slice: true + # v_expression + - column: v_expression.variables + go_type: + type: VExpressionTimeseriesVariable + slice: true + - column: v_expression.opts + go_type: encoding/json.RawMessage + # v_incl_measurement - column: v_incl_measurement.measurements go_type: