diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc1cdabf..f64e1753 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ default: - --character-set-server=utf8 - --collation-server=utf8_unicode_ci - --max_connections=1000 - - name: docker:dind + - name: docker:28.5.0-dind include: - project: toradex/torizon-cloud/ci-container-build diff --git a/build.sbt b/build.sbt index 5634dcbc..6eb098f4 100644 --- a/build.sbt +++ b/build.sbt @@ -15,32 +15,32 @@ scalacOptions := Seq( "-Wconf:cat=other-match-analysis:error" ) -resolvers += "sonatype-snapshots".at("https://s01.oss.sonatype.org/content/repositories/snapshots") -resolvers += "sonatype-releases".at("https://s01.oss.sonatype.org/content/repositories/releases") +resolvers += Resolver.mavenCentral +resolvers += "maven-snapshots"at "https://central.sonatype.com/repository/maven-snapshots" Global / bloopAggregateSourceDependencies := true libraryDependencies ++= { - val akkaV = "2.8.5" - val akkaHttpV = "10.5.2" - val tufV = "3.2.11" + val pekkoV = "1.1.5" + val pekkoHttpV = "1.2.0" + val tufV = "5.0.0" val scalaTestV = "3.2.19" - val bouncyCastleV = "1.83" - val libatsV = "2.6.6" + val bouncyCastleV = "1.80" + val libatsV = "5.0.0" Seq( - "com.typesafe.akka" %% "akka-actor" % akkaV, - "com.typesafe.akka" %% "akka-stream" % akkaV, - "com.typesafe.akka" %% "akka-http" % akkaHttpV, - "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV, - "com.typesafe.akka" %% "akka-stream-testkit" % akkaV, - "com.typesafe.akka" %% "akka-slf4j" % akkaV, + "org.apache.pekko" %% "pekko-actor" % pekkoV, + "org.apache.pekko" %% "pekko-stream" % pekkoV, + "org.apache.pekko" %% "pekko-http" % pekkoHttpV, + "org.apache.pekko" %% "pekko-http-testkit" % pekkoHttpV, + "org.apache.pekko" %% "pekko-stream-testkit" % pekkoV, + "org.apache.pekko" %% "pekko-slf4j" % pekkoV, "org.scalatest" %% "scalatest" % scalaTestV % Test, "org.scalacheck" %% "scalacheck" % "1.19.0" % Test, "io.github.uptane" %% "libats" % libatsV, "io.github.uptane" %% "libats-messaging" % libatsV, "io.github.uptane" %% "libats-messaging-datatype" % libatsV, - "io.github.uptane" %% "libats-metrics-akka" % libatsV, + "io.github.uptane" %% "libats-metrics-pekko" % libatsV, "io.github.uptane" %% "libats-metrics-prometheus" % libatsV, "io.github.uptane" %% "libats-http-tracing" % libatsV, "io.github.uptane" %% "libats-slick" % libatsV, @@ -56,8 +56,8 @@ libraryDependencies ++= { "com.beachape" %% "enumeratum-circe" % "1.9.0", // Device registry specific dependencies - "com.lightbend.akka" %% "akka-stream-alpakka-csv" % "2.0.0", - "io.circe" %% "circe-testing" % "0.14.15", + "org.apache.pekko" %% "pekko-connectors-csv" % "1.0.0", + "io.circe" %% "circe-testing" % "0.14.13", "tech.sparse" %% "toml-scala" % "0.2.2", "org.tpolecat" %% "atto-core" % "0.9.5", "org.scalatestplus" %% "scalacheck-1-16" % "3.2.14.0" % Test diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 44b7ad2d..bac0b11f 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,16 +1,16 @@ -akka { - loggers = ["akka.event.slf4j.Slf4jLogger"] +pekko { + loggers = ["org.apache.pekko.event.slf4j.Slf4jLogger"] loglevel = "DEBUG" - loglevel = ${?AKKA_LOGLEVEL} + loglevel = ${?PEKKO_LOGLEVEL} log-config-on-start = off - log-config-on-start = ${?AKKA_LOG_CONFIG_ON_START} + log-config-on-start = ${?PEKKO_LOG_CONFIG_ON_START} http { server { max-connections = 2048 - max-connections = ${?AKKA_HTTP_MAX_CONNECTIONS} + max-connections = ${?PEKKO_HTTP_MAX_CONNECTIONS} backlog = 10000 - backlog = ${?AKKA_HTTP_BACKLOG} + backlog = ${?PEKKO_HTTP_BACKLOG} } # older versions of RAC send an "invalid" header @@ -20,7 +20,7 @@ akka { # The maximum number of parallel connections that a connection pool to a # single host endpoint is allowed to establish. Must be greater than zero. max-connections = 2048 - max-connections = ${?AKKA_HTTP_CLIENT_MAX_CONNECTIONS} + max-connections = ${?PEKKO_HTTP_CLIENT_MAX_CONNECTIONS} # The maximum number of open requests accepted into the pool across all # materializations of any of its client flows. # Protects against (accidentally) overloading a single pool with too many client flow materializations. @@ -28,7 +28,7 @@ akka { # will never exceed N * max-connections * pipelining-limit. # Must be a power of 2 and > 0! max-open-requests = 4096 - max-open-requests = ${?AKKA_HTTP_CLIENT_MAX_OPEN_REQUESTS} + max-open-requests = ${?PEKKO_HTTP_CLIENT_MAX_OPEN_REQUESTS} } } } diff --git a/src/main/resources/db/migration/director/V16__create_updates_table.sql b/src/main/resources/db/migration/director/V16__create_updates_table.sql new file mode 100644 index 00000000..4d673a83 --- /dev/null +++ b/src/main/resources/db/migration/director/V16__create_updates_table.sql @@ -0,0 +1,66 @@ +CREATE TABLE `updates` ( + `namespace` varchar(255) NOT NULL, + `id` char(36) NOT NULL, + `device_id` char(36) NOT NULL, + `hardware_update_id` char(36) NOT NULL, + `correlation_id` varchar(255) NOT NULL, + `scheduled_for` datetime(3) NULL, + `status` Enum('Assigned', 'Scheduled', 'Completed', 'PartiallyCompleted', 'Cancelled') NOT NULL, + `status_info` json DEFAULT NULL, + `completed_at` datetime(3) DEFAULT NULL, + `created_at` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`, `device_id`), + + CONSTRAINT fk_updates_hardware_update + FOREIGN KEY (hardware_update_id) REFERENCES hardware_updates (id), + FOREIGN KEY (device_id) REFERENCES provisioned_devices(id), + + KEY `updates_status_scheduled_for` (`status`,`scheduled_for`), + KEY `idx_updates_device_id` (`device_id`), + KEY `idx_updates_namespace` (`namespace`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; + +CREATE INDEX `idx_assignments_namespace_correlation_id` ON `assignments` (`namespace`, `correlation_id`); + +INSERT INTO `updates` ( + `namespace`, + `id`, + `device_id`, + `hardware_update_id`, + `correlation_id`, + `scheduled_for`, + `status`, + `status_info`, + `completed_at`, + `created_at`, + `updated_at` +) +SELECT + `namespace`, + `id`, + `device_id`, + `hardware_update_id`, + CONCAT('urn:here-ota:mtu:', `hardware_update_id`), + `scheduled_at`, + `status`, + `status_info`, + CASE + WHEN `status` IN ('Completed', 'Cancelled') THEN `updated_at` + ELSE NULL + END as `completed_at`, + `created_at`, + `updated_at` +FROM `scheduled_updates` +ON DUPLICATE KEY UPDATE + `status` = VALUES(`status`), + `status_info` = VALUES(`status_info`), + `completed_at` = CASE + WHEN VALUES(`status`) IN ('Completed', 'Cancelled') THEN VALUES(`updated_at`) + ELSE NULL + END, + `updated_at` = VALUES(`updated_at`) + ; + +alter table EcuInstallationResult ADD `description` TEXT NULL +; \ No newline at end of file diff --git a/src/main/resources/db/migration/director/V17__device_manifests_checksum.sql b/src/main/resources/db/migration/director/V17__device_manifests_checksum.sql new file mode 100644 index 00000000..a3c156f0 --- /dev/null +++ b/src/main/resources/db/migration/director/V17__device_manifests_checksum.sql @@ -0,0 +1,6 @@ + +ALTER TABLE device_manifests CHANGE COLUMN sha256 checksum char(8) +; + +CREATE INDEX device_manifests_received_at ON device_manifests(device_id, received_at) +; diff --git a/src/main/resources/db/migration/director/V18__device_manifests_index.sql b/src/main/resources/db/migration/director/V18__device_manifests_index.sql new file mode 100644 index 00000000..ed11cc23 --- /dev/null +++ b/src/main/resources/db/migration/director/V18__device_manifests_index.sql @@ -0,0 +1,3 @@ +ALTER TABLE device_manifests DROP INDEX IF EXISTS device_manifests_device_id_received_at_idx; + +ALTER TABLE device_manifests ADD INDEX device_manifests_device_id_received_at_idx (device_id, received_at); diff --git a/src/main/resources/db/migration/director/V19__add_update_seen_col.sql b/src/main/resources/db/migration/director/V19__add_update_seen_col.sql new file mode 100644 index 00000000..dd8ef200 --- /dev/null +++ b/src/main/resources/db/migration/director/V19__add_update_seen_col.sql @@ -0,0 +1,3 @@ +ALTER TABLE `updates` MODIFY COLUMN + `status` Enum('Scheduled', 'Assigned', 'Seen', 'Completed', 'PartiallyCompleted', 'Cancelled') NOT NULL +; \ No newline at end of file diff --git a/src/main/scala/com/advancedtelematic/director/Boot.scala b/src/main/scala/com/advancedtelematic/director/Boot.scala index 6878a353..89e3985c 100644 --- a/src/main/scala/com/advancedtelematic/director/Boot.scala +++ b/src/main/scala/com/advancedtelematic/director/Boot.scala @@ -1,11 +1,11 @@ package com.advancedtelematic.director import com.advancedtelematic.director.http.deviceregistry.TomlSupport.`application/toml` -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.Http.ServerBinding -import akka.http.scaladsl.server.{Directives, Route} -import akka.http.scaladsl.settings.{ParserSettings, ServerSettings} +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.http.scaladsl.Http +import org.apache.pekko.http.scaladsl.Http.ServerBinding +import org.apache.pekko.http.scaladsl.server.{Directives, Route} +import org.apache.pekko.http.scaladsl.settings.{ParserSettings, ServerSettings} import com.advancedtelematic.director.db.deviceregistry.DeviceRepository import com.advancedtelematic.director.deviceregistry.AllowUUIDPath import com.advancedtelematic.director.http.DirectorRoutes @@ -17,23 +17,14 @@ import com.advancedtelematic.libats.http.VersionDirectives.versionHeaders import com.advancedtelematic.libats.http.monitoring.ServiceHealthCheck import com.advancedtelematic.libats.http.tracing.Tracing import com.advancedtelematic.libats.http.tracing.Tracing.ServerRequestTracing -import com.advancedtelematic.libats.http.{ - BootApp, - BootAppDatabaseConfig, - BootAppDefaultConfig, - NamespaceDirectives -} -import com.advancedtelematic.libats.messaging.MessageBus +import com.advancedtelematic.libats.http.{BootApp, BootAppDatabaseConfig, BootAppDefaultConfig, NamespaceDirectives} +import com.advancedtelematic.libats.messaging.{MessageBus, MessageBusPublisher} import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libats.slick.db.{CheckMigrations, DatabaseSupport} import com.advancedtelematic.libats.slick.monitoring.{DatabaseMetrics, DbHealthResource} import com.advancedtelematic.libtuf_server.keyserver.KeyserverHttpClient import com.advancedtelematic.metrics.prometheus.PrometheusMetricsSupport -import com.advancedtelematic.metrics.{ - AkkaHttpConnectionMetrics, - AkkaHttpRequestMetrics, - MetricsSupport -} +import com.advancedtelematic.metrics.{PekkoHttpConnectionMetrics, PekkoHttpRequestMetrics, MetricsSupport} import com.codahale.metrics.MetricRegistry import com.typesafe.config.Config import org.bouncycastle.jce.provider.BouncyCastleProvider @@ -53,8 +44,8 @@ class DirectorBoot(override val globalConfig: Config, with DatabaseSupport with MetricsSupport with DatabaseMetrics - with AkkaHttpRequestMetrics - with AkkaHttpConnectionMetrics + with PekkoHttpRequestMetrics + with PekkoHttpConnectionMetrics with PrometheusMetricsSupport with CheckMigrations { @@ -69,7 +60,7 @@ class DirectorBoot(override val globalConfig: Config, private def keyserverClient(implicit tracing: ServerRequestTracing) = KeyserverHttpClient(tufUri) - private implicit val msgPublisher: com.advancedtelematic.libats.messaging.MessageBusPublisher = + private implicit val msgPublisher: MessageBusPublisher = MessageBus.publisher(system, globalConfig) private lazy val authNamespace = NamespaceDirectives.fromConfig() diff --git a/src/main/scala/com/advancedtelematic/director/Settings.scala b/src/main/scala/com/advancedtelematic/director/Settings.scala index ce8c0374..797fed2f 100644 --- a/src/main/scala/com/advancedtelematic/director/Settings.scala +++ b/src/main/scala/com/advancedtelematic/director/Settings.scala @@ -1,7 +1,7 @@ package com.advancedtelematic.director -import akka.event.Logging -import akka.http.scaladsl.model.Uri +import org.apache.pekko.event.Logging +import org.apache.pekko.http.scaladsl.model.Uri import com.typesafe.config.ConfigFactory trait Settings { diff --git a/src/main/scala/com/advancedtelematic/director/daemon/DaemonBoot.scala b/src/main/scala/com/advancedtelematic/director/daemon/DaemonBoot.scala index 5e1fdc86..3088e155 100644 --- a/src/main/scala/com/advancedtelematic/director/daemon/DaemonBoot.scala +++ b/src/main/scala/com/advancedtelematic/director/daemon/DaemonBoot.scala @@ -1,9 +1,9 @@ package com.advancedtelematic.director.daemon -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.Http.ServerBinding -import akka.http.scaladsl.server.Directives +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.http.scaladsl.Http +import org.apache.pekko.http.scaladsl.Http.ServerBinding +import org.apache.pekko.http.scaladsl.server.Directives import com.advancedtelematic.director.{Settings, VersionInfo} import com.advancedtelematic.libats.http.{BootApp, BootAppDatabaseConfig, BootAppDefaultConfig} import com.advancedtelematic.libats.messaging.{ @@ -68,6 +68,8 @@ class DirectorDaemonBoot(override val globalConfig: Config, startMonitoredListener[DeviceUpdateEvent](new DeviceUpdateEventListener(messageBus)) startMonitoredListener[DeviceMqttLifecycle](new MqttLifecycleListener) + new DeviceManifestReportedListener(globalConfig).start() + val routes = versionHeaders(version) { prometheusMetricsRoutes ~ DbHealthResource( diff --git a/src/main/scala/com/advancedtelematic/director/daemon/DeviceManifestReportedListener.scala b/src/main/scala/com/advancedtelematic/director/daemon/DeviceManifestReportedListener.scala index 35018b78..b27186b0 100644 --- a/src/main/scala/com/advancedtelematic/director/daemon/DeviceManifestReportedListener.scala +++ b/src/main/scala/com/advancedtelematic/director/daemon/DeviceManifestReportedListener.scala @@ -1,17 +1,116 @@ package com.advancedtelematic.director.daemon -import com.advancedtelematic.director.data.Messages.DeviceManifestReported +import org.apache.pekko.NotUsed +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.kafka.CommitterSettings +import org.apache.pekko.kafka.ConsumerMessage.{CommittableMessage, CommittableOffsetBatch} +import org.apache.pekko.kafka.scaladsl.Committer +import org.apache.pekko.stream.scaladsl.{Flow, RestartSource, Sink} +import org.apache.pekko.stream.{Attributes, RestartSettings} +import com.advancedtelematic.director.VersionInfo +import com.advancedtelematic.director.data.Messages +import com.advancedtelematic.director.data.Messages.{ + deviceManifestReportedMsgLike, + DeviceManifestReported +} import com.advancedtelematic.director.db.DeviceManifestRepositorySupport -import com.advancedtelematic.libats.messaging.MsgOperation.MsgOperation -import slick.jdbc.MySQLProfile.api._ +import com.advancedtelematic.libats.messaging.kafka.KafkaClient +import com.advancedtelematic.metrics.MetricsSupport +import com.typesafe.config.Config +import org.slf4j.LoggerFactory +import slick.jdbc.MySQLProfile.api.* +import scala.concurrent.duration.* import scala.concurrent.{ExecutionContext, Future} -class DeviceManifestReportedListener()(implicit val db: Database, val ec: ExecutionContext) - extends MsgOperation[DeviceManifestReported] - with DeviceManifestRepositorySupport { +class DeviceManifestReportedListener(globalConfig: Config)( + implicit val db: Database, + val ec: ExecutionContext, + val system: ActorSystem) + extends DeviceManifestRepositorySupport + with VersionInfo { + + private lazy val log = LoggerFactory.getLogger(this.getClass) + + private lazy val committerSettings = + CommitterSettings(globalConfig.getConfig("ats.messaging.kafka.committer")) + + private lazy val groupInstanceId = + if (globalConfig.hasPath(s"ats.$projectName.consumers.instance-id")) + Some( + s"$projectName-${Messages.deviceManifestReportedMsgLike.streamName}-${globalConfig.getString(s"ats.$projectName.consumers.instance-id")}" + ) + else + None + + protected[director] val processingFlow + : Flow[CommittableMessage[Array[Byte], DeviceManifestReported], Seq[ + CommittableMessage[Array[Byte], DeviceManifestReported] + ], NotUsed] = + Flow[CommittableMessage[Array[Byte], DeviceManifestReported]] + .groupedWithin(100, 3.seconds) + .mapAsync(1) { group => + val manifests = group.map { msg => + val manifestMsg = msg.record.value() + (manifestMsg.deviceId, manifestMsg.manifest.signed, manifestMsg.receivedAt) + } + + system.log.info(s"Received ${manifests.size} manifests") + + deviceManifestRepository + .createOrUpdate(manifests) + .map(_ => group) + } + + def start(): Future[Unit] = { + if (globalConfig.getString("ats.messaging.mode") != "kafka") { + log.warn("device-manifest-reported-listener is disabled, as messaging mode is not kafka") + return Future.successful(()) + } + + val restartSettings = RestartSettings(1.second, 30.seconds, 0.1) + val committerFlow = Committer.flow(committerSettings) + + lazy val counter = MetricsSupport.metricRegistry.counter( + s"director.${deviceManifestReportedMsgLike.streamName}.subscriptions" + ) + + lazy val failureCount = MetricsSupport.metricRegistry.counter( + s"director.${deviceManifestReportedMsgLike.streamName}.failures" + ) + + lazy val batchSizeHist = MetricsSupport.metricRegistry.histogram( + s"director.${deviceManifestReportedMsgLike.streamName}.batch-size" + ) - override def apply(msg: DeviceManifestReported): Future[Unit] = - deviceManifestRepository.createOrUpdate(msg.deviceId, msg.manifest.signed, msg.receivedAt) + RestartSource + .withBackoff(restartSettings) { () => + counter.inc() + KafkaClient + .committableSource[DeviceManifestReported](globalConfig, projectName, groupInstanceId) + .via(processingFlow) + .map { group => + batchSizeHist.update(group.size) + CommittableOffsetBatch(group.map(_.committableOffset)) + } + .log("device-manifest-reported-listener") + .addAttributes( + Attributes + .logLevels(onFinish = Attributes.logLevelWarning) + ) + .wireTap(batch => batch.offsets.size ) + .via(committerFlow) + } + .watchTermination() { (_, done) => + done + .failed + .map { err => failureCount.inc() + log.error("device-manifest-reported-listener failed", err) + } + NotUsed + } + .runWith(Sink.ignore) + .map(_ => ()) + } } diff --git a/src/main/scala/com/advancedtelematic/director/daemon/SignedRolesMigrationBoot.scala b/src/main/scala/com/advancedtelematic/director/daemon/SignedRolesMigrationBoot.scala index 6ca130d0..917e7d01 100644 --- a/src/main/scala/com/advancedtelematic/director/daemon/SignedRolesMigrationBoot.scala +++ b/src/main/scala/com/advancedtelematic/director/daemon/SignedRolesMigrationBoot.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.daemon -import akka.http.scaladsl.server.Directives +import org.apache.pekko.http.scaladsl.server.Directives import com.advancedtelematic.director.db.SignedRoleMigration import com.advancedtelematic.director.{Settings, VersionInfo} import com.advancedtelematic.libats.http.{BootApp, BootAppDatabaseConfig, BootAppDefaultConfig} diff --git a/src/main/scala/com/advancedtelematic/director/daemon/TufTargetAddedListener.scala b/src/main/scala/com/advancedtelematic/director/daemon/TufTargetAddedListener.scala index f04e94ea..b11679c2 100644 --- a/src/main/scala/com/advancedtelematic/director/daemon/TufTargetAddedListener.scala +++ b/src/main/scala/com/advancedtelematic/director/daemon/TufTargetAddedListener.scala @@ -1,24 +1,14 @@ package com.advancedtelematic.director.daemon -import akka.http.scaladsl.model.Uri -import akka.http.scaladsl.util.FastFuture -import com.advancedtelematic.director.data.DbDataType.{ - Assignment, - AutoUpdateDefinition, - EcuTarget, - EcuTargetId -} -import com.advancedtelematic.director.db.{ - AssignmentsRepositorySupport, - AutoUpdateDefinitionRepositorySupport, - EcuTargetsRepositorySupport, - ProvisionedDeviceRepositorySupport -} +import org.apache.pekko.http.scaladsl.model.Uri +import org.apache.pekko.http.scaladsl.util.FastFuture +import com.advancedtelematic.director.data.DbDataType.{Assignment, AutoUpdateDefinition, EcuTarget, EcuTargetId} +import com.advancedtelematic.director.db.{AssignmentsRepositorySupport, AutoUpdateDefinitionRepositorySupport, EcuTargetsRepositorySupport, ProvisionedDeviceRepositorySupport} import com.advancedtelematic.libats.data.DataType.{AutoUpdateId, Namespace} import com.advancedtelematic.libats.messaging.MsgOperation.MsgOperation import com.advancedtelematic.libtuf_server.data.Messages.TufTargetAdded import org.slf4j.LoggerFactory -import slick.jdbc.MySQLProfile.api._ +import slick.jdbc.MySQLProfile.api.* import java.time.Instant import scala.concurrent.{ExecutionContext, Future} diff --git a/src/main/scala/com/advancedtelematic/director/daemon/UpdateScheduler.scala b/src/main/scala/com/advancedtelematic/director/daemon/UpdateScheduler.scala deleted file mode 100644 index 61e8b723..00000000 --- a/src/main/scala/com/advancedtelematic/director/daemon/UpdateScheduler.scala +++ /dev/null @@ -1,134 +0,0 @@ -package com.advancedtelematic.director.daemon - -import cats.implicits.* -import cats.implicits.catsSyntaxOptionId -import com.advancedtelematic.director.data.DataType.{ScheduledUpdate, ScheduledUpdateId, StatusInfo} -import com.advancedtelematic.director.db.{ScheduledUpdatesRepositorySupport, UpdateSchedulerDBIO} -import com.advancedtelematic.director.deviceregistry.daemon.DeviceUpdateStatus -import com.advancedtelematic.director.deviceregistry.data.DeviceStatus -import com.advancedtelematic.libats.data.DataType.{MultiTargetUpdateId, Namespace} -import com.advancedtelematic.libats.messaging.MessageBusPublisher -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} -import com.advancedtelematic.libats.messaging_datatype.Messages.{ - DeviceUpdateAssigned, - DeviceUpdateEvent -} -import io.circe.syntax.EncoderOps -import io.circe.{Encoder, Json} -import org.slf4j.LoggerFactory -import slick.jdbc.MySQLProfile.api.* - -import java.time.Instant -import scala.concurrent.duration.* -import scala.concurrent.{blocking, ExecutionContext, Future} - -class UpdateSchedulerDaemon()( - implicit db: Database, - ec: ExecutionContext, - msgBus: MessageBusPublisher) { - - private val dbio = new UpdateSchedulerDBIO() - - private lazy val log = LoggerFactory.getLogger(this.getClass) - - private val CHECK_INTERVAL = 1.second - - private val BACKOFF_INTERVAL = 3.seconds - - def start(): Future[Unit] = { - log.info( - s"starting update scheduler daemon. CHECK_INTERVAL=$CHECK_INTERVAL BACKOFF_INTERVAL=$BACKOFF_INTERVAL" - ) - run() - } - - def runOnce(): Future[Unit] = { - log.debug("checking for pending scheduled updates") - dbio - .run() - .flatMap { scheduledUpdates => - if (scheduledUpdates.isEmpty) { - log.debug("no scheduled updates pending, trying later") - Future { - blocking(Thread.sleep(CHECK_INTERVAL.toMillis)) - 0 - } - } else { - log.info(s"$scheduledUpdates scheduled updates processed") - - val msgFuture = scheduledUpdates.toList.traverse_ { su => - val correlationId = MultiTargetUpdateId(su.updateId.uuid) - msgBus.publishSafe( - DeviceUpdateAssigned( - su.ns, - Instant.now(), - correlationId, - su.deviceId - ): DeviceUpdateEvent - ) - } - - msgFuture.map(_ => scheduledUpdates.size) - } - } - } - - private def run(): Future[Unit] = - runOnce() - .flatMap { _ => - run() - } - .recoverWith { case ex => - log.error("could not process pending updates", ex) - Future { - blocking(Thread.sleep(BACKOFF_INTERVAL.toMillis)) - }.flatMap { _ => - run() - } - } - -} - -class UpdateScheduler()(implicit db: Database, ec: ExecutionContext, msgBus: MessageBusPublisher) - extends ScheduledUpdatesRepositorySupport { - - private val dbio = new UpdateSchedulerDBIO() - - def create(ns: Namespace, - deviceId: DeviceId, - updateId: UpdateId, - scheduleAt: Instant): Future[ScheduledUpdateId] = { - val scheduledUpdateId = ScheduledUpdateId.generate() - - for { - id <- dbio.validateAndPersist( - ScheduledUpdate( - ns, - scheduledUpdateId, - deviceId, - updateId, - scheduleAt, - ScheduledUpdate.Status.Scheduled - ) - ) - _ <- msgBus.publishSafe( - DeviceUpdateStatus(ns, deviceId, DeviceStatus.UpdateScheduled, Instant.now()) - ) - } yield id - } - - def cancel(ns: Namespace, scheduledUpdateId: ScheduledUpdateId): Future[Unit] = - scheduledUpdatesRepository.setStatus( - ns, - scheduledUpdateId, - ScheduledUpdate.Status.Cancelled, - CancelledByUser.some - ) - - case object CancelledByUser extends StatusInfo - - implicit val cancelledByUserEncoder: Encoder[CancelledByUser.type] = Encoder.instance { _ => - Json.obj("cancelled_by_user" -> "cancelled by user".asJson) - } - -} diff --git a/src/main/scala/com/advancedtelematic/director/daemon/UpdateSchedulerDaemon.scala b/src/main/scala/com/advancedtelematic/director/daemon/UpdateSchedulerDaemon.scala new file mode 100644 index 00000000..33cf5fb3 --- /dev/null +++ b/src/main/scala/com/advancedtelematic/director/daemon/UpdateSchedulerDaemon.scala @@ -0,0 +1,77 @@ +package com.advancedtelematic.director.daemon + +import cats.implicits.* +import com.advancedtelematic.director.db.UpdateSchedulerDBIO +import com.advancedtelematic.libats.messaging.MessageBusPublisher +import com.advancedtelematic.libats.messaging_datatype.Messages.{DeviceUpdateAssigned, DeviceUpdateEvent} +import org.slf4j.LoggerFactory +import slick.jdbc.MySQLProfile.api.* + +import java.time.Instant +import scala.concurrent.duration.* +import scala.concurrent.{ExecutionContext, Future, blocking} + +class UpdateSchedulerDaemon()( + implicit db: Database, + ec: ExecutionContext, + msgBus: MessageBusPublisher) { + + private val dbio = new UpdateSchedulerDBIO() + + private lazy val log = LoggerFactory.getLogger(this.getClass) + + private val CHECK_INTERVAL = 1.second + + private val BACKOFF_INTERVAL = 3.seconds + + def start(): Future[Unit] = { + log.info( + s"starting update scheduler daemon. CHECK_INTERVAL=$CHECK_INTERVAL BACKOFF_INTERVAL=$BACKOFF_INTERVAL" + ) + run() + } + + def runOnce(): Future[Unit] = { + log.debug("checking for pending scheduled updates") + dbio + .run() + .flatMap { scheduledUpdates => + if (scheduledUpdates.isEmpty) { + log.debug("no scheduled updates pending, trying later") + Future { + blocking(Thread.sleep(CHECK_INTERVAL.toMillis)) + } + } else { + log.info(s"$scheduledUpdates scheduled updates processed") + + val msgFuture = scheduledUpdates.toList.traverse_ { su => + msgBus.publishSafe( + DeviceUpdateAssigned( + su.ns, + Instant.now(), + su.correlationId, + su.deviceId + ): DeviceUpdateEvent + ) + } + + msgFuture.map(_ => scheduledUpdates.size) + } + } + } + + private def run(): Future[Unit] = + runOnce() + .flatMap { _ => + run() + } + .recoverWith { case ex => + log.error("could not process pending updates", ex) + Future { + blocking(Thread.sleep(BACKOFF_INTERVAL.toMillis)) + }.flatMap { _ => + run() + } + } + +} diff --git a/src/main/scala/com/advancedtelematic/director/data/Codecs.scala b/src/main/scala/com/advancedtelematic/director/data/Codecs.scala index 822e067d..8ca781ea 100644 --- a/src/main/scala/com/advancedtelematic/director/data/Codecs.scala +++ b/src/main/scala/com/advancedtelematic/director/data/Codecs.scala @@ -9,8 +9,20 @@ import UptaneDataType.* import io.circe.* import AdminDataType.* import com.advancedtelematic.director.http.DeviceAssignments.AssignmentCreateResult -import com.advancedtelematic.director.http.{OfflineUpdateRequest, RemoteSessionRequest} -import com.advancedtelematic.libats.data.EcuIdentifier +import com.advancedtelematic.director.http.{ + CreateDeviceUpdateRequest, + CreateUpdateRequest, + CreateUpdateResult, + OfflineUpdateRequest, + RemoteCommandRequest, + RemoteSessionRequest, + UpdateDetailResponse, + UpdateReportedResult, + UpdateEventResponse, + UpdateResponse, + UpdateResultResponse +} +import com.advancedtelematic.libats.messaging_datatype.DataType.EcuIdentifier import com.advancedtelematic.libtuf.data.ClientCodecs.* import com.advancedtelematic.libtuf.data.TufDataType.SignedPayload import cats.syntax.either.* @@ -108,8 +120,8 @@ object Codecs { implicit val decoderTargetUpdateRequest: Decoder[TargetUpdateRequest] = deriveDecoder implicit val encoderTargetUpdateRequest: Encoder[TargetUpdateRequest] = deriveEncoder - implicit val multiTargetUpdateEncoder: Encoder[MultiTargetUpdate] = deriveEncoder - implicit val multiTargetUpdateDecoder: Decoder[MultiTargetUpdate] = deriveDecoder + implicit val targetUpdateSpecEncoder: Encoder[TargetUpdateSpec] = deriveEncoder + implicit val targetUpdateSpecDecoder: Decoder[TargetUpdateSpec] = deriveDecoder implicit val assignUpdateRequestEncoder: Encoder[AssignUpdateRequest] = deriveEncoder implicit val assignUpdateRequestDecoder: Decoder[AssignUpdateRequest] = deriveDecoder @@ -143,15 +155,16 @@ object Codecs { implicit val remoteSessionRequestCodec: Codec[RemoteSessionRequest] = deriveCodec - implicit val assignmentCreateResultCodec: Codec[AssignmentCreateResult] = deriveCodec + implicit val remoteCommandRequestCodec: Codec[RemoteCommandRequest] = + deriveCodec[RemoteCommandRequest] - implicit val scheduledUpdateStatusDecoder: Decoder[ScheduledUpdate.Status] = - enumeratum.Circe.decoder(ScheduledUpdate.Status) + implicit val assignmentCreateResultCodec: Codec[AssignmentCreateResult] = deriveCodec - implicit val scheduledUpdateStatusEncoder: Encoder[ScheduledUpdate.Status] = - enumeratum.Circe.encoder(ScheduledUpdate.Status) + implicit val updateStatusDecoder: Decoder[Update.Status] = + enumeratum.Circe.decoder(Update.Status) - implicit val scheduledUpdateCodec: Codec[ScheduledUpdate] = deriveCodec + implicit val updateStatusEncoder: Encoder[Update.Status] = + enumeratum.Circe.encoder(Update.Status) implicit val createScheduledUpdateRequestCodec: Codec[CreateScheduledUpdateRequest] = deriveCodec @@ -161,5 +174,24 @@ object Codecs { implicit val processedAssignmentCodec: Codec[ProcessedAssignment] = deriveCodec[ProcessedAssignment] + implicit val updateCode: Codec[Update] = deriveCodec + implicit val deviceKnownStateCodec: Codec[DeviceKnownState] = deriveCodec[DeviceKnownState] + + implicit val updateResultResponseCodec: Codec[UpdateResultResponse] = deriveCodec + + implicit val updateResponseCodec: Codec[UpdateResponse] = deriveCodec + + implicit val createUpdateRequestCodec: Codec[CreateUpdateRequest] = deriveCodec + + implicit val createUpdateResultCodec: Codec[CreateUpdateResult] = deriveCodec + + implicit val createDeviceUpdateRequestCodec: Codec[CreateDeviceUpdateRequest] = deriveCodec + + implicit val updateEcuResultCodec: Codec[UpdateReportedResult] = deriveCodec[UpdateReportedResult] + + implicit val updateDetailResponseCodec: Codec[UpdateDetailResponse] = + deriveCodec[UpdateDetailResponse] + + implicit val updateEventResponseCodec: Codec[UpdateEventResponse] = deriveCodec[UpdateEventResponse] } diff --git a/src/main/scala/com/advancedtelematic/director/data/DataType.scala b/src/main/scala/com/advancedtelematic/director/data/DataType.scala index 65b95324..08b97f8f 100644 --- a/src/main/scala/com/advancedtelematic/director/data/DataType.scala +++ b/src/main/scala/com/advancedtelematic/director/data/DataType.scala @@ -3,47 +3,25 @@ package com.advancedtelematic.director.data import java.security.PublicKey import java.time.{Duration, Instant} import java.util.UUID -import akka.http.scaladsl.model.Uri -import akka.http.scaladsl.server.PathMatcher +import org.apache.pekko.http.scaladsl.model.Uri +import org.apache.pekko.http.scaladsl.server.PathMatcher import cats.implicits.* -import com.advancedtelematic.director.data.DataType.{AdminRoleName, ScheduledUpdate} -import com.advancedtelematic.director.data.DataType.ScheduledUpdate.Status +import com.advancedtelematic.director.data.DataType.{AdminRoleName, TargetSpecId, Update} import com.advancedtelematic.director.data.DbDataType.Ecu import com.advancedtelematic.director.data.UptaneDataType.{Hashes, TargetImage} -import com.advancedtelematic.libats.data.DataType.{ - Checksum, - CorrelationId, - HashMethod, - Namespace, - ValidChecksum -} +import com.advancedtelematic.director.deviceregistry.data.DataType.UpdateTagValue +import com.advancedtelematic.director.deviceregistry.data.TagId +import com.advancedtelematic.libats.data.DataType.{Checksum, CorrelationId, HashMethod, Namespace, UpdateCorrelationId, ValidChecksum} import com.advancedtelematic.libats.data.UUIDKey.{UUIDKey, UUIDKeyObj, UuidKeyObjTimeBased} -import com.advancedtelematic.libats.data.{EcuIdentifier, PaginationResult} -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.data.PaginationResult +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libats.messaging_datatype.MessageLike import com.advancedtelematic.libats.messaging_datatype.Messages.EcuAndHardwareId import com.advancedtelematic.libtuf.crypt.CanonicalJson.* -import com.advancedtelematic.libtuf.data.ClientDataType.{ - ClientHashes, - MetaPath, - TufRole, - ValidMetaPath -} +import com.advancedtelematic.libtuf.data.ClientDataType.{ClientHashes, MetaPath, TufRole, ValidMetaPath} import com.advancedtelematic.libtuf.data.TufDataType.RoleType.RoleType -import com.advancedtelematic.libtuf.data.TufDataType.{ - HardwareIdentifier, - JsonSignedPayload, - KeyType, - RepoId, - SignedPayload, - TargetFilename, - TargetName, - TufKey -} -import com.advancedtelematic.libtuf.data.ValidatedString.{ - ValidatedString, - ValidatedStringValidation -} +import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, JsonSignedPayload, KeyType, RepoId, SignedPayload, TargetFilename, TargetName, TufKey} +import com.advancedtelematic.libtuf.data.ValidatedString.{ValidatedString, ValidatedStringValidation} import com.advancedtelematic.libtuf_server.crypto.Sha256Digest import com.advancedtelematic.libtuf_server.repo.server.DataType.SignedRole import eu.timepit.refined.api.Refined @@ -51,6 +29,11 @@ import io.circe.Json import com.advancedtelematic.libats.data.RefinedUtils.* import com.advancedtelematic.libtuf.data.TufCodecs import enumeratum.EnumEntry.Camelcase +import eu.timepit.refined.boolean.And +import eu.timepit.refined.collection.{NonEmpty, Size} +import eu.timepit.refined.generic.Equal +import eu.timepit.refined.refineV +import eu.timepit.refined.string.{HexStringSpec, MatchesRegex} import scala.annotation.nowarn @@ -64,15 +47,18 @@ object DbDataType { ecuId: EcuIdentifier, targetName: TargetName) - final case class DeviceKnownState(deviceId: DeviceId, - primaryEcu: EcuIdentifier, - ecuStatus: Map[EcuIdentifier, Option[EcuTargetId]], - ecuTargets: Map[EcuTargetId, EcuTarget], - currentAssignments: Set[Assignment], - processedAssignments: Set[ProcessedAssignment], - scheduledUpdates: Set[ScheduledUpdate], - scheduledUpdatesEcuTargetIds: Map[UpdateId, Seq[EcuTargetId]], - generatedMetadataOutdated: Boolean) + final case class DeviceKnownState( + deviceId: DeviceId, + primaryEcu: EcuIdentifier, + ecuStatus: Map[EcuIdentifier, Option[EcuTargetId]], + ecuTargets: Map[EcuTargetId, EcuTarget], + currentAssignments: Set[Assignment], + processedAssignments: Set[ProcessedAssignment], + // scheduledUpdates: Set[ScheduledUpdate], + // scheduledUpdatesEcuTargetIds: Map[TargetSpecId, Seq[EcuTargetId]], + updates: Set[Update], + updatesTargetIds: Map[TargetSpecId, Seq[EcuTargetId]], + generatedMetadataOutdated: Boolean) final case class Device(ns: Namespace, id: DeviceId, @@ -158,7 +144,7 @@ object DbDataType { } final case class HardwareUpdate(namespace: Namespace, - id: UpdateId, + id: TargetSpecId, hardwareId: HardwareIdentifier, fromTarget: Option[EcuTargetId], toTarget: EcuTargetId) @@ -216,6 +202,9 @@ object DbDataType { canceled: Boolean) type SHA256Checksum = Refined[String, ValidChecksum] + + type ValidMurmurHash3Checksum = HexStringSpec And Size[Equal[8]] + type MurmurHash3Checksum = Refined[String, ValidMurmurHash3Checksum] } object AdminDataType { @@ -234,7 +223,7 @@ object AdminDataType { uri: Option[Uri], userDefinedCustom: Option[Json]) - final case class MultiTargetUpdate(targets: Map[HardwareIdentifier, TargetUpdateRequest]) + final case class TargetUpdateSpec(targets: Map[HardwareIdentifier, TargetUpdateRequest]) final case class RegisterEcu(ecu_serial: EcuIdentifier, hardware_identifier: HardwareIdentifier, @@ -253,7 +242,7 @@ object AdminDataType { final case class AssignUpdateRequest(correlationId: CorrelationId, devices: Seq[DeviceId], - mtuId: UpdateId, + mtuId: TargetSpecId, // mtuId for legacy compatibility dryRun: Option[Boolean] = None) final case class QueueResponse(correlationId: CorrelationId, @@ -309,6 +298,8 @@ object Messages { } object DataType { + import enumeratum.* + final case class TargetItemCustomEcuData(hardwareId: HardwareIdentifier) final case class TargetItemCustom(uri: Option[Uri], @@ -344,16 +335,24 @@ object DataType { } - case class ScheduledUpdate(ns: Namespace, - id: ScheduledUpdateId, - deviceId: DeviceId, - updateId: UpdateId, - scheduledAt: Instant, - status: Status) + case class UpdateId(uuid: UUID) extends UUIDKey { + def toCorrelationId: UpdateCorrelationId = UpdateCorrelationId(uuid) + } + + object UpdateId extends UuidKeyObjTimeBased[UpdateId] - object ScheduledUpdate { + case class Update(ns: Namespace, + id: UpdateId, + deviceId: DeviceId, + correlationId: CorrelationId, + targetSpecId: TargetSpecId, + createdAt: Instant, + scheduledFor: Option[Instant], + status: Update.Status, + completedAt: Option[Instant] = None, + ) - import enumeratum.* + object Update { sealed trait Status extends EnumEntry with Camelcase @@ -364,20 +363,20 @@ object DataType { case object Assigned extends Status - case object Completed extends Status + case object Seen extends Status case object PartiallyCompleted extends Status case object Cancelled extends Status + + case object Completed extends Status } } - trait StatusInfo - - case class ScheduledUpdateId(uuid: UUID) extends UUIDKey + case class TargetSpecId(uuid: UUID) extends UUIDKey - object ScheduledUpdateId extends UuidKeyObjTimeBased[ScheduledUpdateId] + object TargetSpecId extends UuidKeyObjTimeBased[TargetSpecId] } object ClientDataType { @@ -437,7 +436,17 @@ object ClientDataType { case class DevicesCurrentTarget(values: Map[DeviceId, Seq[EcuTarget]]) case class CreateScheduledUpdateRequest(device: DeviceId, - updateId: UpdateId, - scheduledAt: Instant) + TargetSpecId: TargetSpecId, + scheduledFor: Instant) + type ValidTagSearchParam = MatchesRegex["[\\w\\-_]{1,20}+=[\\w\\-_/ ]{1,254}"] + type TagSearchParam = Refined[String, ValidTagSearchParam] + + implicit class TagSearchOps(value: TagSearchParam) { + // (0) safe because Refined checks Regex + def tagId: TagId = TagId(value.value.split("=")(0)) + + // (1) safe because Refined checks Regex + def tagValue: String = value.value.split("=")(1) + } } diff --git a/src/main/scala/com/advancedtelematic/director/data/DeviceRequest.scala b/src/main/scala/com/advancedtelematic/director/data/DeviceRequest.scala index b98913d1..eb5f6d9a 100644 --- a/src/main/scala/com/advancedtelematic/director/data/DeviceRequest.scala +++ b/src/main/scala/com/advancedtelematic/director/data/DeviceRequest.scala @@ -1,9 +1,8 @@ package com.advancedtelematic.director.data import com.advancedtelematic.libats.data.DataType.CorrelationId -import com.advancedtelematic.libats.messaging_datatype.DataType.InstallationResult +import com.advancedtelematic.libats.messaging_datatype.DataType.{EcuIdentifier, InstallationResult} import io.circe.Json -import com.advancedtelematic.libats.data.EcuIdentifier import com.advancedtelematic.libtuf.data.TufDataType.SignedPayload object DeviceRequest { diff --git a/src/main/scala/com/advancedtelematic/director/db/AffectedEcusDBIO.scala b/src/main/scala/com/advancedtelematic/director/db/AffectedEcusDBIO.scala new file mode 100644 index 00000000..6dfb2c7c --- /dev/null +++ b/src/main/scala/com/advancedtelematic/director/db/AffectedEcusDBIO.scala @@ -0,0 +1,202 @@ +package com.advancedtelematic.director.db + +import com.advancedtelematic.director.data.DataType.TargetSpecId +import com.advancedtelematic.director.data.DbDataType.{Ecu, EcuTarget, EcuTargetId, HardwareUpdate} +import com.advancedtelematic.director.http.DeviceAssignments.AffectedEcusResult +import com.advancedtelematic.director.http.Errors +import com.advancedtelematic.libats.data.DataType.Namespace +import com.advancedtelematic.libats.messaging_datatype.DataType.* +import com.advancedtelematic.libats.messaging_datatype.DataType.ValidEcuIdentifier +import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier +import eu.timepit.refined.refineV +import io.circe.syntax.EncoderOps +import org.slf4j.LoggerFactory +import slick.dbio.DBIO +import slick.jdbc.MySQLProfile.api.* +import scala.concurrent.{ExecutionContext, Future} + +class AffectedEcusDBIO()(implicit val db: Database, val ec: ExecutionContext) + extends HardwareUpdateRepositorySupport + with EcuTargetsRepositorySupport + with EcuRepositorySupport + with UpdatesRepositorySupport + with AssignmentsRepositorySupport { + + private val _log = LoggerFactory.getLogger(this.getClass) + + def findAffectedEcus(ns: Namespace, + devices: Seq[DeviceId], + targetSpecId: TargetSpecId): Future[AffectedEcusResult] = + db.run(findAffectedEcusAction(ns, devices, targetSpecId).transactionally) + + protected[db] def findAffectedEcusAction( + ns: Namespace, + devices: Seq[DeviceId], + targetSpecId: TargetSpecId, + ignoreScheduledUpdates: Boolean = false): DBIO[AffectedEcusResult] = + for { + // Find hardware updates and their targets + (hardwareUpdates, allTargets) <- findHardwareUpdatesAndTargets(ns, targetSpecId) + + // Filter devices with compatible hardware + (compatible, unaffected) <- filterDevicesWithCompatibleHardware( + ns, + devices, + hardwareUpdates, + targetSpecId + ) + + // Filter out devices with scheduled updates + (compatible, unaffected) <- + if (ignoreScheduledUpdates) + DBIO.successful((compatible, unaffected)) + else + filterDevicesWithActiveUpdates(ns, compatible, unaffected, targetSpecId) + + // Determine which ECUs are affected by the update + affected = determineAffectedEcus( + compatible, + hardwareUpdates, + allTargets, + unaffected, + targetSpecId + ) + + // Filter out ECUs with running assignments + finalResult <- filterEcusWithRunningAssignments(ns, affected) + } yield finalResult + + private def findHardwareUpdatesAndTargets(ns: Namespace, targetSpecId: TargetSpecId) + : DBIO[(Map[HardwareIdentifier, HardwareUpdate], Map[EcuTargetId, EcuTarget])] = + for { + hardwareUpdates <- hardwareUpdateRepository.findByAction(ns, targetSpecId) + allTargetIds = hardwareUpdates.values.flatMap { update => + Seq(update.toTarget) ++ update.fromTarget.toSeq + } + allTargets <- ecuTargetsRepository.findAllAction(ns, allTargetIds.toSeq) + } yield (hardwareUpdates, allTargets) + + private def filterDevicesWithCompatibleHardware( + ns: Namespace, + devices: Seq[DeviceId], + hardwareUpdates: Map[HardwareIdentifier, HardwareUpdate], + targetSpecId: TargetSpecId): DBIO[(Seq[(Ecu, Option[EcuTarget])], AffectedEcusResult)] = + for { + ecusWithCompatibleHardware <- ecuRepository.findEcuWithTargetsAction( + devices.toSet, + hardwareUpdates.keys.toSet + ) + + // Find devices with incompatible hardware + devicesWithIncompatibleHardware = devices.toSet -- ecusWithCompatibleHardware + .map(_._1.deviceId) + .toSet + + // Get primary ECU IDs for devices with incompatible hardware + devicePrimaries <- ecuRepository.findDevicePrimaryIdsAction( + ns, + devicesWithIncompatibleHardware + ) + + // Create result for devices with incompatible hardware + unaffectedDueToHardware = devicesWithIncompatibleHardware.foldLeft( + AffectedEcusResult(Seq.empty, Map.empty) + ) { case (acc, deviceId) => + val error = Errors.DeviceNoCompatibleHardware(deviceId, targetSpecId) + _log.info(error.getMessage) + val primaryEcuId = + devicePrimaries.getOrElse(deviceId, refineV[ValidEcuIdentifier].unsafeFrom("unknown")) + acc.addNotAffected(deviceId, primaryEcuId, error) + } + } yield (ecusWithCompatibleHardware, unaffectedDueToHardware) + + private def filterDevicesWithActiveUpdates( + ns: Namespace, + ecusWithCompatibleHardware: Seq[(Ecu, Option[EcuTarget])], + unaffectedDueToHardware: AffectedEcusResult, + targetSpecId: TargetSpecId): DBIO[(Seq[(Ecu, Option[EcuTarget])], AffectedEcusResult)] = + for { + devicesWithUpdates <- updatesRepository.filterActiveUpdateExistsAction( + ns, + ecusWithCompatibleHardware.map(_._1.deviceId).toSet + ) + + _ = _log + .atDebug() + .addKeyValue("devicesWithActiveUpdates", devicesWithUpdates.asJson) + .log() + + ecusWithoutScheduledUpdates = ecusWithCompatibleHardware.filterNot { case (ecu, _) => + devicesWithUpdates.contains(ecu.deviceId) + } + + unaffectedDueToActiveUpdates = devicesWithUpdates.foldLeft(unaffectedDueToHardware) { + case (acc, deviceId) => + val ecus = ecusWithCompatibleHardware.filter(_._1.deviceId == deviceId) + + ecus.foldLeft(acc) { case (result, (ecu, _)) => + val error = Errors.DeviceHasActiveUpdate(deviceId, targetSpecId) + _log.info(error.getMessage) + result.addNotAffected(deviceId, ecu.ecuSerial, error) + } + } + } yield (ecusWithoutScheduledUpdates, unaffectedDueToActiveUpdates) + + private def determineAffectedEcus(compatibleEcus: Seq[(Ecu, Option[EcuTarget])], + hardwareUpdates: Map[HardwareIdentifier, HardwareUpdate], + allTargets: Map[EcuTargetId, EcuTarget], + notAffected: AffectedEcusResult, + targetSpecId: TargetSpecId): AffectedEcusResult = + + compatibleEcus.foldLeft(notAffected) { case (acc, (ecu, installedTarget)) => + val hwUpdate = hardwareUpdates(ecu.hardwareId) + val updateFrom = hwUpdate.fromTarget.flatMap(allTargets.get) + val updateTo = allTargets(hwUpdate.toTarget) + + if (isEcuUpdateable(hwUpdate, installedTarget, updateFrom)) { + if (isTargetAlreadyInstalled(installedTarget, updateTo)) { + val error = Errors.InstalledTargetIsUpdate(ecu.deviceId, ecu.ecuSerial, hwUpdate) + _log.info(error.getMessage) + acc.addNotAffected(ecu.deviceId, ecu.ecuSerial, error) + } else { + _log.info(s"${ecu.deviceId}/${ecu.ecuSerial} affected for $hwUpdate") + acc.addAffected(ecu, hwUpdate.toTarget) + } + } else { + val error = Errors.NotAffectedByMtu(ecu.deviceId, ecu.ecuSerial, targetSpecId) + _log.info(error.getMessage) + acc.addNotAffected(ecu.deviceId, ecu.ecuSerial, error) + } + } + + private def isEcuUpdateable(hwUpdate: HardwareUpdate, + installedTarget: Option[EcuTarget], + updateFrom: Option[EcuTarget]): Boolean = + hwUpdate.fromTarget.isEmpty || installedTarget.zip(updateFrom).exists { case (a, b) => + a.matches(b) + } + + private def isTargetAlreadyInstalled(installedTarget: Option[EcuTarget], + updateTo: EcuTarget): Boolean = + installedTarget.exists(_.matches(updateTo)) + + private def filterEcusWithRunningAssignments( + ns: Namespace, + ecusResult: AffectedEcusResult): DBIO[AffectedEcusResult] = { + val ecuIds = ecusResult.affected.map { case (ecu, _) => + ecu.deviceId -> ecu.ecuSerial + }.toSet + + assignmentsRepository.withAssignmentsAction(ns, ecuIds).map { ecusWithAssignments => + ecusResult.affected.foldLeft(AffectedEcusResult(Seq.empty, ecusResult.notAffected)) { + case (acc, (ecu, _)) if ecusWithAssignments.contains(ecu.deviceId -> ecu.ecuSerial) => + val error = Errors.NotAffectedRunningAssignment(ecu.deviceId, ecu.ecuSerial) + _log.info(error.getMessage) + acc.addNotAffected(ecu.deviceId, ecu.ecuSerial, error) + case (acc, (ecu, target)) => + acc.addAffected(ecu, target) + } + } + } + +} diff --git a/src/main/scala/com/advancedtelematic/director/db/CompiledManifestExecutor.scala b/src/main/scala/com/advancedtelematic/director/db/CompiledManifestExecutor.scala index 1124aa67..8c96ed39 100644 --- a/src/main/scala/com/advancedtelematic/director/db/CompiledManifestExecutor.scala +++ b/src/main/scala/com/advancedtelematic/director/db/CompiledManifestExecutor.scala @@ -1,17 +1,16 @@ package com.advancedtelematic.director.db -import com.advancedtelematic.director.data.DataType.ScheduledUpdate.Status +import com.advancedtelematic.director.data.DataType.Update.Status import com.advancedtelematic.director.data.DbDataType.{Device, DeviceKnownState, EcuTargetId} import com.advancedtelematic.director.manifest.ManifestCompiler.ManifestCompileResult -import com.advancedtelematic.libats.data.EcuIdentifier import com.advancedtelematic.libats.http.Errors.MissingEntity -import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId -import com.advancedtelematic.libats.slick.db.SlickAnyVal.* +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* -import SlickMapping.scheduledUpdatesMapper +import SlickMapping.updateStatusMapper import org.slf4j.LoggerFactory import slick.jdbc.MySQLProfile.api.* import slick.jdbc.TransactionIsolation +import com.advancedtelematic.libats.slick.codecs.SlickRefined.* import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} @@ -33,12 +32,12 @@ class CompiledManifestExecutor()(implicit val db: Database, val ec: ExecutionCon .filter(_.id === deviceId) .result .failIfNotSingle(MissingEntity[Device]()) - scheduledUpdates <- Schema.scheduledUpdates + updates <- Schema.updates .filter(_.deviceId === deviceId) .filterNot(_.status.inSet(Set(Status.Completed, Status.Cancelled))) .result hardwareUpdatesEcuTargetIds <- Schema.hardwareUpdates - .filter(_.id.inSet(scheduledUpdates.map(_.updateId).toSet)) + .filter(_.id.inSet(updates.map(_.targetSpecId).toSet)) .map(t => t.id -> t.toTarget) .result ecuTargetIds = ecuStatus.flatMap(_._2) ++ assignments.map( @@ -52,7 +51,7 @@ class CompiledManifestExecutor()(implicit val db: Database, val ec: ExecutionCon ecuTargets.toMap, assignments.toSet, processed.toSet, - scheduledUpdates.toSet, + updates.toSet, hardwareUpdatesEcuTargetIds.groupBy(_._1).view.mapValues(_.map(_._2)).toMap, device.generatedMetadataOutdated ) @@ -90,7 +89,7 @@ class CompiledManifestExecutor()(implicit val db: Database, val ec: ExecutionCon .filter(_.ecuId.inSet(assignmentsToDelete)) .delete - val newScheduledUpdates = newStatus.scheduledUpdates -- oldStatus.scheduledUpdates + val newUpdates = newStatus.updates -- oldStatus.updates for { _ <- DBIO.sequence(newEcuTargets.values.map(Schema.ecuTargets.insertOrUpdate)) @@ -100,7 +99,7 @@ class CompiledManifestExecutor()(implicit val db: Database, val ec: ExecutionCon _ <- deleteAssignmentsIO _ <- DBIO.sequence(newProcessedAssignments.map(Schema.processedAssignments += _).toList) _ <- updateMetadataOutdatedFlagAction(deviceId, oldStatus, newStatus) - _ <- DBIO.sequence(newScheduledUpdates.map(Schema.scheduledUpdates.insertOrUpdate).toList) + _ <- DBIO.sequence(newUpdates.map(Schema.updates.insertOrUpdate).toList) } yield () } diff --git a/src/main/scala/com/advancedtelematic/director/db/DbDebug.scala b/src/main/scala/com/advancedtelematic/director/db/DbDebug.scala index dab826ec..6d67c730 100644 --- a/src/main/scala/com/advancedtelematic/director/db/DbDebug.scala +++ b/src/main/scala/com/advancedtelematic/director/db/DbDebug.scala @@ -1,5 +1,6 @@ package com.advancedtelematic.director.db +import com.advancedtelematic.director.data.DbDataType.MurmurHash3Checksum import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import slick.jdbc.MySQLProfile.api.* @@ -114,10 +115,10 @@ class DirectorDbDebug()(implicit val db: Database, ec: ExecutionContext) { readTable("DeviceType", id => sql"""select * from DeviceType where id = $id""") ), TableResource( - "scheduled_updates", + "device_manifests", readTable( - "scheduled_updates", - id => sql"""select * from scheduled_updates where device_id = $id""" + "device_manifests", + id => sql"""select * from device_manifests where device_id = $id order by received_at desc limit 20""" ) ) ) diff --git a/src/main/scala/com/advancedtelematic/director/db/DeviceRegistration.scala b/src/main/scala/com/advancedtelematic/director/db/DeviceRegistration.scala index 71fdef06..bb361845 100644 --- a/src/main/scala/com/advancedtelematic/director/db/DeviceRegistration.scala +++ b/src/main/scala/com/advancedtelematic/director/db/DeviceRegistration.scala @@ -1,12 +1,8 @@ package com.advancedtelematic.director.db -import akka.http.scaladsl.util.FastFuture +import org.apache.pekko.http.scaladsl.util.FastFuture import cats.implicits.toShow -import com.advancedtelematic.director.data.AdminDataType.{ - EcuInfoImage, - EcuInfoResponse, - RegisterEcu -} +import com.advancedtelematic.director.data.AdminDataType.{EcuInfoImage, EcuInfoResponse, RegisterEcu} import com.advancedtelematic.director.data.UptaneDataType.Hashes import com.advancedtelematic.director.db.ProvisionedDeviceRepository.DeviceCreateResult import com.advancedtelematic.director.db.deviceregistry.{DeviceRepository, EcuReplacementRepository} @@ -16,8 +12,7 @@ import com.advancedtelematic.director.http.Errors.AssignmentExistsError import com.advancedtelematic.director.http.deviceregistry.Errors.MissingDevice import com.advancedtelematic.director.repo.DeviceRoleGeneration import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.data.EcuIdentifier -import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libtuf.data.TufDataType.RepoId import com.advancedtelematic.libtuf_server.keyserver.KeyserverClient import org.slf4j.LoggerFactory diff --git a/src/main/scala/com/advancedtelematic/director/db/Repository.scala b/src/main/scala/com/advancedtelematic/director/db/Repository.scala index c1195edf..197fd444 100644 --- a/src/main/scala/com/advancedtelematic/director/db/Repository.scala +++ b/src/main/scala/com/advancedtelematic/director/db/Repository.scala @@ -1,8 +1,9 @@ package com.advancedtelematic.director.db -import java.time.Instant -import java.util.UUID import cats.Show +import cats.data.NonEmptyList +import cats.syntax.show.* +import com.advancedtelematic.director.data.DataType.{AdminRoleName, TargetSpecId, Update, UpdateId} import com.advancedtelematic.director.data.DbDataType.{ Assignment, AutoUpdateDefinition, @@ -14,44 +15,47 @@ import com.advancedtelematic.director.data.DbDataType.{ EcuTarget, EcuTargetId, HardwareUpdate, - ProcessedAssignment + MurmurHash3Checksum, + ProcessedAssignment, + ValidMurmurHash3Checksum +} +import com.advancedtelematic.director.db.AdminRolesRepository.{ + Deleted, + FindLatestResult, + NotDeleted } import com.advancedtelematic.director.db.ProvisionedDeviceRepository.DeviceCreateResult +import com.advancedtelematic.director.db.Schema.{adminRoles, notDeletedAdminRoles, AssignmentsTable} +import com.advancedtelematic.director.db.SlickMapping.* +import com.advancedtelematic.director.http.Errors import com.advancedtelematic.libats.data.DataType.{CorrelationId, Namespace} -import com.advancedtelematic.libats.data.{EcuIdentifier, PaginationResult} +import com.advancedtelematic.libats.data.PaginationResult +import com.advancedtelematic.libats.data.PaginationResult.{Limit, LongAsParam, Offset} +import com.advancedtelematic.libats.data.RefinedUtils.* import com.advancedtelematic.libats.http.Errors.{ EntityAlreadyExists, MissingEntity, MissingEntityId } -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} -import cats.syntax.either.* -import com.advancedtelematic.libats.slick.db.SlickExtensions.* -import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* -import com.advancedtelematic.libats.slick.codecs.SlickRefined.* -import slick.jdbc.MySQLProfile.api.* -import akka.http.scaladsl.util.FastFuture -import com.advancedtelematic.director.data.DataType.{ - AdminRoleName, - ScheduledUpdate, - ScheduledUpdateId, - StatusInfo -} -import com.advancedtelematic.director.db.AdminRolesRepository.{ - Deleted, - FindLatestResult, - NotDeleted +import com.advancedtelematic.libats.messaging_datatype.DataType.{ + DeviceId, + EcuIdentifier, + ValidEcuIdentifier } -import com.advancedtelematic.director.db.Schema.{adminRoles, notDeletedAdminRoles} -import com.advancedtelematic.director.http.Errors import com.advancedtelematic.libats.messaging_datatype.Messages.{ EcuAndHardwareId, EcuReplaced, EcuReplacement } +import com.advancedtelematic.libats.slick.codecs.SlickRefined.* import com.advancedtelematic.libats.slick.db.SlickAnyVal.* import com.advancedtelematic.libats.slick.db.SlickCirceMapper.jsonMapper +import com.advancedtelematic.libats.slick.db.SlickExtensions.* +import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* +import com.advancedtelematic.libats.slick.db.SlickUrnMapper.* +import com.advancedtelematic.libtuf.crypt.CanonicalJson.* import com.advancedtelematic.libtuf.data.ClientDataType.TufRole +import com.advancedtelematic.libtuf.data.TufDataType.RoleType.RoleType import com.advancedtelematic.libtuf.data.TufDataType.{ HardwareIdentifier, RepoId, @@ -59,16 +63,17 @@ import com.advancedtelematic.libtuf.data.TufDataType.{ TargetFilename, TargetName } -import com.advancedtelematic.libtuf_server.crypto.Sha256Digest import com.advancedtelematic.libtuf_server.data.TufSlickMappings.* -import io.circe.{Encoder, Json} -import com.advancedtelematic.libtuf.crypt.CanonicalJson.* -import com.advancedtelematic.libtuf.data.TufDataType.RoleType.RoleType import io.circe.syntax.EncoderOps +import io.circe.{Encoder, Json} +import org.apache.pekko.http.scaladsl.util.FastFuture import slick.jdbc.GetResult +import slick.jdbc.MySQLProfile.api.* +import java.time.Instant +import java.util.UUID import scala.concurrent.{ExecutionContext, Future} -import cats.syntax.show.* +import scala.util.hashing.MurmurHash3 protected trait DatabaseSupport { implicit val ec: ExecutionContext @@ -130,6 +135,12 @@ protected class ProvisionedDeviceRepository()(implicit val db: Database, val ec: db.run(io.transactionally) } + protected[db] def ensureExistsIO(device: DeviceId): DBIO[Unit] = + existsIO(device).flatMap { + case true => DBIO.successful(()) + case false => DBIO.failed(MissingEntity[Device]()) + } + private def existsIO(deviceId: DeviceId): DBIO[Boolean] = Schema.allProvisionedDevices.filter(_.id === deviceId).exists.result @@ -137,8 +148,8 @@ protected class ProvisionedDeviceRepository()(implicit val db: Database, val ec: db.run(existsIO(deviceId)) def findAllDeviceIds(ns: Namespace, - offset: Long, - limit: Long): Future[PaginationResult[DeviceId]] = db.run { + offset: Offset, + limit: Limit): Future[PaginationResult[DeviceId]] = db.run { Schema.activeProvisionedDevices .filter(_.namespace === ns) .map(d => (d.id, d.createdAt)) @@ -148,8 +159,8 @@ protected class ProvisionedDeviceRepository()(implicit val db: Database, val ec: def findDevices(ns: Namespace, hardwareIdentifier: HardwareIdentifier, - offset: Long, - limit: Long): Future[PaginationResult[(Instant, Device)]] = db.run { + offset: Offset, + limit: Limit): Future[PaginationResult[(Instant, Device)]] = db.run { Schema.activeProvisionedDevices .filter(_.namespace === ns) .join(Schema.activeEcus.filter(_.hardwareId === hardwareIdentifier)) @@ -291,17 +302,13 @@ protected[db] class RepoNamespaceRepository()(implicit val db: Database, val ec: object HardwareUpdateRepository { - implicit val showHardwareUpdateId: cats.Show[ - ( - com.advancedtelematic.libats.data.DataType.Namespace, - com.advancedtelematic.libats.messaging_datatype.DataType.UpdateId - ) - ] = Show.show[(Namespace, UpdateId)] { case (ns, id) => - s"($ns, $id)" - } + implicit val showHardwareTargetSpecId: cats.Show[(Namespace, TargetSpecId)] = + Show.show[(Namespace, TargetSpecId)] { case (ns, id) => + s"($ns, $id)" + } - def MissingHardwareUpdate(namespace: Namespace, id: UpdateId) = - MissingEntityId[(Namespace, UpdateId)](namespace -> id) + def MissingHardwareUpdate(namespace: Namespace, id: TargetSpecId) = + MissingEntityId[(Namespace, TargetSpecId)](namespace -> id) } @@ -310,25 +317,29 @@ trait HardwareUpdateRepositorySupport extends DatabaseSupport { } protected class HardwareUpdateRepository()(implicit val db: Database, val ec: ExecutionContext) { - import HardwareUpdateRepository._ + import HardwareUpdateRepository.* protected[db] def persistAction(hardwareUpdate: HardwareUpdate): DBIO[Unit] = (Schema.hardwareUpdates += hardwareUpdate).map(_ => ()) - def findBy(ns: Namespace, id: UpdateId): Future[Map[HardwareIdentifier, HardwareUpdate]] = - db.run { - Schema.hardwareUpdates - .filter(_.namespace === ns) - .filter(_.id === id) - .result - .failIfEmpty(MissingHardwareUpdate(ns, id)) - .map { hwUpdates => - hwUpdates.map(hwUpdate => hwUpdate.hardwareId -> hwUpdate).toMap - } - } + def findAll(ns: Namespace): Future[Seq[HardwareUpdate]] = db.run { + Schema.hardwareUpdates.filter(_.namespace === ns).result + } - def findUpdateTargets(ns: Namespace, - id: UpdateId): Future[Seq[(HardwareUpdate, Option[EcuTarget], EcuTarget)]] = + protected[db] def findByAction(ns: Namespace, + id: TargetSpecId): DBIO[Map[HardwareIdentifier, HardwareUpdate]] = + Schema.hardwareUpdates + .filter(_.namespace === ns) + .filter(_.id === id) + .result + .failIfEmpty(MissingHardwareUpdate(ns, id)) + .map { hwUpdates => + hwUpdates.map(hwUpdate => hwUpdate.hardwareId -> hwUpdate).toMap + } + + def findUpdateTargets( + ns: Namespace, + id: TargetSpecId): Future[Seq[(HardwareUpdate, Option[EcuTarget], EcuTarget)]] = db.run { val io = Schema.hardwareUpdates .filter(_.namespace === ns) @@ -364,12 +375,16 @@ protected class EcuTargetsRepository()(implicit val db: Database, val ec: Execut } def findAll(ns: Namespace, ids: Seq[EcuTargetId]): Future[Map[EcuTargetId, EcuTarget]] = db.run { + findAllAction(ns, ids) + } + + protected[db] def findAllAction(ns: Namespace, + ids: Seq[EcuTargetId]): DBIO[Map[EcuTargetId, EcuTarget]] = Schema.ecuTargets .filter(_.namespace === ns) .filter(_.id.inSet(ids)) .result .map(_.map(e => e.id -> e).toMap) - } } @@ -416,6 +431,14 @@ protected class AssignmentsRepository()(implicit val db: Database, val ec: Execu }.toMap } + protected[db] def findByCorrelationIdsAction( + ns: Namespace, + correlationIds: Set[CorrelationId]): DBIO[Seq[Assignment]] = + Schema.assignments + .filter(_.namespace === ns) + .filter(_.correlationId.inSet(correlationIds)) + .result + def existsForDevices(deviceIds: Set[DeviceId]): Future[Map[DeviceId, Boolean]] = db.run { Schema.assignments .filter(_.deviceId.inSet(deviceIds)) @@ -424,26 +447,25 @@ protected class AssignmentsRepository()(implicit val db: Database, val ec: Execu .map(existing => deviceIds.map(_ -> false).toMap ++ existing.toMap) } - def withAssignments(ids: Set[(DeviceId, EcuIdentifier)]): Future[Set[(DeviceId, EcuIdentifier)]] = + protected[db] def withAssignmentsAction( + ns: Namespace, + ids: Set[(DeviceId, EcuIdentifier)]): DBIO[Set[(DeviceId, EcuIdentifier)]] = if (ids.isEmpty) { - FastFuture.successful(Set.empty) + DBIO.successful(Set.empty) } else { // raw sql is workaround for https://github.com/slick/slick/pull/995 implicit val getResult = GetResult { r => - DeviceId(UUID.fromString(r.nextString())) -> EcuIdentifier - .from(r.nextString()) - .valueOr(throw _) + DeviceId(UUID.fromString(r.nextString())) -> + r.nextString().refineTry[ValidEcuIdentifier].get } val elems = ids .map { case (d, e) => "('" + d.uuid.toString + "','" + e.value + "')" } .mkString("(", ",", ")") - db.run { - sql"select device_id, ecu_serial from assignments where (device_id, ecu_serial) in #$elems" - .as[(DeviceId, EcuIdentifier)] - .map(_.toSet) - } + sql"select device_id, ecu_serial from assignments where (device_id, ecu_serial) in #$elems AND namespace = ${ns.get}" + .as[(DeviceId, EcuIdentifier)] + .map(_.toSet) } def markRegenerated(deviceRepository: ProvisionedDeviceRepository)( @@ -452,6 +474,11 @@ protected class AssignmentsRepository()(implicit val db: Database, val ec: Execu val io = for { assignments <- deviceAssignments.forUpdate.result + _ <- Schema.updates + .filter(_.correlationId.inSet(assignments.map(_.correlationId).toSet)) + .filter(_.status === (Update.Status.Assigned: Update.Status)) + .map(_.status) + .update(Update.Status.Seen) _ <- deviceAssignments.map(_.inFlight).update(true) _ <- deviceRepository.setMetadataOutdatedAction(Set(deviceId), outdated = false) } yield assignments @@ -459,33 +486,48 @@ protected class AssignmentsRepository()(implicit val db: Database, val ec: Execu io.transactionally } - def processDeviceCancellation(deviceRepository: ProvisionedDeviceRepository)( - ns: Namespace, - deviceId: DeviceId, - allowInFlightCancellation: Boolean): Future[List[CorrelationId]] = db.run { - val assignmentQuery = - Schema.assignments.filter(_.namespace === ns).filter(_.deviceId === deviceId) - - val action = assignmentQuery.forUpdate.result.flatMap { assignments => - if (assignments.isEmpty) - DBIO.failed(MissingEntity[Assignment]()) - else if (assignments.exists(_.inFlight) && !allowInFlightCancellation) - DBIO.failed(Errors.AssignmentInFlight(deviceId)) - else + protected[db] def processDeviceCancellationAction(deviceRepository: ProvisionedDeviceRepository)( + assignmentsQuery: Query[AssignmentsTable, Assignment, Seq], + allowInFlightCancellation: Boolean = false): DBIO[Map[CorrelationId, List[DeviceId]]] = + assignmentsQuery.forUpdate.result.flatMap { assignments => + if (assignments.exists(_.inFlight) && !allowInFlightCancellation) { + // safe because of above `exists` + val deviceIds = + NonEmptyList.fromListUnsafe(assignments.filter(_.inFlight).map(_.deviceId).toSet.toList) + DBIO.failed(Errors.AssignmentInFlight(deviceIds)) + } else DBIO .seq( Schema.processedAssignments ++= assignments .map(_.toProcessedAssignment(successful = true, canceled = true)), - assignmentQuery.delete, - deviceRepository.setMetadataOutdatedAction(Set(deviceId), outdated = true) + assignmentsQuery.delete, + deviceRepository + .setMetadataOutdatedAction(assignments.map(_.deviceId).toSet, outdated = true) ) - .map(_ => assignments.map(_.correlationId).toList) + .map(_ => assignments.map(a => a.correlationId -> a.deviceId).toList) + .map(_.groupMap(_._1)(_._2)) } - action.transactionally + def processDeviceCancellation(deviceRepository: ProvisionedDeviceRepository, + updatesRepository: UpdatesRepository)( + ns: Namespace, + deviceId: DeviceId, + allowInFlightCancellation: Boolean): Future[List[CorrelationId]] = db.run { + val assignmentsToCancelQuery = Schema.assignments.filter(_.deviceId === deviceId) + + val io = for { + ids <- processDeviceCancellationAction(deviceRepository)( + assignmentsToCancelQuery, + allowInFlightCancellation + ) + _ <- updatesRepository.ensureNoUpdateFor(ns, ids.keySet).map(_ => ids.keySet) + } yield ids.keySet.toList + + io.transactionally } - def processCancellation(deviceRepository: ProvisionedDeviceRepository)( + def processCancellation(deviceRepository: ProvisionedDeviceRepository, + updatesRepository: UpdatesRepository)( ns: Namespace, deviceIds: Seq[DeviceId]): Future[Seq[Assignment]] = db.run { val assignmentQuery = Schema.assignments @@ -498,6 +540,8 @@ protected class AssignmentsRepository()(implicit val db: Database, val ec: Execu _ <- Schema.processedAssignments ++= assignments.map( _.toProcessedAssignment(successful = true, canceled = true) ) + correlationIds = assignments.map(_.correlationId).toSet + _ <- updatesRepository.ensureNoUpdateFor(ns, correlationIds) _ <- assignmentQuery.delete _ <- deviceRepository.setMetadataOutdatedAction(deviceIds.toSet, outdated = true) } yield assignments @@ -509,6 +553,16 @@ protected class AssignmentsRepository()(implicit val db: Database, val ec: Execu Schema.processedAssignments.filter(_.namespace === ns).filter(_.deviceId === deviceId).result } + protected[db] def findAllProcessedByCorrelatioIds( + ns: Namespace, + deviceId: DeviceId, + correlationIds: Set[CorrelationId]): DBIO[Seq[ProcessedAssignment]] = + Schema.processedAssignments + .filter(_.namespace === ns) + .filter(_.deviceId === deviceId) + .filter(_.correlationId.inSet(correlationIds)) + .result + } trait EcuRepositorySupport extends DatabaseSupport { @@ -558,8 +612,8 @@ protected class EcuRepository()(implicit val db: Database, val ec: ExecutionCont } def findAllHardwareIdentifiers(ns: Namespace, - offset: Long, - limit: Long): Future[PaginationResult[HardwareIdentifier]] = + offset: Offset, + limit: Limit): Future[PaginationResult[HardwareIdentifier]] = db.run { Schema.activeEcus .filter(_.namespace === ns) @@ -585,16 +639,15 @@ protected class EcuRepository()(implicit val db: Database, val ec: ExecutionCont .result } - def findEcuWithTargets( + def findEcuWithTargetsAction( devices: Set[DeviceId], - hardwareIds: Set[HardwareIdentifier]): Future[Seq[(Ecu, Option[EcuTarget])]] = db.run { + hardwareIds: Set[HardwareIdentifier]): DBIO[Seq[(Ecu, Option[EcuTarget])]] = Schema.activeEcus .filter(_.deviceId.inSet(devices)) .filter(_.hardwareId.inSet(hardwareIds)) .joinLeft(Schema.ecuTargets) .on(_.installedTarget === _.id) .result - } private[db] def findDevicePrimaryAction(ns: Namespace, deviceId: DeviceId): DBIO[Ecu] = Schema.allProvisionedDevices @@ -610,8 +663,9 @@ protected class EcuRepository()(implicit val db: Database, val ec: ExecutionCont def findDevicePrimary(ns: Namespace, deviceId: DeviceId): Future[Ecu] = db.run(findDevicePrimaryAction(ns, deviceId)) - def findDevicePrimaryIds(ns: Namespace, - deviceId: Set[DeviceId]): Future[Map[DeviceId, EcuIdentifier]] = db.run { + protected[db] def findDevicePrimaryIdsAction( + ns: Namespace, + deviceId: Set[DeviceId]): DBIO[Map[DeviceId, EcuIdentifier]] = Schema.allProvisionedDevices .filter(_.namespace === ns) .filter(_.id.inSet(deviceId)) @@ -622,7 +676,6 @@ protected class EcuRepository()(implicit val db: Database, val ec: ExecutionCont .map { case (_, ecu) => ecu.deviceId -> ecu.ecuSerial } .result .map(_.toMap) - } protected[db] def setActiveEcus(ns: Namespace, deviceId: DeviceId, @@ -742,8 +795,8 @@ protected[db] class AdminRolesRepository()(implicit val db: Database, val ec: Ex select max(ar0.expires_at) from #${Schema.adminRoles.baseTableRow.tableName} ar0 join (select max(version) version, repo_id, name from #${Schema.adminRoles.baseTableRow.tableName} group by repo_id, name) ar USING (name, version, repo_id) - where ar0.repo_id = '${repoId.show} - AND deleted = 0' + where ar0.repo_id = ${repoId.show} + AND deleted = 0 """.as[Option[Instant]].headOption db.run(sql).map(_.flatten) @@ -917,99 +970,182 @@ trait DeviceManifestRepositorySupport { protected class DeviceManifestRepository()(implicit db: Database, ec: ExecutionContext) { - def find(deviceId: DeviceId): Future[Option[(Json, Instant)]] = db.run { + def findLatest(deviceId: DeviceId): Future[Option[(Json, Instant)]] = db.run { Schema.deviceManifests .filter(_.deviceId === deviceId) + .sortBy(_.receivedAt.desc) + .take(1) .map(r => r.manifest -> r.receivedAt) .result .headOption } - def findAll(deviceId: DeviceId): Future[Seq[(Json, Instant)]] = db.run { + def findAll(deviceId: DeviceId, + offset: Offset = 0L.toOffset, + limit: Limit = 50L.toLimit): Future[PaginationResult[(Json, Instant)]] = db.run { Schema.deviceManifests .filter(_.deviceId === deviceId) + .sortBy(_.receivedAt.desc) .map(r => r.manifest -> r.receivedAt) - .result + .paginateResult(offset, limit) } - def createOrUpdate(device: DeviceId, jsonManifest: Json, receivedAt: Instant): Future[Unit] = - db.run { - val checksum = Sha256Digest.digest(jsonManifest.canonical.getBytes).hash - Schema.deviceManifests - .insertOrUpdate((device, jsonManifest, checksum, receivedAt)) - .map(_ => ()) - } - -} + // Calculate checksum after removing unstable fields like signatures + private def calculateJsonChecksum(json: Json): MurmurHash3Checksum = { + def removeUnstableFieldsRecursively(json: Json): Json = json.arrayOrObject( + json, + jsonArray = arr => Json.fromValues(arr.map(removeUnstableFieldsRecursively)), + jsonObject = obj => + Json.fromJsonObject( + obj + .remove("signatures") + .remove("report_counter") + .mapValues(removeUnstableFieldsRecursively) + ) + ) -trait ScheduledUpdatesRepositorySupport { + f"${MurmurHash3.bytesHash(removeUnstableFieldsRecursively(json).canonical.getBytes)}%08x" + .refineTry[ValidMurmurHash3Checksum] + .get + } - def scheduledUpdatesRepository(implicit db: Database, ec: ExecutionContext) = - new ScheduledUpdatesRepository() + def createOrUpdate(manifests: Seq[(DeviceId, Json, Instant)]): Future[Unit] = db.run { + val items = manifests + .map { case (deviceId, json, receivedAt) => + val checksum = calculateJsonChecksum(json) + (deviceId, json, checksum, receivedAt) + } + .sortBy(_._4) -} + // Using string concat to set a list but currently mariadb doesn't have native arrays so this is the best we can do + val deviceIdStr = items.map(_._1).map(d => s"'${d.show}'").toSet.mkString(",") -protected class ScheduledUpdatesRepository()(implicit db: Database, ec: ExecutionContext) { - import SlickMapping.* + val deleteSql = + sql""" + DELETE dm1 + FROM device_manifests AS dm1 + JOIN ( + SELECT device_id, checksum + FROM ( + SELECT device_id, checksum, ROW_NUMBER() OVER (PARTITION BY device_id ORDER BY received_at DESC) as row_num + FROM device_manifests + WHERE device_id IN (#$deviceIdStr) + ) ranked + WHERE ranked.row_num > 200 + ) AS dm2 USING (device_id, checksum) + """.asUpdate - def persist(scheduledUpdate: ScheduledUpdate): Future[ScheduledUpdateId] = db.run { - persistAction(scheduledUpdate) + Schema.deviceManifests + .insertOrUpdateAll(items) + .andThen(deleteSql) + .transactionally + .map(_ => ()) } - protected[db] def persistAction(scheduledUpdate: ScheduledUpdate): DBIO[ScheduledUpdateId] = - (Schema.scheduledUpdates += scheduledUpdate).map(_ => scheduledUpdate.id) +} + +protected class UpdatesRepository()(implicit db: Database, ec: ExecutionContext) { + + import PaginationResult.* + import SlickMapping.* - def findFor(ns: Namespace, device: DeviceId): Future[PaginationResult[ScheduledUpdate]] = db.run { - Schema.scheduledUpdates + protected[db] def findAction(ns: Namespace, + deviceId: DeviceId, + offset: Offset = 0L.toOffset, + limit: Limit = 10L.toLimit): DBIO[PaginationResult[Update]] = + Schema.updates .filter(_.namespace === ns) - .filter(_.deviceId === device) - .result - .map(seq => PaginationResult(seq, seq.length, 0, seq.length)) - } + .filter(_.deviceId === deviceId) + .paginateAndSortResult(_.createdAt.desc, offset, limit) - def setStatus[T <: StatusInfo: Encoder](ns: Namespace, - id: ScheduledUpdateId, - newStatus: ScheduledUpdate.Status, - info: Option[T] = None): Future[Unit] = db.run { - setStatusAction(ns, id, newStatus, info) + def persist(update: Update): Future[Unit] = db.run { + persistMany(Seq(update)) } - def filterActiveUpdateExists(ns: Namespace, devices: Set[DeviceId]): Future[Set[DeviceId]] = - db.run { - Schema.scheduledUpdates - .filter(_.namespace === ns) - .filter(_.deviceId.inSet(devices)) - .filterNot( - _.status.inSet(Set(ScheduledUpdate.Status.Completed, ScheduledUpdate.Status.Cancelled)) - ) - .map(_.deviceId) - .result - .map(_.toSet) - } + protected[db] def persistMany(updates: Seq[Update]): DBIO[Unit] = + (Schema.updates ++= updates).map(_ => ()) + + protected[db] def cancelUpdateAction( + ns: Namespace, + updateId: UpdateId, + deviceId: Option[DeviceId], + allowInFlightCancellation: Boolean): DBIO[NonEmptyList[(CorrelationId, DeviceId)]] = - protected[db] def scheduledUpdateExistsFor(ns: Namespace, - deviceId: DeviceId): DBIO[Option[ScheduledUpdateId]] = - Schema.scheduledUpdates + Schema.updates .filter(_.namespace === ns) - .filter(_.deviceId === deviceId) - .filterNot( - _.status.inSet(Set(ScheduledUpdate.Status.Completed, ScheduledUpdate.Status.Cancelled)) - ) - .map(_.id) - .take(1) + .filter(_.id === updateId) + .maybeFilter(_.deviceId === deviceId) + .forUpdate .result - .headOption - - protected[db] def setStatusAction[T <: StatusInfo: Encoder](ns: Namespace, - id: ScheduledUpdateId, - newStatus: ScheduledUpdate.Status, - info: Option[T] = None): DBIO[Unit] = - Schema.scheduledUpdates + .flatMap { + case updates if updates.isEmpty => DBIO.failed(MissingEntity[Update]()) + case updates + if updates.forall(u => + u.status == Update.Status.Assigned || u.status == Update.Status.Scheduled || + (u.status == Update.Status.Seen && allowInFlightCancellation) + ) => + DBIO.successful(updates) + case _ => DBIO.failed(Errors.UpdateCannotBeCancelled(updateId)) + } + .flatMap { updates => + Schema.updates + .filter(_.namespace === ns) + .filter(_.id === updateId) + .maybeFilter(_.deviceId === deviceId) + .map(r => (r.status, r.completedAt)) + .update((Update.Status.Cancelled, Some(Instant.now))) + .map(_ => updates.map(u => u.correlationId -> u.deviceId)) + } + .map { updates => + NonEmptyList.fromListUnsafe(updates.toList) + } // safe because we check if `updates` is empty above + + protected[db] def setStatusAction[T: Encoder](ns: Namespace, + id: UpdateId, + newStatus: Update.Status, + info: Option[T] = None): DBIO[Unit] = + Schema.updates .filter(_.namespace === ns) .filter(_.id === id) .map(r => (r.status, r.statusInfo)) .update(newStatus -> info.map(_.asJson)) - .handleSingleUpdateError(MissingEntity[ScheduledUpdate]()) + .handleSingleUpdateError(MissingEntity[Update]()) .map(_ => ()) + protected[db] def filterActiveUpdateExistsAction(ns: Namespace, + devices: Set[DeviceId]): DBIO[Set[DeviceId]] = + Schema.updates + .filter(_.namespace === ns) + .filter(_.deviceId.inSet(devices)) + .filterNot(_.status.inSet(Set(Update.Status.Completed, Update.Status.Cancelled))) + .map(_.deviceId) + .result + .map(_.toSet) + + def findFor(ns: Namespace, device: DeviceId): Future[Seq[Update]] = db.run { + Schema.updates + .filter(_.namespace === ns) + .filter(_.deviceId === device) + .result + } + + def ensureNoUpdateFor(ns: Namespace, correlationIds: Set[CorrelationId]): DBIO[Unit] = + Schema.updates + .filter(_.namespace === ns) + .filter(_.correlationId.inSet(correlationIds)) + .map(_.correlationId) + .result + .flatMap { + case id +: ids => + DBIO.failed(Errors.AssignmentBelongsToUpdate(NonEmptyList(id, ids.toList))) + case _ => DBIO.successful(()) + } + +} + +trait UpdatesRepositorySupport { + + def updatesRepository(implicit db: Database, ec: ExecutionContext) = + new UpdatesRepository() + } diff --git a/src/main/scala/com/advancedtelematic/director/db/Schema.scala b/src/main/scala/com/advancedtelematic/director/db/Schema.scala index 7e9fda13..730c6fb6 100644 --- a/src/main/scala/com/advancedtelematic/director/db/Schema.scala +++ b/src/main/scala/com/advancedtelematic/director/db/Schema.scala @@ -1,28 +1,18 @@ package com.advancedtelematic.director.db -import akka.http.scaladsl.model.Uri -import com.advancedtelematic.director.data.DataType.{ - AdminRoleName, - ScheduledUpdate, - ScheduledUpdateId -} +import org.apache.pekko.http.scaladsl.model.Uri +import com.advancedtelematic.director.data.DataType.{AdminRoleName, TargetSpecId, Update, UpdateId} import com.advancedtelematic.director.data.DbDataType.* import com.advancedtelematic.libats.data.DataType.{Checksum, CorrelationId, Namespace} -import com.advancedtelematic.libats.data.EcuIdentifier -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libats.slick.db.SlickCirceMapper.jsonMapper import com.advancedtelematic.libtuf.data.TufDataType.RoleType.RoleType -import com.advancedtelematic.libtuf.data.TufDataType.{ - HardwareIdentifier, - JsonSignedPayload, - RepoId, - TargetFilename, - TargetName, - TufKey -} +import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, JsonSignedPayload, RepoId, TargetFilename, TargetName, TufKey} import io.circe.Json import slick.jdbc.MySQLProfile.api.* import SlickMapping.* +import com.advancedtelematic.libats.slick.codecs.SlickRefined +import eu.timepit.refined.string.HexStringSpec import java.time.Instant @@ -199,7 +189,7 @@ object Schema { class HardwareUpdatesTable(tag: Tag) extends Table[HardwareUpdate](tag, "hardware_updates") { def namespace = column[Namespace]("namespace") - def id = column[UpdateId]("id") + def id = column[TargetSpecId]("id") def hardwareId = column[HardwareIdentifier]("hardware_identifier") def toTarget = column[EcuTargetId]("to_target_id") def fromTarget = column[Option[EcuTargetId]]("from_target_id") @@ -242,40 +232,49 @@ object Schema { protected[db] val autoUpdates = TableQuery[AutoUpdateDefinitionTable] class DeviceManifestsTable(tag: Tag) - extends Table[(DeviceId, Json, SHA256Checksum, Instant)](tag, "device_manifests") { + extends Table[(DeviceId, Json, MurmurHash3Checksum, Instant)](tag, "device_manifests") { def deviceId = column[DeviceId]("device_id") def targetName = column[TargetName]("target_name") def receivedAt = column[Instant]("received_at")(javaInstantMapping) - def sha256 = column[SHA256Checksum]("sha256") + def checksum = column[MurmurHash3Checksum]("checksum") def manifest = column[Json]("manifest") - def pk = primaryKey("device-manifests-pk", (deviceId, sha256)) + def pk = primaryKey("device-manifests-pk", (deviceId, checksum)) - override def * = (deviceId, manifest, sha256, receivedAt) + override def * = (deviceId, manifest, checksum, receivedAt) } protected[db] val deviceManifests = TableQuery[DeviceManifestsTable] - class ScheduledUpdatesTable(tag: Tag) extends Table[ScheduledUpdate](tag, "scheduled_updates") { + class UpdatesTable(tag: Tag) extends Table[Update](tag, "updates") { def namespace = column[Namespace]("namespace") - def id = column[ScheduledUpdateId]("id") + def id = column[UpdateId]("id") def deviceId = column[DeviceId]("device_id") - def updateId = column[UpdateId]("hardware_update_id") - def scheduledAt = column[Instant]("scheduled_at")(javaInstantMapping) - def status = column[ScheduledUpdate.Status]("status") + def correlationId = column[CorrelationId]("correlation_id") + def targetSpecId = column[TargetSpecId]("hardware_update_id") + def scheduledFor = column[Option[Instant]]("scheduled_for")(javaInstantMapping.optionType) + def status = column[Update.Status]("status") def statusInfo = column[Option[Json]]("status_info") - + def completedAt = column[Option[Instant]]("completed_at")(javaInstantMapping.optionType) def createdAt = column[Instant]("created_at")(javaInstantMapping) - def updatedAt = column[Instant]("updated_at")(javaInstantMapping) - def pk = primaryKey("scheduled-updates-pk", id) + def pk = primaryKey("updates-pk", id -> deviceId) - override def * = (namespace, id, deviceId, updateId, scheduledAt, status) <> ( - (ScheduledUpdate.apply _).tupled, - ScheduledUpdate.unapply - ) + override def * = + ( + namespace, + id, + deviceId, + correlationId, + targetSpecId, + createdAt, + scheduledFor, + status, + completedAt + ) <> ((Update.apply _).tupled, Update.unapply) } - protected[db] val scheduledUpdates = TableQuery[ScheduledUpdatesTable] + protected[db] val updates = TableQuery[UpdatesTable] + } diff --git a/src/main/scala/com/advancedtelematic/director/db/SignedRoleMigration.scala b/src/main/scala/com/advancedtelematic/director/db/SignedRoleMigration.scala index edb7744c..99328939 100644 --- a/src/main/scala/com/advancedtelematic/director/db/SignedRoleMigration.scala +++ b/src/main/scala/com/advancedtelematic/director/db/SignedRoleMigration.scala @@ -4,10 +4,10 @@ import java.sql.Timestamp import java.time.Instant import java.util.UUID -import akka.http.scaladsl.util.FastFuture -import akka.stream.Materializer -import akka.stream.scaladsl.{Flow, Sink, Source} -import akka.{Done, NotUsed} +import org.apache.pekko.http.scaladsl.util.FastFuture +import org.apache.pekko.stream.Materializer +import org.apache.pekko.stream.scaladsl.{Flow, Sink, Source} +import org.apache.pekko.{Done, NotUsed} import com.advancedtelematic.libats.codecs.CirceCodecs.checkSumCodec import com.advancedtelematic.libats.data.DataType.Checksum import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId diff --git a/src/main/scala/com/advancedtelematic/director/db/SlickMapping.scala b/src/main/scala/com/advancedtelematic/director/db/SlickMapping.scala index 724d6faa..8c8c84da 100644 --- a/src/main/scala/com/advancedtelematic/director/db/SlickMapping.scala +++ b/src/main/scala/com/advancedtelematic/director/db/SlickMapping.scala @@ -2,15 +2,12 @@ package com.advancedtelematic.director.db import com.advancedtelematic.director.data.AdminDataType.TargetUpdate import com.advancedtelematic.director.data.Codecs.* -import com.advancedtelematic.director.data.DataType.{AdminRoleName, ScheduledUpdate} +import com.advancedtelematic.director.data.DataType.{AdminRoleName, Update} import com.advancedtelematic.libats.data.DataType.HashMethod import com.advancedtelematic.libats.data.DataType.HashMethod.HashMethod import com.advancedtelematic.libats.slick.codecs.SlickEnumeratum.enumeratumMapper import com.advancedtelematic.libats.slick.db.SlickCirceMapper -import com.advancedtelematic.libtuf.data.ValidatedString.{ - ValidatedString, - ValidatedStringValidation -} +import com.advancedtelematic.libtuf.data.ValidatedString.{ValidatedString, ValidatedStringValidation} import slick.jdbc.MySQLProfile.api.* import java.time.Instant @@ -40,9 +37,7 @@ object SlickMapping { implicit val adminRoleNameMapper: BaseColumnType[AdminRoleName] = validatedStringMapper[AdminRoleName] - implicit val scheduledUpdatesMapper: BaseColumnType[ScheduledUpdate.Status] = enumeratumMapper( - ScheduledUpdate.Status - ) + implicit val updateStatusMapper: BaseColumnType[Update.Status] = enumeratumMapper(Update.Status) implicit def instantOrdering: Ordering[Instant] = Ordering.fromLessThan(_ isBefore _) } diff --git a/src/main/scala/com/advancedtelematic/director/db/MultiTargetUpdates.scala b/src/main/scala/com/advancedtelematic/director/db/TargetUpdateSpecs.scala similarity index 62% rename from src/main/scala/com/advancedtelematic/director/db/MultiTargetUpdates.scala rename to src/main/scala/com/advancedtelematic/director/db/TargetUpdateSpecs.scala index d6aa4466..de95f06a 100644 --- a/src/main/scala/com/advancedtelematic/director/db/MultiTargetUpdates.scala +++ b/src/main/scala/com/advancedtelematic/director/db/TargetUpdateSpecs.scala @@ -1,28 +1,34 @@ package com.advancedtelematic.director.db import com.advancedtelematic.director.data.AdminDataType.{ - MultiTargetUpdate, TargetUpdate, - TargetUpdateRequest + TargetUpdateRequest, + TargetUpdateSpec } +import com.advancedtelematic.director.data.DataType.TargetSpecId import com.advancedtelematic.director.data.DbDataType.{EcuTarget, EcuTargetId, HardwareUpdate} +import com.advancedtelematic.director.http.Errors.InvalidMtu import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.messaging_datatype.DataType.UpdateId import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier -import slick.jdbc.MySQLProfile.api._ +import slick.jdbc.MySQLProfile.api.* import scala.concurrent.{ExecutionContext, Future} -class MultiTargetUpdates(implicit val db: Database, val ec: ExecutionContext) +class TargetUpdateSpecs(implicit val db: Database, val ec: ExecutionContext) extends HardwareUpdateRepositorySupport with EcuTargetsRepositorySupport { - def create(ns: Namespace, multiTargetUpdate: MultiTargetUpdate): Future[UpdateId] = { - require(multiTargetUpdate.targets.nonEmpty, "multiTargetUpdate.targets cannot be empty") + def create(ns: Namespace, targetUpdateSpec: TargetUpdateSpec): Future[TargetSpecId] = + db.run(createAction(ns, targetUpdateSpec).transactionally) - val updateId = UpdateId.generate() + protected[db] def createAction(ns: Namespace, + targetUpdateSpec: TargetUpdateSpec): DBIO[TargetSpecId] = { + if (targetUpdateSpec.targets.isEmpty) + throw InvalidMtu("multiTargetUpdate.targets cannot be empty") - val hardwareUpdates = multiTargetUpdate.targets.map { case (hwId, targetUpdateReq) => + val targetSpecId = TargetSpecId.generate() + + val hardwareUpdates = targetUpdateSpec.targets.map { case (hwId, targetUpdateReq) => val toId = EcuTargetId.generate() val t = targetUpdateReq.to @@ -55,17 +61,17 @@ class MultiTargetUpdates(implicit val db: Database, val ec: ExecutionContext) _ <- from.map(ecuTargetsRepository.persistAction).getOrElse(DBIO.successful(())) _ <- ecuTargetsRepository.persistAction(to) _ <- hardwareUpdateRepository.persistAction( - HardwareUpdate(ns, updateId, hwId, from.map(_.id), to.id) + HardwareUpdate(ns, targetSpecId, hwId, from.map(_.id), to.id) ) } yield () }.toVector - db.run(DBIO.sequence(hardwareUpdates).transactionally).map(_ => updateId) + DBIO.sequence(hardwareUpdates).map(_ => targetSpecId) } - def find(ns: Namespace, updateId: UpdateId): Future[MultiTargetUpdate] = + def find(ns: Namespace, TargetSpecId: TargetSpecId): Future[TargetUpdateSpec] = hardwareUpdateRepository - .findUpdateTargets(ns, updateId) + .findUpdateTargets(ns, TargetSpecId) .map { hardwareUpdates => hardwareUpdates.foldLeft(Map.empty[HardwareIdentifier, TargetUpdateRequest]) { case (acc, (hu, fromO, toU)) => @@ -79,6 +85,6 @@ class MultiTargetUpdates(implicit val db: Database, val ec: ExecutionContext) acc + (hu.hardwareId -> TargetUpdateRequest(from, to)) } } - .map(targets => MultiTargetUpdate(targets)) + .map(targets => TargetUpdateSpec(targets)) } diff --git a/src/main/scala/com/advancedtelematic/director/db/UpdateSchedulerDBIO.scala b/src/main/scala/com/advancedtelematic/director/db/UpdateSchedulerDBIO.scala index 0e12e241..a1af1045 100644 --- a/src/main/scala/com/advancedtelematic/director/db/UpdateSchedulerDBIO.scala +++ b/src/main/scala/com/advancedtelematic/director/db/UpdateSchedulerDBIO.scala @@ -1,215 +1,99 @@ package com.advancedtelematic.director.db -import cats.data.Validated.{Invalid, Valid} -import cats.data.{NonEmptyList, Validated, ValidatedNel} import cats.implicits.* -import com.advancedtelematic.director.data.DataType.{ScheduledUpdate, ScheduledUpdateId, StatusInfo} -import com.advancedtelematic.director.deviceregistry.data.DeviceStatus -import com.advancedtelematic.director.db.deviceregistry.DeviceRepository -import com.advancedtelematic.director.db.Schema -import com.advancedtelematic.director.data.DbDataType.{Assignment, EcuTargetId} -import com.advancedtelematic.director.http.Errors.UpdateScheduleError -import com.advancedtelematic.libats.data.DataType.MultiTargetUpdateId -import com.advancedtelematic.libats.data.EcuIdentifier -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} -import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier -import eu.timepit.refined.api.RefType +import com.advancedtelematic.director.data.DataType.Update +import com.advancedtelematic.director.data.DbDataType.Assignment +import com.advancedtelematic.director.http.Errors +import io.circe.Json import io.circe.syntax.EncoderOps -import io.circe.{Codec, Encoder, Json} import org.slf4j.LoggerFactory import slick.jdbc.MySQLProfile.api.* -import slick.jdbc.{GetResult, SetParameter, TransactionIsolation} - +import slick.jdbc.TransactionIsolation +import com.advancedtelematic.libats.codecs.CirceRefined.* +import com.advancedtelematic.libats.data.ErrorRepresentation.* import java.time.Instant -import java.util.UUID import scala.concurrent.{ExecutionContext, Future} class UpdateSchedulerDBIO()(implicit val db: Database, val ec: ExecutionContext) - extends ScheduledUpdatesRepositorySupport + extends UpdatesRepositorySupport with AssignmentsRepositorySupport with ProvisionedDeviceRepositorySupport { - import UpdateSchedulerDBIO.* private lazy val log = LoggerFactory.getLogger(this.getClass) - private implicit val setUpdateId: SetParameter[UpdateId] = - SetParameter.SetString.contramap(_.uuid.toString) - - private implicit val setDeviceId: SetParameter[DeviceId] = - SetParameter.SetString.contramap(_.uuid.toString) - // Go to db, lock and fetch updates to assign - private def findScheduledUpdates(): DBIO[Seq[ScheduledUpdate]] = { + private def findScheduledUpdates(): DBIO[Seq[Update]] = { // TODO: To run more than one instance of the scheduler we need to use `FOR UPDATE SKIP LOCKED` but this requires mariadb 10.6 import SlickMapping.* - Schema.scheduledUpdates - .filter(_.status === (ScheduledUpdate.Status.Scheduled: ScheduledUpdate.Status)) - .filter(_.scheduledAt < Instant.now()) - .sortBy(_.scheduledAt.desc) + Schema.updates + .filter(_.status === (Update.Status.Scheduled: Update.Status)) + .filter(_.scheduledFor < Instant.now()) + .sortBy(_.scheduledFor.desc) .take(10) .forUpdate .result } - case class EcuInfoRow(ecuTargetId: EcuTargetId, - ecuIdentifier: Option[EcuIdentifier], - updateHardwareId: HardwareIdentifier, - ecuHardwareId: Option[HardwareIdentifier], - fromTargetId: Option[EcuTargetId], - currentTargetId: Option[EcuTargetId], - assignmentExists: Boolean) - - private def findEcuSerialsForUpdate(deviceId: DeviceId, updateId: UpdateId) - : DBIO[ValidatedNel[InvalidReason, NonEmptyList[(EcuTargetId, EcuIdentifier)]]] = { - - implicit val getResult: GetResult[EcuInfoRow] = GetResult { pr => - EcuInfoRow( - EcuTargetId(UUID.fromString(pr.nextString())), - pr.nextStringOption().map(str => EcuIdentifier.from(str).valueOr(throw _)), - RefType - .applyRef[HardwareIdentifier](pr.nextString()) - .valueOr(err => throw new IllegalArgumentException(err)), - pr.nextStringOption().flatMap(str => RefType.applyRef[HardwareIdentifier](str).toOption), - pr.nextStringOption().map(str => EcuTargetId(UUID.fromString(str))), - pr.nextStringOption().map(str => EcuTargetId(UUID.fromString(str))), - pr.nextStringOption().isDefined + private def startUpdate(update: Update): DBIO[Unit] = + new AffectedEcusDBIO() + .findAffectedEcusAction( + update.ns, + Seq(update.deviceId), + update.targetSpecId, + ignoreScheduledUpdates = true ) - } - - val ecusio = - sql""" - SELECT ecu_targets.id, ecus.ecu_serial, hu.hardware_identifier, ecus.hardware_identifier, hu.from_target_id, ecus.current_target, a.ecu_serial - FROM #${Schema.hardwareUpdates.baseTableRow.tableName} hu - JOIN #${Schema.ecuTargets.baseTableRow.tableName} ecu_targets ON hu.to_target_id = ecu_targets.id - LEFT JOIN #${Schema.allEcus.baseTableRow.tableName} ecus ON ecus.hardware_identifier = hu.hardware_identifier AND ecus.deleted = false AND ecus.device_id = $deviceId - LEFT JOIN #${Schema.assignments.baseTableRow.tableName} a ON a.ecu_serial = ecus.ecu_serial - WHERE - hu.id = $updateId - """.as[EcuInfoRow] - - ecusio.map { ecus => - if (ecus.isEmpty) - HardwareUpdateMissing(updateId).invalidNel - else - NonEmptyList - .fromListUnsafe(ecus.toList) - .map { - case EcuInfoRow( - ecuTargetId, - Some(ecuSerial), - _, - _, - fromTargetId, - currentTargetId, - false - ) if fromTargetId == currentTargetId || fromTargetId.isEmpty => - (ecuTargetId, ecuSerial).validNel[InvalidReason] - case EcuInfoRow( - _, - Some(_), - _, - _, - fromTargetId, - currentTargetId, - false - ) => // incompatible fromTarget in compatible ecu - IncompatibleFromTarget(updateId, currentTargetId, fromTargetId).invalidNel - case EcuInfoRow( - _, - None, - updateHardwareId, - ecuHardwareId, - _, - _, - _ - ) => // no compatible ecu found - IncompatibleEcuHardware(updateId, updateHardwareId, ecuHardwareId).invalidNel - case EcuInfoRow(_, _, _, _, _, _, true) => // an assignment already exists for ecu - EcuAssignmentExists(updateId).invalidNel - } - .sequence - } - } - - private def startUpdate(update: ScheduledUpdate): DBIO[Unit] = - findEcuSerialsForUpdate(update.deviceId, update.updateId).flatMap { - case Validated.Valid(ecuTargetIds) => - val correlationId = MultiTargetUpdateId(update.updateId.uuid) - - val assignments = ecuTargetIds.map { case (ecuTargetId, ecuSerial) => - Assignment( - update.ns, - update.deviceId, - ecuSerial, - ecuTargetId, - correlationId, - inFlight = false, - createdAt = Instant.now() - ) - } - - log - .atDebug() - .addKeyValue("scheduled-update-id", update.id.uuid.toString) - .addKeyValue("device-id", update.deviceId.uuid.toString) - .log(s"creating ${assignments.length} for ecu ${update.deviceId}") - - DBIO.seq( - assignmentsRepository.persistManyDBIO(provisionedDeviceRepository)(assignments.toList), - scheduledUpdatesRepository.setStatusAction( - update.ns, - update.id, - ScheduledUpdate.Status.Assigned - ) - ) - case Validated.Invalid(errors) => - log - .atInfo() - .addKeyValue("scheduled-update-id", update.id.uuid.toString) - .addKeyValue("device-id", update.deviceId.uuid.toString) - .addKeyValue("errors", errors.asJson) - .log(s"cancelling scheduled update for ${update.deviceId}") - - scheduledUpdatesRepository.setStatusAction( - update.ns, - update.id, - ScheduledUpdate.Status.Cancelled, - InvalidEcuStatus(errors).some - ) - } + .flatMap { assignmentResult => + val notAffected = assignmentResult.notAffected.get(update.deviceId) + + notAffected match { + case Some(errors) => + log + .atInfo() + .addKeyValue("update-id", update.id.uuid.toString) + .addKeyValue("device-id", update.deviceId.uuid.toString) + .addKeyValue("errors", errors.view.mapValues(_.toErrorRepr).toMap.asJson) + .log(s"cancelling scheduled update for ${update.deviceId}") + + updatesRepository.setStatusAction( + update.ns, + update.id, + Update.Status.Cancelled, + Errors.DeviceCannotBeUpdated(update.deviceId, errors).toErrorRepr.some + ) - def validateAndPersist(scheduledUpdate: ScheduledUpdate): Future[ScheduledUpdateId] = db.run { - scheduledUpdatesRepository - .scheduledUpdateExistsFor(scheduledUpdate.ns, scheduledUpdate.deviceId) - .flatMap { - case Some(id) => - DBIO.failed( - UpdateScheduleError( - scheduledUpdate.deviceId, - NonEmptyList.of(ScheduledUpdateExists(id): InvalidReason) + case None => + val ecuTargetIds = assignmentResult.affected + + val assignments = ecuTargetIds.map { case (ecu, ecuSerial) => + Assignment( + update.ns, + update.deviceId, + ecu.ecuSerial, + ecuSerial, + update.correlationId, + inFlight = false, + createdAt = Instant.now() + ) + } + + log + .atDebug() + .addKeyValue("update-id", update.id.uuid.toString) + .addKeyValue("device-id", update.deviceId.uuid.toString) + .log(s"creating ${assignments.length} for ecu ${update.deviceId}") + + DBIO.seq( + assignmentsRepository + .persistManyDBIO(provisionedDeviceRepository)(assignments.toList), + updatesRepository + .setStatusAction[Json](update.ns, update.id, Update.Status.Assigned, None) ) - ) - case None => - findEcuSerialsForUpdate(scheduledUpdate.deviceId, scheduledUpdate.updateId).flatMap { - case Valid(_) => - for { - id <- scheduledUpdatesRepository.persistAction(scheduledUpdate) - _ <- DeviceRepository.setDeviceStatusAction( - scheduledUpdate.deviceId, - DeviceStatus.UpdateScheduled - ) - } yield id - case Invalid(errors) => - DBIO.failed(UpdateScheduleError(scheduledUpdate.deviceId, errors)) - } + } } - .withTransactionIsolation(TransactionIsolation.Serializable) - .transactionally - } - def run(): Future[Seq[ScheduledUpdate]] = { + def run(): Future[Seq[Update]] = { val scheduled = findScheduledUpdates() val io = scheduled.flatMap { seq => @@ -220,42 +104,3 @@ class UpdateSchedulerDBIO()(implicit val db: Database, val ec: ExecutionContext) } } - -object UpdateSchedulerDBIO { - - import com.advancedtelematic.libats.codecs.CirceRefined.* - import io.circe.generic.auto.* - import io.circe.syntax.* - - sealed trait InvalidReason - - case class IncompatibleFromTarget(updateId: UpdateId, - ecuFromTarget: Option[EcuTargetId], - updateFromTarget: Option[EcuTargetId]) - extends InvalidReason - - case class ScheduledUpdateExists(scheduledUpdateId: ScheduledUpdateId) extends InvalidReason - - case class IncompatibleEcuHardware(updateId: UpdateId, - updateHardware: HardwareIdentifier, - ecuHardware: Option[HardwareIdentifier]) - extends InvalidReason - - case class EcuAssignmentExists(updateId: UpdateId) extends InvalidReason - - case class HardwareUpdateMissing(updateId: UpdateId) extends InvalidReason - - implicit val ecuStatusForUpdateInvalidReasonEncoder: Encoder[InvalidReason] = Encoder.instance { - case e: IncompatibleFromTarget => Json.obj("incompatible_from_target" -> e.asJson) - case e: IncompatibleEcuHardware => Json.obj("incompatible_ecu_hardware" -> e.asJson) - case e: EcuAssignmentExists => Json.obj("ecu_assignment_exists" -> e.asJson) - case e: HardwareUpdateMissing => Json.obj("hardware_update_missing" -> e.asJson) - case e: ScheduledUpdateExists => Json.obj("scheduled_update_exists" -> e.asJson) - } - - case class InvalidEcuStatus(reasons: NonEmptyList[InvalidReason]) extends StatusInfo - - implicit val invalidEcuStatusCodec: Codec[InvalidEcuStatus] = - io.circe.generic.semiauto.deriveCodec[InvalidEcuStatus] - -} diff --git a/src/main/scala/com/advancedtelematic/director/db/UpdatesDBIO.scala b/src/main/scala/com/advancedtelematic/director/db/UpdatesDBIO.scala new file mode 100644 index 00000000..762ddc3c --- /dev/null +++ b/src/main/scala/com/advancedtelematic/director/db/UpdatesDBIO.scala @@ -0,0 +1,400 @@ +package com.advancedtelematic.director.db + +import cats.Traverse +import cats.data.NonEmptyList +import cats.syntax.functor.* +import cats.syntax.foldable.* +import com.advancedtelematic.libats.data.PaginationResult.* +import cats.implicits.catsSyntaxOptionId +import com.advancedtelematic.libats.messaging_datatype.MessageCodecs.deviceUpdateCompletedCodec +import com.advancedtelematic.libats.codecs.CirceCodecs.* +import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* +import com.advancedtelematic.director.data.AdminDataType.TargetUpdateSpec +import com.advancedtelematic.director.data.DataType.{TargetSpecId, Update, UpdateId} +import com.advancedtelematic.director.data.DbDataType.{Assignment, Ecu, EcuTarget, EcuTargetId} +import com.advancedtelematic.director.db.deviceregistry.{EventJournal, InstallationReportRepository} +import com.advancedtelematic.director.http.DeviceAssignments.AssignmentCreateResult +import com.advancedtelematic.director.http.{ + Errors, + UpdateDetailResponse, + UpdateEventResponse, + UpdateReportedResult, + UpdateResponse, + UpdateResultResponse +} +import com.advancedtelematic.libats.data.DataType.{CorrelationId, Namespace, UpdateCorrelationId} +import com.advancedtelematic.libats.data.PaginationResult +import com.advancedtelematic.libats.http.Errors.MissingEntity +import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.messaging_datatype.Messages.DeviceUpdateCompleted +import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier +import org.slf4j.LoggerFactory +import slick.jdbc.MySQLProfile.api.* +import slick.jdbc.TransactionIsolation + +import java.time.Instant +import scala.concurrent.{ExecutionContext, Future} +import scala.language.implicitConversions + +class UpdatesDBIO()(implicit val db: Database, val ec: ExecutionContext) + extends AssignmentsRepositorySupport + with UpdatesRepositorySupport + with ProvisionedDeviceRepositorySupport { + + private val targetUpdateSpecs = new TargetUpdateSpecs() + private val affectedEcusDBIO = new AffectedEcusDBIO() + private val eventJournal = new EventJournal() + + private lazy val log = LoggerFactory.getLogger(this.getClass) + + import com.advancedtelematic.libats.slick.db.SlickUrnMapper.* + + def cancel(ns: Namespace, + updateId: UpdateId, + deviceId: DeviceId, + allowInFlightCancellation: Boolean): Future[CorrelationId] = { + val io = for { + ids <- updatesRepository.cancelUpdateAction( + ns, + updateId, + Option(deviceId), + allowInFlightCancellation + ) + assignmentsToCancelQuery = Schema.assignments + .filter(_.deviceId === deviceId) + .filter(_.correlationId.inSet(ids.map(_._1).toList.toSet)) + _ <- assignmentsRepository + .processDeviceCancellationAction(provisionedDeviceRepository)( + assignmentsToCancelQuery, + allowInFlightCancellation + ) + } yield ids.head._1 + + db.run(io.withTransactionIsolation(TransactionIsolation.Serializable).transactionally) + } + + def createFor(ns: Namespace, + deviceId: DeviceId, + targetsSpec: TargetUpdateSpec, + scheduledFor: Option[Instant]): Future[UpdateId] = { + val io = for { + _ <- provisionedDeviceRepository.ensureExistsIO(deviceId) + targetSpecId <- targetUpdateSpecs.createAction(ns, targetsSpec) + affectedResult <- affectedEcusDBIO.findAffectedEcusAction(ns, Seq(deviceId), targetSpecId) + _ <- affectedResult.notAffected.get(deviceId) match { + case Some(errorMap) => + DBIO.failed(Errors.DeviceCannotBeUpdated(deviceId, errorMap)) + case None => + DBIO.successful(()) + } + status = + if (scheduledFor.isDefined) + Update.Status.Scheduled + else + Update.Status.Assigned + updateId <- updatesDBIO( + ns, + targetSpecId, + affectedResult.affected.map(_._1.deviceId).toSet, + status, + scheduledFor + ) + _ <- + if (status == Update.Status.Assigned) + assignmentsDBIO(ns, updateId.toCorrelationId, affectedResult.affected) + else + DBIO.successful(()) + } yield updateId + + // we could use FOR UPDATE on the devices table to lock the rows + // (like an advisory lock) instead of using SERIALIZABLE if performance becomes an issue + db.run(io.withTransactionIsolation(TransactionIsolation.Serializable).transactionally) + } + + def createMany(ns: Namespace, + targetsSpec: TargetUpdateSpec, + devices: Seq[DeviceId]): Future[(UpdateId, AssignmentCreateResult)] = { + val io = for { + targetSpecId <- targetUpdateSpecs.createAction(ns, targetsSpec) + affectedResult <- affectedEcusDBIO.findAffectedEcusAction(ns, devices, targetSpecId) + _ = if (affectedResult.affected.isEmpty) { + log.warn(s"no devices affected by this assignment") + } + updateId <- updatesDBIO( + ns, + targetSpecId, + affectedResult.affected.map(_._1.deviceId).toSet, + Update.Status.Assigned, + None + ) + devices <- assignmentsDBIO(ns, updateId.toCorrelationId, affectedResult.affected) + } yield (updateId, AssignmentCreateResult(devices, affectedResult.notAffectedSerializable)) + + db.run(io.withTransactionIsolation(TransactionIsolation.Serializable).transactionally) + } + + def cancelAll(ns: Namespace, + updateId: UpdateId): Future[NonEmptyList[(CorrelationId, DeviceId)]] = { + val io = for { + updatesCorrelationIds <- updatesRepository.cancelUpdateAction( + ns, + updateId, + deviceId = None, + allowInFlightCancellation = false + ) + query = Schema.assignments + .filter(_.deviceId.inSet(updatesCorrelationIds.map(_._2).toList.toSet)) + .filter(_.correlationId.inSet(updatesCorrelationIds.map(_._1).toList.toSet)) + _ <- assignmentsRepository + .processDeviceCancellationAction(provisionedDeviceRepository)( + query, + allowInFlightCancellation = false + ) + } yield updatesCorrelationIds + + db.run(io.withTransactionIsolation(TransactionIsolation.Serializable).transactionally) + } + + private def updatesDBIO(ns: Namespace, + targetSpecId: TargetSpecId, + devices: Set[DeviceId], + status: Update.Status, + scheduledFor: Option[Instant]): DBIO[UpdateId] = { + + val id = UpdateId.generate() + + val updates = devices.map { deviceId => + Update(ns, id, deviceId, id.toCorrelationId, targetSpecId, Instant.now, scheduledFor, status) + } + + updatesRepository.persistMany(updates.toSeq).map(_ => id) + } + + private def assignmentsDBIO(ns: Namespace, + correlationId: UpdateCorrelationId, + ecus: Seq[(Ecu, EcuTargetId)]): DBIO[Seq[DeviceId]] = { + val assignments = ecus.map { case (ecu, toTargetId) => + Assignment( + ns, + ecu.deviceId, + ecu.ecuSerial, + toTargetId, + correlationId, + inFlight = false, + createdAt = Instant.now + ) + } + + assignmentsRepository.persistManyDBIO(provisionedDeviceRepository)(assignments).map { _ => + assignments.map(_.deviceId).distinct + } + } + + def find(ns: Namespace, updateId: UpdateId, deviceId: DeviceId): Future[UpdateDetailResponse] = { + val io = for { + (update, targets) <- findSingleUpdateEcuTargets(ns, deviceId, updateId) + _ <- assignmentsRepository.findByCorrelationIdsAction(ns, Set(update.correlationId)) + processedAssignments <- assignmentsRepository.findAllProcessedByCorrelatioIds( + ns, + deviceId, + Set(update.correlationId) + ) + ecuReports <- InstallationReportRepository.fetchEcuInstallationReport( + deviceId, + update.correlationId + ) + deviceInstallationReport <- InstallationReportRepository + .fetchDeviceInstallationResultByCorrelationId(deviceId, update.correlationId) + } yield UpdateDetailResponse( + update.id, + update.status, + update.createdAt, + scheduledFor = update.scheduledFor, + packages = targets.view.mapValues(_.filename).toMap, + completedAt = update.completedAt, + deviceResult = deviceInstallationReport.map { r => + UpdateReportedResult(r.resultCode, r.success, r.description) + }, + ecuResults = targets.flatMap { case (hwId, ecuTarget) => + val processedAssignment = processedAssignments.find(a => + a.correlationId == update.correlationId && a.ecuTargetId == ecuTarget.id + ) + + // `.description` is only in ecuReport for newer updates, so we need to fetch it from + // deviceInstallationReport.installationReport.ecuReports[ecuId].result.description + // until we migrate them to EcuInstallationReport + val ecuReportsOnDeviceReport = deviceInstallationReport + .flatMap { report => + report.installationReport.as[DeviceUpdateCompleted].toOption.map { msg => + msg.ecuReports + } + } + .getOrElse(Map.empty) + + processedAssignment.map { pa => + val updateEcuReport = ecuReports + .get(pa.ecuId) + .map { er => + val ecuReportOnDeviceReport = + ecuReportsOnDeviceReport.get(er.ecuId).map(_.result.description.value) + val desc = er.description.orElse(ecuReportOnDeviceReport) + UpdateReportedResult(er.resultCode, er.success, desc) + } + + pa.ecuId -> UpdateResultResponse( + hwId, + ecuTarget.filename, + pa.successful, + pa.result.getOrElse(""), + updateEcuReport + ) + } + } + ) + + db.run(io.transactionally) + } + + import com.advancedtelematic.libats.slick.db.SlickAnyVal.* + import com.advancedtelematic.libats.slick.db.SlickExtensions.* + + private def findSingleUpdateEcuTargets( + ns: Namespace, + deviceId: DeviceId, + updateId: UpdateId): DBIO[(Update, Map[HardwareIdentifier, EcuTarget])] = + Schema.updates + .filter(_.namespace === ns) + .filter(_.deviceId === deviceId) + .filter(_.id === updateId) + .resultHead(MissingEntity[Update]()) + .flatMap { update => + Schema.hardwareUpdates + .filter(_.id === update.targetSpecId) + .join(Schema.ecuTargets) + .on { case (hwUpdates, ecuTargets) => hwUpdates.toTarget === ecuTargets.id } + .result + .map { + _.map { case (hwUpdates, ecuTarget) => + hwUpdates.hardwareId -> ecuTarget + }.toMap + } + .map { ecuTargets => + update -> ecuTargets + } + } + + private def findUpdateEcuTargets[S[_]: Traverse]( + updates: S[Update]): DBIO[S[(Update, Map[HardwareIdentifier, EcuTarget])]] = { + val io = + Schema.hardwareUpdates + .filter(_.id.inSet(updates.map(_.targetSpecId).toList.toSet)) + .join(Schema.ecuTargets) + .on { case (hwUpdates, ecuTargets) => ecuTargets.id === hwUpdates.toTarget } + .result + .map { + _.foldLeft(Map.empty[TargetSpecId, Map[HardwareIdentifier, EcuTarget]]) { + case (acc, (hwUpdate, ecuTarget)) => + val existing = acc.getOrElse(hwUpdate.id, Map.empty) + acc + (hwUpdate.id -> (existing + (hwUpdate.hardwareId -> ecuTarget))) + } + } + .map { ecuTargets => + updates.map { update => + val targets = ecuTargets.getOrElse(update.targetSpecId, Map.empty) + (update, targets) + } + } + + io + } + + def findAll(ns: Namespace, + offset: Offset, + limit: Limit): Future[PaginationResult[UpdateResponse]] = { + val io = for { + updates <- Schema.updates + .filter(_.namespace === ns) + .distinctOn(_.id) + .paginateResult(offset, limit) + ids = updates.values.map(u => u.deviceId -> u.correlationId).toSet + deviceResults <- InstallationReportRepository.fetchManyDevicesInstallationResults(ids) + updateTargets <- findUpdateEcuTargets(updates) + } yield updateTargets.map { case (update, targets) => + UpdateResponse( + update.id, + update.status, + update.createdAt, + scheduledFor = update.scheduledFor, + packages = targets.view.mapValues(_.filename).toMap, + completedAt = update.completedAt, + deviceResult = deviceResults + .find(r => r.deviceId == update.deviceId && r.correlationId == update.correlationId) + .map(r => UpdateReportedResult(r.resultCode, r.success, r.description)) + ) + } + + db.run(io.transactionally) + } + + def findUpdateDevices(ns: Namespace, + updateId: UpdateId, + offset: Offset, + limit: Limit): Future[PaginationResult[DeviceId]] = db.run { + Schema.updates + .filter(_.namespace === ns) + .filter(_.id === updateId) + .map(_.deviceId) + .paginateResult(offset, limit) + } + + def findFor(ns: Namespace, + deviceId: DeviceId, + offset: Offset, + limit: Limit): Future[PaginationResult[UpdateResponse]] = { + val io = for { + updates <- updatesRepository.findAction(ns, deviceId, offset, limit) + updateTargets <- findUpdateEcuTargets(updates) + ids = updates.values.map(_.correlationId).toSet + deviceResults <- InstallationReportRepository.fetchManyByDevice(deviceId, ids) + } yield updateTargets.map { case (update, targets) => + UpdateResponse( + update.id, + update.status, + update.createdAt, + scheduledFor = update.scheduledFor, + packages = targets.view.mapValues(_.filename).toMap, + completedAt = update.completedAt, + deviceResult = deviceResults + .find(r => r.deviceId == update.deviceId && r.correlationId == update.correlationId) + .map(r => UpdateReportedResult(r.resultCode, r.success, r.description)) + ) + } + + db.run(io.transactionally) + } + + def findEvents(ns: Namespace, + deviceId: DeviceId, + updateId: UpdateId): Future[Seq[UpdateEventResponse]] = + db.run( + Schema.updates + .filter(_.namespace === ns) + .filter(_.deviceId === deviceId) + .filter(_.id === updateId) + .resultHead(MissingEntity[Update]()) + ).flatMap { update => + eventJournal.getEvents(deviceId, update.correlationId.some) + }.map { + _.map { event => + UpdateEventResponse( + event.deviceUuid, + event.eventType, + event.deviceTime, + event.receivedAt, + event.payload.hcursor.downField("success").as[Boolean].toOption, + event.ecu + ) + } + } + +} diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/DbOps.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/DbOps.scala index 4cfa87c6..554539f6 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/DbOps.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/DbOps.scala @@ -7,6 +7,7 @@ import com.advancedtelematic.director.deviceregistry.data.{DeviceSortBy, GroupSo import com.advancedtelematic.director.deviceregistry.data.SortDirection.SortDirection import Schema.DeviceTable import GroupInfoRepository.GroupInfoTable +import com.advancedtelematic.libats.data.PaginationResult.{Offset, Limit} import slick.ast.Ordering import slick.jdbc.MySQLProfile.api.* import slick.lifted.ColumnOrdered @@ -59,9 +60,4 @@ object DbOps { } - implicit class PaginationResultOps(x: Option[Long]) { - def orDefaultOffset: Long = x.getOrElse(0L) - def orDefaultLimit: Long = x.getOrElse(50L) - } - } diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/DeviceRepository.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/DeviceRepository.scala index 26182769..9d764314 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/DeviceRepository.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/DeviceRepository.scala @@ -206,6 +206,18 @@ object DeviceRepository { .update(status) .handleSingleUpdateError(Errors.MissingDevice) + // updates device status if current status matches `currentStatus` + protected[db] def compareAndSetDeviceStatusAction(uuid: DeviceId, + currentStatus: DeviceStatus, + status: DeviceStatus)( + implicit ec: ExecutionContext): DBIO[Unit] = + devices + .filter(_.id === uuid) + .filter(_.deviceStatus === currentStatus) + .map(_.deviceStatus) + .update(status) + .map(_ => ()) + // Returns the previous hibernation status def setHibernationStatus(ns: Namespace, id: DeviceId, status: HibernationStatus)( implicit ec: ExecutionContext): DBIO[HibernationStatus] = diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EcuReplacementRepository.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EcuReplacementRepository.scala index 5912b819..4f5622f6 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EcuReplacementRepository.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EcuReplacementRepository.scala @@ -1,30 +1,26 @@ package com.advancedtelematic.director.db.deviceregistry +import com.advancedtelematic.libats.data.PaginationResult.* import java.time.Instant - -import cats.instances.option._ -import cats.syntax.apply._ -import cats.syntax.option._ +import cats.instances.option.* +import cats.syntax.apply.* +import cats.syntax.option.* import com.advancedtelematic.director.http.deviceregistry.Errors -import com.advancedtelematic.libats.data.{EcuIdentifier, PaginationResult} -import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.data.PaginationResult +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libats.messaging_datatype.MessageCodecs.ecuReplacementCodec -import com.advancedtelematic.libats.messaging_datatype.Messages.{ - EcuAndHardwareId, - EcuReplaced, - EcuReplacement, - EcuReplacementFailed -} +import com.advancedtelematic.libats.messaging_datatype.Messages.{EcuAndHardwareId, EcuReplaced, EcuReplacement, EcuReplacementFailed} import com.advancedtelematic.libats.slick.codecs.SlickRefined.refinedMappedType import com.advancedtelematic.libats.slick.db.SlickExtensions.javaInstantMapping -import com.advancedtelematic.libats.slick.db.SlickResultExtensions._ +import com.advancedtelematic.libats.slick.db.SlickResultExtensions.* import com.advancedtelematic.libats.slick.db.SlickUUIDKey.dbMapping import com.advancedtelematic.libats.slick.db.SlickValidatedGeneric.validatedStringMapper import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, ValidHardwareIdentifier} import eu.timepit.refined.refineV import io.circe.Json -import io.circe.syntax._ -import slick.jdbc.MySQLProfile.api._ +import io.circe.syntax.* +import slick.jdbc.MySQLProfile.api.* import scala.concurrent.ExecutionContext @@ -97,7 +93,7 @@ object EcuReplacementRepository { .filter(_.deviceId === deviceId) .result - def deviceHistory(deviceId: DeviceId, offset: Long, limit: Long)( + def deviceHistory(deviceId: DeviceId, offset: Offset, limit: Limit)( implicit ec: ExecutionContext): DBIO[PaginationResult[Json]] = for { installations <- InstallationReportRepository.queryInstallationHistory(deviceId).result diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EventIndex.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EventIndex.scala index be497ddb..20bb9da6 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EventIndex.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EventIndex.scala @@ -3,10 +3,9 @@ package com.advancedtelematic.director.db.deviceregistry import cats.syntax.either._ import cats.syntax.option._ import cats.syntax.show._ -import com.advancedtelematic.libats.data.DataType.{CampaignId, CorrelationId} +import com.advancedtelematic.libats.data.DataType.CorrelationId import com.advancedtelematic.libats.messaging_datatype.DataType.{Event, EventType} import com.advancedtelematic.director.deviceregistry.data.DataType.{IndexedEvent, _} -import java.util.UUID object EventIndex { type EventIndexResult = Either[String, IndexedEvent] @@ -22,17 +21,6 @@ object EventIndex { IndexedEvent(event.deviceUuid, event.eventId, indexedEventType, correlationId.some) } - private def parseEventOfTypeWithCampaignId( - event: Event, - indexedEventType: IndexedEventType.Value): EventIndexResult = - event.payload.hcursor - .downField("campaignId") - .as[UUID] - .leftMap(err => s"Could not parse payload for event ${event.show}: $err") - .map { campaignId => - IndexedEvent(event.deviceUuid, event.eventId, indexedEventType, CampaignId(campaignId).some) - } - private def parseEventOfType(event: Event, indexedEventType: IndexedEventType.Value): EventIndexResult = IndexedEvent(event.deviceUuid, event.eventId, indexedEventType, None).asRight @@ -56,12 +44,6 @@ object EventIndex { parseEventOfTypeWithCorrelationId(event, IndexedEventType.DevicePaused) case EventType("DeviceResumed", 0) => parseEventOfTypeWithCorrelationId(event, IndexedEventType.DeviceResumed) - case EventType("campaign_accepted", 0) => - parseEventOfTypeWithCampaignId(event, IndexedEventType.CampaignAccepted) - case EventType("campaign_declined", 0) => - parseEventOfTypeWithCampaignId(event, IndexedEventType.CampaignDeclined) - case EventType("campaign_postponed", 0) => - parseEventOfTypeWithCampaignId(event, IndexedEventType.CampaignPostponed) case eventType => s"Unknown event type $eventType".asLeft } diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EventJournal.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EventJournal.scala index 384661d3..451d6d49 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EventJournal.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/EventJournal.scala @@ -11,7 +11,12 @@ package com.advancedtelematic.director.db.deviceregistry import java.time.Instant import cats.syntax.show.* import com.advancedtelematic.libats.data.DataType.CorrelationId -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, Event, EventType} +import com.advancedtelematic.libats.messaging_datatype.DataType.{ + DeviceId, + EcuIdentifier, + Event, + EventType +} import com.advancedtelematic.libats.slick.db.SlickCirceMapper.* import com.advancedtelematic.libats.slick.db.SlickExtensions.javaInstantMapping import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* @@ -25,6 +30,7 @@ import slick.jdbc.MySQLProfile.api.* import com.advancedtelematic.libats.slick.db.SlickExtensions.* import scala.concurrent.{ExecutionContext, Future} +import com.advancedtelematic.libats.codecs.CirceRefined.* object EventJournal { @@ -39,7 +45,7 @@ object EventJournal { def pk = primaryKey("events_pk", (deviceUuid, eventId)) - private def fromEvent(e: Event) = + private def toRow(e: Event) = Some( e.deviceUuid, e.eventId, @@ -50,8 +56,10 @@ object EventJournal { e.payload ) - private def toEvent(x: (DeviceId, String, String, Int, Instant, Instant, Json)): Event = - Event(x._1, x._2, EventType(x._3, x._4), x._5, x._6, x._7) + private def toEvent(x: (DeviceId, String, String, Int, Instant, Instant, Json)): Event = { + val ecu = x._7.hcursor.downField("ecu").as[Option[EcuIdentifier]].toOption.flatten + Event(x._1, x._2, EventType(x._3, x._4), x._5, x._6, ecu, x._7) + } override def * = ( @@ -62,7 +70,7 @@ object EventJournal { deviceTime, receivedAt, event - ).shaped <> (toEvent, fromEvent) + ).shaped <> (toEvent, toRow) } diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/GroupInfoRepository.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/GroupInfoRepository.scala index 812f7439..d7a8c724 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/GroupInfoRepository.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/GroupInfoRepository.scala @@ -8,13 +8,7 @@ package com.advancedtelematic.director.db.deviceregistry -import com.advancedtelematic.director.deviceregistry.data.{ - Group, - GroupExpression, - GroupName, - GroupType, - TagId -} +import com.advancedtelematic.director.deviceregistry.data.{Group, GroupExpression, GroupName, GroupType, TagId} import com.advancedtelematic.director.deviceregistry.data.Group.GroupId import com.advancedtelematic.director.deviceregistry.data.GroupSortBy.GroupSortBy @@ -26,9 +20,10 @@ import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* import com.advancedtelematic.libats.slick.db.SlickValidatedGeneric.validatedStringMapper import com.advancedtelematic.director.deviceregistry.data import com.advancedtelematic.director.deviceregistry.data.GroupType.GroupType -import DbOps.{PaginationResultOps, SortBySlickOrderedGroupConversion} +import DbOps.SortBySlickOrderedGroupConversion import SlickMappings.* import com.advancedtelematic.director.http.deviceregistry.{ErrorHandlers, Errors} +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} import slick.jdbc.MySQLProfile.api.* import scala.concurrent.{ExecutionContext, Future} @@ -56,15 +51,15 @@ object GroupInfoRepository { val groupInfos = TableQuery[GroupInfoTable] def list(namespace: Namespace, - offset: Option[Long], - limit: Option[Long], + offset: Offset, + limit: Limit, sortBy: GroupSortBy, nameContains: Option[String])( implicit ec: ExecutionContext): DBIO[PaginationResult[Group]] = groupInfos .filter(_.namespace === namespace) .maybeContains(_.groupName, nameContains) - .paginateAndSortResult(sortBy.orderedConv(), offset.orDefaultOffset, limit.orDefaultLimit) + .paginateAndSortResult(sortBy.orderedConv(), offset, limit) def findById(id: GroupId)(implicit db: Database, ec: ExecutionContext): Future[Group] = db.run(findByIdAction(id)) diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/GroupMemberRepository.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/GroupMemberRepository.scala index e4d14d9e..4ea0c182 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/GroupMemberRepository.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/GroupMemberRepository.scala @@ -17,16 +17,10 @@ import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* import com.advancedtelematic.director.http.deviceregistry.Errors.MemberAlreadyExists import com.advancedtelematic.director.deviceregistry.data.DataType.HibernationStatus import com.advancedtelematic.director.deviceregistry.data.Group.GroupId -import com.advancedtelematic.director.deviceregistry.data.{ - Device, - DeviceDB, - GroupExpression, - GroupExpressionAST, - GroupType, - TagId -} -import DbOps.PaginationResultOps +import com.advancedtelematic.director.deviceregistry.data.{Device, DeviceDB, GroupExpression, GroupExpressionAST, GroupType, TagId} +import DbOps.* import com.advancedtelematic.director.http.deviceregistry.Errors +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} import slick.jdbc.{PositionedParameters, SetParameter} import slick.jdbc.MySQLProfile.api.* import slick.lifted.Tag @@ -89,16 +83,16 @@ object GroupMemberRepository { .filter(_.deviceUuid === deviceUuid) .delete - def listDevicesInGroup(groupId: GroupId, offset: Option[Long], limit: Option[Long])( + def listDevicesInGroup(groupId: GroupId, offset: Offset, limit: Limit)( implicit ec: ExecutionContext): DBIO[PaginationResult[DeviceId]] = listDevicesInGroupAction(groupId, offset, limit) - def listDevicesInGroupAction(groupId: GroupId, offset: Option[Long], limit: Option[Long])( + def listDevicesInGroupAction(groupId: GroupId, offset: Offset, limit: Limit)( implicit ec: ExecutionContext): DBIO[PaginationResult[DeviceId]] = groupMembers .filter(_.groupId === groupId) .map(_.deviceUuid) - .paginateResult(offset.orDefaultOffset, limit.orDefaultLimit) + .paginateResult(offset, limit) def countDevicesInGroup(groupIds: Set[GroupId])( implicit ec: ExecutionContext): DBIO[Map[GroupId, Long]] = @@ -150,12 +144,12 @@ object GroupMemberRepository { _ <- GroupMemberRepository.addDeviceToDynamicGroups(namespace, device, tags.toMap) } yield () - def listGroupsForDevice(deviceUuid: DeviceId, offset: Option[Long], limit: Option[Long])( + def listGroupsForDevice(deviceUuid: DeviceId, offset: Offset, limit: Limit)( implicit ec: ExecutionContext): DBIO[PaginationResult[GroupId]] = groupMembers .filter(_.deviceUuid === deviceUuid) .map(_.groupId) - .paginateResult(offset.orDefaultOffset, limit.orDefaultLimit) + .paginateResult(offset, limit) def listGroupsForDevices(deviceUuids: Seq[DeviceId])( implicit ec: ExecutionContext): DBIO[Map[DeviceId, Seq[GroupId]]] = { diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/InstallationReportRepository.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/InstallationReportRepository.scala index 42a1b4f3..95c70dc0 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/InstallationReportRepository.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/InstallationReportRepository.scala @@ -1,22 +1,23 @@ package com.advancedtelematic.director.db.deviceregistry -import java.time.Instant +import cats.implicits.toShow +import java.time.Instant import com.advancedtelematic.libats.data.DataType.{CorrelationId, ResultCode} -import com.advancedtelematic.libats.data.{EcuIdentifier, PaginationResult} -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuInstallationReport} -import com.advancedtelematic.director.deviceregistry.data.DataType.{ - DeviceInstallationResult, - EcuInstallationResult, - InstallationStat -} +import com.advancedtelematic.libats.data.PaginationResult +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier, EcuInstallationReport} +import com.advancedtelematic.director.deviceregistry.data.DataType.{DeviceInstallationResult, EcuInstallationResult, InstallationStat} +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} import io.circe.Json -import slick.jdbc.MySQLProfile.api._ -import com.advancedtelematic.libats.slick.db.SlickAnyVal._ -import com.advancedtelematic.libats.slick.db.SlickCirceMapper._ -import com.advancedtelematic.libats.slick.db.SlickExtensions._ -import com.advancedtelematic.libats.slick.db.SlickUUIDKey._ +import slick.jdbc.MySQLProfile.api.* +import com.advancedtelematic.libats.slick.db.SlickAnyVal.* +import com.advancedtelematic.libats.slick.codecs.SlickRefined.* +import com.advancedtelematic.libats.slick.db.SlickCirceMapper.* +import com.advancedtelematic.libats.slick.db.SlickExtensions.* +import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* +import com.advancedtelematic.libats.slick.db.SlickUrnMapper import com.advancedtelematic.libats.slick.db.SlickUrnMapper.correlationIdMapper +import slick.jdbc.GetResult import slick.lifted.AbstractTable import scala.concurrent.ExecutionContext @@ -58,12 +59,13 @@ object InstallationReportRepository { def deviceUuid = column[DeviceId]("device_uuid") def ecuId = column[EcuIdentifier]("ecu_id") def success = column[Boolean]("success") + def description = column[Option[String]]("description") def * = - (correlationId, resultCode, deviceUuid, ecuId, success) <> + (correlationId, resultCode, deviceUuid, ecuId, success, description) <> ((EcuInstallationResult.apply _).tupled, EcuInstallationResult.unapply) - def pk = primaryKey("pk_ecu_report", (deviceUuid, ecuId)) + def pk = primaryKey("pk_ecu_report", (correlationId, deviceUuid, ecuId)) } private val ecuInstallationResults = TableQuery[EcuInstallationResultTable] @@ -91,7 +93,8 @@ object InstallationReportRepository { ecuReport.result.code, deviceUuid, ecuId, - ecuReport.result.success + ecuReport.result.success, + Option(ecuReport.result.description.value) ) } val q = @@ -122,28 +125,68 @@ object InstallationReportRepository { implicit ec: ExecutionContext): DBIO[Seq[InstallationStat]] = statsQuery(ecuInstallationResults, correlationId) - def fetchDeviceInstallationResult( - correlationId: CorrelationId): DBIO[Seq[DeviceInstallationResult]] = - deviceInstallationResults.filter(_.correlationId === correlationId).result +// def fetchDeviceInstallationResult( +// correlationId: CorrelationId): DBIO[Seq[DeviceInstallationResult]] = +// deviceInstallationResults.filter(_.correlationId === correlationId).result + + import cats.syntax.either.* + + def fetchManyDevicesInstallationResults(ids: Set[(DeviceId, CorrelationId)]): DBIO[Vector[DeviceInstallationResult]] = { + val idsStr = ids.map { case (d, c) => s"(${d.show}, ${c.toString})"}.mkString(",") - def fetchDeviceInstallationResultFor( - deviceId: DeviceId, - correlationId: CorrelationId): DBIO[Seq[DeviceInstallationResult]] = + implicit val getDeviceInstallationResult: GetResult[DeviceInstallationResult] = GetResult { r => + DeviceInstallationResult( + correlationId = CorrelationId.fromString(r.nextString()).valueOr(err => throw new IllegalArgumentException(err)), + resultCode = ResultCode(r.nextString()), + deviceId = DeviceId(java.util.UUID.fromString(r.nextString())), + success = r.nextBoolean(), + receivedAt = r.nextTimestamp().toInstant, + installationReport = io.circe.parser.parse(r.nextString()).getOrElse(Json.Null) + ) + } + + sql"""SELECT correlation_id, result_code, device_uuid, success, received_at FROM #${deviceInstallationResults.baseTableRow.tableName} + WHERE (device_id, correlation_id) IN (#$idsStr) + """.as[DeviceInstallationResult] + } + + def fetchManyByDevice(deviceId: DeviceId, correlationIds: Set[CorrelationId]): DBIO[Seq[DeviceInstallationResult]] = + deviceInstallationResults + .filter(_.deviceUuid === deviceId) + .filter(_.correlationId.inSet(correlationIds)) + .result + + def fetchDeviceInstallationResultByCorrelationId(deviceId: DeviceId, + correlationId: CorrelationId): DBIO[Option[DeviceInstallationResult]] = deviceInstallationResults .filter(_.deviceUuid === deviceId) .filter(_.correlationId === correlationId) .result + .headOption - def fetchEcuInstallationReport(correlationId: CorrelationId): DBIO[Seq[EcuInstallationResult]] = - ecuInstallationResults.filter(_.correlationId === correlationId).result + def fetchEcuInstallationReport(deviceId: DeviceId, correlationId: CorrelationId)( + implicit ec: ExecutionContext): DBIO[Map[EcuIdentifier, EcuInstallationResult]] = + ecuInstallationResults + .filter(_.deviceUuid === deviceId) + .filter(_.correlationId === correlationId) + .result + .map { seq => + seq.map { res => + res.ecuId -> res + }.toMap + } private[db] def queryInstallationHistory(deviceId: DeviceId): Query[Rep[Json], Json, Seq] = deviceInstallationResults .filter(_.deviceUuid === deviceId) .sortBy(_.receivedAt.desc) .map(_.installationReport) + // TODO: Returning or even storing an installationReport here doesn't make sense, since it just contains + // the same as ecuInstallationResults. installationReport is just the serialized form of the DeviceUpdateEvent + // that originated this deviceInstallationResult + // We cannot change this now as this is used by the frontend currently - def installationReports(deviceId: DeviceId, offset: Long, limit: Long)( + def installationReports(deviceId: DeviceId, offset: Offset, limit: Limit)( implicit ec: ExecutionContext): DBIO[PaginationResult[Json]] = queryInstallationHistory(deviceId).paginateResult(offset, limit) diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/InstalledPackages.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/InstalledPackages.scala index 9e3cb4cd..25f8aa9d 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/InstalledPackages.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/InstalledPackages.scala @@ -9,18 +9,18 @@ package com.advancedtelematic.director.db.deviceregistry import java.time.Instant - import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libats.data.PaginationResult import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.slick.db.SlickExtensions._ -import SlickMappings._ -import com.advancedtelematic.libats.slick.db.SlickUUIDKey._ +import com.advancedtelematic.libats.slick.db.SlickExtensions.* +import SlickMappings.* +import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* import com.advancedtelematic.director.deviceregistry.data.Group.GroupId import com.advancedtelematic.director.deviceregistry.data.{PackageId, PackageStat} import com.advancedtelematic.director.deviceregistry.data.PackageId.Name -import DbOps.PaginationResultOps -import slick.jdbc.MySQLProfile.api._ +import DbOps.* +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} +import slick.jdbc.MySQLProfile.api.* import scala.concurrent.ExecutionContext @@ -79,8 +79,8 @@ object InstalledPackages { def installedOn(device: DeviceId, nameContains: Option[String], - offset: Option[Long], - limit: Option[Long])( + offset: Offset, + limit: Limit)( implicit ec: ExecutionContext): DBIO[PaginationResult[InstalledPackage]] = installedPackages .filter(_.device === device) @@ -88,7 +88,7 @@ object InstalledPackages { ip => ip.name.mappedTo[String] ++ "-" ++ ip.version.mappedTo[String], nameContains ) - .paginateResult(offset.orDefaultOffset, limit.orDefaultLimit) + .paginateResult(offset, limit) def getDevicesCount(pkg: PackageId, ns: Namespace)( implicit ec: ExecutionContext): DBIO[DevicesCount] = @@ -132,10 +132,10 @@ object InstalledPackages { PackageId(name, version) }) - def getInstalledForAllDevices(ns: Namespace, offset: Option[Long], limit: Option[Long])( + def getInstalledForAllDevices(ns: Namespace, offset: Offset, limit: Limit)( implicit ec: ExecutionContext): DBIO[PaginationResult[PackageId]] = { val query = installedForAllDevicesQuery(ns) - .paginateAndSortResult(identity, offset.orDefaultOffset, limit.orDefaultLimit) + .paginateAndSortResult(identity, offset, limit) query.map { nameVersionResult => PaginationResult( nameVersionResult.values.map(nameVersion => PackageId(nameVersion._1, nameVersion._2)), @@ -164,8 +164,8 @@ object InstalledPackages { def listAllWithPackageByName(ns: Namespace, name: Name, - offset: Option[Long], - limit: Option[Long])( + offset: Offset, + limit: Limit)( implicit ec: ExecutionContext): DBIO[PaginationResult[PackageStat]] = { val query = installedPackages .filter(_.name === name) @@ -176,12 +176,12 @@ object InstalledPackages { .map { case (version, installedPkg) => (version, installedPkg.length) } val pkgResult = query - .paginate(offset.orDefaultOffset, limit.orDefaultLimit) + .paginate(offset, limit) .result .map(_.map { case (version, count) => PackageStat(version, count) }) query.length.result.zip(pkgResult).map { case (total, values) => - PaginationResult(values, total, offset.orDefaultOffset, limit.orDefaultLimit) + PaginationResult(values, total, offset, limit) } } diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/Schema.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/Schema.scala index 9b8c1f35..c61d7ca8 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/Schema.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/Schema.scala @@ -16,9 +16,6 @@ import java.time.Instant object Schema { - protected[db] implicit val DeviceStatusColumnType: BaseColumnType[DeviceStatus.Value] = - MappedColumnType.base[DeviceStatus.Value, String](_.toString, DeviceStatus.withName) - // scalastyle:off class DeviceTable(tag: Tag) extends Table[DeviceDB](tag, "Device") { def namespace = column[Namespace]("namespace") diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SearchDBIO.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SearchDBIO.scala index 303caa8e..3653cbd7 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SearchDBIO.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SearchDBIO.scala @@ -1,28 +1,30 @@ package com.advancedtelematic.director.db.deviceregistry +import com.advancedtelematic.director.data.ClientDataType.TagSearchOps import com.advancedtelematic.director.db -import com.advancedtelematic.director.db.deviceregistry.DbOps.{ - deviceTableToSlickOrder, - PaginationResultOps -} +import com.advancedtelematic.director.db.deviceregistry.DbOps.deviceTableToSlickOrder import com.advancedtelematic.director.db.deviceregistry.GroupInfoRepository.groupInfos import com.advancedtelematic.director.db.deviceregistry.GroupMemberRepository.groupMembers import com.advancedtelematic.director.db.deviceregistry.Schema.* import com.advancedtelematic.director.db.deviceregistry.SlickMappings.* +import com.advancedtelematic.director.db.deviceregistry.TaggedDeviceRepository.taggedDevices import com.advancedtelematic.director.deviceregistry.data.* import com.advancedtelematic.director.deviceregistry.data.DataType.{ DeviceCountParams, DeviceStatusCounts, SearchParams } +import com.advancedtelematic.director.deviceregistry.data.Device.DeviceOemId import com.advancedtelematic.director.deviceregistry.data.Group.GroupId import com.advancedtelematic.director.deviceregistry.data.GroupType.GroupType import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.data.PaginationResult +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libats.slick.codecs.SlickRefined.* import com.advancedtelematic.libats.slick.db.SlickExtensions.* import com.advancedtelematic.libats.slick.db.SlickUUIDKey.* +import com.advancedtelematic.libats.slick.db.{SlickAnyVal, SlickUUIDKey} import slick.jdbc.GetResult import slick.jdbc.MySQLProfile.api.* import slick.lifted.Rep @@ -79,9 +81,9 @@ object SearchDBIO { .filter(notSeenSinceFilter) } - private def runQueryFilteringByName(ns: Namespace, - query: Query[DeviceTable, DeviceDB, Seq], - nameContains: Option[String]) = { + private def searchQueryByName(ns: Namespace, + query: Query[DeviceTable, DeviceDB, Seq], + nameContains: Option[String]) = { val deviceIdsByName = searchQuery(ns, nameContains, None, None).map(_.id) query.filter(_.id in deviceIdsByName) } @@ -98,9 +100,56 @@ object SearchDBIO { .map(_._2) .distinct + private def applyCommonSearchFiltersTo(params: SearchParams, + query: Query[DeviceTable, DeviceDB, Seq])( + implicit ec: ExecutionContext): DBIO[PaginationResult[DeviceDB]] = { + val sortBy = params.sortBy.getOrElse(DeviceSortBy.Name) + val sortDirection = params.sortDirection.getOrElse(SortDirection.Asc) + + val activatedAfterFilter = optionalFilter(params.activatedAfter) { (dt, from) => + dt.activatedAt.map(i => i >= from).getOrElse(false.bind) + } + + val activatedBeforeFilter = optionalFilter(params.activatedBefore) { (dt, to) => + dt.activatedAt.map(i => i < to).getOrElse(false.bind) + } + + val lastSeenStartFilter = optionalFilter(params.lastSeenStart) { (dt, lastSeen) => + dt.lastSeen.map(i => i > lastSeen).getOrElse(false.bind) + } + + val lastSeenEndFilter = optionalFilter(params.lastSeenEnd) { (dt, lastSeen) => + dt.lastSeen.map(i => i < lastSeen).getOrElse(false.bind) + } + + val hardwareIdFilter: DeviceTable => Rep[Boolean] = params.hardwareId match { + case x :: xs => + dt => { + val hardwareIdsQuery = + db.Schema.activeEcus.filter(_.hardwareId.inSet(x :: xs)).map(_.deviceId) + dt.id.in(hardwareIdsQuery) + } + case _ => + _ => true.bind + } + + query + .maybeFilter(r => r.deviceStatus === params.status) + .maybeFilter(_.hibernated === params.hibernated) + .maybeFilter(_.createdAt > params.createdAtStart) + .maybeFilter(_.createdAt < params.createdAtEnd) + .filter(activatedAfterFilter) + .filter(activatedBeforeFilter) + .filter(lastSeenStartFilter) + .filter(lastSeenEndFilter) + .filter(hardwareIdFilter) + .sortBy(devices => devices.ordered(sortBy, sortDirection)) + .paginateResult(params.offset, params.limit) + } + def search(ns: Namespace, params: SearchParams)( implicit ec: ExecutionContext): DBIO[PaginationResult[DeviceDB]] = { - val deviceTableQuery = params match { + params match { case SearchParams( Some(oemId), @@ -121,9 +170,10 @@ object SearchDBIO { _, _, _, + _, _ ) => - DeviceRepository.findByDeviceIdQuery(ns, oemId) + applyCommonSearchFiltersTo(params, DeviceRepository.findByDeviceIdQuery(ns, oemId)) case SearchParams( None, @@ -144,9 +194,13 @@ object SearchDBIO { _, _, _, + _, _ ) => - runQueryFilteringByName(ns, groupedDevicesQuery(ns, gt), nameContains) + applyCommonSearchFiltersTo( + params, + searchQueryByName(ns, groupedDevicesQuery(ns, gt), nameContains) + ) case SearchParams( None, @@ -167,11 +221,19 @@ object SearchDBIO { _, _, _, + _, _ ) => val ungroupedDevicesQuery = devices.filterNot(_.id.in(groupedDevicesQuery(ns, gt).map(_.id))) - runQueryFilteringByName(ns, ungroupedDevicesQuery, nameContains) + applyCommonSearchFiltersTo( + params, + searchQueryByName(ns, ungroupedDevicesQuery, nameContains) + ) + + case SearchParams(None, _, _, _, _, _, _, _, _, _, _, _, _, _, _, deviceTags, _, _, _, _) + if deviceTags.nonEmpty => + tagsSearch(ns, params) case SearchParams( None, @@ -192,55 +254,84 @@ object SearchDBIO { _, _, _, + _, _ ) => - searchQuery(ns, nameContains, gid, notSeenSinceHours) + applyCommonSearchFiltersTo(params, searchQuery(ns, nameContains, gid, notSeenSinceHours)) case _ => throw new IllegalArgumentException("Invalid parameter combination.") } + } - val sortBy = params.sortBy.getOrElse(DeviceSortBy.Name) - val sortDirection = params.sortDirection.getOrElse(SortDirection.Asc) + private def tagsSearch(ns: Namespace, params: SearchParams)( + implicit ec: ExecutionContext): DBIO[PaginationResult[DeviceDB]] = { - val activatedAfterFilter = optionalFilter(params.activatedAfter) { (dt, from) => - dt.activatedAt.map(i => i >= from).getOrElse(false.bind) + trait SearchQuery { + type Result + val query: String + val limit: Limit + val offset: Offset + implicit val getResult: GetResult[Result] } - val activatedBeforeFilter = optionalFilter(params.activatedBefore) { (dt, to) => - dt.activatedAt.map(i => i < to).getOrElse(false.bind) + object Count extends SearchQuery { + override type Result = Long + override val query: String = "SELECT COUNT(*)" + override val limit: Limit = Limit(Long.MaxValue) + override val offset: Offset = Offset(0) + override implicit val getResult: GetResult[Long] = GetResult.GetLong } - val lastSeenStartFilter = optionalFilter(params.lastSeenStart) { (dt, lastSeen) => - dt.lastSeen.map(i => i > lastSeen).getOrElse(false.bind) + object Results extends SearchQuery { + override type Result = DeviceDB + override val query: String = """SELECT DISTINCT uuid, device_name, device_id, device_type, + last_seen, devices.created_at, activated_at, device_status, notes, + hibernated, mqtt_status, mqtt_last_seen""" + override val limit: Limit = params.limit + override val offset: Offset = params.offset + override implicit val getResult: GetResult[DeviceDB] = pr => + DeviceDB( + ns, + SlickUUIDKey.dbMapping[DeviceId].getValue(pr.rs, 1), + SlickAnyVal.stringAnyValSerializer[DeviceName].getValue(pr.rs, 2), + SlickAnyVal.stringAnyValSerializer[DeviceOemId].getValue(pr.rs, 3), + SlickMappings.deviceTypeMaper.getValue(pr.rs, 4), + Option(javaInstantMapping.getValue(pr.rs, 5)), + javaInstantMapping.getValue(pr.rs, 6), + Option(javaInstantMapping.getValue(pr.rs, 7)), + SlickMappings.deviceStatusColumnType.getValue(pr.rs, 8), + Option(pr.rs.getString(9)), + pr.rs.getBoolean(10), + SlickMappings.mqttStatusMapper.getValue(pr.rs, 11), + Option(javaInstantMapping.getValue(pr.rs, 12)) + ) } - val lastSeenEndFilter = optionalFilter(params.lastSeenEnd) { (dt, lastSeen) => - dt.lastSeen.map(i => i < lastSeen).getOrElse(false.bind) - } + def buildQuery(squery: SearchQuery) = { + import squery.* - val hardwareIdFilter: DeviceTable => Rep[Boolean] = params.hardwareId match { - case x :: xs => - dt => { - val hardwareIdsQuery = - db.Schema.activeEcus.filter(_.hardwareId.inSet(x :: xs)).map(_.deviceId) - dt.id.in(hardwareIdsQuery) - } - case _ => - _ => true.bind + // this is SQL injection safe only because we validate the parameters format + val tags = params.deviceTags.map(t => s"('${t.tagId.value}','${t.tagValue}')").mkString(",") + + sql""" + #${squery.query} FROM Device devices + JOIN #${taggedDevices.baseTableRow.tableName} tags ON devices.uuid = tags.device_uuid + WHERE + devices.namespace = ${ns.get} AND + (tags.tag_id, tags.tag_value) IN (#$tags) + ORDER BY ${params.sortBy.map(_.columnName).getOrElse("uuid")} ${params.sortDirection + .getOrElse(SortDirection.Asc) + .toString} + LIMIT ${squery.limit.value} + OFFSET ${squery.offset.value} + """.as[squery.Result] } - deviceTableQuery - .maybeFilter(r => r.deviceStatus === params.status) - .maybeFilter(_.hibernated === params.hibernated) - .maybeFilter(_.createdAt > params.createdAtStart) - .maybeFilter(_.createdAt < params.createdAtEnd) - .filter(activatedAfterFilter) - .filter(activatedBeforeFilter) - .filter(lastSeenStartFilter) - .filter(lastSeenEndFilter) - .filter(hardwareIdFilter) - .sortBy(devices => devices.ordered(sortBy, sortDirection)) - .paginateResult(params.offset.orDefaultOffset, params.limit.orDefaultLimit) + buildQuery(Count).flatMap { count => + buildQuery(Results).map { d => + PaginationResult(d, count.headOption.getOrElse(0L), params.offset, params.limit) + } + } } def countByStatus(ns: Namespace, params: DeviceCountParams): DBIO[DeviceStatusCounts] = { diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SlickMappings.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SlickMappings.scala index f9c85571..ab6f36a4 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SlickMappings.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SlickMappings.scala @@ -10,12 +10,9 @@ package com.advancedtelematic.director.db.deviceregistry import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.slick.codecs.{SlickEnumMapper, SlickEnumeratum} -import com.advancedtelematic.director.deviceregistry.data.DataType.{ - IndexedEventType, - MqttStatus, - PackageListItemCount -} -import com.advancedtelematic.director.deviceregistry.data.{CredentialsType, GroupType, PackageId} +import com.advancedtelematic.director.deviceregistry.data.DataType.{IndexedEventType, MqttStatus, PackageListItemCount} +import com.advancedtelematic.director.deviceregistry.data.Device.DeviceType +import com.advancedtelematic.director.deviceregistry.data.{CredentialsType, DeviceStatus, GroupType, PackageId} import slick.jdbc.MySQLProfile.api.* object SlickMappings { @@ -53,4 +50,13 @@ object SlickMappings { implicit val mqttStatusMapper: BaseColumnType[MqttStatus] = SlickEnumeratum.enumeratumMapper(MqttStatus) + implicit val deviceStatusColumnType: BaseColumnType[DeviceStatus.Value] = + MappedColumnType.base[DeviceStatus.Value, String](_.toString, DeviceStatus.withName) + + // TODO: We should encode Enums as strings, not Ints + // Moved this from SlickEnum, because this should **NOT** be used + // It's difficult to read this when reading from the database and the Id is not stable when we add/remove + // values from the enum + implicit val deviceTypeMaper: slick.jdbc.MySQLProfile.BaseColumnType[DeviceType.Value] = + MappedColumnType.base[DeviceType.Value, Int](_.id, DeviceType.apply) } diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SystemInfoRepository.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SystemInfoRepository.scala index cbbef5c8..7ee604ac 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SystemInfoRepository.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/SystemInfoRepository.scala @@ -8,7 +8,7 @@ package com.advancedtelematic.director.db.deviceregistry -import akka.Done +import org.apache.pekko.Done import cats.data.State import cats.implicits._ import com.advancedtelematic.director.http.deviceregistry.Errors diff --git a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/TaggedDeviceRepository.scala b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/TaggedDeviceRepository.scala index eaae2dd6..c52be134 100644 --- a/src/main/scala/com/advancedtelematic/director/db/deviceregistry/TaggedDeviceRepository.scala +++ b/src/main/scala/com/advancedtelematic/director/db/deviceregistry/TaggedDeviceRepository.scala @@ -24,6 +24,7 @@ object TaggedDeviceRepository { class TaggedDeviceTable(tag: Tag) extends Table[TaggedDevice](tag, "TaggedDevice") { def namespace = column[Namespace]("namespace") def deviceUuid = column[DeviceId]("device_uuid") + def tagId = column[TagId]("tag_id")(validatedStringMapper) def tagValue = column[String]("tag_value") diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/AllowUUIDPath.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/AllowUUIDPath.scala index 70d41f32..f217f78a 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/AllowUUIDPath.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/AllowUUIDPath.scala @@ -8,11 +8,11 @@ package com.advancedtelematic.director.deviceregistry -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.{AuthorizationFailedRejection, Directive1, Directives} +import org.apache.pekko.http.scaladsl.server.Directives._ +import org.apache.pekko.http.scaladsl.server.{AuthorizationFailedRejection, Directive1, Directives} import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.data.UUIDKey.{UUIDKey, UUIDKeyObj} -import com.advancedtelematic.libats.http.UUIDKeyAkka._ +import com.advancedtelematic.libats.http.UUIDKeyPekko._ import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import scala.concurrent.Future diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/GroupMembership.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/GroupMembership.scala index 0f2b2d06..8442665b 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/GroupMembership.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/GroupMembership.scala @@ -1,22 +1,14 @@ package com.advancedtelematic.director.deviceregistry -import akka.http.scaladsl.util.FastFuture -import com.advancedtelematic.director.db.deviceregistry.{ - GroupInfoRepository, - GroupMemberRepository, - SearchDBIO -} +import org.apache.pekko.http.scaladsl.util.FastFuture +import com.advancedtelematic.director.db.deviceregistry.{GroupInfoRepository, GroupMemberRepository, SearchDBIO} import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.data.PaginationResult import com.advancedtelematic.director.deviceregistry.data.Group.GroupId import com.advancedtelematic.director.deviceregistry.data.GroupType.GroupType -import com.advancedtelematic.director.deviceregistry.data.{ - Group, - GroupExpression, - GroupName, - GroupType -} +import com.advancedtelematic.director.deviceregistry.data.{Group, GroupExpression, GroupName, GroupType} import com.advancedtelematic.director.http.deviceregistry.Errors +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} import slick.jdbc.MySQLProfile.api.* import scala.concurrent.{ExecutionContext, Future} @@ -95,8 +87,8 @@ class GroupMembership(implicit val db: Database, ec: ExecutionContext) { } def listDevices(groupId: GroupId, - offset: Option[Long], - limit: Option[Long]): Future[PaginationResult[DeviceId]] = + offset: Offset, + limit: Limit): Future[PaginationResult[DeviceId]] = db.run(GroupMemberRepository.listDevicesInGroup(groupId, offset, limit)) def addGroupMember(groupId: GroupId, deviceId: DeviceId): Future[Unit] = diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceEventListener.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceEventListener.scala index 07c499b0..5ff5c5e2 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceEventListener.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceEventListener.scala @@ -8,7 +8,7 @@ package com.advancedtelematic.director.deviceregistry.daemon -import akka.Done +import org.apache.pekko.Done import com.advancedtelematic.director.db.deviceregistry.EventJournal import com.advancedtelematic.libats.messaging.MsgOperation.MsgOperation import com.advancedtelematic.libats.messaging_datatype.Messages.DeviceEventMessage diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceUpdateEventListener.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceUpdateEventListener.scala index 2e2da903..1eb7e766 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceUpdateEventListener.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceUpdateEventListener.scala @@ -1,7 +1,7 @@ package com.advancedtelematic.director.deviceregistry.daemon import java.time.Instant -import akka.http.scaladsl.util.FastFuture +import org.apache.pekko.http.scaladsl.util.FastFuture import com.advancedtelematic.director.db.deviceregistry.{ DeviceRepository, InstallationReportRepository @@ -80,14 +80,16 @@ class DeviceUpdateEventListener(messageBus: MessageBusPublisher)( private def wasCompleted(deviceId: DeviceId, correlationId: CorrelationId): Future[Boolean] = { val existingReport = - db.run(InstallationReportRepository.fetchDeviceInstallationResultFor(deviceId, correlationId)) + db.run(InstallationReportRepository.fetchDeviceInstallationResultByCorrelationId(deviceId, correlationId)) existingReport.map(_.exists(_.success)) // if we handle success event - other should be ignored } private def handleEvent(event: DeviceUpdateEvent): Future[DeviceStatus] = event match { - case _: DeviceUpdateAssigned => FastFuture.successful(DeviceStatus.Outdated) - case _: DeviceUpdateCanceled => FastFuture.successful(DeviceStatus.UpToDate) - case _: DeviceUpdateInFlight => FastFuture.successful(DeviceStatus.UpdatePending) + case msg: DeviceUpdateAssigned if msg.scheduledFor.isEmpty => + FastFuture.successful(DeviceStatus.Outdated) + case _: DeviceUpdateAssigned => FastFuture.successful(DeviceStatus.UpdateScheduled) + case _: DeviceUpdateCanceled => FastFuture.successful(DeviceStatus.UpToDate) + case _: DeviceUpdateInFlight => FastFuture.successful(DeviceStatus.UpdatePending) case msg: DeviceUpdateCompleted => db.run { InstallationReportRepository @@ -98,7 +100,7 @@ class DeviceUpdateEventListener(messageBus: MessageBusPublisher)( msg.result.success, msg.ecuReports, msg.eventTime, - msg.asJson + msg.asJson // TODO: Should not be here. if we need data from here, provide it parsed ) .map(_ => if (msg.result.success) DeviceStatus.UpToDate else DeviceStatus.Error) } diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/DataType.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/DataType.scala index ff086453..2a86298e 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/DataType.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/DataType.scala @@ -2,9 +2,9 @@ package com.advancedtelematic.director.deviceregistry.data import java.time.Instant import cats.Show +import com.advancedtelematic.director.data.ClientDataType.TagSearchParam import com.advancedtelematic.libats.data.DataType.{CorrelationId, Namespace, ResultCode} -import com.advancedtelematic.libats.data.EcuIdentifier -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, Event} +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier, Event} import com.advancedtelematic.libats.messaging_datatype.Messages.DeviceMetricsObservation import com.advancedtelematic.director.deviceregistry.data.CredentialsType.CredentialsType import com.advancedtelematic.director.deviceregistry.data.DataType.IndexedEventType.IndexedEventType @@ -14,6 +14,7 @@ import com.advancedtelematic.director.deviceregistry.data.DeviceStatus.DeviceSta import com.advancedtelematic.director.deviceregistry.data.Group.GroupId import com.advancedtelematic.director.deviceregistry.data.GroupType.GroupType import com.advancedtelematic.director.deviceregistry.data.SortDirection.SortDirection +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier import enumeratum.EnumEntry import enumeratum.EnumEntry.Camelcase @@ -78,17 +79,24 @@ object DataType { deviceId: DeviceId, success: Boolean, receivedAt: Instant, - installationReport: Json) + installationReport: Json) { + + // TODO: parse this when writing to db (migrate old rows) + def description: Option[String] = + installationReport.hcursor.downField("result").downField("description").as[String].toOption + + } final case class EcuInstallationResult(correlationId: CorrelationId, resultCode: ResultCode, deviceId: DeviceId, ecuId: EcuIdentifier, - success: Boolean) + success: Boolean, + description: Option[String]) object SearchParams { - def all(limit: Option[Long], offset: Option[Long]) = SearchParams( + def all(limit: Limit, offset: Offset) = SearchParams( None, None, None, @@ -104,6 +112,7 @@ object DataType { None, None, List.empty, + Set.empty, Some(DeviceSortBy.CreatedAt), Some(SortDirection.Asc), offset, @@ -123,7 +132,7 @@ object DataType { updateScheduled: Long) final case class SearchParams(oemId: Option[DeviceOemId], - grouped: Option[HibernationStatus], + grouped: Option[Boolean], groupType: Option[GroupType], groupId: Option[GroupId], nameContains: Option[String], @@ -137,10 +146,59 @@ object DataType { createdAtStart: Option[Instant], createdAtEnd: Option[Instant], hardwareId: Seq[HardwareIdentifier], + deviceTags: Set[TagSearchParam], sortBy: Option[DeviceSortBy], sortDirection: Option[SortDirection], - offset: Option[Long], - limit: Option[Long]) { + offset: Offset, + limit: Limit) { + + if(deviceTags.nonEmpty) { + require( + oemId.isEmpty, + "Invalid parameters: oemId must be empty when searching by deviceTags" + ) + + require( + nameContains.isEmpty, + "Invalid parameters: nameContains must be empty when searching by deviceTags" + ) + + require( + grouped.isEmpty, + "Invalid parameters: grouped must be empty when searching by deviceTags" + ) + + require( + groupType.isEmpty, + "Invalid parameters: groupType must be empty when searching by deviceTags" + ) + + require( + hardwareId.isEmpty, + "Invalid parameters: hardwareId must be empty when searching by deviceTags" + ) + + require( + activatedAfter.isEmpty, + "Invalid parameters: activatedAfter must be empty when searching by deviceTags" + ) + + require( + activatedBefore.isEmpty, + "Invalid parameters: activatedBefore must be empty when searching by deviceTags" + ) + + require( + lastSeenStart.isEmpty, + "Invalid parameters: lastSeenStart must be empty when searching by deviceTags" + ) + + require( + lastSeenEnd.isEmpty, + "Invalid parameters: lastSeenEnd must be empty when searching by deviceTags" + ) + + } if (oemId.isDefined) { require( diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/Device.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/Device.scala index ccbe0c56..47c80812 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/Device.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/Device.scala @@ -19,6 +19,10 @@ import com.advancedtelematic.director.deviceregistry.data.Device.{DeviceOemId, D import com.advancedtelematic.director.deviceregistry.data.DeviceStatus.* import io.circe.{Decoder, Encoder, KeyDecoder, KeyEncoder} import DataType.{mqttStatusDecoder, mqttStatusEncoder} +import com.advancedtelematic.director.db.deviceregistry.Schema +import com.advancedtelematic.director.db.deviceregistry.Schema.DeviceTable +import slick.ast.{FieldSymbol, Select} +import slick.lifted.{Rep, TableQuery} final case class DeviceDB(namespace: Namespace, uuid: DeviceId, @@ -82,16 +86,6 @@ object Device { final object DeviceType extends Enumeration { - // TODO: We should encode Enums as strings, not Ints - // Moved this from SlickEnum, because this should **NOT** be used - // It's difficult to read this when reading from the database and the Id is not stable when we add/remove - // values from the enum - import slick.jdbc.MySQLProfile.MappedJdbcType - import slick.jdbc.MySQLProfile.api._ - - implicit val enumMapper: slick.jdbc.MySQLProfile.BaseColumnType[Value] = - MappedJdbcType.base[Value, Int](_.id, this.apply) - type DeviceType = Value val Other, Vehicle = Value @@ -158,11 +152,17 @@ object SortDirection { } object DeviceSortBy { - sealed trait DeviceSortBy - case object Name extends DeviceSortBy - case object CreatedAt extends DeviceSortBy - case object DeviceId extends DeviceSortBy - case object Uuid extends DeviceSortBy - case object ActivatedAt extends DeviceSortBy - case object LastSeen extends DeviceSortBy + sealed abstract class DeviceSortBy(column: DeviceTable => Rep[?]) { + def columnName: String = + column(TableQuery[Schema.DeviceTable].baseTableRow).toNode match { + case Select(_, FieldSymbol(name)) => name + case col => throw new IllegalArgumentException(s"$col cannot be used as device sort column") + } + } + case object Name extends DeviceSortBy(_.deviceName) + case object CreatedAt extends DeviceSortBy(_.createdAt) + case object DeviceId extends DeviceSortBy(_.oemId) + case object Uuid extends DeviceSortBy(_.id) + case object ActivatedAt extends DeviceSortBy(_.activatedAt) + case object LastSeen extends DeviceSortBy(_.lastSeen) } diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/DeviceStatus.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/DeviceStatus.scala index 8d5babdf..89feac11 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/DeviceStatus.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/DeviceStatus.scala @@ -8,7 +8,7 @@ package com.advancedtelematic.director.deviceregistry.data -import akka.http.scaladsl.unmarshalling.Unmarshaller +import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshaller import io.circe.{Decoder, Encoder} object DeviceStatus extends Enumeration { diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/Group.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/Group.scala index cf7d2552..efe30ae8 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/Group.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/Group.scala @@ -10,7 +10,7 @@ package com.advancedtelematic.director.deviceregistry.data import java.time.Instant import java.util.UUID -import akka.http.scaladsl.unmarshalling.Unmarshaller +import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshaller import com.advancedtelematic.libats.codecs.CirceCodecs._ import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.data.UUIDKey.{UUIDKey, UUIDKeyObj} diff --git a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/TagId.scala b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/TagId.scala index 228222bc..ae0d7a27 100644 --- a/src/main/scala/com/advancedtelematic/director/deviceregistry/data/TagId.scala +++ b/src/main/scala/com/advancedtelematic/director/deviceregistry/data/TagId.scala @@ -20,7 +20,7 @@ object TagId { } def from(s: String): Either[ValidationError, TagId] = - if (s.length <= 20 && s.matches("[\\w\\-_ ]+")) + if (s.length <= 20 && s.matches("[\\w\\-_]+")) Right(new TagId(s)) else Left( diff --git a/src/main/scala/com/advancedtelematic/director/http/AdminResource.scala b/src/main/scala/com/advancedtelematic/director/http/AdminResource.scala index 3fba90c8..af89565e 100644 --- a/src/main/scala/com/advancedtelematic/director/http/AdminResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/AdminResource.scala @@ -1,29 +1,29 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.* -import akka.http.scaladsl.server.Directives.* +import com.advancedtelematic.libats.data.PaginationResult.* +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.* +import org.apache.pekko.http.scaladsl.server.Directives.* import cats.syntax.option.* -import com.advancedtelematic.director.daemon.UpdateScheduler import com.advancedtelematic.director.data.AdminDataType.{FindImageCount, RegisterDevice} import com.advancedtelematic.director.data.ClientDataType.* import com.advancedtelematic.director.data.Codecs.* import com.advancedtelematic.director.data.DataType.AdminRoleName.AdminRoleNamePathMatcher -import com.advancedtelematic.director.data.DataType.ScheduledUpdateId import com.advancedtelematic.director.db.* import com.advancedtelematic.director.http.PaginationParametersDirectives.* import com.advancedtelematic.director.repo.{DeviceRoleGeneration, OfflineUpdates, RemoteSessions} import com.advancedtelematic.libats.codecs.CirceCodecs.* import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.data.{EcuIdentifier, PaginationResult} +import com.advancedtelematic.libats.data.PaginationResult import com.advancedtelematic.libats.data.RefinedUtils.RefineTry import com.advancedtelematic.libats.http.RefinedMarshallingSupport.* -import com.advancedtelematic.libats.http.UUIDKeyAkka.* +import com.advancedtelematic.libats.http.UUIDKeyPekko.* import com.advancedtelematic.libats.messaging.MessageBusPublisher -import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, ValidEcuIdentifier} import com.advancedtelematic.libtuf.data.ClientCodecs.* import com.advancedtelematic.libtuf.data.ClientDataType.{ ClientTargetItem, + RemoteCommandsPayload, RemoteSessionsPayload, RootRole } @@ -38,7 +38,7 @@ import com.advancedtelematic.libtuf.data.TufDataType.{ } import com.advancedtelematic.libtuf_server.data.Marshalling.jsonSignedPayloadMarshaller import com.advancedtelematic.libtuf_server.keyserver.KeyserverClient -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import slick.jdbc.MySQLProfile.api.* import java.time.Instant @@ -47,7 +47,9 @@ import scala.concurrent.{ExecutionContext, Future} case class OfflineUpdateRequest(values: Map[TargetFilename, ClientTargetItem], expiresAt: Option[Instant]) -case class RemoteSessionRequest(remoteSessions: RemoteSessionsPayload, previousVersion: Int) +case class RemoteSessionRequest(remoteSessions: RemoteSessionsPayload) + +case class RemoteCommandRequest(remoteCommands: RemoteCommandsPayload) class AdminResource(extractNamespace: Directive1[Namespace], val keyserverClient: KeyserverClient)( implicit val db: Database, @@ -60,9 +62,9 @@ class AdminResource(extractNamespace: Directive1[Namespace], val keyserverClient with EcuRepositorySupport with ProvisionedDeviceRepositorySupport with AutoUpdateDefinitionRepositorySupport - with ScheduledUpdatesRepositorySupport { + with UpdatesRepositorySupport { - private val EcuIdPath = Segment.flatMap(EcuIdentifier.from(_).toOption) + private val EcuIdPath = Segment.flatMap(_.refineTry[ValidEcuIdentifier].toOption) private val KeyIdPath = Segment.flatMap(_.refineTry[ValidKeyId].toOption) private val TargetNamePath: PathMatcher1[TargetName] = Segment.map(TargetName.apply) @@ -71,7 +73,6 @@ class AdminResource(extractNamespace: Directive1[Namespace], val keyserverClient val deviceRoleGeneration = new DeviceRoleGeneration(keyserverClient) val offlineUpdates = new OfflineUpdates(keyserverClient) val remoteSessions = new RemoteSessions(keyserverClient) - val updateScheduler = new UpdateScheduler() private def findDevicesCurrentTarget(ns: Namespace, devices: Seq[DeviceId]): Future[DevicesCurrentTarget] = { @@ -176,60 +177,39 @@ class AdminResource(extractNamespace: Directive1[Namespace], val keyserverClient def devicePath(ns: Namespace): Route = pathPrefix(DeviceId.Path) { device => - pathPrefix("scheduled-updates") { - pathEnd { - (post & entity(as[CreateScheduledUpdateRequest])) { req => - val f = updateScheduler - .create(ns, req.device, req.updateId, req.scheduledAt) - .map(id => StatusCodes.Created -> id) - complete(f) - } ~ - get { - val f = scheduledUpdatesRepository.findFor(ns, device) - complete(f) - } - } ~ - path(ScheduledUpdateId.Path) { scheduledId => - delete { - val f = updateScheduler.cancel(ns, scheduledId) - complete(f) - } - } - - } ~ - pathPrefix("ecus") { - pathPrefix(EcuIdPath) { ecuId => - pathPrefix("auto_update") { - (pathEnd & get) { - complete( - autoUpdateDefinitionRepository - .findOnDevice(ns, device, ecuId) - .map(_.map(_.targetName)) - ) - } ~ - path(TargetNamePath) { targetName => - put { + pathPrefix("ecus") { + pathPrefix(EcuIdPath) { ecuId => + pathPrefix("auto_update") { + (pathEnd & get) { + complete( + autoUpdateDefinitionRepository + .findOnDevice(ns, device, ecuId) + .map(_.map(_.targetName)) + ) + } ~ + path(TargetNamePath) { targetName => + put { + complete( + autoUpdateDefinitionRepository + .persist(ns, device, ecuId, targetName) + .map(_ => StatusCodes.NoContent) + ) + } ~ + delete { complete( autoUpdateDefinitionRepository - .persist(ns, device, ecuId, targetName) + .remove(ns, device, ecuId, targetName) .map(_ => StatusCodes.NoContent) ) - } ~ - delete { - complete( - autoUpdateDefinitionRepository - .remove(ns, device, ecuId, targetName) - .map(_ => StatusCodes.NoContent) - ) - } - } - } ~ - (path("public_key") & get) { - val key = ecuRepository.findBySerial(ns, device, ecuId).map(_.publicKey) - complete(key) + } } - } - } ~ + } ~ + (path("public_key") & get) { + val key = ecuRepository.findBySerial(ns, device, ecuId).map(_.publicKey) + complete(key) + } + } + } ~ get { val f = deviceRegistration.findDeviceEcuInfo(ns, device) complete(f) @@ -251,12 +231,20 @@ class AdminResource(extractNamespace: Directive1[Namespace], val keyserverClient } } }, - (path("remote-sessions") & UserRepoId(ns)) { repoId => - (post & entity(as[RemoteSessionRequest])) { req => - val f = remoteSessions.set(repoId, req.remoteSessions, req.previousVersion) - complete(f.map(_.content)) + (path("remote-commands") & UserRepoId(ns)) { repoId => + (post & entity(as[RemoteCommandRequest])) { req => + val f = remoteSessions + .setRemoteCommands(repoId, req.remoteCommands) + .map(_ => StatusCodes.Accepted) + complete(f) } - }, + } ~ + (path("remote-sessions") & UserRepoId(ns)) { repoId => + (post & entity(as[RemoteSessionRequest])) { req => + val f = remoteSessions.setRemoteSessions(repoId, req.remoteSessions) + complete(f.map(_.content)) + } + }, (path("remote-sessions.json") & UserRepoId(ns)) { repoId => get { val f = remoteSessions.find(repoId) @@ -306,7 +294,7 @@ class AdminResource(extractNamespace: Directive1[Namespace], val keyserverClient * [[LegacyRoutes.route]] */ parameter(Symbol("primaryHardwareId").as[HardwareIdentifier]) { hardwareId => - PaginationParameters { (limit, offset) => + PaginationParameters { (offset, limit) => val f = provisionedDeviceRepository .findDevices(ns, hardwareId, offset, limit) .map(_.toClient) @@ -315,13 +303,13 @@ class AdminResource(extractNamespace: Directive1[Namespace], val keyserverClient } }, path("hardware_identifiers") { - PaginationParameters { (limit, offset) => - val f = ecuRepository.findAllHardwareIdentifiers(ns, offset, limit) + PaginationParameters { (offset, limit) => + val f = ecuRepository. findAllHardwareIdentifiers(ns, offset, limit) complete(f) } }, path("ecus") { - PaginationParameters { (limit, offset) => + PaginationParameters { (offset, limit) => val pagedEcus = ecuRepository.findAll(ns).map { ecuTuples => val uniqueDeviceIds = ecuTuples.map(_._1.deviceId).toSet val s = uniqueDeviceIds.map { deviceId => diff --git a/src/main/scala/com/advancedtelematic/director/http/AssignmentsResource.scala b/src/main/scala/com/advancedtelematic/director/http/AssignmentsResource.scala index 3bef6eb3..bd479bd7 100644 --- a/src/main/scala/com/advancedtelematic/director/http/AssignmentsResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/AssignmentsResource.scala @@ -1,22 +1,24 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.marshalling.ToResponseMarshallable -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.* -import akka.http.scaladsl.unmarshalling.PredefinedFromStringUnmarshallers.CsvSeq +import org.apache.pekko.http.scaladsl.marshalling.ToResponseMarshallable +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.* +import org.apache.pekko.http.scaladsl.unmarshalling.PredefinedFromStringUnmarshallers.CsvSeq import cats.implicits.* import com.advancedtelematic.director.data.AdminDataType.AssignUpdateRequest import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.DataType.TargetSpecId import com.advancedtelematic.director.http.DeviceAssignments.AssignmentCreateResult import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.http.UUIDKeyAkka.* +import com.advancedtelematic.libats.http.UUIDKeyPekko.* import com.advancedtelematic.libats.messaging.MessageBusPublisher -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libats.messaging_datatype.Messages.{ DeviceUpdateAssigned, DeviceUpdateEvent } -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* +import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshaller import slick.jdbc.MySQLProfile.api.Database import java.time.Instant @@ -47,22 +49,19 @@ class AssignmentsResource(extractNamespace: Directive1[Namespace])( } } - private implicit val updateIdUnmarshaller: akka.http.scaladsl.unmarshalling.Unmarshaller[ - String, - com.advancedtelematic.libats.messaging_datatype.DataType.UpdateId - ] = UpdateId.unmarshaller + private implicit val TargetSpecIdUnmarshaller: Unmarshaller[String, TargetSpecId] = + TargetSpecId.unmarshaller - private implicit val deviceIdUnmarshaller: akka.http.scaladsl.unmarshalling.Unmarshaller[ - String, - com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId - ] = DeviceId.unmarshaller + private implicit val deviceIdUnmarshaller + : Unmarshaller[String, com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId] = + DeviceId.unmarshaller val route = extractNamespace { ns => pathPrefix("assignments") { - (path("devices") & parameter(Symbol("mtuId").as[UpdateId]) & parameter( + (path("devices") & parameter(Symbol("targetSpecId").as[TargetSpecId]) & parameter( Symbol("ids").as(CsvSeq[DeviceId]) - )) { (mtuId, deviceIds) => - val f = deviceAssignments.findAffectedDevices(ns, deviceIds, mtuId) + )) { (targetSpecId, deviceIds) => + val f = deviceAssignments.findAffectedDevices(ns, deviceIds, targetSpecId) complete(f) } ~ pathEnd { diff --git a/src/main/scala/com/advancedtelematic/director/http/DeviceAssignments.scala b/src/main/scala/com/advancedtelematic/director/http/DeviceAssignments.scala index 6697bbb7..cb5a447d 100644 --- a/src/main/scala/com/advancedtelematic/director/http/DeviceAssignments.scala +++ b/src/main/scala/com/advancedtelematic/director/http/DeviceAssignments.scala @@ -2,20 +2,20 @@ package com.advancedtelematic.director.http import cats.implicits.* import com.advancedtelematic.director.data.AdminDataType.QueueResponse +import com.advancedtelematic.director.data.DataType.TargetSpecId import com.advancedtelematic.director.data.DbDataType.{Assignment, Ecu, EcuTarget, EcuTargetId} import com.advancedtelematic.director.data.UptaneDataType.* import com.advancedtelematic.director.db.* import com.advancedtelematic.director.http.Errors.InvalidAssignment import com.advancedtelematic.libats.data.DataType.{CorrelationId, Namespace} -import com.advancedtelematic.libats.data.{EcuIdentifier, ErrorRepresentation} +import com.advancedtelematic.libats.data.ErrorRepresentation import com.advancedtelematic.libats.http.Errors.Error import com.advancedtelematic.libats.messaging.MessageBusPublisher -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libats.messaging_datatype.Messages.{ DeviceUpdateCanceled, DeviceUpdateEvent } -import io.circe.syntax.* import org.slf4j.LoggerFactory import slick.jdbc.MySQLProfile.api.* @@ -57,7 +57,7 @@ class DeviceAssignments(implicit val db: Database, val ec: ExecutionContext) with AssignmentsRepositorySupport with EcuTargetsRepositorySupport with ProvisionedDeviceRepositorySupport - with ScheduledUpdatesRepositorySupport { + with UpdatesRepositorySupport { import DeviceAssignments.* @@ -65,6 +65,8 @@ class DeviceAssignments(implicit val db: Database, val ec: ExecutionContext) import scala.async.Async.* + private val affectDBIO = new AffectedEcusDBIO() + private def assignmentsToQueueResponse( ns: Namespace, idAssignments: Map[CorrelationId, Seq[Assignment]]): Future[Vector[QueueResponse]] = async { @@ -117,126 +119,41 @@ class DeviceAssignments(implicit val db: Database, val ec: ExecutionContext) def findAffectedDevices(ns: Namespace, deviceIds: Seq[DeviceId], - mtuId: UpdateId): Future[Seq[DeviceId]] = - findAffectedEcus(ns, deviceIds, mtuId).map(_.affected.map(_._1.deviceId)) - - import cats.syntax.option.* - - private def findAffectedEcus(ns: Namespace, devices: Seq[DeviceId], mtuId: UpdateId) = async { - val hardwareUpdates = await(hardwareUpdateRepository.findBy(ns, mtuId)) - - val allTargetIds = - hardwareUpdates.values.flatMap(v => List(v.toTarget.some, v.fromTarget).flatten) - val allTargets = await(ecuTargetsRepository.findAll(ns, allTargetIds.toSeq)) - - val ecusWithCompatibleHardware = - await(ecuRepository.findEcuWithTargets(devices.toSet, hardwareUpdates.keys.toSet)) - - val devicesWithIncompatibleHardware = - devices.toSet -- ecusWithCompatibleHardware.map(_._1.deviceId).toSet - val devicePrimaries = - await(ecuRepository.findDevicePrimaryIds(ns, devicesWithIncompatibleHardware)) - - val unaffectedDueToHardware = - devicesWithIncompatibleHardware.foldLeft(AffectedEcusResult(Seq.empty, Map.empty)) { - case (acc, deviceId) => - val error = Errors.DeviceNoCompatibleHardware(deviceId, mtuId) - _log.info(error.getMessage) - val primaryEcuId = devicePrimaries.getOrElse(deviceId, EcuIdentifier("unknown")) - acc.addNotAffected(deviceId, primaryEcuId, error) - } - - val devicesWithScheduledUpdates = await( - scheduledUpdatesRepository - .filterActiveUpdateExists(ns, ecusWithCompatibleHardware.map(_._1.deviceId).toSet) - ) - - _log - .atDebug() - .addKeyValue("devicesWithScheduledUpdates", devicesWithScheduledUpdates.asJson) - .log() - - val ecusWithoutScheduledUpdates = ecusWithCompatibleHardware.filterNot { case (ecu, _) => - devicesWithScheduledUpdates.contains(ecu.deviceId) - } - - val unaffectedDueToScheduledUpdates = - devicesWithScheduledUpdates.foldLeft(unaffectedDueToHardware) { case (acc, deviceId) => - val error = Errors.DeviceHasScheduledUpdate(deviceId, mtuId) - _log.info(error.getMessage) - acc.addNotAffected(deviceId, EcuIdentifier("unknown"), error) - } - - val ecus = ecusWithoutScheduledUpdates.foldLeft(unaffectedDueToScheduledUpdates) { - case (acc, (ecu, installedTarget)) => - val hwUpdate = hardwareUpdates(ecu.hardwareId) - val updateFrom = hwUpdate.fromTarget.flatMap(allTargets.get) - val updateTo = allTargets(hwUpdate.toTarget) - - if ( - hwUpdate.fromTarget.isEmpty || installedTarget.zip(updateFrom).exists { case (a, b) => - a.matches(b) - } - ) { - if (installedTarget.exists(_.matches(updateTo))) { - val error = Errors.InstalledTargetIsUpdate(ecu.deviceId, ecu.ecuSerial, hwUpdate) - _log.info(error.getMessage) - acc.addNotAffected(ecu.deviceId, ecu.ecuSerial, error) - } else { - _log.info(s"${ecu.deviceId}/${ecu.ecuSerial} affected for $hwUpdate") - acc.addAffected(ecu, hwUpdate.toTarget) - } - } else { - val error = Errors.NotAffectedByMtu(ecu.deviceId, ecu.ecuSerial, mtuId) - _log.info(error.getMessage) - acc.addNotAffected(ecu.deviceId, ecu.ecuSerial, error) - } - } - - val ecuIds = ecus.affected.map { case (ecu, _) => ecu.deviceId -> ecu.ecuSerial }.toSet - val ecusWithAssignments = await(assignmentsRepository.withAssignments(ecuIds)) - - ecus.affected.foldLeft(AffectedEcusResult(Seq.empty, ecus.notAffected)) { - case (acc, (ecu, _)) if ecusWithAssignments.contains(ecu.deviceId -> ecu.ecuSerial) => - val error = Errors.NotAffectedRunningAssignment(ecu.deviceId, ecu.ecuSerial) - _log.info(error.getMessage) - acc.addNotAffected(ecu.deviceId, ecu.ecuSerial, error) - case (acc, (ecu, target)) => - acc.addAffected(ecu, target) - } - } + targetSpecId: TargetSpecId): Future[Seq[DeviceId]] = + affectDBIO.findAffectedEcus(ns, deviceIds, targetSpecId).map(_.affected.map(_._1.deviceId)) def createForDevice(ns: Namespace, correlationId: CorrelationId, deviceId: DeviceId, - mtuId: UpdateId): Future[DeviceId] = - createForDevices(ns, correlationId, List(deviceId), mtuId).map( + targetSpecId: TargetSpecId): Future[DeviceId] = + createForDevices(ns, correlationId, List(deviceId), targetSpecId).map( _.affected.head ) // TODO: This HEAD is problematic def createForDevices(ns: Namespace, correlationId: CorrelationId, devices: Seq[DeviceId], - mtuId: UpdateId): Future[AssignmentCreateResult] = async { - val ecus = await(findAffectedEcus(ns, devices, mtuId)) + targetSpecId: TargetSpecId): Future[AssignmentCreateResult] = async { + val ecus = await(affectDBIO.findAffectedEcus(ns, devices, targetSpecId)) - _log.debug(s"$ns $correlationId $devices $mtuId") + _log.debug(s"$ns $correlationId $devices $targetSpecId") if (ecus.affected.isEmpty) { - _log.warn(s"No devices affected for this assignment: $ns, $correlationId, $devices, $mtuId") + _log.warn( + s"No devices affected for this assignment: $ns, $correlationId, $devices, $targetSpecId" + ) AssignmentCreateResult(Seq.empty, ecus.notAffectedSerializable) } else { - val assignments = ecus.affected.foldLeft(List.empty[Assignment]) { - case (acc, (ecu, toTargetId)) => - Assignment( - ns, - ecu.deviceId, - ecu.ecuSerial, - toTargetId, - correlationId, - inFlight = false, - createdAt = Instant.now - ) :: acc + val assignments = ecus.affected.map { case (ecu, toTargetId) => + Assignment( + ns, + ecu.deviceId, + ecu.ecuSerial, + toTargetId, + correlationId, + inFlight = false, + createdAt = Instant.now + ) } await(assignmentsRepository.persistMany(provisionedDeviceRepository)(assignments)) @@ -248,7 +165,11 @@ class DeviceAssignments(implicit val db: Database, val ec: ExecutionContext) def cancel(namespace: Namespace, deviceId: DeviceId, cancelInFlight: Boolean = false)( implicit messageBusPublisher: MessageBusPublisher): Future[Unit] = assignmentsRepository - .processDeviceCancellation(provisionedDeviceRepository)(namespace, deviceId, cancelInFlight) + .processDeviceCancellation(provisionedDeviceRepository, updatesRepository)( + namespace, + deviceId, + cancelInFlight + ) .flatMap { ids => ids .map[DeviceUpdateEvent](ci => DeviceUpdateCanceled(namespace, Instant.now, ci, deviceId)) @@ -259,7 +180,7 @@ class DeviceAssignments(implicit val db: Database, val ec: ExecutionContext) def cancel(namespace: Namespace, devices: Seq[DeviceId])( implicit messageBusPublisher: MessageBusPublisher): Future[Seq[Assignment]] = assignmentsRepository - .processCancellation(provisionedDeviceRepository)(namespace, devices) + .processCancellation(provisionedDeviceRepository, updatesRepository)(namespace, devices) .flatMap { canceledAssignments => Future.traverse(canceledAssignments) { canceledAssignment => val ev: DeviceUpdateEvent = diff --git a/src/main/scala/com/advancedtelematic/director/http/DeviceResource.scala b/src/main/scala/com/advancedtelematic/director/http/DeviceResource.scala index 1c32ce92..2fd09a90 100644 --- a/src/main/scala/com/advancedtelematic/director/http/DeviceResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/DeviceResource.scala @@ -1,8 +1,8 @@ package com.advancedtelematic.director.http import java.time.Instant -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.{Directive, Directive0, Directive1, Route} +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.{Directive, Directive0, Directive1, Route} import cats.data.Validated.{Invalid, Valid} import cats.implicits.* import com.advancedtelematic.director.data.AdminDataType.RegisterDevice @@ -15,7 +15,7 @@ import com.advancedtelematic.director.manifest.{DeviceManifestProcess, ManifestC import com.advancedtelematic.director.repo.{DeviceRoleGeneration, OfflineUpdates, RemoteSessions} import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.data.ErrorRepresentation.* -import com.advancedtelematic.libats.http.UUIDKeyAkka.* +import com.advancedtelematic.libats.http.UUIDKeyPekko.* import com.advancedtelematic.libats.messaging.MessageBusPublisher import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libats.messaging_datatype.Messages.{ @@ -28,7 +28,7 @@ import com.advancedtelematic.libtuf.data.TufCodecs.* import com.advancedtelematic.libtuf.data.TufDataType.{RoleType, SignedPayload} import com.advancedtelematic.libtuf_server.data.Marshalling.JsonRoleTypeMetaPath import com.advancedtelematic.libtuf_server.keyserver.KeyserverClient -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.Json import slick.jdbc.MySQLProfile.api.* @@ -55,7 +55,7 @@ class DeviceResource(extractNamespace: Directive1[Namespace], with AdminRolesRepositorySupport with RootFetching { - import akka.http.scaladsl.server.Directives.* + import org.apache.pekko.http.scaladsl.server.Directives.* val deviceRegistration = new DeviceRegistration(keyserverClient) val deviceManifestProcess = new DeviceManifestProcess() diff --git a/src/main/scala/com/advancedtelematic/director/http/DirectorDebugResource.scala b/src/main/scala/com/advancedtelematic/director/http/DirectorDebugResource.scala index 8309b822..06cb7dca 100644 --- a/src/main/scala/com/advancedtelematic/director/http/DirectorDebugResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/DirectorDebugResource.scala @@ -1,26 +1,32 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.server.{Directives, Route} +import org.apache.pekko.http.scaladsl.server.{Directives, Route} import com.advancedtelematic.director.data.DeviceRequest.DeviceManifest -import com.advancedtelematic.director.db.{CompiledManifestExecutor, DirectorDbDebug} +import com.advancedtelematic.director.db.{ + CompiledManifestExecutor, + DeviceManifestRepositorySupport, + DirectorDbDebug +} import com.advancedtelematic.director.manifest.ManifestCompiler import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.debug.DebugRoutes import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import slick.jdbc.MySQLProfile.api.* import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.http.PaginationParametersDirectives.PaginationParameters import scala.concurrent.ExecutionContext -class DirectorDebugResource()(implicit val db: Database, val ec: ExecutionContext) { +class DirectorDebugResource()(implicit val db: Database, val ec: ExecutionContext) + extends DeviceManifestRepositorySupport { import com.advancedtelematic.libats.debug.DebugDatatype.* - import com.advancedtelematic.libats.http.UUIDKeyAkka.* + import com.advancedtelematic.libats.http.UUIDKeyPekko.* val debug = new DirectorDbDebug() import Directives.* - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* val route: Route = DebugRoutes.routes { Directives.concat( @@ -31,17 +37,22 @@ class DirectorDebugResource()(implicit val db: Database, val ec: ExecutionContex val f = db.run(new CompiledManifestExecutor().findStateAction(deviceId)) complete(f) }, - (put & path("run-manifest" / DeviceId.Path) & entity(as[DeviceManifest])) { - (deviceId, manifest) => - onSuccess(db.run(new CompiledManifestExecutor().findStateAction(deviceId))) { - currentState => - complete( - ManifestCompiler(Namespace("notused"), manifest) - .apply(currentState) - .map(_.knownState) - ) - } - } + (get & path("device-manifests" / DeviceId.Path) & PaginationParameters) { + (deviceId, offset, limit) => + val f = deviceManifestRepository.findAll(deviceId, offset, limit).map(_.map(_._1)) + complete(f) + } ~ + (put & path("run-manifest" / DeviceId.Path) & entity(as[DeviceManifest])) { + (deviceId, manifest) => + onSuccess(db.run(new CompiledManifestExecutor().findStateAction(deviceId))) { + currentState => + complete( + ManifestCompiler(Namespace("notused"), manifest) + .apply(currentState) + .map(_.knownState) + ) + } + } ) } diff --git a/src/main/scala/com/advancedtelematic/director/http/DirectorRoutes.scala b/src/main/scala/com/advancedtelematic/director/http/DirectorRoutes.scala index 99f01f55..ac2fd58d 100644 --- a/src/main/scala/com/advancedtelematic/director/http/DirectorRoutes.scala +++ b/src/main/scala/com/advancedtelematic/director/http/DirectorRoutes.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.server.{Directives, _} +import org.apache.pekko.http.scaladsl.server.{Directives, _} import com.advancedtelematic.libats.http.{ErrorHandler, NamespaceDirectives} import com.advancedtelematic.libats.messaging.MessageBusPublisher import com.advancedtelematic.libtuf_server.keyserver.KeyserverClient @@ -24,8 +24,9 @@ class DirectorRoutes(keyserverClient: KeyserverClient, allowEcuReplacement: Bool new AdminResource(extractNamespace, keyserverClient).route ~ new AssignmentsResource(extractNamespace).route ~ new DeviceResource(extractNamespace, keyserverClient, allowEcuReplacement).route ~ - new MultiTargetUpdatesResource(extractNamespace).route ~ - new LegacyRoutes(extractNamespace).route + new TargetUpdateSpecsResource(extractNamespace).route ~ + new LegacyRoutes(extractNamespace).route ~ + new UpdateResource(extractNamespace).route } ~ new DirectorDebugResource().route } diff --git a/src/main/scala/com/advancedtelematic/director/http/Errors.scala b/src/main/scala/com/advancedtelematic/director/http/Errors.scala index 60eb4660..de55e03b 100644 --- a/src/main/scala/com/advancedtelematic/director/http/Errors.scala +++ b/src/main/scala/com/advancedtelematic/director/http/Errors.scala @@ -1,18 +1,18 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import cats.Show -import cats.data.NonEmptyList -import com.advancedtelematic.director.data.DataType.AdminRoleName +import cats.data.{NonEmptyList, NonEmptySet} +import com.advancedtelematic.director.data.DataType.{AdminRoleName, TargetSpecId, UpdateId} import com.advancedtelematic.director.data.DbDataType.{EcuTargetId, HardwareUpdate} import com.advancedtelematic.libats.data.DataType.CorrelationId -import com.advancedtelematic.libats.data.{EcuIdentifier, ErrorCode} +import com.advancedtelematic.libats.data.ErrorCode import com.advancedtelematic.libats.http.Errors.{Error, JsonError, MissingEntityId, RawError} -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libtuf.data.ClientDataType.TufRole import com.advancedtelematic.libtuf.data.TufDataType.RepoId import com.advancedtelematic.libtuf.data.TufDataType.RoleType.RoleType -import io.circe.Encoder +import io.circe.{Encoder, Json} import io.circe.syntax.* object ErrorCodes { @@ -49,7 +49,13 @@ object ErrorCodes { val UpdateScheduleError = ErrorCode("update_schedule_error") - val DeviceHasScheduledUpdate = ErrorCode("update_already_scheduled_error") + val DeviceHasActiveUpdate = ErrorCode("device_has_active_update") + + val UpdateCannotBeCancelled = ErrorCode("update_cannot_be_cancelled") + + val DeviceCannotBeUpdated = ErrorCode("device_cannot_be_updated") + + val AssignmentBelongsToUpdate = ErrorCode("assignment_belongs_to_update") } object Errors { @@ -76,25 +82,27 @@ object Errors { s"Ecu $deviceId/$ecuIdentifier not affected for $update, installed target is already the target update" ) - case class DeviceNoCompatibleHardware(deviceId: DeviceId, mtuId: UpdateId) + case class DeviceNoCompatibleHardware(deviceId: DeviceId, targetSpecId: TargetSpecId) extends Error( ErrorCodes.DeviceNoCompatibleHardware, StatusCodes.BadRequest, - s"$deviceId not affected for $mtuId, device does not have any ecu with compatible hardware" + s"$deviceId not affected for $targetSpecId, device does not have any ecu with compatible hardware" ) - case class DeviceHasScheduledUpdate(deviceId: DeviceId, mtuId: UpdateId) + case class DeviceHasActiveUpdate(deviceId: DeviceId, targetSpecId: TargetSpecId) extends Error( - ErrorCodes.DeviceHasScheduledUpdate, + ErrorCodes.DeviceHasActiveUpdate, StatusCodes.BadRequest, - s"$deviceId not affected for $mtuId, there is an update scheduled for the device" + s"$deviceId not affected for $targetSpecId, there is an update scheduled for the device" ) - case class NotAffectedByMtu(deviceId: DeviceId, ecuIdentifier: EcuIdentifier, mtuId: UpdateId) + case class NotAffectedByMtu(deviceId: DeviceId, + ecuIdentifier: EcuIdentifier, + targetSpecId: TargetSpecId) extends Error( ErrorCodes.NotAffectedByMtu, StatusCodes.BadRequest, - s"ecu $deviceId$ecuIdentifier not affected by $mtuId" + s"ecu $deviceId$ecuIdentifier not affected by $targetSpecId" ) case class InvalidAssignment(targetId: EcuTargetId, correlationId: CorrelationId) @@ -115,6 +123,17 @@ object Errors { s"admin role $repoId/$name not found" ) + import com.advancedtelematic.libats.data.ErrorRepresentation.* + import com.advancedtelematic.libats.codecs.CirceRefined.* + + def DeviceCannotBeUpdated(deviceId: DeviceId, reasons: Map[EcuIdentifier, Error]) = JsonError( + ErrorCodes.DeviceCannotBeUpdated, + StatusCodes.BadRequest, + reasons.view.mapValues(_.toErrorRepr).toMap.asJson, + msg = + s"Device $deviceId cannot be updated: ${reasons.view.mapValues(_.msg).toList.mkString(", ")}" + ) + def DeviceMissingPrimaryEcu(deviceId: DeviceId) = RawError( ErrorCodes.DeviceMissingPrimaryEcu, StatusCodes.NotFound, @@ -182,6 +201,13 @@ object Errors { s"there is an assignmement in flight for $deviceId" ) + def AssignmentInFlight(deviceIds: NonEmptyList[DeviceId]) = + RawError( + ErrorCode("assignment_in_flight_devices"), + StatusCodes.Conflict, + s"there is an assignment in flight for $deviceIds" + ) + def TooManyOfflineRoles(max: Int) = RawError( ErrorCodes.TooManyOfflineRoles, @@ -202,7 +228,20 @@ object Errors { ErrorCodes.UpdateScheduleError, StatusCodes.BadRequest, reasons.asJson, - msg = "Invalid ecu status for scheduled update" + msg = s"Invalid ecu ($deviceId) status for scheduled update" ) + def UpdateCannotBeCancelled(updateId: UpdateId) = + RawError( + ErrorCodes.UpdateCannotBeCancelled, + StatusCodes.Conflict, + s"Update $updateId cannot be cancelled because it is not in the Assigned status" + ) + + def AssignmentBelongsToUpdate(correlationIds: NonEmptyList[CorrelationId]) = + RawError( + ErrorCodes.AssignmentBelongsToUpdate, + StatusCodes.Conflict, + s"The assignment cannot be cancelled using the assignments API because it belongs to an update. Use the updates API" + ) } diff --git a/src/main/scala/com/advancedtelematic/director/http/ForceHeader.scala b/src/main/scala/com/advancedtelematic/director/http/ForceHeader.scala index 61d58898..6f2e2835 100644 --- a/src/main/scala/com/advancedtelematic/director/http/ForceHeader.scala +++ b/src/main/scala/com/advancedtelematic/director/http/ForceHeader.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.headers.{ModeledCustomHeader, ModeledCustomHeaderCompanion} +import org.apache.pekko.http.scaladsl.model.headers.{ModeledCustomHeader, ModeledCustomHeaderCompanion} import scala.util.Try diff --git a/src/main/scala/com/advancedtelematic/director/http/LegacyRoutes.scala b/src/main/scala/com/advancedtelematic/director/http/LegacyRoutes.scala index f33395eb..558b555d 100644 --- a/src/main/scala/com/advancedtelematic/director/http/LegacyRoutes.scala +++ b/src/main/scala/com/advancedtelematic/director/http/LegacyRoutes.scala @@ -1,22 +1,19 @@ package com.advancedtelematic.director.http import java.time.Instant - -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.{Directive1, Route} +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.Directives.* +import org.apache.pekko.http.scaladsl.server.{Directive1, Route} +import com.advancedtelematic.director.data.DataType.TargetSpecId import com.advancedtelematic.director.db.{EcuRepositorySupport, ProvisionedDeviceRepositorySupport} -import com.advancedtelematic.director.http.PaginationParametersDirectives._ -import com.advancedtelematic.libats.data.DataType.{MultiTargetUpdateId, Namespace} -import com.advancedtelematic.libats.http.UUIDKeyAkka._ +import com.advancedtelematic.director.http.PaginationParametersDirectives.* +import com.advancedtelematic.libats.data.DataType.{MultiTargetUpdateCorrelationId, Namespace} +import com.advancedtelematic.libats.http.UUIDKeyPekko.* import com.advancedtelematic.libats.messaging.MessageBusPublisher -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} -import com.advancedtelematic.libats.messaging_datatype.Messages.{ - DeviceUpdateAssigned, - DeviceUpdateEvent -} -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._ -import slick.jdbc.MySQLProfile.api._ +import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.messaging_datatype.Messages.{DeviceUpdateAssigned, DeviceUpdateEvent} +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* +import slick.jdbc.MySQLProfile.api.* import scala.concurrent.{ExecutionContext, Future} @@ -30,11 +27,12 @@ class LegacyRoutes(extractNamespace: Directive1[Namespace])( private val deviceAssignments = new DeviceAssignments() + // TODO: Remove this, and its endpoint, no longer used private def createDeviceAssignment(ns: Namespace, deviceId: DeviceId, - mtuId: UpdateId): Future[Unit] = { - val correlationId = MultiTargetUpdateId(mtuId.uuid) - val assignment = deviceAssignments.createForDevice(ns, correlationId, deviceId, mtuId) + targetSpecId: TargetSpecId): Future[Unit] = { + val correlationId = MultiTargetUpdateCorrelationId(targetSpecId.uuid) + val assignment = deviceAssignments.createForDevice(ns, correlationId, deviceId, targetSpecId) assignment.map { d => val msg: DeviceUpdateEvent = DeviceUpdateAssigned(ns, Instant.now(), correlationId, d) @@ -45,10 +43,10 @@ class LegacyRoutes(extractNamespace: Directive1[Namespace])( val route: Route = extractNamespace { ns => concat( - path("admin" / "devices" / DeviceId.Path / "multi_target_update" / UpdateId.Path) { - (deviceId, updateId) => + path("admin" / "devices" / DeviceId.Path / "multi_target_update" / TargetSpecId.Path) { + (deviceId, TargetSpecId) => put { - val f = createDeviceAssignment(ns, deviceId, updateId).map(_ => StatusCodes.OK) + val f = createDeviceAssignment(ns, deviceId, TargetSpecId).map(_ => StatusCodes.OK) complete(f) } }, @@ -58,7 +56,7 @@ class LegacyRoutes(extractNamespace: Directive1[Namespace])( complete(a.map(_.map(_.deviceId))) } }, - (path("admin" / "devices") & PaginationParameters) { (limit, offset) => + (path("admin" / "devices") & PaginationParameters) { (offset, limit) => get { complete(provisionedDeviceRepository.findAllDeviceIds(ns, offset, limit)) } diff --git a/src/main/scala/com/advancedtelematic/director/http/MultiTargetUpdatesResource.scala b/src/main/scala/com/advancedtelematic/director/http/MultiTargetUpdatesResource.scala deleted file mode 100644 index 55ec51b0..00000000 --- a/src/main/scala/com/advancedtelematic/director/http/MultiTargetUpdatesResource.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.advancedtelematic.director.http - -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server._ -import com.advancedtelematic.director.data.AdminDataType.MultiTargetUpdate -import com.advancedtelematic.libats.messaging_datatype.DataType.UpdateId -import slick.jdbc.MySQLProfile.api.Database -import com.advancedtelematic.libats.http.UUIDKeyAkka._ -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._ -import com.advancedtelematic.director.data.Codecs._ -import com.advancedtelematic.director.db.MultiTargetUpdates -import com.advancedtelematic.director.http.Errors.InvalidMtu -import com.advancedtelematic.libats.data.DataType.Namespace - -import scala.concurrent.ExecutionContext -import com.advancedtelematic.libats.codecs.CirceCodecs._ - -class MultiTargetUpdatesResource(extractNamespace: Directive1[Namespace])( - implicit val db: Database, - val ec: ExecutionContext) { - import Directives._ - - val multiTargetUpdates = new MultiTargetUpdates() - - val route = extractNamespace { ns => - pathPrefix("multi_target_updates") { - (get & pathPrefix(UpdateId.Path)) { uid => - // For some reason director-v1 accepts `{targets: ...}` but returns `{...}` - // To make app compatible with director-v2, for now we do the same, but we should be returning what we accept: - // complete(multiTargetUpdates.find(ns, uid)) - complete(multiTargetUpdates.find(ns, uid).map(_.targets)) - } ~ - (post & pathEnd) { - entity(as[MultiTargetUpdate]) { mtuRequest => - if (mtuRequest.targets.isEmpty) { - throw InvalidMtu("targets cannot be empty") - } - - val f = multiTargetUpdates.create(ns, mtuRequest).map { - StatusCodes.Created -> _ - } - complete(f) - } - } - } - } - -} diff --git a/src/main/scala/com/advancedtelematic/director/http/NamespaceRepoId.scala b/src/main/scala/com/advancedtelematic/director/http/NamespaceRepoId.scala index 7d9afbc9..d323a1ca 100644 --- a/src/main/scala/com/advancedtelematic/director/http/NamespaceRepoId.scala +++ b/src/main/scala/com/advancedtelematic/director/http/NamespaceRepoId.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.server._ +import org.apache.pekko.http.scaladsl.server._ import com.advancedtelematic.director.db.RepoNamespaceRepositorySupport import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libtuf.data.TufDataType.RepoId diff --git a/src/main/scala/com/advancedtelematic/director/http/PaginationParametersDirectives.scala b/src/main/scala/com/advancedtelematic/director/http/PaginationParametersDirectives.scala index 3830d8b3..27b6962f 100644 --- a/src/main/scala/com/advancedtelematic/director/http/PaginationParametersDirectives.scala +++ b/src/main/scala/com/advancedtelematic/director/http/PaginationParametersDirectives.scala @@ -1,16 +1,21 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.server.Directive -import akka.http.scaladsl.server.Directives._ +import org.apache.pekko.http.scaladsl.server.{Directive, MalformedQueryParamRejection} +import org.apache.pekko.http.scaladsl.server.Directives.* +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} object PaginationParametersDirectives { - val PaginationParameters: Directive[(Long, Long)] = - (parameters(Symbol("limit").as[Long].?) & parameters(Symbol("offset").as[Long].?)).tmap { + val PaginationParameters: Directive[(Offset, Limit)] = + (parameters(Symbol("limit").as[Long].?) & parameters(Symbol("offset").as[Long].?)).tflatMap { + case (Some(mlimit), _) if mlimit < 0 => + reject(MalformedQueryParamRejection("limit", "limit cannot be negative")) + case (_, Some(mOffset)) if mOffset < 0 => + reject(MalformedQueryParamRejection("offset", "offset cannot be negative")) case (mLimit, mOffset) => val limit = mLimit.getOrElse(50L).min(1000) val offset = mOffset.getOrElse(0L) - (limit, offset) + tprovide(Offset(offset), Limit(limit)) } } diff --git a/src/main/scala/com/advancedtelematic/director/http/RootFetching.scala b/src/main/scala/com/advancedtelematic/director/http/RootFetching.scala index 1a249f2a..caf01697 100644 --- a/src/main/scala/com/advancedtelematic/director/http/RootFetching.scala +++ b/src/main/scala/com/advancedtelematic/director/http/RootFetching.scala @@ -5,9 +5,10 @@ import com.advancedtelematic.director.db.{ RepoNamespaceRepositorySupport } import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libtuf.data.ClientDataType.RootRole -import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, SignedPayload} +import com.advancedtelematic.libtuf.data.ClientCodecs.* +import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, JsonSignedPayload} import com.advancedtelematic.libtuf_server.keyserver.KeyserverClient +import io.circe.syntax.* import java.time.Instant import java.time.temporal.ChronoUnit @@ -18,7 +19,7 @@ trait RootFetching { val keyserverClient: KeyserverClient - def fetchRoot(ns: Namespace, version: Option[Int]): Future[SignedPayload[RootRole]] = { + def fetchRoot(ns: Namespace, version: Option[Int]): Future[JsonSignedPayload] = { val fetchFn = version .map(v => (r: RepoId, _: Option[Instant]) => keyserverClient.fetchRootRole(r, v)) .getOrElse((r: RepoId, i: Option[Instant]) => @@ -30,7 +31,7 @@ trait RootFetching { latestExpiringRole <- adminRolesRepository.findLatestExpireDate(repoId) latestExpire = latestExpiringRole.map(_.plus(180, ChronoUnit.DAYS)) root <- fetchFn(repoId, latestExpire) - } yield root + } yield JsonSignedPayload(root.signatures, root.signed.asJson) } } diff --git a/src/main/scala/com/advancedtelematic/director/http/TargetUpdateSpecsResource.scala b/src/main/scala/com/advancedtelematic/director/http/TargetUpdateSpecsResource.scala new file mode 100644 index 00000000..eb9b52ed --- /dev/null +++ b/src/main/scala/com/advancedtelematic/director/http/TargetUpdateSpecsResource.scala @@ -0,0 +1,47 @@ +package com.advancedtelematic.director.http + +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.* +import com.advancedtelematic.director.data.AdminDataType.TargetUpdateSpec +import slick.jdbc.MySQLProfile.api.Database +import com.advancedtelematic.libats.http.UUIDKeyPekko.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* +import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.DataType.TargetSpecId +import com.advancedtelematic.director.db.TargetUpdateSpecs +import com.advancedtelematic.director.http.Errors.InvalidMtu +import com.advancedtelematic.libats.data.DataType.Namespace + +import scala.concurrent.ExecutionContext +import com.advancedtelematic.libats.codecs.CirceCodecs.* + +class TargetUpdateSpecsResource(extractNamespace: Directive1[Namespace])( + implicit val db: Database, + val ec: ExecutionContext) { + import Directives.* + + private val targetUpdateSpecs = new TargetUpdateSpecs() + + val route = extractNamespace { ns => + pathPrefix("multi_target_updates") { + (get & pathPrefix(TargetSpecId.Path)) { uid => + // For some reason director-v1 accepts `{targets: ...}` but returns `{...}` + // To make app compatible with director-v2, for now we do the same, but we should be returning what we accept: + complete(targetUpdateSpecs.find(ns, uid).map(_.targets)) + } ~ + (post & pathEnd) { + entity(as[TargetUpdateSpec]) { mtuRequest => + if (mtuRequest.targets.isEmpty) { + throw InvalidMtu("targets cannot be empty") + } + + val f = targetUpdateSpecs.create(ns, mtuRequest).map { + StatusCodes.Created -> _ + } + complete(f) + } + } + } + } + +} diff --git a/src/main/scala/com/advancedtelematic/director/http/UpdateResource.scala b/src/main/scala/com/advancedtelematic/director/http/UpdateResource.scala new file mode 100644 index 00000000..d19c0ea3 --- /dev/null +++ b/src/main/scala/com/advancedtelematic/director/http/UpdateResource.scala @@ -0,0 +1,188 @@ +package com.advancedtelematic.director.http + +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.{Directive1, Route} +import cats.implicits.* +import com.advancedtelematic.director.data.AdminDataType.{TargetUpdateRequest, TargetUpdateSpec} +import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.DataType.{Update, UpdateId} +import com.advancedtelematic.director.db.UpdatesDBIO +import com.advancedtelematic.director.http.PaginationParametersDirectives.PaginationParameters +import com.advancedtelematic.libats.data.DataType.{Namespace, ResultCode} +import com.advancedtelematic.libats.data.ErrorRepresentation +import com.advancedtelematic.libats.http.UUIDKeyPekko.UUIDKeyPathOp +import com.advancedtelematic.libats.messaging.MessageBusPublisher +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier, EventType} +import com.advancedtelematic.libats.messaging_datatype.Messages.{ + DeviceUpdateAssigned, + DeviceUpdateCanceled, + DeviceUpdateEvent +} +import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, TargetFilename} +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* +import slick.jdbc.MySQLProfile.api.* + +import java.time.Instant +import scala.concurrent.ExecutionContext + +case class CreateDeviceUpdateRequest(targets: Map[HardwareIdentifier, TargetUpdateRequest], + scheduledFor: Option[Instant] = None) + +case class CreateUpdateRequest(targets: Map[HardwareIdentifier, TargetUpdateRequest], + devices: Seq[DeviceId]) + +case class CreateUpdateResult(affected: Seq[DeviceId], + notAffected: Map[DeviceId, Map[EcuIdentifier, ErrorRepresentation]]) + +case class UpdateReportedResult(resultCode: ResultCode, success: Boolean, description: Option[String]) + +case class UpdateResultResponse(hardwareId: HardwareIdentifier, + id: TargetFilename, + success: Boolean, + description: String, + result: Option[UpdateReportedResult]) + +case class UpdateDetailResponse(updateId: UpdateId, + status: Update.Status, + createdAt: Instant, + scheduledFor: Option[Instant], + completedAt: Option[Instant], + packages: Map[HardwareIdentifier, TargetFilename], + deviceResult: Option[UpdateReportedResult], + ecuResults: Map[EcuIdentifier, UpdateResultResponse]) + +case class UpdateResponse(updateId: UpdateId, + status: Update.Status, + createdAt: Instant, + scheduledFor: Option[Instant], + completedAt: Option[Instant], + deviceResult: Option[UpdateReportedResult], + packages: Map[HardwareIdentifier, TargetFilename]) + +case class UpdateEventResponse(deviceId: DeviceId, + eventType: EventType, + deviceTime: Instant, + receivedAt: Instant, + success: Option[Boolean], + ecu: Option[EcuIdentifier]) + +class UpdateResource(extractNamespace: Directive1[Namespace])( + implicit val db: Database, + val ec: ExecutionContext, + val messageBusPublisher: MessageBusPublisher) { + + import org.apache.pekko.http.scaladsl.server.Directives.* + + private val updates = new UpdatesDBIO() + + private def createDeviceUpdate(ns: Namespace, + targets: Map[HardwareIdentifier, TargetUpdateRequest], + deviceId: DeviceId, + scheduledFor: Option[Instant]): Route = complete(for { + updateId <- updates.createFor(ns, deviceId, TargetUpdateSpec(targets), scheduledFor) + _ <- messageBusPublisher.publishSafe( + DeviceUpdateAssigned( + ns, + Instant.now(), + updateId.toCorrelationId, + deviceId, + scheduledFor + ): DeviceUpdateEvent + ) + } yield updateId) + + val route = extractNamespace { ns => + concat( + path("updates") { + (get & pathEnd & PaginationParameters) { case (offset, limit) => + complete(updates.findAll(ns, offset, limit)) + } + }, + path("updates" / UpdateId.Path) { updateId => + concat( + patch { + val f = updates + .cancelAll(ns, updateId) + .flatMap { ids => + ids.map { case (id, deviceId) => + messageBusPublisher.publishSafe( + DeviceUpdateCanceled(ns, Instant.now(), id, deviceId): DeviceUpdateEvent + ) + }.sequence_ + } + complete(f.map(_ => StatusCodes.NoContent)) + }, + path("devices") { + (get & PaginationParameters) { case (offset, limit) => + complete(updates.findUpdateDevices(ns, updateId, offset, limit)) + } + } + ) + }, + pathPrefix("updates" / "devices") { + path(DeviceId.Path) { deviceId => + concat( + (get & pathEnd & PaginationParameters) { (offset, limit) => + val f = + updates.findFor(ns, deviceId, offset, limit) + complete(f) + }, + (post & pathEnd) { + entity(as[CreateDeviceUpdateRequest]) { req => + createDeviceUpdate(ns, req.targets, deviceId, req.scheduledFor) + } + } + ) + } ~ + pathPrefix(DeviceId.Path / UpdateId.Path) { (deviceId, updateId) => + pathEnd { + concat( + (patch & optionalHeaderValueByType(ForceHeader)) { force => + val f = updates + .cancel(ns, updateId, deviceId, force.exists(_.asBoolean)) + .flatMap { id => + messageBusPublisher.publishSafe( + DeviceUpdateCanceled(ns, Instant.now(), id, deviceId): DeviceUpdateEvent + ) + } + complete(f.map(_ => StatusCodes.NoContent)) + }, + get { + val f = updates.find(ns, updateId, deviceId) + complete(f) + } + ) + } ~ + path("events") { + val f = updates.findEvents(ns, deviceId, updateId) + complete(f) + } + } ~ + (post & pathEnd) { + entity(as[CreateUpdateRequest]) { req => + val f = + updates + .createMany(ns, TargetUpdateSpec(req.targets), req.devices) + .flatMap { case (updateId, createResult) => + createResult.affected.toList + .traverse_ { deviceId => + messageBusPublisher.publishSafe( + DeviceUpdateAssigned( + ns, + Instant.now(), + updateId.toCorrelationId, + deviceId + ): DeviceUpdateEvent + ) + } + .map(_ => createResult) + } + + complete(f) + } + } + } + ) + } + +} diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DeviceMonitoringResource.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DeviceMonitoringResource.scala index 9654cf7d..2160ffde 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DeviceMonitoringResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DeviceMonitoringResource.scala @@ -1,8 +1,8 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.actor.ActorSystem -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.{Directive1, Route} +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.{Directive1, Route} import com.advancedtelematic.director.deviceregistry.data.Codecs.ObservationPublishResultCodec import com.advancedtelematic.director.deviceregistry.data.DataType.ObservationPublishResult import com.advancedtelematic.libats.data.DataType.Namespace @@ -12,7 +12,7 @@ import com.advancedtelematic.libats.messaging_datatype.Messages.{ deviceMetricsObservationMessageLike, DeviceMetricsObservation } -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.{Decoder, Json} import org.slf4j.LoggerFactory @@ -40,7 +40,7 @@ class DeviceMonitoringResource(namespaceExtractor: Directive1[Namespace], deviceNamespaceAuthorizer: Directive1[DataType.DeviceId], messageBus: MessageBusPublisher)(implicit system: ActorSystem) { - import akka.http.scaladsl.server.Directives.* + import org.apache.pekko.http.scaladsl.server.Directives.* import system.dispatcher private lazy val log = LoggerFactory.getLogger(this.getClass) diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DeviceRegistryRoutes.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DeviceRegistryRoutes.scala index 40798c3b..54fab48a 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DeviceRegistryRoutes.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DeviceRegistryRoutes.scala @@ -1,8 +1,8 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.actor.ActorSystem -import akka.http.scaladsl.server.{Directive1, Directives, Route} -import akka.stream.Materializer +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.http.scaladsl.server.{Directive1, Directives, Route} +import org.apache.pekko.stream.Materializer import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.http.DefaultRejectionHandler.rejectionHandler import com.advancedtelematic.libats.http.ErrorHandler diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DevicesResource.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DevicesResource.scala index eba7ee18..5b3065d9 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DevicesResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/DevicesResource.scala @@ -8,34 +8,25 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.* -import akka.http.scaladsl.model.headers.* -import akka.http.scaladsl.server.* -import akka.http.scaladsl.unmarshalling.PredefinedFromStringUnmarshallers.CsvSeq -import akka.http.scaladsl.unmarshalling.{FromStringUnmarshaller, Unmarshaller} -import akka.stream.Materializer -import akka.stream.alpakka.csv.scaladsl.{CsvParsing, CsvToMap} -import akka.stream.scaladsl.{Sink, Source} -import akka.util.ByteString +import com.advancedtelematic.libats.codecs.CirceRefined.* +import org.apache.pekko.http.scaladsl.model.* +import org.apache.pekko.http.scaladsl.model.headers.* +import org.apache.pekko.http.scaladsl.server.* +import org.apache.pekko.http.scaladsl.unmarshalling.PredefinedFromStringUnmarshallers.CsvSeq +import org.apache.pekko.http.scaladsl.unmarshalling.{FromStringUnmarshaller, Unmarshaller} +import org.apache.pekko.stream.Materializer +import org.apache.pekko.stream.connectors.csv.scaladsl.{CsvParsing, CsvToMap} +import org.apache.pekko.stream.scaladsl.{Sink, Source} +import org.apache.pekko.util.ByteString import cats.syntax.either.* import cats.syntax.show.* +import com.advancedtelematic.director.data.ClientDataType.TagSearchParam import com.advancedtelematic.director.db.deviceregistry.* -import com.advancedtelematic.director.db.deviceregistry.DbOps.PaginationResultOps +import com.advancedtelematic.director.db.deviceregistry.DbOps.* import com.advancedtelematic.director.deviceregistry.data.* import com.advancedtelematic.director.deviceregistry.data.Codecs.* import com.advancedtelematic.director.deviceregistry.data.DataType.InstallationStatsLevel.InstallationStatsLevel -import com.advancedtelematic.director.deviceregistry.data.DataType.{ - DeviceCountParams, - DeviceT, - DevicesQuery, - InstallationStatsLevel, - RenameTagId, - SearchParams, - SetDevice, - UpdateDevice, - UpdateHibernationStatusRequest, - UpdateTagValue -} +import com.advancedtelematic.director.deviceregistry.data.DataType.{DeviceCountParams, DeviceT, DevicesQuery, InstallationStatsLevel, RenameTagId, SearchParams, SetDevice, UpdateDevice, UpdateHibernationStatusRequest, UpdateTagValue} import com.advancedtelematic.director.deviceregistry.data.Device.{ActiveDeviceCount, DeviceOemId} import com.advancedtelematic.director.deviceregistry.data.DeviceSortBy.DeviceSortBy import com.advancedtelematic.director.deviceregistry.data.DeviceStatus.DeviceStatus @@ -49,31 +40,30 @@ import com.advancedtelematic.director.http.deviceregistry.Errors.{Codes, Missing import com.advancedtelematic.libats.data.DataType.{CorrelationId, Namespace, ResultCode} import com.advancedtelematic.libats.http.Errors.JsonError import com.advancedtelematic.libats.http.RefinedMarshallingSupport.* -import com.advancedtelematic.libats.http.UUIDKeyAkka.* +import com.advancedtelematic.libats.http.UUIDKeyPekko.* import com.advancedtelematic.libats.messaging.MessageBusPublisher import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId.* -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, Event, EventType} +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier, Event, EventType} import com.advancedtelematic.libats.messaging_datatype.MessageCodecs.* -import com.advancedtelematic.libats.messaging_datatype.Messages.{ - DeleteDeviceRequest, - DeviceEventMessage -} +import com.advancedtelematic.libats.messaging_datatype.Messages.{DeleteDeviceRequest, DeviceEventMessage} import com.advancedtelematic.libats.slick.db.SlickExtensions.* import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier import io.circe.Json import io.circe.syntax.EncoderOps import slick.jdbc.MySQLProfile.api.* -import Unmarshallers.nonNegativeLong import com.advancedtelematic.director.db.DeleteDeviceDBIO +import com.advancedtelematic.director.http.PaginationParametersDirectives +import com.advancedtelematic.director.http.PaginationParametersDirectives.PaginationParameters import java.time.{Instant, OffsetDateTime} import java.util.concurrent.TimeUnit import scala.concurrent.duration.Duration import scala.concurrent.{ExecutionContext, Future} import scala.util.Failure +import com.advancedtelematic.libats.data.PaginationResult.* object DevicesResource { - import akka.http.scaladsl.server.PathMatchers.Segment + import org.apache.pekko.http.scaladsl.server.PathMatchers.Segment type EventPayload = (DeviceId, Instant) => Event @@ -84,8 +74,9 @@ object DevicesResource { deviceTime <- c.get[Instant]("deviceTime")(io.circe.Decoder.decodeInstant) eventType <- c.get[EventType]("eventType") payload <- c.get[Json]("event") + ecu <- c.get[Option[EcuIdentifier]]("ecu") } yield (deviceUuid: DeviceId, receivedAt: Instant) => - Event(deviceUuid, id, eventType, deviceTime, receivedAt, payload) + Event(deviceUuid, id, eventType, deviceTime, receivedAt, ecu, payload) } implicit val groupIdUnmarshaller: Unmarshaller[String, GroupId] = GroupId.unmarshaller @@ -154,7 +145,7 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], import Directives.* import StatusCodes.* import com.advancedtelematic.libats.http.AnyvalMarshallingSupport.* - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* val extractPackageId: Directive1[PackageId] = pathPrefix(Segment / Segment).as(PackageId.apply) @@ -174,7 +165,7 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], complete(db.run(SearchDBIO.countByStatus(ns, params))) def searchDevice(ns: Namespace): Route = - parameters( + (parameters( Symbol("deviceId").as[DeviceOemId].?, Symbol("grouped").as[Boolean].?, Symbol("groupType").as[GroupType].?, @@ -190,11 +181,10 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], Symbol("createdAtStart").as[Instant].?, Symbol("createdAtEnd").as[Instant].?, Symbol("hardwareIds").as(CsvSeq[HardwareIdentifier]).?, + Symbol("tags").as(CsvSeq[TagSearchParam]).?, Symbol("sortBy").as[DeviceSortBy].?, Symbol("sortDirection").as[SortDirection].?, - Symbol("offset").as(nonNegativeLong).?, - Symbol("limit").as(nonNegativeLong).? - ).as( + ) & PaginationParameters).as( (oemId, grouped, groupType, @@ -210,6 +200,7 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], createdAtStart, createdAtEnd, hardwareId: Option[Seq[HardwareIdentifier]], + tags: Option[Seq[TagSearchParam]], sortBy, sortDirection, offset, @@ -230,10 +221,11 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], createdAtStart, createdAtEnd, hardwareId.getOrElse(Seq.empty), + tags.map(_.toSet).getOrElse(Set.empty), sortBy, sortDirection, offset, - limit + limit, ) ) { params => complete(for { @@ -300,28 +292,15 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], for { foundOemDevices <- db.run(DeviceRepository.findByOemIds(ns, oemDevices)) foundUuidDevices <- db.run(DeviceRepository.findByDeviceUuids(ns, uuidDevices)) - } yield { - val foundDevices = (foundOemDevices ++ foundUuidDevices).toSet.toList - val missingOemDevices = - oemDevices.filterNot(expected => foundOemDevices.map(_.deviceId).contains(expected)) - val missingUuidDevices = - uuidDevices.filterNot(expected => foundUuidDevices.map(_.uuid).contains(expected)) - if (missingOemDevices.nonEmpty || missingUuidDevices.nonEmpty) { - val msg = Map( - "missingOemIds" -> missingOemDevices.map(_.underlying).mkString(","), - "missingDeviceUuids" -> missingUuidDevices.map(_.uuid).mkString(",") - ) - throw JsonError(Codes.MissingDevice, StatusCodes.NotFound, msg.asJson, "Devices not found") - } - foundDevices.map(DeviceDB.toDevice(_)) - } + foundDevices = (foundOemDevices ++ foundUuidDevices).toSet.toList + } yield foundDevices.map(DeviceDB.toDevice(_)) } def countDynamicGroupCandidates(ns: Namespace, expression: GroupExpression): Route = complete(db.run(SearchDBIO.countDevicesForExpression(ns, expression))) def getGroupsForDevice(uuid: DeviceId): Route = - parameters(Symbol("offset").as(nonNegativeLong).?, Symbol("limit").as(nonNegativeLong).?) { + PaginationParameters { (offset, limit) => complete(db.run(GroupMemberRepository.listGroupsForDevice(uuid, offset, limit))) } @@ -336,11 +315,9 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], complete(db.run(InstalledPackages.getDevicesCount(pkg, ns))) def listPackagesOnDevice(device: DeviceId): Route = - parameters( - Symbol("nameContains").as[String].?, - Symbol("offset").as(nonNegativeLong).?, - Symbol("limit").as(nonNegativeLong).? - ) { (nameContains, offset, limit) => + (parameters( + Symbol("nameContains").as[String].? + ) & PaginationParameters) { (nameContains, offset, limit) => complete(db.run(InstalledPackages.installedOn(device, nameContains, offset, limit))) } @@ -360,7 +337,7 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], } def getDistinctPackages(ns: Namespace): Route = - parameters(Symbol("offset").as(nonNegativeLong).?, Symbol("limit").as(nonNegativeLong).?) { + PaginationParameters { (offset, limit) => complete(db.run(InstalledPackages.getInstalledForAllDevices(ns, offset, limit))) } @@ -374,27 +351,27 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], } def getPackageStats(ns: Namespace, name: PackageId.Name): Route = - parameters(Symbol("offset").as(nonNegativeLong).?, Symbol("limit").as(nonNegativeLong).?) { + PaginationParameters { (offset, limit) => val f = db.run(InstalledPackages.listAllWithPackageByName(ns, name, offset, limit)) complete(f) } def fetchInstallationHistory(deviceId: DeviceId, - offset: Option[Long], - limit: Option[Long]): Route = + offset: Offset, + limit: Limit): Route = complete( db.run( EcuReplacementRepository - .deviceHistory(deviceId, offset.orDefaultOffset, limit.orDefaultLimit) + .deviceHistory(deviceId, offset, limit) ) ) - def installationReports(deviceId: DeviceId, offset: Option[Long], limit: Option[Long]): Route = + def installationReports(deviceId: DeviceId, offset: Offset, limit: Limit): Route = complete( db.run( InstallationReportRepository - .installationReports(deviceId, offset.orDefaultOffset, limit.orDefaultLimit) + .installationReports(deviceId, offset, limit) ) ) @@ -514,16 +491,10 @@ class DevicesResource(namespaceExtractor: Directive1[Namespace], path("active_device_count") { getActiveDeviceCount(ns) } ~ - (path("installation_reports") & parameters( - Symbol("offset").as(nonNegativeLong).?, - Symbol("limit").as(nonNegativeLong).? - )) { (offset, limit) => + (path("installation_reports") & PaginationParameters) { (offset, limit) => installationReports(uuid, offset, limit) } ~ - (path("installation_history") & parameters( - Symbol("offset").as(nonNegativeLong).?, - Symbol("limit").as(nonNegativeLong).? - )) { (offset, limit) => + (path("installation_history") & PaginationParameters) { (offset, limit) => fetchInstallationHistory(uuid, offset, limit) } ~ (pathPrefix("device_count") & extractPackageId) { pkg => diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/Errors.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/Errors.scala index 9d7d54e2..6d721cb9 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/Errors.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/Errors.scala @@ -27,7 +27,7 @@ import java.sql.SQLSyntaxErrorException object Errors { - import akka.http.scaladsl.model.StatusCodes + import org.apache.pekko.http.scaladsl.model.StatusCodes object Codes { val MissingDevice = ErrorCode("missing_device") diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/GroupsResource.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/GroupsResource.scala index 9328a1e6..5e1f95d5 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/GroupsResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/GroupsResource.scala @@ -8,22 +8,17 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.marshalling.Marshaller.* -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.* - -import akka.http.scaladsl.unmarshalling.{FromStringUnmarshaller, Unmarshaller} -import akka.http.scaladsl.util.FastFuture -import akka.stream.Materializer -import akka.stream.scaladsl.Framing.FramingException -import akka.stream.scaladsl.{Framing, Sink, Source} -import akka.util.ByteString +import org.apache.pekko.http.scaladsl.marshalling.Marshaller.* +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.* +import org.apache.pekko.http.scaladsl.unmarshalling.{FromStringUnmarshaller, Unmarshaller} +import org.apache.pekko.http.scaladsl.util.FastFuture +import org.apache.pekko.stream.Materializer +import org.apache.pekko.stream.scaladsl.Framing.FramingException +import org.apache.pekko.stream.scaladsl.{Framing, Sink, Source} +import org.apache.pekko.util.ByteString import cats.syntax.either.* -import com.advancedtelematic.director.db.deviceregistry.{ - DeviceRepository, - GroupInfoRepository, - GroupMemberRepository -} +import com.advancedtelematic.director.db.deviceregistry.{DeviceRepository, GroupInfoRepository, GroupMemberRepository} import com.advancedtelematic.director.deviceregistry.data.* import com.advancedtelematic.director.deviceregistry.data.Codecs.* import com.advancedtelematic.director.deviceregistry.data.DataType.UpdateHibernationStatusRequest @@ -33,9 +28,11 @@ import com.advancedtelematic.director.deviceregistry.data.Group.GroupId import com.advancedtelematic.director.deviceregistry.data.GroupSortBy.GroupSortBy import com.advancedtelematic.director.deviceregistry.data.GroupType.GroupType import com.advancedtelematic.director.deviceregistry.{AllowUUIDPath, GroupMembership} +import com.advancedtelematic.director.http.PaginationParametersDirectives.PaginationParameters import com.advancedtelematic.libats.data.DataType.Namespace +import com.advancedtelematic.libats.data.PaginationResult.{Limit, Offset} import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.{Codec, Decoder, Encoder, Json, KeyDecoder, KeyEncoder} import slick.jdbc.MySQLProfile.api.* @@ -65,9 +62,8 @@ object DeviceGroupStats { } -import Unmarshallers.nonNegativeLong -import akka.http.scaladsl.unmarshalling.PredefinedFromStringUnmarshallers.CsvSeq -import com.advancedtelematic.libats.http.UUIDKeyAkka.* +import org.apache.pekko.http.scaladsl.unmarshalling.PredefinedFromStringUnmarshallers.CsvSeq +import com.advancedtelematic.libats.http.UUIDKeyPekko.* import GroupId.* import io.circe.syntax.* @@ -107,14 +103,14 @@ class GroupsResource(namespaceExtractor: Directive1[Namespace], val groupMembership = new GroupMembership() def getDevicesInGroup(groupId: GroupId): Route = - parameters(Symbol("offset").as(nonNegativeLong).?, Symbol("limit").as(nonNegativeLong).?) { + PaginationParameters { (offset, limit) => complete(groupMembership.listDevices(groupId, offset, limit)) } def listGroups(ns: Namespace, - offset: Option[Long], - limit: Option[Long], + offset: Offset, + limit: Limit, sortBy: GroupSortBy, nameContains: Option[String]): Route = complete(db.run(GroupInfoRepository.list(ns, offset, limit, sortBy, nameContains))) @@ -209,9 +205,7 @@ class GroupsResource(namespaceExtractor: Directive1[Namespace], val route: Route = (pathPrefix("device_groups") & namespaceExtractor) { ns => pathEnd { - (get & parameters( - Symbol("offset").as(nonNegativeLong).?, - Symbol("limit").as(nonNegativeLong).?, + (get & PaginationParameters & parameters( Symbol("sortBy").as[GroupSortBy].?, Symbol("nameContains").as[String].? )) { (offset, limit, sortBy, nameContains) => diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/PackageListsResource.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/PackageListsResource.scala index dae94e45..35674191 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/PackageListsResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/PackageListsResource.scala @@ -1,8 +1,8 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.StatusCodes.{Created, NoContent} -import akka.http.scaladsl.server.* -import akka.http.scaladsl.server.Directives.* +import org.apache.pekko.http.scaladsl.model.StatusCodes.{Created, NoContent} +import org.apache.pekko.http.scaladsl.server.* +import org.apache.pekko.http.scaladsl.server.Directives.* import com.advancedtelematic.director.db.deviceregistry.PackageListItemRepository import com.advancedtelematic.director.deviceregistry.data.Codecs.* import com.advancedtelematic.director.deviceregistry.data.DataType.{ @@ -11,7 +11,7 @@ import com.advancedtelematic.director.deviceregistry.data.DataType.{ } import com.advancedtelematic.director.deviceregistry.data.PackageId import com.advancedtelematic.libats.data.DataType.Namespace -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.generic.auto.* import slick.jdbc.MySQLProfile.api.* diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsResource.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsResource.scala index a2e352be..d6d778d7 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsResource.scala @@ -8,10 +8,10 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.marshalling.Marshaller.* -import akka.http.scaladsl.server.Directives.* -import akka.http.scaladsl.server.{Directive1, Route} -import akka.http.scaladsl.util.FastFuture +import org.apache.pekko.http.scaladsl.marshalling.Marshaller.* +import org.apache.pekko.http.scaladsl.server.Directives.* +import org.apache.pekko.http.scaladsl.server.{Directive1, Route} +import org.apache.pekko.http.scaladsl.util.FastFuture import com.advancedtelematic.director.db.deviceregistry.{ DeviceRepository, PublicCredentialsRepository @@ -51,7 +51,7 @@ class PublicCredentialsResource( deviceNamespaceAuthorizer: Directive1[DeviceId])(implicit db: Database, ec: ExecutionContext) { import PublicCredentialsResource.* - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* lazy val base64Decoder = Base64.getDecoder() lazy val base64Encoder = Base64.getEncoder() diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/SystemInfoResource.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/SystemInfoResource.scala index 71d4027d..1c449f88 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/SystemInfoResource.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/SystemInfoResource.scala @@ -8,12 +8,12 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.Done -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.model.StatusCodes.* -import akka.http.scaladsl.server.Directives.* -import akka.http.scaladsl.server.{Directive1, Route} -import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} +import org.apache.pekko.Done +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes.* +import org.apache.pekko.http.scaladsl.server.Directives.* +import org.apache.pekko.http.scaladsl.server.{Directive1, Route} +import org.apache.pekko.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} import cats.syntax.option.* import com.advancedtelematic.director.db.deviceregistry.SystemInfoRepository import com.advancedtelematic.director.db.deviceregistry.SystemInfoRepository.NetworkInfo @@ -21,7 +21,7 @@ import com.advancedtelematic.director.deviceregistry.SystemInfoUpdatePublisher import com.advancedtelematic.director.http.deviceregistry.Errors.{Codes, MissingSystemInfo} import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.http.Errors.RawError -import com.advancedtelematic.libats.http.UUIDKeyAkka.* +import com.advancedtelematic.libats.http.UUIDKeyPekko.* import com.advancedtelematic.libats.messaging.MessageBusPublisher import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libats.messaging_datatype.Messages.{ @@ -74,7 +74,7 @@ class SystemInfoResource( deviceNamespaceAuthorizer: Directive1[DeviceId])(implicit db: Database, ec: ExecutionContext) { import SystemInfoResource.* - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* private val systemInfoUpdatePublisher = new SystemInfoUpdatePublisher(messageBus) diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/TomlSupport.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/TomlSupport.scala index 58f49107..7454eb97 100644 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/TomlSupport.scala +++ b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/TomlSupport.scala @@ -1,8 +1,8 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.ContentType.WithFixedCharset -import akka.http.scaladsl.model.HttpCharsets.`UTF-8` -import akka.http.scaladsl.model.MediaType +import org.apache.pekko.http.scaladsl.model.ContentType.WithFixedCharset +import org.apache.pekko.http.scaladsl.model.HttpCharsets.`UTF-8` +import org.apache.pekko.http.scaladsl.model.MediaType object TomlSupport { diff --git a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/Unmarshallers.scala b/src/main/scala/com/advancedtelematic/director/http/deviceregistry/Unmarshallers.scala deleted file mode 100644 index abc031df..00000000 --- a/src/main/scala/com/advancedtelematic/director/http/deviceregistry/Unmarshallers.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.advancedtelematic.director.http.deviceregistry - -import akka.http.scaladsl.unmarshalling.{PredefinedFromStringUnmarshallers, Unmarshaller} -import akka.http.scaladsl.util.FastFuture - -object Unmarshallers { - - val nonNegativeLong: Unmarshaller[String, Long] = - PredefinedFromStringUnmarshallers.longFromStringUnmarshaller - .flatMap { _ => _ => value => - if (value < 0) FastFuture.failed(new IllegalArgumentException("Value cannot be negative")) - else FastFuture.successful(value) - } - -} diff --git a/src/main/scala/com/advancedtelematic/director/manifest/DeviceManifestProcess.scala b/src/main/scala/com/advancedtelematic/director/manifest/DeviceManifestProcess.scala index 13d12e58..91f1bfca 100644 --- a/src/main/scala/com/advancedtelematic/director/manifest/DeviceManifestProcess.scala +++ b/src/main/scala/com/advancedtelematic/director/manifest/DeviceManifestProcess.scala @@ -10,8 +10,8 @@ import com.advancedtelematic.director.data.DeviceRequest.{ } import com.advancedtelematic.director.db.EcuRepositorySupport import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.data.EcuIdentifier import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.messaging_datatype.DataType.EcuIdentifier import com.advancedtelematic.libtuf.data.TufDataType.{SignedPayload, TufKey} import io.circe.Decoder.Result import io.circe.{Decoder, Json} @@ -33,6 +33,7 @@ object DeviceManifestProcess { def fromJsonV1(manifest: Json): Result[DeviceManifest] = { import com.advancedtelematic.libtuf.data.TufCodecs.signedPayloadDecoder + import com.advancedtelematic.libats.codecs.CirceRefined.* import cats.implicits._ final case class DeviceManifestV1(primary_ecu_serial: EcuIdentifier, diff --git a/src/main/scala/com/advancedtelematic/director/manifest/ManifestCompiler.scala b/src/main/scala/com/advancedtelematic/director/manifest/ManifestCompiler.scala index cd00c763..fdfebbfb 100644 --- a/src/main/scala/com/advancedtelematic/director/manifest/ManifestCompiler.scala +++ b/src/main/scala/com/advancedtelematic/director/manifest/ManifestCompiler.scala @@ -1,7 +1,8 @@ package com.advancedtelematic.director.manifest import cats.syntax.option.* -import com.advancedtelematic.director.data.DataType.ScheduledUpdate +import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.DataType.Update import com.advancedtelematic.director.data.DbDataType.{ Assignment, DeviceKnownState, @@ -20,9 +21,9 @@ import com.advancedtelematic.libats.data.DataType.{ ResultCode, ResultDescription } -import com.advancedtelematic.libats.data.EcuIdentifier import com.advancedtelematic.libats.messaging_datatype.DataType.{ DeviceId, + EcuIdentifier, EcuInstallationReport, InstallationResult } @@ -36,7 +37,6 @@ import org.slf4j.LoggerFactory import java.time.Instant import scala.util.{Failure, Success, Try} -import com.advancedtelematic.director.data.Codecs.* object ManifestCompiler { private val _log = LoggerFactory.getLogger(this.getClass) @@ -60,92 +60,87 @@ object ManifestCompiler { assignmentTarget.checksum.hash == installedChecksum } - private def compileScheduleUpdatesChanges( + private def compileUpdatesChanges( knownState: DeviceKnownState, msgs: List[DeviceUpdateEvent]): (DeviceKnownState, List[DeviceUpdateEvent]) = { /* This is not ideal because in compileManifest we match existing assignments using `ecu_version_manifests` and checking what is installed on the device. We then do the same again here, to match against the - scheduled updates ecu targets. - - We cannot rely on processed assignments **only** because scheduled updates might not have generated assignments yet. - Processed assignments still need to be checked in case the assignment was processed, but not installed, for example, - if the update as cancelled. + updates ecu targets. - We cannot rely on correlation id, because we might not have an assignment, which contains the correlation id. + We cannot rely on processed assignments **only** because updates might not have generated assignments yet. + Processed assignments still need to be checked in case the assignment was processed, but not installed, + for example, if the update was cancelled. Here we use knownStatus.ecuStatus rather than `ecu_version_manifests` as we don't have the manifest at this point, but knownStatus.ecuStatus is populated based on matching `ecu_version_manifests` so that is what would need to end up doing here anyway. */ - if (knownState.scheduledUpdates.nonEmpty) + if (knownState.updates.nonEmpty) _log .atInfo() - .addKeyValue("scheduledUpdatesMtuIds", knownState.scheduledUpdates.map(_.updateId).asJson) .addKeyValue("deviceId", knownState.deviceId.asJson) - .log("calculating scheduled updates changes") + .log("calculating updates changes") else _log .atDebug() .addKeyValue("deviceId", knownState.deviceId.asJson) - .log("no scheduled updates") + .log("no updates scheduled") - val newScheduledUpdates = knownState.scheduledUpdates.map { su => - val scheduledUpdateEcuTargets = knownState.scheduledUpdatesEcuTargetIds - .get(su.updateId) + val newUpdates = knownState.updates.map { u => + val updatesEcuTargets = knownState.updatesTargetIds + .get(u.targetSpecId) .toList .flatten .flatMap(knownState.ecuTargets.get) - if (scheduledUpdateEcuTargets.isEmpty) { + if (updatesEcuTargets.isEmpty) { _log .atWarn() .addKeyValue("deviceId", knownState.deviceId.asJson) - .addKeyValue("scheduledUpdateId", su.id.asJson) - .log("no ecu targets for scheduled update") + .addKeyValue("updateTargetSpecId", u.id.asJson) + .log("no ecu targets for update") } - val totalTargetsForScheduledUpdate = - knownState.scheduledUpdatesEcuTargetIds.get(su.updateId).toList.flatten.size + val totalTargetsForUpdate = + knownState.updatesTargetIds.get(u.targetSpecId).toList.flatten.size val processedInEcuStatus = knownState.ecuStatus.values.flatten.filter { ecuTargetId => - scheduledUpdateEcuTargets.exists { ecuTarget => + updatesEcuTargets.exists { ecuTarget => knownState.ecuTargets.get(ecuTargetId).exists(_.matches(ecuTarget)) } } val processedInAssignment = knownState.processedAssignments + .filter(_.correlationId == u.correlationId) .map(_.ecuTargetId) - .filter(scheduledUpdateEcuTargets.map(_.id).contains) val alreadyProcessedIds = (processedInEcuStatus ++ processedInAssignment).toSet val updated = - if ( - alreadyProcessedIds.nonEmpty && (alreadyProcessedIds.size >= totalTargetsForScheduledUpdate) - ) - su.copy(status = ScheduledUpdate.Status.Completed) + if (alreadyProcessedIds.nonEmpty && (alreadyProcessedIds.size >= totalTargetsForUpdate)) + u.copy(status = Update.Status.Completed, completedAt = Some(Instant.now)) else if (alreadyProcessedIds.nonEmpty) - su.copy(status = ScheduledUpdate.Status.PartiallyCompleted) + u.copy(status = Update.Status.PartiallyCompleted) else - su + u _log .atInfo() .addKeyValue("deviceId", knownState.deviceId.asJson) - .addKeyValue("totalTargetsForScheduledUpdate", totalTargetsForScheduledUpdate) + .addKeyValue("totalTargetsForUpdate", totalTargetsForUpdate) .addKeyValue("alreadyProcessedCount", alreadyProcessedIds.size) .addKeyValue("processedAsAssignment", processedInAssignment.asJson) .addKeyValue("processedInEcuStatus", processedInEcuStatus.asJson) - .addKeyValue("scheduledUpdateEcuTargets", scheduledUpdateEcuTargets.map(_.id).asJson) + .addKeyValue("updatesEcuTargets", updatesEcuTargets.map(_.id).asJson) .addKeyValue("result", updated.asJson) - .log(s"updating scheduled updates for ${su.id}") + .log(s"updating updates for ${u.id}") updated } - (knownState.copy(scheduledUpdates = newScheduledUpdates), msgs) + (knownState.copy(updates = newUpdates), msgs) } def apply(ns: Namespace, @@ -153,7 +148,7 @@ object ManifestCompiler { (beforeState: DeviceKnownState) => validateManifest(manifest, beforeState).map { _ => val (nextStatus, msgs) = compileManifest(ns, manifest) - .andThen(compileScheduleUpdatesChanges.tupled) + .andThen(compileUpdatesChanges.tupled) .apply(beforeState) ManifestCompileResult(nextStatus, msgs) } @@ -231,8 +226,8 @@ object ManifestCompiler { knownStatus.ecuTargets ++ newEcuTargets, currentAssignments = Set.empty, processedAssignments = Set.empty, - scheduledUpdates = knownStatus.scheduledUpdates, - knownStatus.scheduledUpdatesEcuTargetIds, + knownStatus.updates, + knownStatus.updatesTargetIds, generatedMetadataOutdated = knownStatus.generatedMetadataOutdated ) diff --git a/src/main/scala/com/advancedtelematic/director/repo/DeviceDataProviders.scala b/src/main/scala/com/advancedtelematic/director/repo/DeviceDataProviders.scala index 2f6b4ea6..c5c68260 100644 --- a/src/main/scala/com/advancedtelematic/director/repo/DeviceDataProviders.scala +++ b/src/main/scala/com/advancedtelematic/director/repo/DeviceDataProviders.scala @@ -1,7 +1,7 @@ package com.advancedtelematic.director.repo -import akka.http.scaladsl.model.Uri -import akka.http.scaladsl.util.FastFuture +import org.apache.pekko.http.scaladsl.model.Uri +import org.apache.pekko.http.scaladsl.util.FastFuture import cats.implicits._ import com.advancedtelematic.director.data.Codecs._ import com.advancedtelematic.director.data.DataType.{ @@ -17,8 +17,7 @@ import com.advancedtelematic.director.db.{ EcuTargetsRepositorySupport } import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.data.EcuIdentifier -import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.messaging_datatype.DataType.* import com.advancedtelematic.libtuf.data.ClientDataType.{ ClientHashes, ClientTargetItem, diff --git a/src/main/scala/com/advancedtelematic/director/repo/RemoteSessions.scala b/src/main/scala/com/advancedtelematic/director/repo/RemoteSessions.scala index 0c3be825..f1ac4aeb 100644 --- a/src/main/scala/com/advancedtelematic/director/repo/RemoteSessions.scala +++ b/src/main/scala/com/advancedtelematic/director/repo/RemoteSessions.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.repo -import akka.http.scaladsl.util.FastFuture +import org.apache.pekko.http.scaladsl.util.FastFuture import cats.data.Validated.{Invalid, Valid} import cats.data.ValidatedNel import com.advancedtelematic.director.data.DataType.AdminRoleName @@ -9,6 +9,7 @@ import com.advancedtelematic.director.db.AdminRolesRepositorySupport import com.advancedtelematic.director.http.Errors import com.advancedtelematic.libtuf.data.ClientCodecs.* import com.advancedtelematic.libtuf.data.ClientDataType.{ + RemoteCommandsPayload, RemoteSessionsPayload, RemoteSessionsRole, TufRole, @@ -62,36 +63,64 @@ class RemoteSessions(keyserverClient: KeyserverClient)( existing.content } - def set(repoId: RepoId, - remoteSessions: RemoteSessionsPayload, - previousVersion: Int): Future[SignedRole[RemoteSessionsRole]] = async { - val existing = await( - adminRolesRepository - .findLatestOpt(repoId, RoleType.REMOTE_SESSIONS, REMOTE_SESSIONS_ADMIN_NAME) - ) + def setRemoteCommands( + repoId: RepoId, + remoteCommands: RemoteCommandsPayload): Future[SignedRole[RemoteSessionsRole]] = + set(repoId) { + case Some(rs) => + rs.copy(remote_commands = Some(remoteCommands)) + case None => + RemoteSessionsRole ( + remote_sessions = RemoteSessionsPayload.empty, + remote_commands = Some(remoteCommands), + version = 1, + expires = nextExpires + ) + } - val expireAt = nextExpires - - val newRole = existing match { - case Some(r) => - val role = r.value.toSignedRole[RemoteSessionsRole].role - val newRole = role.copy( - remoteSessions, - expireAt, - version = previousVersion + 1 - ) // persist will check this bump is valid and does not conflict - newRole + def setRemoteSessions( + repoId: RepoId, + remoteSessions: RemoteSessionsPayload): Future[SignedRole[RemoteSessionsRole]] = + set(repoId) { + case Some(rs) => + rs.copy(remote_sessions = remoteSessions) case None => - await(keyserverClient.addRemoteSessionsRole(repoId)) - RemoteSessionsRole(remoteSessions, expireAt, version = 1) + RemoteSessionsRole( + remote_sessions = remoteSessions, + None, + version = 1, + expires = nextExpires + ) } - val signedRole = await(sign(repoId, newRole)) - await( - adminRolesRepository.persistAll(signedRole.toDbAdminRole(repoId, REMOTE_SESSIONS_ADMIN_NAME)) - ) - signedRole - } + private def set(repoId: RepoId)(updateFn: Option[RemoteSessionsRole] => RemoteSessionsRole) + : Future[SignedRole[RemoteSessionsRole]] = + async { + val existing = await( + adminRolesRepository + .findLatestOpt(repoId, RoleType.REMOTE_SESSIONS, REMOTE_SESSIONS_ADMIN_NAME) + ) + + val newRole = existing match { + case Some(r) => + val role = r.value.toSignedRole[RemoteSessionsRole].role + val newRole = updateFn(Some(role)).copy( + expires = nextExpires, + version = role.version + 1 + ) // persist will check this bump is valid and does not conflict + newRole + case None => + await(keyserverClient.addRemoteSessionsRole(repoId)) + updateFn(None) + } + + val signedRole = await(sign(repoId, newRole)) + await( + adminRolesRepository + .persistAll(signedRole.toDbAdminRole(repoId, REMOTE_SESSIONS_ADMIN_NAME)) + ) + signedRole + } private def sign[T: Codec: TufRole](repoId: RepoId, role: T): Future[SignedRole[T]] = async { val signedPayload = await(keyserverClient.sign(repoId, role)) diff --git a/src/main/scala/com/advancedtelematic/libats/debug/Debug.scala b/src/main/scala/com/advancedtelematic/libats/debug/Debug.scala index 3b7d61e3..f1b51b43 100644 --- a/src/main/scala/com/advancedtelematic/libats/debug/Debug.scala +++ b/src/main/scala/com/advancedtelematic/libats/debug/Debug.scala @@ -1,12 +1,12 @@ package com.advancedtelematic.libats.debug -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.Directives.JavaUUID -import akka.http.scaladsl.server.{PathMatcher1, Route} +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.Directives.JavaUUID +import org.apache.pekko.http.scaladsl.server.{PathMatcher1, Route} import cats.Show import cats.syntax.show.* import com.advancedtelematic.libats.data.DataType.Namespace -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.{Codec, Json} import scala.concurrent.Future @@ -41,7 +41,7 @@ object DebugDatatype { object DebugRoutes { import DebugDatatype.* - import akka.http.scaladsl.server.Directives.* + import org.apache.pekko.http.scaladsl.server.Directives.* def buildNavigation(resourceGroup: ResourceGroup[?]*): Route = path("navigation.json") { diff --git a/src/test/scala/com/advancedtelematic/director/client/FakeKeyserverClient.scala b/src/test/scala/com/advancedtelematic/director/client/FakeKeyserverClient.scala index 014f40c1..d2d5afb6 100644 --- a/src/test/scala/com/advancedtelematic/director/client/FakeKeyserverClient.scala +++ b/src/test/scala/com/advancedtelematic/director/client/FakeKeyserverClient.scala @@ -5,7 +5,7 @@ import io.circe.Codec import java.security.PublicKey import java.time.Instant import java.util.concurrent.ConcurrentHashMap -import akka.http.scaladsl.util.FastFuture +import org.apache.pekko.http.scaladsl.util.FastFuture import com.advancedtelematic.libtuf.crypt.TufCrypto import com.advancedtelematic.libtuf.data.ClientCodecs._ import com.advancedtelematic.libtuf.data.ClientDataType.{RoleKeys, RootRole, TufRole} @@ -101,8 +101,15 @@ class FakeKeyserverClient extends KeyserverClient { throw KeyserverClient.RootRoleNotFound } }.flatMap { role => - sign(repoId, role).map { jsonSigned => - SignedPayload(jsonSigned.signatures, role, jsonSigned.json) + val expireNotBefore = _expireNotBefore.getOrElse(role.expires) + + val role2 = if(role.expires.isBefore(expireNotBefore)) + role.copy(expires = expireNotBefore) + else + role + + sign(repoId, role2).map { jsonSigned => + SignedPayload(jsonSigned.signatures, role2, jsonSigned.json) } } diff --git a/src/test/scala/com/advancedtelematic/director/daemon/DeviceManifestGenerator.scala b/src/test/scala/com/advancedtelematic/director/daemon/DeviceManifestGenerator.scala new file mode 100644 index 00000000..800d58a2 --- /dev/null +++ b/src/test/scala/com/advancedtelematic/director/daemon/DeviceManifestGenerator.scala @@ -0,0 +1,68 @@ +package com.advancedtelematic.director.daemon + +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.stream.scaladsl.{Flow, Sink, Source} +import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.GeneratorOps.* +import com.advancedtelematic.director.data.Generators.* +import com.advancedtelematic.director.data.Messages +import com.advancedtelematic.director.data.Messages.{ + deviceManifestReportedMsgLike, + DeviceManifestReported +} +import com.advancedtelematic.libats.data.DataType +import com.advancedtelematic.libats.messaging.{MessageBus, MessageBusPublisher} +import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libtuf.data.TufDataType.SignedPayload +import com.typesafe.config.ConfigFactory +import io.circe.syntax.* + +import java.time.Instant +import java.util.UUID +import scala.concurrent.duration.* +import scala.concurrent.{Await, ExecutionContext} + +object DeviceManifestGenerator { + + def main(args: Array[String]): Unit = { + if (args.length < 1) { + println("Usage: DeviceManifestGenerator [device-id]") + System.exit(1) + } + + val numManifests = args(0).toInt + + val deviceId = + Option(args(1)).map(d => DeviceId(UUID.fromString(d))) + + implicit val system: ActorSystem = ActorSystem("DeviceManifestGenerator") + implicit val ec: ExecutionContext = system.dispatcher + + val config = ConfigFactory.load() + val namespace = DataType.Namespace("device-manifest-generator") + + implicit val msgPublisher: MessageBusPublisher = MessageBus.publisher(system, config) + + println(s"Generating $numManifests random DeviceManifestReported messages...") + + val manifests = (1 to numManifests).map { _ => + val id = deviceId.getOrElse(DeviceId.generate()) + val manifest = GenDeviceManifest.generate + val signedManifest = SignedPayload(Seq.empty, manifest.asJson, manifest.asJson) + + Messages.DeviceManifestReported(namespace, id, signedManifest, Instant.now()) + } + + val result = Source(manifests) + .via(Flow[DeviceManifestReported].mapAsync(3) { msg => + println(s"Publishing message for device: ${msg.deviceId}") + msgPublisher.publish(msg) + }) + .runWith(Sink.ignore) + + Await.result(result, Duration.Inf) + + Await.result(system.terminate(), 5.seconds) + } + +} diff --git a/src/test/scala/com/advancedtelematic/director/daemon/DeviceManifestReportedListenerSpec.scala b/src/test/scala/com/advancedtelematic/director/daemon/DeviceManifestReportedListenerSpec.scala index e79518a7..73d04eba 100644 --- a/src/test/scala/com/advancedtelematic/director/daemon/DeviceManifestReportedListenerSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/daemon/DeviceManifestReportedListenerSpec.scala @@ -1,32 +1,71 @@ package com.advancedtelematic.director.daemon -import java.time.Instant -import com.advancedtelematic.director.data.Codecs._ -import com.advancedtelematic.director.data.GeneratorOps._ -import com.advancedtelematic.director.data.Generators._ +import cats.implicits.toShow +import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.GeneratorOps.* +import com.advancedtelematic.director.data.Generators.* import com.advancedtelematic.director.data.Messages +import com.advancedtelematic.director.data.Messages.DeviceManifestReported import com.advancedtelematic.director.db.DeviceManifestRepositorySupport import com.advancedtelematic.director.util.DirectorSpec import com.advancedtelematic.libats.data.DataType +import com.advancedtelematic.libats.data.PaginationResult.LongAsParam import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libats.test.MysqlDatabaseSpec import com.advancedtelematic.libtuf.data.TufDataType.SignedPayload -import io.circe.syntax._ -import org.scalatest.OptionValues._ +import com.typesafe.config.ConfigFactory +import io.circe.Json +import io.circe.syntax.* +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.apache.pekko.Done +import org.apache.pekko.actor.ActorSystem +import org.apache.pekko.kafka.ConsumerMessage.CommittableMessage +import org.apache.pekko.stream.scaladsl.{Sink, Source} +import org.apache.pekko.testkit.TestKitBase +import org.scalatest.LoneElement.convertToCollectionLoneElementWrapper +import org.scalatest.OptionValues.* +import java.time.Instant import java.time.temporal.ChronoUnit -import scala.concurrent.ExecutionContext +import scala.concurrent.{ExecutionContext, Future} +import scala.util.Random class DeviceManifestReportedListenerSpec extends DirectorSpec + with TestKitBase with MysqlDatabaseSpec with DeviceManifestRepositorySupport { + override implicit def system: ActorSystem = ActorSystem(this.getClass.getSimpleName) + val defaultNs = DataType.Namespace(this.getClass.getName) implicit lazy val ec: scala.concurrent.ExecutionContextExecutor = ExecutionContext.global - lazy val listener = new DeviceManifestReportedListener() + lazy val listener = new DeviceManifestReportedListener(ConfigFactory.load()) + + private def runListener(msgs: Seq[DeviceManifestReported]): Future[Done] = { + val cm = msgs.map { msg => + CommittableMessage( + new ConsumerRecord[Array[Byte], DeviceManifestReported]( + "topic", + 0, + 0, + msg.deviceId.show.getBytes, + msg + ), + null + ) + } + + Source + .fromIterator(() => cm.iterator) + .via(listener.processingFlow) + .runWith(Sink.ignore) + } + + private def runListener(msg: DeviceManifestReported): Future[Done] = + runListener(List(msg)) test("it saves manifest to database") { val manifest = GenDeviceManifest.generate @@ -35,9 +74,9 @@ class DeviceManifestReportedListenerSpec val msg = Messages.DeviceManifestReported(defaultNs, DeviceId.generate(), signedManifest, Instant.now()) - listener.apply(msg).futureValue + runListener(msg).futureValue - val (saved, receivedAt) = deviceManifestRepository.find(msg.deviceId).futureValue.value + val (saved, receivedAt) = deviceManifestRepository.findLatest(msg.deviceId).futureValue.value saved shouldBe msg.manifest.signed receivedAt.truncatedTo(ChronoUnit.SECONDS) shouldBe msg.receivedAt.truncatedTo( @@ -52,10 +91,10 @@ class DeviceManifestReportedListenerSpec val msg = Messages.DeviceManifestReported(defaultNs, DeviceId.generate(), signedManifest, Instant.now()) - listener.apply(msg).futureValue - listener.apply(msg.copy(receivedAt = Instant.now().plusSeconds(30))).futureValue + runListener(msg).futureValue + runListener(msg.copy(receivedAt = Instant.now().plusSeconds(30))).futureValue - val all = deviceManifestRepository.findAll(msg.deviceId).futureValue + val all = deviceManifestRepository.findAll(msg.deviceId).futureValue.values all should have size 1 all.head._1 shouldBe msg.manifest.json @@ -67,14 +106,14 @@ class DeviceManifestReportedListenerSpec val manifest = GenDeviceManifest.generate val signedManifest = SignedPayload(Seq.empty, manifest.asJson, manifest.asJson) val msg = Messages.DeviceManifestReported(defaultNs, device, signedManifest, Instant.now()) - listener.apply(msg).futureValue + runListener(msg).futureValue val manifest2 = GenDeviceManifest.generate val signedManifest2 = SignedPayload(Seq.empty, manifest2.asJson, manifest2.asJson) val msg2 = Messages.DeviceManifestReported(defaultNs, device, signedManifest2, Instant.now()) - listener.apply(msg2).futureValue + runListener(msg2).futureValue - val all = deviceManifestRepository.findAll(device).futureValue.map(_._1) + val all = deviceManifestRepository.findAll(device).futureValue.values.map(_._1) all should have size 2 @@ -82,4 +121,150 @@ class DeviceManifestReportedListenerSpec all should contain(manifest2.asJson) } + test("it keeps only the latest 200 manifests in database") { + val device = DeviceId.generate() + val now = Instant.now().truncatedTo(ChronoUnit.SECONDS) + val manifests = (1 to 250).map { i => + val manifest = GenDeviceManifest.generate + val signedManifest = SignedPayload(Seq.empty, manifest.asJson, manifest.asJson) + Messages.DeviceManifestReported(defaultNs, device, signedManifest, now.plusSeconds(i)) + } + + runListener(manifests).futureValue + + val savedManifests = + deviceManifestRepository.findAll(device, 0L.toOffset, 1000L.toLimit).futureValue.values + val lastManifests = manifests.map(_.manifest.json).takeRight(200) + + savedManifests should have size 200 + savedManifests.map(_._2).reverse should be(sorted) + savedManifests.map(_._1) should contain theSameElementsAs lastManifests + } + + test("it keeps only the latest 200 manifests per device in database") { + val devices = List(DeviceId.generate(), DeviceId.generate()) + val now = Instant.now().truncatedTo(ChronoUnit.SECONDS) + val manifests = devices.flatMap { device => + (1 to 250).map { i => + val manifest = GenDeviceManifest.generate + val signedManifest = SignedPayload(Seq.empty, manifest.asJson, manifest.asJson) + Messages.DeviceManifestReported(defaultNs, device, signedManifest, now.plusSeconds(i)) + } + } + + runListener(manifests).futureValue + + devices.foreach { device => + val savedManifests = + deviceManifestRepository.findAll(device, 0.toOffset, 1000.toLimit).futureValue.values + val deviceManifests = + manifests.filter(_.deviceId == device).map(_.manifest.json).takeRight(200) + + savedManifests should have size 200 + savedManifests.map(_._2).reverse should be(sorted) + savedManifests.map(_._1) should contain theSameElementsAs deviceManifests + } + } + + test("when sending same manifest twice, only latest version is saved if in same batch") { + val device = DeviceId.generate() + val manifest = GenDeviceManifest.generate + val signedManifest = SignedPayload(Seq.empty, manifest.asJson, manifest.asJson) + val firstTime = Instant.now().truncatedTo(ChronoUnit.SECONDS) + val secondTime = firstTime.plusSeconds(30) + + val msg1 = DeviceManifestReported(defaultNs, device, signedManifest, firstTime) + val msg2 = DeviceManifestReported(defaultNs, device, signedManifest, secondTime) + + runListener(List(msg2, msg1)).futureValue + + val saved = deviceManifestRepository.findAll(device).futureValue.values + + saved should have size 1 + saved.head._1 shouldBe manifest.asJson + saved.head._2 shouldBe secondTime + } + + test("when sending same manifest twice, only latest processed is saved") { + val device = DeviceId.generate() + val manifest = GenDeviceManifest.generate + val signedManifest = SignedPayload(Seq.empty, manifest.asJson, manifest.asJson) + val firstTime = Instant.now().truncatedTo(ChronoUnit.SECONDS) + val secondTime = firstTime.plusSeconds(30) + + val msg1 = DeviceManifestReported(defaultNs, device, signedManifest, firstTime) + val msg2 = DeviceManifestReported(defaultNs, device, signedManifest, secondTime) + + runListener(msg2).futureValue + runListener(msg1).futureValue + + val saved = deviceManifestRepository.findAll(device).futureValue.values + + saved should have size 1 + saved.head._1 shouldBe manifest.asJson + saved.head._2 shouldBe firstTime + } + + test( + "manifests with identical content but different signatures and report counters have the same checksum" + ) { + val device = DeviceId.generate() + val manifest = GenDeviceManifest.generate + + def setSignatures(json: Json, newValue: String): Json = json.arrayOrObject( + json, + jsonArray = arr => Json.fromValues(arr.map(setSignatures(_, newValue))), + jsonObject = obj => + obj.toMap.map { + case ("signed", signed) => + "signed" -> signed.deepMerge( + Json.obj("report_counter" -> newValue.asJson) + ) + case ("signatures", _) => + "signatures" -> Json.arr(Json.obj("sig" -> newValue.asJson)) + case (key, value) => + key -> setSignatures(value, newValue) + }.asJson + ) + + val manifest1 = setSignatures(manifest.asJson, "sig1") + val signedManifest1 = SignedPayload(Seq.empty, manifest1, manifest1) + + val manifest2 = setSignatures(manifest.asJson, "sig2") + val signedManifest2 = SignedPayload(Seq.empty, manifest2, manifest2) + + val msg1 = DeviceManifestReported(defaultNs, device, signedManifest1, Instant.now()) + runListener(msg1).futureValue + + val manifests1 = deviceManifestRepository.findAll(device).futureValue + manifests1.values.size shouldBe 1 + val (storedManifest1, storedTs1) = manifests1.values.loneElement + + val msg2 = + DeviceManifestReported(defaultNs, device, signedManifest2, Instant.now().plusSeconds(10)) + runListener(msg2).futureValue + + val manifests2 = deviceManifestRepository.findAll(device).futureValue + manifests2.values.size shouldBe 1 + + val (storedManifest2, storedTs2) = manifests2.values.loneElement + setSignatures(storedManifest2, "") shouldBe setSignatures(storedManifest1, "") + storedTs2.isAfter(storedTs1) shouldBe true + + val differentManifest = GenDeviceManifest.generate + val signedDifferentManifest = + SignedPayload(Seq.empty, differentManifest.asJson, differentManifest.asJson) + + val msg3 = DeviceManifestReported( + defaultNs, + device, + signedDifferentManifest, + Instant.now().plusSeconds(20) + ) + runListener(msg3).futureValue + + val manifests3 = deviceManifestRepository.findAll(device).futureValue + manifests3.values.size shouldBe 2 + } + } diff --git a/src/test/scala/com/advancedtelematic/director/data/Generators.scala b/src/test/scala/com/advancedtelematic/director/data/Generators.scala index 6435dc52..319ecf20 100644 --- a/src/test/scala/com/advancedtelematic/director/data/Generators.scala +++ b/src/test/scala/com/advancedtelematic/director/data/Generators.scala @@ -1,49 +1,18 @@ package com.advancedtelematic.director.data -import com.advancedtelematic.libats.data.EcuIdentifier import org.scalacheck.Gen import GeneratorOps.* -import akka.http.scaladsl.model.Uri -import com.advancedtelematic.director.data.AdminDataType.{ - MultiTargetUpdate, - RegisterEcu, - TargetUpdate, - TargetUpdateRequest -} -import com.advancedtelematic.director.data.DeviceRequest.{ - DeviceManifest, - EcuManifest, - InstallationItem, - InstallationReport, - MissingInstallationReport -} +import org.apache.pekko.http.scaladsl.model.Uri +import com.advancedtelematic.director.data.AdminDataType.{RegisterEcu, TargetUpdate, TargetUpdateRequest, TargetUpdateSpec} +import com.advancedtelematic.director.data.DeviceRequest.{DeviceManifest, EcuManifest, InstallationItem, InstallationReport, MissingInstallationReport} import com.advancedtelematic.director.data.UptaneDataType.* -import com.advancedtelematic.libats.data.DataType.{ - Checksum, - CorrelationId, - HashMethod, - MultiTargetUpdateId, - OfflineUpdateId, - ResultCode, - ResultDescription, - ValidChecksum, - ValidLockboxHash -} -import com.advancedtelematic.libats.messaging_datatype.DataType.InstallationResult -import com.advancedtelematic.libtuf.data.TufDataType.{ - Ed25519KeyType, - HardwareIdentifier, - KeyType, - RsaKeyType, - SignedPayload, - TargetFilename, - TufKey, - TufKeyPair, - ValidTargetFilename -} +import com.advancedtelematic.libats.data.DataType.{Checksum, CorrelationId, HashMethod, MultiTargetUpdateCorrelationId, OfflineUpdateId, ResultCode, ResultDescription, ValidChecksum, ValidLockboxHash} +import com.advancedtelematic.libats.messaging_datatype.DataType.{EcuIdentifier, InstallationResult, ValidEcuIdentifier} +import com.advancedtelematic.libtuf.data.TufDataType.{Ed25519KeyType, HardwareIdentifier, KeyType, RsaKeyType, SignedPayload, TargetFilename, TufKey, TufKeyPair, ValidTargetFilename} import eu.timepit.refined.api.{RefType, Refined} import io.circe.Json import Codecs.* +import com.advancedtelematic.libats.data.RefinedUtils.RefineTry import com.advancedtelematic.libtuf.data.ClientDataType.ClientTargetItem trait Generators { @@ -53,7 +22,7 @@ trait Generators { Gen .choose(10, 64) .flatMap(GenStringByCharN(_, Gen.alphaChar)) - .map(EcuIdentifier.from(_).toOption.get) + .map(_.refineTry[ValidEcuIdentifier].get) lazy val GenHardwareIdentifier: Gen[HardwareIdentifier] = Gen.choose(10, 200).flatMap(GenRefinedStringByCharN(_, Gen.alphaChar)) @@ -137,9 +106,9 @@ trait Generators { targetUpdate <- GenTargetUpdate } yield TargetUpdateRequest(None, targetUpdate) - val GenMultiTargetUpdateRequest: Gen[MultiTargetUpdate] = for { + val GenMultiTargetUpdateRequest: Gen[TargetUpdateSpec] = for { targets <- Gen.nonEmptyMap(Gen.zip(GenHardwareIdentifier, GenTargetUpdateRequest)) - } yield MultiTargetUpdate(targets) + } yield TargetUpdateSpec(targets) lazy val GenKeyType: Gen[KeyType] = Gen.oneOf(RsaKeyType, Ed25519KeyType) @@ -158,7 +127,7 @@ trait Generators { } yield RegisterEcu(ecu, hwId, crypto) lazy val GenCorrelationId = - Gen.uuid.map(u => MultiTargetUpdateId(u)) + Gen.uuid.map(u => MultiTargetUpdateCorrelationId(u)) lazy val GenOffLineCorrelationId = for { hash <- Gen diff --git a/src/test/scala/com/advancedtelematic/director/db/SignedRoleMigrationSpec.scala b/src/test/scala/com/advancedtelematic/director/db/SignedRoleMigrationSpec.scala index b33a2935..26b52973 100644 --- a/src/test/scala/com/advancedtelematic/director/db/SignedRoleMigrationSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/db/SignedRoleMigrationSpec.scala @@ -1,7 +1,7 @@ package com.advancedtelematic.director.db import java.util.UUID -import akka.actor.ActorSystem +import org.apache.pekko.actor.ActorSystem import com.advancedtelematic.director.util.DirectorSpec import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libats.test.MysqlDatabaseSpec diff --git a/src/test/scala/com/advancedtelematic/director/db/UpdateSchedulerDBIOSpec.scala b/src/test/scala/com/advancedtelematic/director/db/UpdateSchedulerDBIOSpec.scala index 1ee4207d..85b38f55 100644 --- a/src/test/scala/com/advancedtelematic/director/db/UpdateSchedulerDBIOSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/db/UpdateSchedulerDBIOSpec.scala @@ -1,30 +1,26 @@ package com.advancedtelematic.director.db -import com.advancedtelematic.director.daemon.UpdateScheduler +import cats.implicits.catsSyntaxOptionId import com.advancedtelematic.director.data.AdminDataType.{ - MultiTargetUpdate, RegisterEcu, - TargetUpdateRequest + TargetUpdateRequest, + TargetUpdateSpec } -import com.advancedtelematic.director.data.DataType.{ScheduledUpdate, ScheduledUpdateId} +import com.advancedtelematic.director.data.DataType.{Update, UpdateId} import com.advancedtelematic.director.data.DbDataType.Ecu import com.advancedtelematic.director.data.GeneratorOps.* import com.advancedtelematic.director.data.Generators.* import com.advancedtelematic.director.db.ProvisionedDeviceRepository.DeviceCreateResult -import com.advancedtelematic.director.db.UpdateSchedulerDBIO.invalidEcuStatusCodec import com.advancedtelematic.director.db.deviceregistry.DeviceRepository import com.advancedtelematic.director.deviceregistry.data.Device.DeviceOemId import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.genDeviceT -import com.advancedtelematic.director.deviceregistry.data.DeviceStatus import com.advancedtelematic.director.http.DeviceAssignments import com.advancedtelematic.director.util.DirectorSpec -import com.advancedtelematic.libats.data.DataType.{MultiTargetUpdateId, Namespace} -import com.advancedtelematic.libats.data.EcuIdentifier -import com.advancedtelematic.libats.messaging.MessageBusPublisher -import com.advancedtelematic.libats.messaging.test.MockMessageBus -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.data.DataType.{MultiTargetUpdateCorrelationId, Namespace} +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libats.test.MysqlDatabaseSpec import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier +import io.circe.Json import org.scalatest.LoneElement.* import java.time.Instant @@ -33,35 +29,28 @@ import scala.concurrent.{ExecutionContext, Future} class UpdateSchedulerDBIOSpec extends DirectorSpec with MysqlDatabaseSpec - with ScheduledUpdatesRepositorySupport + with UpdatesRepositorySupport with AssignmentsRepositorySupport with EcuRepositorySupport with ProvisionedDeviceRepositorySupport { implicit val ec: ExecutionContext = ExecutionContext.Implicits.global - private implicit val msgPub: MessageBusPublisher = new MockMessageBus - - val multiTargetUpdates = new MultiTargetUpdates() + val targetUpdateSpecs = new TargetUpdateSpecs() val deviceAssignments = new DeviceAssignments() val updateSchedulerIO = new UpdateSchedulerDBIO() - val updateScheduler = new UpdateScheduler() + val updatesDBIO = new UpdatesDBIO() def createScheduledUpdate(device: DeviceId, - _mtu: Map[HardwareIdentifier, TargetUpdateRequest], + mtu: Map[HardwareIdentifier, TargetUpdateRequest], ecuId: EcuIdentifier, - registerEcus: RegisterEcu*)(implicit ns: Namespace) = { - val mtu = MultiTargetUpdate(_mtu) - val updateId = multiTargetUpdates.create(ns, mtu).futureValue - + registerEcus: RegisterEcu*)(implicit ns: Namespace): UpdateId = { createDevice(device, ecuId, registerEcus.map(_.toEcu(ns, device))*).futureValue - updateScheduler.create(ns, device, updateId, Instant.now).futureValue - - updateId + updatesDBIO.createFor(ns, device, TargetUpdateSpec(mtu), Instant.now.some).futureValue } private def createDevice(deviceId: DeviceId, ecuId: EcuIdentifier, ecus: Ecu*)( @@ -76,10 +65,8 @@ class UpdateSchedulerDBIOSpec .create(ecuRepository)(ns, deviceId, ecuId, ecus) } - private def createMtu(hardwareId: HardwareIdentifier)(implicit ns: Namespace): UpdateId = { - val mtu = MultiTargetUpdate(Map(hardwareId -> GenTargetUpdateRequest.generate)) - multiTargetUpdates.create(ns, mtu).futureValue - } + private def buildMtu(hardwareId: HardwareIdentifier): TargetUpdateSpec = + TargetUpdateSpec(Map(hardwareId -> GenTargetUpdateRequest.generate)) testWithNamespace("creates assignment for a device") { implicit ns => val device = DeviceId.generate() @@ -96,14 +83,13 @@ class UpdateSchedulerDBIOSpec assignments.loneElement.deviceId shouldBe device assignments.loneElement.ecuId shouldBe registerEcu.ecu_serial - assignments.loneElement.correlationId shouldBe MultiTargetUpdateId(updateId.uuid) + assignments.loneElement.correlationId shouldBe updateId.toCorrelationId - scheduledUpdatesRepository + updatesRepository .findFor(ns, device) .futureValue - .values .loneElement - .status shouldBe ScheduledUpdate.Status.Assigned + .status shouldBe Update.Status.Assigned } testWithNamespace("creates assignment for all ecus in an MTU") { implicit ns => @@ -136,14 +122,13 @@ class UpdateSchedulerDBIOSpec primaryRegisterEcu.ecu_serial, secondaryRegisterEcu.ecu_serial ) - assignments.map(_.correlationId) should contain only MultiTargetUpdateId(updateId.uuid) + assignments.map(_.correlationId) should contain only updateId.toCorrelationId - scheduledUpdatesRepository + updatesRepository .findFor(ns, device) .futureValue - .values .loneElement - .status shouldBe ScheduledUpdate.Status.Assigned + .status shouldBe Update.Status.Assigned } testWithNamespace("cancels/terminates scheduled update when device not compatible") { @@ -152,13 +137,13 @@ class UpdateSchedulerDBIOSpec val primaryHardwareId = primaryRegisterEcu.hardware_identifier val device = DeviceId.generate() - val mtu = MultiTargetUpdate( + val mtu = TargetUpdateSpec( Map( primaryHardwareId -> GenTargetUpdateRequest.generate .copy(from = Some(GenTargetUpdate.generate)) ) ) - val updateId = multiTargetUpdates.create(ns, mtu).futureValue + val targetSpecId = targetUpdateSpecs.create(ns, mtu).futureValue provisionedDeviceRepository .create(ecuRepository)( @@ -169,15 +154,19 @@ class UpdateSchedulerDBIOSpec ) .futureValue - val scheduledUpdate = ScheduledUpdate( + val id = UpdateId.generate() + + val scheduledUpdate = Update( ns, - ScheduledUpdateId.generate(), + id, device, - updateId, + id.toCorrelationId, + targetSpecId, Instant.now(), - ScheduledUpdate.Status.Scheduled + Instant.now().some, + Update.Status.Scheduled ) - scheduledUpdatesRepository.persist(scheduledUpdate).futureValue + updatesRepository.persist(scheduledUpdate).futureValue updateSchedulerIO.run().futureValue @@ -186,16 +175,16 @@ class UpdateSchedulerDBIOSpec assignments shouldBe empty val scheduledUpdateAfter = - scheduledUpdatesRepository.findFor(ns, device).futureValue.values.loneElement - scheduledUpdateAfter.status shouldBe ScheduledUpdate.Status.Cancelled + updatesRepository.findFor(ns, device).futureValue.loneElement + scheduledUpdateAfter.status shouldBe Update.Status.Cancelled } testWithNamespace("cancels terminates when device does not have compatible ecu hardware at all") { implicit ns => val primaryRegisterEcu = GenRegisterEcu.generate val mtu = - MultiTargetUpdate(Map(GenHardwareIdentifier.generate -> GenTargetUpdateRequest.generate)) - val updateId = multiTargetUpdates.create(ns, mtu).futureValue + TargetUpdateSpec(Map(GenHardwareIdentifier.generate -> GenTargetUpdateRequest.generate)) + val targetSpecId = targetUpdateSpecs.create(ns, mtu).futureValue val device = DeviceId.generate() createDevice( @@ -204,15 +193,19 @@ class UpdateSchedulerDBIOSpec primaryRegisterEcu.toEcu(ns, device) ).futureValue - val scheduledUpdate = ScheduledUpdate( + val id = UpdateId.generate() + + val scheduledUpdate = Update( ns, - ScheduledUpdateId.generate(), + id, device, - updateId, - Instant.now(), - ScheduledUpdate.Status.Scheduled + id.toCorrelationId, + targetSpecId, + Instant.now, + Instant.now().some, + Update.Status.Scheduled ) - scheduledUpdatesRepository.persist(scheduledUpdate).futureValue + updatesRepository.persist(scheduledUpdate).futureValue updateSchedulerIO.run().futureValue @@ -220,12 +213,11 @@ class UpdateSchedulerDBIOSpec assignments shouldBe empty - scheduledUpdatesRepository + updatesRepository .findFor(ns, device) .futureValue - .values .loneElement - .status shouldBe ScheduledUpdate.Status.Cancelled + .status shouldBe Update.Status.Cancelled } testWithNamespace( @@ -234,29 +226,38 @@ class UpdateSchedulerDBIOSpec val registerEcu = GenRegisterEcu.generate val mtu = - MultiTargetUpdate(Map(registerEcu.hardware_identifier -> GenTargetUpdateRequest.generate)) - val updateId = multiTargetUpdates.create(ns, mtu).futureValue + TargetUpdateSpec(Map(registerEcu.hardware_identifier -> GenTargetUpdateRequest.generate)) + val targetSpecId = targetUpdateSpecs.create(ns, mtu).futureValue val device = DeviceId.generate() createDevice(device, registerEcu.ecu_serial, registerEcu.toEcu(ns, device)).futureValue val mtuExisting = - MultiTargetUpdate(Map(registerEcu.hardware_identifier -> GenTargetUpdateRequest.generate)) - val updateIdExisting = multiTargetUpdates.create(ns, mtuExisting).futureValue + TargetUpdateSpec(Map(registerEcu.hardware_identifier -> GenTargetUpdateRequest.generate)) + val TargetSpecIdExisting = targetUpdateSpecs.create(ns, mtuExisting).futureValue val assignedTo = deviceAssignments - .createForDevice(ns, MultiTargetUpdateId(updateIdExisting.uuid), device, updateIdExisting) + .createForDevice( + ns, + MultiTargetUpdateCorrelationId(TargetSpecIdExisting.uuid), + device, + TargetSpecIdExisting + ) .futureValue assignedTo shouldBe device - val scheduledUpdate = ScheduledUpdate( + val id = UpdateId.generate() + + val scheduledUpdate = Update( ns, - ScheduledUpdateId.generate(), + id, device, - updateId, - Instant.now(), - ScheduledUpdate.Status.Scheduled + id.toCorrelationId, + targetSpecId, + Instant.now, + Instant.now().some, + Update.Status.Scheduled ) - scheduledUpdatesRepository.persist(scheduledUpdate).futureValue + updatesRepository.persist(scheduledUpdate).futureValue updateSchedulerIO.run().futureValue @@ -266,14 +267,15 @@ class UpdateSchedulerDBIOSpec createdAssignment.deviceId shouldBe device createdAssignment.ecuId shouldBe registerEcu.ecu_serial - createdAssignment.correlationId shouldBe MultiTargetUpdateId(updateIdExisting.uuid) + createdAssignment.correlationId shouldBe MultiTargetUpdateCorrelationId( + TargetSpecIdExisting.uuid + ) - scheduledUpdatesRepository + updatesRepository .findFor(ns, device) .futureValue - .values .loneElement - .status shouldBe ScheduledUpdate.Status.Cancelled + .status shouldBe Update.Status.Cancelled } testWithNamespace("creates assignments for future schedules only") { implicit ns => @@ -282,37 +284,38 @@ class UpdateSchedulerDBIOSpec val device = DeviceId.generate() val device2 = DeviceId.generate() - val updateId = createMtu(registerEcu.hardware_identifier) - val updateId2 = createMtu(registerEcu2.hardware_identifier) + val mtu1 = buildMtu(registerEcu.hardware_identifier) + val mtu2 = buildMtu(registerEcu2.hardware_identifier) createDevice(device, registerEcu.ecu_serial, registerEcu.toEcu(ns, device)).futureValue createDevice(device2, registerEcu2.ecu_serial, registerEcu2.toEcu(ns, device2)).futureValue - val scheduledUpdateId = updateScheduler.create(ns, device, updateId, Instant.now).futureValue + val scheduledUpdateId = + updatesDBIO.createFor(ns, device, mtu1, Instant.now.some).futureValue val scheduledUpdateId2 = - updateScheduler.create(ns, device2, updateId2, Instant.now.plusSeconds(360)).futureValue + updatesDBIO.createFor(ns, device2, mtu2, Instant.now.plusSeconds(360).some).futureValue updateSchedulerIO.run().futureValue val assignment = assignmentsRepository.findBy(device).futureValue.loneElement assignment.deviceId shouldBe device - assignment.correlationId shouldBe MultiTargetUpdateId(updateId.uuid) + assignment.correlationId shouldBe scheduledUpdateId.toCorrelationId - val updatedScheduledUpdates = scheduledUpdatesRepository.findFor(ns, device).futureValue.values + val updatedScheduledUpdates = updatesRepository.findFor(ns, device).futureValue updatedScheduledUpdates should have size 1 updatedScheduledUpdates.map(u => u.id -> u.status).toMap shouldBe Map( - scheduledUpdateId -> ScheduledUpdate.Status.Assigned + scheduledUpdateId -> Update.Status.Assigned ) val updatedScheduledUpdates2 = - scheduledUpdatesRepository.findFor(ns, device2).futureValue.values + updatesRepository.findFor(ns, device2).futureValue updatedScheduledUpdates2 should have size 1 updatedScheduledUpdates2.map(u => u.id -> u.status).toMap shouldBe Map( - scheduledUpdateId2 -> ScheduledUpdate.Status.Scheduled + scheduledUpdateId2 -> Update.Status.Scheduled ) } @@ -320,55 +323,40 @@ class UpdateSchedulerDBIOSpec val registerEcu = GenRegisterEcu.generate val device = DeviceId.generate() - val updateId = createMtu(registerEcu.hardware_identifier) - val updateId1 = createMtu(registerEcu.hardware_identifier) + val mtu = buildMtu(registerEcu.hardware_identifier) + val mtu1 = buildMtu(registerEcu.hardware_identifier) createDevice(device, registerEcu.ecu_serial, registerEcu.toEcu(ns, device)).futureValue - val cancelledUpdateId = updateScheduler.create(ns, device, updateId1, Instant.now).futureValue + val cancelledTargetSpecId = + updatesDBIO.createFor(ns, device, mtu1, Instant.now.some).futureValue db.run( - scheduledUpdatesRepository - .setStatusAction(ns, cancelledUpdateId, ScheduledUpdate.Status.Cancelled) + updatesRepository + .setStatusAction[Json](ns, cancelledTargetSpecId, Update.Status.Cancelled) ).futureValue - val scheduledUpdateId = updateScheduler.create(ns, device, updateId, Instant.now).futureValue + val scheduledUpdateId = + updatesDBIO.createFor(ns, device, mtu, Instant.now.some).futureValue updateSchedulerIO.run().futureValue val assignment = assignmentsRepository.findBy(device).futureValue.loneElement assignment.deviceId shouldBe device - assignment.correlationId shouldBe MultiTargetUpdateId(updateId.uuid) + assignment.correlationId shouldBe scheduledUpdateId.toCorrelationId - val scheduledUpdates = scheduledUpdatesRepository.findFor(ns, device).futureValue.values + val scheduledUpdates = updatesRepository.findFor(ns, device).futureValue scheduledUpdates should have size 2 scheduledUpdates.map(_.id) should contain theSameElementsAs List( scheduledUpdateId, - cancelledUpdateId + cancelledTargetSpecId ) scheduledUpdates.map(_.status) should contain theSameElementsAs List( - ScheduledUpdate.Status.Assigned, - ScheduledUpdate.Status.Cancelled + Update.Status.Assigned, + Update.Status.Cancelled ) } - testWithNamespace("updates device status to UpdateScheduled when assignment is created") { - implicit ns => - val device = DeviceId.generate() - val mtu = Map(GenHardwareIdentifier.generate -> GenTargetUpdateRequest.generate) - val registerEcu = GenRegisterEcu.generate.copy(hardware_identifier = mtu.keys.head) - - createScheduledUpdate(device, mtu, registerEcu.ecu_serial, registerEcu) - - updateSchedulerIO.run().futureValue - - val deviceStatus = db.run(DeviceRepository.findByUuid(device)).futureValue.deviceStatus - deviceStatus shouldBe DeviceStatus.UpdateScheduled - - val assignments = assignmentsRepository.findBy(device).futureValue - assignments.loneElement.deviceId shouldBe device - } - test("works for a larger group of devices")(pending) } diff --git a/src/test/scala/com/advancedtelematic/director/db/deviceregistry/EventIndexSpec.scala b/src/test/scala/com/advancedtelematic/director/db/deviceregistry/EventIndexSpec.scala index ddc9e88c..104627e6 100644 --- a/src/test/scala/com/advancedtelematic/director/db/deviceregistry/EventIndexSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/db/deviceregistry/EventIndexSpec.scala @@ -4,8 +4,18 @@ import cats.syntax.option.* import com.advancedtelematic.director.deviceregistry.data.DataType.{IndexedEvent, IndexedEventType} import com.advancedtelematic.director.deviceregistry.data.GeneratorOps.* import com.advancedtelematic.director.util.DirectorSpec -import com.advancedtelematic.libats.data.DataType.{CampaignId, CorrelationId, MultiTargetUpdateId} -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, Event, EventType} +import com.advancedtelematic.libats.data.DataType.{ + CorrelationId, + MultiTargetUpdateCorrelationId, + TargetSpecCorrelationId +} +import com.advancedtelematic.libats.messaging_datatype.DataType.{ + DeviceId, + Event, + EventType, + ValidEcuIdentifier +} +import eu.timepit.refined.refineMV import io.circe.Json import io.circe.syntax.* import org.scalacheck.Gen @@ -13,18 +23,30 @@ import org.scalatest.EitherValues.* import java.time.Instant import java.util.UUID +import com.advancedtelematic.director.data.Generators.GenEcuIdentifier class EventIndexSpec extends DirectorSpec { val genCorrelationId: Gen[CorrelationId] = - Gen.uuid.flatMap(uuid => Gen.oneOf(CampaignId(uuid), MultiTargetUpdateId(uuid))) + Gen.uuid.flatMap(uuid => + Gen.oneOf(TargetSpecCorrelationId(uuid), MultiTargetUpdateCorrelationId(uuid)) + ) val eventGen: Gen[Event] = for { device <- Gen.uuid.map(DeviceId.apply) eventId <- Gen.uuid.map(_.toString) eventType = EventType("", 0) + ecu <- Gen.option(GenEcuIdentifier) json = Json.obj() - } yield Event(device, eventId, eventType, Instant.now, Instant.now, json) + } yield Event( + device, + eventId, + eventType, + Instant.now, + Instant.now, + ecu, + json + ) val downloadCompleteEventGen: Gen[Event] = eventGen.map(_.copy(eventType = EventType("DownloadComplete", 0))) @@ -65,25 +87,6 @@ class EventIndexSpec extends DirectorSpec { } } - test("indexes an event with campaign ID by type") { - val eventTypeMap = Map( - EventType("campaign_accepted", 0) -> IndexedEventType.CampaignAccepted, - EventType("campaign_declined", 0) -> IndexedEventType.CampaignDeclined, - EventType("campaign_postponed", 0) -> IndexedEventType.CampaignPostponed - ) - eventTypeMap.foreach { case (eventType, indexedEventType) => - val (event, campaignId) = eventWithCampaignIdGen(eventType).generate - val correlationId = CampaignId(campaignId) - val indexedEvent = EventIndex.index(event).value - indexedEvent shouldBe IndexedEvent( - event.deviceUuid, - event.eventId, - indexedEventType, - correlationId.some - ) - } - } - test("indexes a DownloadComplete event by type") { val event = downloadCompleteEventGen.generate diff --git a/src/test/scala/com/advancedtelematic/director/db/deviceregistry/EventJournalSpec.scala b/src/test/scala/com/advancedtelematic/director/db/deviceregistry/EventJournalSpec.scala index 26505284..008405fc 100644 --- a/src/test/scala/com/advancedtelematic/director/db/deviceregistry/EventJournalSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/db/deviceregistry/EventJournalSpec.scala @@ -11,11 +11,11 @@ package com.advancedtelematic.director.db.deviceregistry import java.time.Instant import java.time.temporal.ChronoUnit import java.util.UUID -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import cats.syntax.option.* import com.advancedtelematic.libats.codecs.CirceCodecs.* -import com.advancedtelematic.libats.data.DataType.{CampaignId, CorrelationId, MultiTargetUpdateId} -import com.advancedtelematic.libats.messaging_datatype.DataType.{Event, EventType} +import com.advancedtelematic.libats.data.DataType.{CorrelationId, MultiTargetUpdateCorrelationId, TargetSpecCorrelationId} +import com.advancedtelematic.libats.messaging_datatype.DataType.{Event, EventType, ValidEcuIdentifier} import com.advancedtelematic.libats.messaging_datatype.MessageCodecs.* import com.advancedtelematic.libats.messaging_datatype.Messages.DeviceEventMessage import EventJournalSpec.EventPayload @@ -30,9 +30,10 @@ import io.circe.{Decoder, Json} import org.scalacheck.{Arbitrary, Gen, Shrink} import org.scalatest.concurrent.{Eventually, ScalaFutures} import org.scalatest.time.SpanSugar.* -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import org.scalatest.time.{Millis, Seconds, Span} import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.* +import eu.timepit.refined.refineMV object EventJournalSpec { @@ -82,7 +83,7 @@ class EventJournalSpec } yield EventType(id, ver) val genCorrelationId: Gen[CorrelationId] = - Gen.uuid.flatMap(uuid => Gen.oneOf(CampaignId(uuid), MultiTargetUpdateId(uuid))) + Gen.uuid.flatMap(uuid => Gen.oneOf(TargetSpecCorrelationId(uuid), MultiTargetUpdateCorrelationId(uuid))) implicit val EventGen: org.scalacheck.Gen[EventJournalSpec.EventPayload] = for { id <- Gen.uuid @@ -116,7 +117,7 @@ class EventJournalSpec events .map(ep => - Event(deviceUuid, ep.id.toString, ep.eventType, ep.deviceTime, Instant.now, ep.event) + Event(deviceUuid, ep.id.toString, ep.eventType, ep.deviceTime, Instant.now, refineMV[ValidEcuIdentifier]("ecuId").some, ep.event) ) .map(DeviceEventMessage(defaultNs, _)) .map(listener.apply) @@ -140,7 +141,7 @@ class EventJournalSpec List(event0, event1) .map(ep => - Event(deviceUuid, ep.id.toString, ep.eventType, ep.deviceTime, Instant.now, ep.event) + Event(deviceUuid, ep.id.toString, ep.eventType, ep.deviceTime, Instant.now, refineMV[ValidEcuIdentifier]("ecuId").some, ep.event) ) .map(DeviceEventMessage(defaultNs, _)) .map(listener.apply) @@ -164,13 +165,13 @@ class EventJournalSpec List(event0, event1) .map(ep => - Event(deviceUuid, ep.id.toString, ep.eventType, ep.deviceTime, Instant.now, ep.event) + Event(deviceUuid, ep.id.toString, ep.eventType, ep.deviceTime, Instant.now, refineMV[ValidEcuIdentifier]("ecuId").some, ep.event) ) .map(DeviceEventMessage(defaultNs, _)) .map(listener.apply) eventually(timeout(3.seconds), interval(100.millis)) { - getEvents(deviceUuid, CampaignId(UUID.randomUUID()).some) ~> routes ~> check { + getEvents(deviceUuid, TargetSpecCorrelationId(UUID.randomUUID()).some) ~> routes ~> check { status should equal(StatusCodes.OK) val events = responseAs[List[EventPayload]] events shouldBe empty @@ -181,7 +182,7 @@ class EventJournalSpec test("DELETE device archives its indexed events") { val uuid = createDeviceOk(genDeviceT.generate) val (e, _) = installCompleteEventGen.generate - val event = Event(uuid, e.id.toString, e.eventType, e.deviceTime, Instant.now, e.event) + val event = Event(uuid, e.id.toString, e.eventType, e.deviceTime, Instant.now, refineMV[ValidEcuIdentifier]("ecuId").some, e.event) val deviceEventMessage = DeviceEventMessage(defaultNs, event) val journal = new EventJournal() diff --git a/src/test/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceInstallationReportListenerSpec.scala b/src/test/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceInstallationReportListenerSpec.scala index e1173044..f74ed5f9 100644 --- a/src/test/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceInstallationReportListenerSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/deviceregistry/daemon/DeviceInstallationReportListenerSpec.scala @@ -1,5 +1,6 @@ package com.advancedtelematic.director.deviceregistry.daemon +import cats.implicits.catsSyntaxOptionId import com.advancedtelematic.director.db.deviceregistry.InstallationReportRepository import com.advancedtelematic.director.deviceregistry.data.DataType.{ DeviceInstallationResult, @@ -38,7 +39,7 @@ class DeviceInstallationReportListenerSpec listener.apply(message).futureValue shouldBe (()) val expectedDeviceReports = - Seq( + Option( DeviceInstallationResult( correlationId, message.result.code, @@ -49,29 +50,31 @@ class DeviceInstallationReportListenerSpec ) ) val deviceReports = - db.run(InstallationReportRepository.fetchDeviceInstallationResult(correlationId)) + db.run(InstallationReportRepository.fetchDeviceInstallationResultByCorrelationId(deviceUuid, correlationId)) deviceReports.futureValue shouldBe expectedDeviceReports val expectedEcuReports = message.ecuReports.map { case (ecuId, ecuReport) => - EcuInstallationResult( + ecuId -> EcuInstallationResult( correlationId, ecuReport.result.code, deviceUuid, ecuId, - message.result.success + ecuReport.result.success, + ecuReport.result.description.value.some ) - }.toSeq - val ecuReports = db.run(InstallationReportRepository.fetchEcuInstallationReport(correlationId)) + } + val ecuReports = + db.run(InstallationReportRepository.fetchEcuInstallationReport(deviceUuid, correlationId)) ecuReports.futureValue shouldBe expectedEcuReports // Saving the reports is idempotent listener.apply(message).futureValue shouldBe (()) val deviceReportsAgain = - db.run(InstallationReportRepository.fetchDeviceInstallationResult(correlationId)) + db.run(InstallationReportRepository.fetchDeviceInstallationResultByCorrelationId(deviceUuid, correlationId)) deviceReportsAgain.futureValue shouldBe expectedDeviceReports val ecuReportsAgain = - db.run(InstallationReportRepository.fetchEcuInstallationReport(correlationId)) + db.run(InstallationReportRepository.fetchEcuInstallationReport(deviceUuid, correlationId)) ecuReportsAgain.futureValue shouldBe expectedEcuReports } @@ -87,7 +90,7 @@ class DeviceInstallationReportListenerSpec listener.apply(messageFailed).futureValue shouldBe (()) val expectedDeviceReportsFailed = - Seq( + Option( DeviceInstallationResult( correlationId, messageFailed.result.code, @@ -98,7 +101,7 @@ class DeviceInstallationReportListenerSpec ) ) val expectedDeviceReportsSuccess = - Seq( + Option( DeviceInstallationResult( correlationId, messageSuccess.result.code, @@ -110,13 +113,13 @@ class DeviceInstallationReportListenerSpec ) val deviceReports = - db.run(InstallationReportRepository.fetchDeviceInstallationResult(correlationId)) + db.run(InstallationReportRepository.fetchDeviceInstallationResultByCorrelationId(deviceUuid, correlationId)) deviceReports.futureValue shouldBe expectedDeviceReportsFailed listener.apply(messageSuccess).futureValue shouldBe (()) val deviceReportsAgain = - db.run(InstallationReportRepository.fetchDeviceInstallationResult(correlationId)) + db.run(InstallationReportRepository.fetchDeviceInstallationResultByCorrelationId(deviceUuid, correlationId)) deviceReportsAgain.futureValue shouldBe expectedDeviceReportsSuccess } diff --git a/src/test/scala/com/advancedtelematic/director/deviceregistry/data/GroupExpressionParserSpec.scala b/src/test/scala/com/advancedtelematic/director/deviceregistry/data/GroupExpressionParserSpec.scala index e4c96324..21d15f65 100644 --- a/src/test/scala/com/advancedtelematic/director/deviceregistry/data/GroupExpressionParserSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/deviceregistry/data/GroupExpressionParserSpec.scala @@ -235,7 +235,7 @@ class GroupExpressionParserSpec extends AnyFunSuite with Matchers { test("parses 'tag() contains' and 'tag() position is'") { runParser("tag(-market-) contains erma") shouldBe TagContains("-market-", "erma") runParser("tag(_trim_) position(1) is P") shouldBe TagCharAt("_trim_", 'P', 0) - runParser("tag( mar ket ) contains erma") shouldBe TagContains("mar ket ", "erma") + runParser("tag(mar_ket) contains erma") shouldBe TagContains("mar_ket", "erma") } test("parses 'tag() position is not'") { diff --git a/src/test/scala/com/advancedtelematic/director/deviceregistry/data/InstallationReportGenerators.scala b/src/test/scala/com/advancedtelematic/director/deviceregistry/data/InstallationReportGenerators.scala index dc666d3c..ce9df9a9 100644 --- a/src/test/scala/com/advancedtelematic/director/deviceregistry/data/InstallationReportGenerators.scala +++ b/src/test/scala/com/advancedtelematic/director/deviceregistry/data/InstallationReportGenerators.scala @@ -1,29 +1,10 @@ package com.advancedtelematic.director.deviceregistry.data import java.time.Instant - -import com.advancedtelematic.libats.data.DataType.{ - CampaignId, - CorrelationId, - MultiTargetUpdateId, - Namespace, - ResultCode, - ResultDescription -} -import com.advancedtelematic.libats.data.EcuIdentifier -import com.advancedtelematic.libats.data.EcuIdentifier.validatedEcuIdentifier -import com.advancedtelematic.libats.messaging_datatype.DataType.{ - DeviceId, - EcuInstallationReport, - InstallationResult -} -import com.advancedtelematic.libats.messaging_datatype.Messages.{ - DeviceUpdateCompleted, - EcuAndHardwareId, - EcuReplaced, - EcuReplacement, - EcuReplacementFailed -} +import com.advancedtelematic.libats.data.DataType.{CorrelationId, MultiTargetUpdateCorrelationId, Namespace, ResultCode, ResultDescription, TargetSpecCorrelationId} +import com.advancedtelematic.libats.data.RefinedUtils.RefineTry +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier, EcuInstallationReport, InstallationResult, ValidEcuIdentifier} +import com.advancedtelematic.libats.messaging_datatype.Messages.{DeviceUpdateCompleted, EcuAndHardwareId, EcuReplaced, EcuReplacement, EcuReplacementFailed} import org.scalacheck.Gen import scala.util.{Success, Try} @@ -32,13 +13,13 @@ object InstallationReportGenerators { import DeviceGenerators.* val genCorrelationId: Gen[CorrelationId] = - Gen.uuid.flatMap(uuid => Gen.oneOf(CampaignId(uuid), MultiTargetUpdateId(uuid))) + Gen.uuid.flatMap(uuid => Gen.oneOf(TargetSpecCorrelationId(uuid), MultiTargetUpdateCorrelationId(uuid))) val genEcuIdentifier: Gen[EcuIdentifier] = Gen .listOfN(64, Gen.alphaNumChar) .map(_.mkString("")) - .map(validatedEcuIdentifier.from(_).toOption.get) + .map(_.refineTry[ValidEcuIdentifier].get) private def genInstallationResult( resultCode: ResultCode, diff --git a/src/test/scala/com/advancedtelematic/director/deviceregistry/data/SystemInfoUpdatePublisherSpec.scala b/src/test/scala/com/advancedtelematic/director/deviceregistry/data/SystemInfoUpdatePublisherSpec.scala index 603097ae..e23d7652 100644 --- a/src/test/scala/com/advancedtelematic/director/deviceregistry/data/SystemInfoUpdatePublisherSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/deviceregistry/data/SystemInfoUpdatePublisherSpec.scala @@ -2,7 +2,7 @@ package com.advancedtelematic.director.deviceregistry.data import java.nio.file.Paths -import akka.dispatch.ExecutionContexts +import org.apache.pekko.dispatch.ExecutionContexts import com.advancedtelematic.director.deviceregistry.SystemInfoUpdatePublisher import io.circe.syntax._ import org.scalatest.concurrent.ScalaFutures diff --git a/src/test/scala/com/advancedtelematic/director/deviceregistry/device_monitoring/DeviceMonitoringResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/deviceregistry/device_monitoring/DeviceMonitoringResourceSpec.scala index c72ed08e..5d6edf14 100644 --- a/src/test/scala/com/advancedtelematic/director/deviceregistry/device_monitoring/DeviceMonitoringResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/deviceregistry/device_monitoring/DeviceMonitoringResourceSpec.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.deviceregistry.device_monitoring -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import cats.syntax.show.* import com.advancedtelematic.director.deviceregistry.data.DataType.ObservationPublishResult import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators @@ -13,7 +13,7 @@ import com.advancedtelematic.libats.messaging.test.MockMessageBus import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId.* import com.advancedtelematic.libats.messaging_datatype.MessageLike import com.advancedtelematic.libats.messaging_datatype.Messages.DeviceMetricsObservation -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import org.scalatest.EitherValues.* import org.scalatest.OptionValues.* import org.scalatest.time.{Seconds, Span} diff --git a/src/test/scala/com/advancedtelematic/director/http/AdminResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/AdminResourceSpec.scala index b25a2903..6705ba2f 100644 --- a/src/test/scala/com/advancedtelematic/director/http/AdminResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/AdminResourceSpec.scala @@ -1,42 +1,30 @@ package com.advancedtelematic.director.http +import com.advancedtelematic.libats.data.PaginationResult.* import java.time.Instant import java.time.temporal.ChronoUnit -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import cats.syntax.option.* import cats.syntax.show.* import com.advancedtelematic.director.data.ClientDataType -import com.advancedtelematic.director.data.AdminDataType.{ - EcuInfoResponse, - FindImageCount, - MultiTargetUpdate, - RegisterDevice -} +import com.advancedtelematic.director.data.AdminDataType.{EcuInfoResponse, FindImageCount, RegisterDevice, TargetUpdateSpec} import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.DataType.TargetSpecId import com.advancedtelematic.director.data.DbDataType.Ecu import com.advancedtelematic.director.data.GeneratorOps.* import com.advancedtelematic.director.data.Generators.* -import com.advancedtelematic.director.db.{ - DbDeviceRoleRepositorySupport, - RepoNamespaceRepositorySupport -} +import com.advancedtelematic.director.db.{DbDeviceRoleRepositorySupport, RepoNamespaceRepositorySupport} import com.advancedtelematic.director.http.AdminResources.RegisterDeviceResult import com.advancedtelematic.director.util.* import com.advancedtelematic.libats.codecs.CirceCodecs.* import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.data.{EcuIdentifier, PaginationResult} -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.data.PaginationResult +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import com.advancedtelematic.libtuf.data.ClientCodecs.* import com.advancedtelematic.libtuf.data.ClientDataType.{RootRole, TargetsRole} import com.advancedtelematic.libtuf.data.TufCodecs.* -import com.advancedtelematic.libtuf.data.TufDataType.{ - HardwareIdentifier, - SignedPayload, - TargetFilename, - TufKey, - TufKeyPair -} -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, SignedPayload, TargetFilename, TufKey, TufKeyPair} +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import org.scalactic.source.Position import org.scalatest.Assertion import io.circe.syntax.* @@ -118,15 +106,15 @@ trait AdminResources { status shouldBe StatusCodes.Created } - def createMtu(mtu: MultiTargetUpdate)(implicit ns: Namespace, pos: Position): RouteTestResult = + def createMtu(mtu: TargetUpdateSpec)(implicit ns: Namespace): RouteTestResult = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes - def createMtuOk()(implicit ns: Namespace, pos: Position): UpdateId = { + def createMtuOk()(implicit ns: Namespace, pos: Position): TargetSpecId = { val mtu = GenMultiTargetUpdateRequest.generate createMtu(mtu) ~> check { status shouldBe StatusCodes.Created - responseAs[UpdateId] + responseAs[TargetSpecId] } } @@ -338,7 +326,7 @@ class AdminResourceSpec Get(apiUri(s"admin/devices?primaryHardwareId=foo")).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK val page = responseAs[PaginationResult[DeviceId]] - page shouldBe PaginationResult(Seq.empty, 0, 0, 50) + page shouldBe PaginationResult(Seq.empty, 0, 0L.toOffset, 50L.toLimit) } } diff --git a/src/test/scala/com/advancedtelematic/director/http/AssignmentsResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/AssignmentsResourceSpec.scala index 28f42a7c..6fd23d72 100644 --- a/src/test/scala/com/advancedtelematic/director/http/AssignmentsResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/AssignmentsResourceSpec.scala @@ -3,33 +3,27 @@ package com.advancedtelematic.director.http import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.genDeviceT import com.advancedtelematic.director.http.deviceregistry.RegistryDeviceRequests import org.scalatest.LoneElement.* -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import cats.syntax.option.* import cats.syntax.show.* -import com.advancedtelematic.director.daemon.UpdateScheduler import com.advancedtelematic.director.data.AdminDataType.* import com.advancedtelematic.director.data.Codecs.* -import com.advancedtelematic.director.data.DataType.TargetItemCustom +import com.advancedtelematic.director.data.DataType.{TargetItemCustom, TargetSpecId} import com.advancedtelematic.director.data.GeneratorOps.* import com.advancedtelematic.director.data.Generators.* -import com.advancedtelematic.director.db.{ - DbDeviceRoleRepositorySupport, - RepoNamespaceRepositorySupport -} -import com.advancedtelematic.director.deviceregistry.daemon.DeviceUpdateStatus -import com.advancedtelematic.director.deviceregistry.data.DeviceStatus +import com.advancedtelematic.director.db.{DbDeviceRoleRepositorySupport, RepoNamespaceRepositorySupport, UpdatesDBIO} import com.advancedtelematic.director.http.DeviceAssignments.AssignmentCreateResult import com.advancedtelematic.director.util.* -import com.advancedtelematic.libats.data.DataType.{CorrelationId, MultiTargetUpdateId, Namespace} +import com.advancedtelematic.libats.data.DataType.{CorrelationId, MultiTargetUpdateCorrelationId, Namespace} import com.advancedtelematic.libats.data.ErrorRepresentation import com.advancedtelematic.libats.messaging.test.MockMessageBus -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} -import com.advancedtelematic.libats.messaging_datatype.Messages.{DeviceUpdateEvent, *} +import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.messaging_datatype.Messages.* import com.advancedtelematic.libtuf.data.ClientCodecs.* import com.advancedtelematic.libtuf.data.ClientDataType.TargetsRole import com.advancedtelematic.libtuf.data.TufCodecs.* import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, SignedPayload} -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import org.scalactic.source.Position import org.scalatest.Inspectors import org.scalatest.OptionValues.* @@ -55,14 +49,14 @@ trait AssignmentResources { val correlationId = correlationIdO.getOrElse(GenCorrelationId.generate) val targetUpdate = targetUpdateO.getOrElse(GenTargetUpdateRequest.generate) - val mtu = MultiTargetUpdate(Map(hwId -> targetUpdate)) + val mtu = TargetUpdateSpec(Map(hwId -> targetUpdate)) - val mtuId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { + val targetSpecId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { status shouldBe StatusCodes.Created - responseAs[UpdateId] + responseAs[TargetSpecId] } - val assignment = AssignUpdateRequest(correlationId, deviceIds, mtuId) + val assignment = AssignUpdateRequest(correlationId, deviceIds, targetSpecId) Post(apiUri("assignments"), assignment).namespaced ~> routes ~> check(checkV) @@ -131,11 +125,12 @@ class AssignmentsResourceSpec with ProvisionedDevicesRequests with DeviceManifestSpec with RegistryDeviceRequests + with UpdatesResources with Inspectors { override implicit val msgPub: MockMessageBus = new MockMessageBus() - val updateScheduler = new UpdateScheduler() + val updatesDBIO = new UpdatesDBIO() testWithRepo("Can create an assignment for existing devices") { implicit ns => val regDev = registerAdminDeviceOk() @@ -155,6 +150,7 @@ class AssignmentsResourceSpec } } + // TODO: Should be moved to AdminResourceSpec testWithRepo( "returns PrimaryIsNotListedForDevice when ecus to register do not include primary ecu" ) { implicit ns => @@ -175,16 +171,16 @@ class AssignmentsResourceSpec val regDev1 = registerAdminDeviceOk() val targetUpdate = GenTargetUpdateRequest.generate - val mtu = MultiTargetUpdate(Map(regDev0.primary.hardwareId -> targetUpdate)) + val mtu = TargetUpdateSpec(Map(regDev0.primary.hardwareId -> targetUpdate)) - val mtuId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { + val targetSpecId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { status shouldBe StatusCodes.Created - responseAs[UpdateId] + responseAs[TargetSpecId] } Get( apiUri( - s"assignments/devices?mtuId=${mtuId.show}&ids=${regDev0.deviceId.show},${regDev1.deviceId.show}" + s"assignments/devices?targetSpecId=${targetSpecId.show}&ids=${regDev0.deviceId.show},${regDev1.deviceId.show}" ) ).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK @@ -198,17 +194,17 @@ class AssignmentsResourceSpec val regDev1 = registerAdminDeviceOk() val targetUpdate = GenTargetUpdateRequest.generate - val mtu = MultiTargetUpdate(Map(regDev0.primary.hardwareId -> targetUpdate)) + val mtu = TargetUpdateSpec(Map(regDev0.primary.hardwareId -> targetUpdate)) - val mtuId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { + val targetSpecId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { status shouldBe StatusCodes.Created - responseAs[UpdateId] + responseAs[TargetSpecId] } val assignment = AssignUpdateRequest( - MultiTargetUpdateId(mtuId.uuid), + MultiTargetUpdateCorrelationId(targetSpecId.uuid), Seq(regDev0.deviceId, regDev1.deviceId), - mtuId, + targetSpecId, dryRun = Some(true) ) @@ -303,12 +299,8 @@ class AssignmentsResourceSpec val deviceT = genDeviceT.generate.copy(uuid = Some(regDev0.deviceId)) createDeviceInNamespaceOk(deviceT, ns) - val mtu = MultiTargetUpdate(Map(regDev0.primary.hardwareId -> GenTargetUpdateRequest.generate)) - val scheduledUpdate = createMtu(mtu) ~> check { - response.status shouldBe StatusCodes.Created - responseAs[UpdateId] - } - updateScheduler.create(ns, regDev0.deviceId, scheduledUpdate, Instant.now).futureValue + val mtu = TargetUpdateSpec(Map(regDev0.primary.hardwareId -> GenTargetUpdateRequest.generate)) + updatesDBIO.createFor(ns, regDev0.deviceId, mtu, Instant.now.some).futureValue val existingUpdate = GenTargetUpdate.generate putManifestOk( @@ -331,7 +323,7 @@ class AssignmentsResourceSpec val (deviceId, errors) = response.notAffected.loneElement val (_, error) = errors.loneElement deviceId shouldBe regDev0.deviceId - error.code shouldBe ErrorCodes.DeviceHasScheduledUpdate + error.code shouldBe ErrorCodes.DeviceHasActiveUpdate } val queue0 = getDeviceAssignmentOk(regDev0.deviceId) @@ -586,25 +578,55 @@ class AssignmentsResourceSpec } } - testWithRepo("publishes DeviceUpdateStatus when creating scheduled update") { implicit ns => + + testWithRepo("PATCH assignments cannot cancel an assignment that belongs to an update") { implicit ns => val regDev = registerAdminDeviceOk() - val deviceT = genDeviceT.generate.copy(uuid = Some(regDev.deviceId)) - createDeviceInNamespaceOk(deviceT, ns) + val hardwareId = regDev.primary.hardwareId + val targetUpdate = GenTargetUpdateRequest.generate + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetUpdate), + devices = Seq(regDev.deviceId) + ) - val mtu = MultiTargetUpdate(Map(regDev.primary.hardwareId -> GenTargetUpdateRequest.generate)) - val scheduledUpdate = createMtu(mtu) ~> check { - response.status shouldBe StatusCodes.Created - responseAs[UpdateId] + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(regDev.deviceId) } - updateScheduler.create(ns, regDev.deviceId, scheduledUpdate, Instant.now).futureValue + val queue = getDeviceAssignmentOk(regDev.deviceId) + queue shouldNot be(empty) - val msg = msgPub.findReceived[DeviceUpdateStatus] { (msg: DeviceUpdateStatus) => - msg.device == regDev.deviceId && - msg.status == DeviceStatus.UpdateScheduled + Patch(apiUri(s"assignments"), Seq(regDev.deviceId)).namespaced ~> routes ~> check { + status shouldBe StatusCodes.Conflict + val error = responseAs[ErrorRepresentation] + error.code shouldBe ErrorCodes.AssignmentBelongsToUpdate } + } - msg shouldBe defined + testWithRepo("PATCH on assignments/device-id cannot cancel an assignment that belongs to an update") { implicit ns => + val regDev = registerAdminDeviceOk() + val hardwareId = regDev.primary.hardwareId + val targetUpdate = GenTargetUpdateRequest.generate + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetUpdate), + devices = Seq(regDev.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(regDev.deviceId) + } + + val queue = getDeviceAssignmentOk(regDev.deviceId) + queue shouldNot be(empty) + + Patch(apiUri(s"assignments/${regDev.deviceId.show}")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.Conflict + val error = responseAs[ErrorRepresentation] + error.code shouldBe ErrorCodes.AssignmentBelongsToUpdate + } } } diff --git a/src/test/scala/com/advancedtelematic/director/http/AutoUpdateResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/AutoUpdateResourceSpec.scala index cf5eedc4..88984fd2 100644 --- a/src/test/scala/com/advancedtelematic/director/http/AutoUpdateResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/AutoUpdateResourceSpec.scala @@ -1,11 +1,11 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import cats.syntax.show.* import com.advancedtelematic.director.util.{DirectorSpec, RepositorySpec, ResourceSpec} import com.advancedtelematic.libtuf.data.ClientCodecs.* import com.advancedtelematic.libtuf.data.TufDataType.TargetName -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* class AutoUpdateResourceSpec extends DirectorSpec diff --git a/src/test/scala/com/advancedtelematic/director/http/DeviceResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/DeviceResourceSpec.scala index 4c5ca822..1f4d9f14 100644 --- a/src/test/scala/com/advancedtelematic/director/http/DeviceResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/DeviceResourceSpec.scala @@ -1,28 +1,15 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.util.ByteString import cats.syntax.option.* import cats.syntax.show.* import com.advancedtelematic.director.daemon.UpdateSchedulerDaemon import com.advancedtelematic.director.data.AdminDataType -import com.advancedtelematic.director.data.AdminDataType.{ - EcuInfoResponse, - MultiTargetUpdate, - QueueResponse, - RegisterDevice, - TargetUpdate -} +import com.advancedtelematic.director.data.AdminDataType.{EcuInfoResponse, QueueResponse, RegisterDevice, TargetUpdate, TargetUpdateSpec} import com.advancedtelematic.director.data.Codecs.* import com.advancedtelematic.director.data.DataType.* -import com.advancedtelematic.director.data.DeviceRequest.{ - DeviceManifest, - EcuManifest, - EcuManifestCustom, - InstallationReport, - InstallationReportEntity, - MissingInstallationReport, - OperationResult -} +import com.advancedtelematic.director.data.DeviceRequest.{DeviceManifest, EcuManifest, EcuManifestCustom, InstallationReport, InstallationReportEntity, MissingInstallationReport, OperationResult} import com.advancedtelematic.director.data.GeneratorOps.* import com.advancedtelematic.director.data.Generators.* import com.advancedtelematic.director.data.Messages.DeviceManifestReported @@ -38,37 +25,28 @@ import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.genDe import com.advancedtelematic.director.http.deviceregistry.RegistryDeviceRequests import com.advancedtelematic.director.manifest.ResultCodes import com.advancedtelematic.director.util.* -import com.advancedtelematic.libats.data.DataType.{ - CorrelationId, - HashMethod, - MultiTargetUpdateId, - Namespace, - ResultCode, - ResultDescription -} +import com.advancedtelematic.libats.data.DataType.{CorrelationId, HashMethod, MultiTargetUpdateCorrelationId, Namespace, ResultCode, ResultDescription, UpdateCorrelationId} import com.advancedtelematic.libats.data.ErrorRepresentation import com.advancedtelematic.libats.messaging.test.MockMessageBus import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, InstallationResult} import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId.* import com.advancedtelematic.libats.messaging_datatype.Messages.* import com.advancedtelematic.libtuf.data.ClientCodecs.* -import com.advancedtelematic.libtuf.data.ClientDataType.{ - RootRole, - SnapshotRole, - TargetsRole, - TimestampRole, - TufRole -} +import com.advancedtelematic.libtuf.data.ClientDataType.{RemoteSessionsPayload, RootRole, SnapshotRole, SshSessionProperties, TargetsRole, TimestampRole, TufRole} +import com.advancedtelematic.director.http.RemoteSessionRequest import com.advancedtelematic.libtuf.data.TufCodecs.* import com.advancedtelematic.libtuf.data.TufDataType.SignedPayload -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.Json +import io.circe.parser.parse +import com.advancedtelematic.libtuf.crypt.CanonicalJson._ import org.scalatest.Inspectors import org.scalatest.LoneElement.* import org.scalatest.OptionValues.* import io.circe.syntax.* import org.scalatest.EitherValues.* +import java.time.Instant import scala.concurrent.Future class DeviceResourceSpec @@ -83,7 +61,7 @@ class DeviceResourceSpec with RegistryDeviceRequests with ProvisionedDevicesRequests with AssignmentsRepositorySupport - with ScheduledUpdatesResources { + with UpdatesResources { override implicit val msgPub: MockMessageBus = new MockMessageBus() @@ -704,6 +682,91 @@ class DeviceResourceSpec fail(s"targets.json $firstTargets is not the same as $secondTargets") } + testWithRepo("returns canonicalized targets.json") { implicit ns => + val regDev = registerAdminDeviceOk() + val targetUpdate = GenTargetUpdateRequest.generate + createDeviceAssignmentOk(regDev.deviceId, regDev.primary.hardwareId, targetUpdate.some) + + val responseBody = + Get(apiUri(s"device/${regDev.deviceId.show}/targets.json")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[ByteString].utf8String + } + + val parsedJson = parse(responseBody).fold( + error => fail(s"Failed to parse targets.json response: ${error.getMessage}"), + identity + ) + + val canonicalForm = parsedJson.canonical + responseBody shouldBe canonicalForm + } + + testWithRepo("returns canonicalized root.json") { implicit ns => + val deviceId = registerDeviceOk() + + val responseBody = + Get(apiUri(s"device/${deviceId.show}/root.json")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[ByteString].utf8String + } + + val parsedJson = parse(responseBody).fold( + error => fail(s"Failed to parse root.json response: ${error.getMessage}"), + identity + ) + + val canonicalForm = parsedJson.canonical + responseBody shouldBe canonicalForm + } + + testWithRepo("returns canonicalized n.root.json") { implicit ns => + val deviceId = registerDeviceOk() + + val responseBody = + Get(apiUri(s"device/${deviceId.show}/1.root.json")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[ByteString].utf8String + } + + val parsedJson = parse(responseBody).fold( + error => fail(s"Failed to parse 1.root.json response: ${error.getMessage}"), + identity + ) + + val canonicalForm = parsedJson.canonical + responseBody shouldBe canonicalForm + } + + testWithRepo("returns canonicalized remote-sessions.json") { implicit ns => + val regDev = registerAdminDeviceOk() + val deviceId = regDev.deviceId + + // Create a remote session first so the endpoint has something to return + val session = RemoteSessionsPayload( + SshSessionProperties("someapiversion", Map.empty, Vector.empty, Vector.empty), + "someapiversion" + ) + val body = RemoteSessionRequest(session) + Post(apiUri(s"admin/remote-sessions"), body).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + } + + val responseBody = + Get(apiUri(s"device/${deviceId.show}/remote-sessions.json")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[ByteString].utf8String + } + + val parsedJson = parse(responseBody).fold( + error => fail(s"Failed to parse remote-sessions.json response: ${error.getMessage}"), + identity + ) + + val canonicalForm = parsedJson.canonical + responseBody shouldBe canonicalForm + } + testWithRepo("returns a refreshed version of targets if it expires") { implicit ns => val regDev = registerAdminDeviceOk() @@ -1612,7 +1675,7 @@ class DeviceResourceSpec } testWithRepo( - "installing assignments created via a scheduled update updates scheduled update status" + "installing assignments created via a scheduled update updates update status" ) { implicit ns => val dev = registerAdminDeviceWithSecondariesOk() val currentUpdate = GenTargetUpdateRequest.generate @@ -1629,11 +1692,11 @@ class DeviceResourceSpec ) putManifestOk(dev.deviceId, deviceManifest) - val mtu = MultiTargetUpdate( + val mtu = TargetUpdateSpec( Map(dev.secondaries.values.head.hardwareId -> newUpdate, dev.primary.hardwareId -> newUpdate) ) - createScheduledUpdateOk(dev.deviceId, mtu) + createUpdateOk(dev.deviceId, mtu) updateSchedulerIO.run().futureValue @@ -1647,8 +1710,8 @@ class DeviceResourceSpec putManifestOk(dev.deviceId, newManifest) - val scheduledUpdate = listScheduledUpdatesOK(dev.deviceId).values.loneElement - scheduledUpdate.status shouldBe ScheduledUpdate.Status.Completed + val scheduledUpdate = listUpdatesOK(dev.deviceId).values.loneElement + scheduledUpdate.status shouldBe Update.Status.Completed } testWithRepo( @@ -1661,9 +1724,9 @@ class DeviceResourceSpec val deviceManifest = buildPrimaryManifest(dev.primary, dev.primaryKey, currentUpdate.to) putManifestOk(dev.deviceId, deviceManifest) - val mtu = MultiTargetUpdate(Map(dev.primary.hardwareId -> newUpdate)) + val mtu = TargetUpdateSpec(Map(dev.primary.hardwareId -> newUpdate)) - createScheduledUpdateOk(dev.deviceId, mtu) + createUpdateOk(dev.deviceId, mtu, Instant.now().some) val scheduledUpdate = updateSchedulerIO.run().futureValue.loneElement @@ -1672,7 +1735,7 @@ class DeviceResourceSpec dev.primaryKey, currentUpdate.to, InstallationReport( - MultiTargetUpdateId(scheduledUpdate.updateId.uuid), + scheduledUpdate.id.toCorrelationId, InstallationResult( success = false, code = ResultCode("download_failed"), @@ -1685,8 +1748,8 @@ class DeviceResourceSpec putManifestOk(dev.deviceId, newManifest) - val apiScheduledUpdate = listScheduledUpdatesOK(dev.deviceId).values.loneElement - apiScheduledUpdate.status shouldBe ScheduledUpdate.Status.Completed + val apiScheduledUpdate = listUpdatesOK(dev.deviceId).values.loneElement + apiScheduledUpdate.status shouldBe Update.Status.Completed } testWithRepo("partially installing a scheduled update sets status to PartiallyCompleted") { @@ -1707,14 +1770,14 @@ class DeviceResourceSpec ) putManifestOk(dev.deviceId, deviceManifest) - val mtu = MultiTargetUpdate( + val mtu = TargetUpdateSpec( Map( dev.secondaries.values.head.hardwareId -> newUpdatePrimary, dev.primary.hardwareId -> newUpdateSecondary ) ) - createScheduledUpdateOk(dev.deviceId, mtu) + createUpdateOk(dev.deviceId, mtu) updateSchedulerIO.run().futureValue @@ -1728,8 +1791,8 @@ class DeviceResourceSpec putManifestOk(dev.deviceId, newManifest) - val scheduledUpdate = listScheduledUpdatesOK(dev.deviceId).values.loneElement - scheduledUpdate.status shouldBe ScheduledUpdate.Status.PartiallyCompleted + val scheduledUpdate = listUpdatesOK(dev.deviceId).values.loneElement + scheduledUpdate.status shouldBe Update.Status.PartiallyCompleted val newManifest2 = buildSecondaryManifest( dev.primary.ecuSerial, @@ -1741,8 +1804,8 @@ class DeviceResourceSpec putManifestOk(dev.deviceId, newManifest2) - val scheduledUpdate2 = listScheduledUpdatesOK(dev.deviceId).values.loneElement - scheduledUpdate2.status shouldBe ScheduledUpdate.Status.Completed + val scheduledUpdate2 = listUpdatesOK(dev.deviceId).values.loneElement + scheduledUpdate2.status shouldBe Update.Status.Completed } testWithRepo( @@ -1758,9 +1821,9 @@ class DeviceResourceSpec val previousTargets = getTargetsOk(dev.deviceId).signed previousTargets.targets shouldBe empty - val mtu = MultiTargetUpdate(Map(dev.primary.hardwareId -> newUpdate)) + val mtu = TargetUpdateSpec(Map(dev.primary.hardwareId -> newUpdate)) - createScheduledUpdateOk(dev.deviceId, mtu) + createUpdateOk(dev.deviceId, mtu) updateSchedulerIO.run().futureValue @@ -1780,11 +1843,11 @@ class DeviceResourceSpec val previousTargets = getTargetsOk(dev.deviceId).signed previousTargets.targets shouldBe empty - val mtu = MultiTargetUpdate(Map(dev.primary.hardwareId -> newUpdate)) + val mtu = TargetUpdateSpec(Map(dev.primary.hardwareId -> newUpdate)) - createScheduledUpdateOk(dev.deviceId, mtu) + createUpdateOk(dev.deviceId, mtu) - val mtuId = listScheduledUpdatesOK(dev.deviceId).values.loneElement.updateId + val updateId = listUpdatesOK(dev.deviceId).values.loneElement.updateId // TODO: runs full daemon instead of IO val daemon = new UpdateSchedulerDaemon() @@ -1801,7 +1864,7 @@ class DeviceResourceSpec .value assignedMsg.deviceUuid shouldBe dev.deviceId - assignedMsg.correlationId shouldBe MultiTargetUpdateId(mtuId.uuid) + assignedMsg.correlationId shouldBe updateId.toCorrelationId } testWithRepo( @@ -1822,16 +1885,16 @@ class DeviceResourceSpec ) putManifestOk(dev.deviceId, deviceManifest) - val mtu = MultiTargetUpdate( + val mtu = TargetUpdateSpec( Map(dev.secondaries.values.head.hardwareId -> newUpdate, dev.primary.hardwareId -> newUpdate) ) - val id = createScheduledUpdateOk(dev.deviceId, mtu) + val id = createUpdateOk(dev.deviceId, mtu) - cancelScheduledUpdateOK(dev.deviceId, id) + cancelUpdateOK(dev.deviceId, id) - val scheduledUpdate0 = listScheduledUpdatesOK(dev.deviceId).values.loneElement - scheduledUpdate0.status shouldBe ScheduledUpdate.Status.Cancelled + val scheduledUpdate0 = listUpdatesOK(dev.deviceId).values.loneElement + scheduledUpdate0.status shouldBe Update.Status.Cancelled val newManifest = buildSecondaryManifest( dev.primary.ecuSerial, @@ -1843,8 +1906,8 @@ class DeviceResourceSpec putManifestOk(dev.deviceId, newManifest) - val scheduledUpdate = listScheduledUpdatesOK(dev.deviceId).values.loneElement - scheduledUpdate.status shouldBe ScheduledUpdate.Status.Cancelled + val scheduledUpdate = listUpdatesOK(dev.deviceId).values.loneElement + scheduledUpdate.status shouldBe Update.Status.Cancelled } } diff --git a/src/test/scala/com/advancedtelematic/director/http/LegacyApiResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/LegacyApiResourceSpec.scala index 513d18ab..783fb41c 100644 --- a/src/test/scala/com/advancedtelematic/director/http/LegacyApiResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/LegacyApiResourceSpec.scala @@ -1,15 +1,16 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.StatusCodes -import com.advancedtelematic.director.data.AdminDataType.{MultiTargetUpdate, QueueResponse} +import org.apache.pekko.http.scaladsl.model.StatusCodes +import com.advancedtelematic.director.data.AdminDataType.{TargetUpdateSpec, QueueResponse} import com.advancedtelematic.director.util.{DirectorSpec, RepositorySpec, ResourceSpec} -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.director.data.Generators.* -import com.advancedtelematic.libats.data.DataType.MultiTargetUpdateId +import com.advancedtelematic.libats.data.DataType.MultiTargetUpdateCorrelationId import com.advancedtelematic.director.data.GeneratorOps.* import com.advancedtelematic.director.data.Codecs.* -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import cats.syntax.show.* +import com.advancedtelematic.director.data.DataType.TargetSpecId import com.advancedtelematic.libats.data.PaginationResult import org.scalatest.OptionValues.* import com.advancedtelematic.libats.messaging_datatype.Messages.* @@ -27,15 +28,15 @@ class LegacyApiResourceSpec val regDev = registerAdminDeviceWithSecondariesOk() val targetUpdate = GenTargetUpdateRequest.generate - val mtu = MultiTargetUpdate(Map(regDev.primary.hardwareId -> targetUpdate)) + val mtu = TargetUpdateSpec(Map(regDev.primary.hardwareId -> targetUpdate)) - val mtuId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { + val targetSpecId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { status shouldBe StatusCodes.Created - responseAs[UpdateId] + responseAs[TargetSpecId] } Put( - apiUri(s"admin/devices/${regDev.deviceId.show}/multi_target_update/${mtuId.show}") + apiUri(s"admin/devices/${regDev.deviceId.show}/multi_target_update/${targetSpecId.show}") ).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK } @@ -46,7 +47,7 @@ class LegacyApiResourceSpec responseAs[List[QueueResponse]] } - queue.head.correlationId shouldBe MultiTargetUpdateId(mtuId.uuid) + queue.head.correlationId shouldBe MultiTargetUpdateCorrelationId(targetSpecId.uuid) queue.head.targets .get(regDev.primary.ecuSerial) .value @@ -91,8 +92,8 @@ class LegacyApiResourceSpec status shouldBe StatusCodes.OK val devices = responseAs[PaginationResult[DeviceId]] devices.total shouldBe 1 - devices.offset shouldBe 0 - devices.limit shouldBe 50 + devices.offset.toLong shouldBe 0L + devices.limit.toLong shouldBe 50L devices.values.loneElement shouldBe regDev.deviceId } } diff --git a/src/test/scala/com/advancedtelematic/director/http/OfflineUpdatesRoutesSpec.scala b/src/test/scala/com/advancedtelematic/director/http/OfflineUpdatesRoutesSpec.scala index 355ea360..ccda7d74 100644 --- a/src/test/scala/com/advancedtelematic/director/http/OfflineUpdatesRoutesSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/OfflineUpdatesRoutesSpec.scala @@ -2,19 +2,20 @@ package com.advancedtelematic.director.http import io.circe.syntax.* import com.advancedtelematic.libtuf.crypt.CanonicalJson.* -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import com.advancedtelematic.director.data.Generators import com.advancedtelematic.director.db.RepoNamespaceRepositorySupport import com.advancedtelematic.director.util.{DirectorSpec, RepositorySpec, ResourceSpec} import com.advancedtelematic.libtuf.data.ClientCodecs.* import com.advancedtelematic.libtuf.data.TufCodecs.* -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import com.advancedtelematic.director.data.GeneratorOps.* import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.data.ErrorRepresentation import com.advancedtelematic.libtuf.data.ClientDataType.{ OfflineSnapshotRole, OfflineUpdatesRole, + RootRole, TufRole, ValidMetaPath } @@ -298,6 +299,12 @@ class OfflineUpdatesRoutesSpec resp.signed.version shouldBe 2 resp.signed.expires shouldBe expiresAt } + + Get(apiUri("admin/repo/root.json")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val resp = responseAs[SignedPayload[RootRole]] + resp.signed.expires shouldBe expiresAt.plus(180, ChronoUnit.DAYS) + } } testWithRepo("returns 4xx when user has too many offline roles") { implicit ns => diff --git a/src/test/scala/com/advancedtelematic/director/http/ProvisionedDevicesRequests.scala b/src/test/scala/com/advancedtelematic/director/http/ProvisionedDevicesRequests.scala index 248ec97a..75bea3a8 100644 --- a/src/test/scala/com/advancedtelematic/director/http/ProvisionedDevicesRequests.scala +++ b/src/test/scala/com/advancedtelematic/director/http/ProvisionedDevicesRequests.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import com.advancedtelematic.director.data.AdminDataType.RegisterDevice import com.advancedtelematic.director.data.DeviceRequest.DeviceManifest import com.advancedtelematic.director.data.Generators.GenRegisterEcu @@ -14,7 +14,7 @@ import org.scalactic.source.Position import com.advancedtelematic.director.data.GeneratorOps.* import cats.syntax.show.* import cats.syntax.option.* -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import com.advancedtelematic.director.data.Codecs.* import com.advancedtelematic.libtuf.data.TufCodecs.* diff --git a/src/test/scala/com/advancedtelematic/director/http/RemoteSessionsRoutesSpec.scala b/src/test/scala/com/advancedtelematic/director/http/RemoteSessionsRoutesSpec.scala index e9ba432f..d1cc1b3c 100644 --- a/src/test/scala/com/advancedtelematic/director/http/RemoteSessionsRoutesSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/RemoteSessionsRoutesSpec.scala @@ -1,16 +1,19 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.StatusCodes +import eu.timepit.refined.auto.* +import org.apache.pekko.http.scaladsl.model.StatusCodes import cats.syntax.show.* import com.advancedtelematic.director.data.Codecs.* import com.advancedtelematic.director.data.Generators import com.advancedtelematic.director.db.RepoNamespaceRepositorySupport import com.advancedtelematic.director.util.{DirectorSpec, RepositorySpec, ResourceSpec} -import com.advancedtelematic.libats.data.ErrorRepresentation import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libtuf.crypt.TufCrypto import com.advancedtelematic.libtuf.data.ClientCodecs.* +import com.advancedtelematic.libtuf.data.ClientDataType.CommandName.{Reboot, RestartService} import com.advancedtelematic.libtuf.data.ClientDataType.{ + CommandParameters, + RemoteCommandsPayload, RemoteSessionsPayload, RemoteSessionsRole, RootRole, @@ -18,7 +21,7 @@ import com.advancedtelematic.libtuf.data.ClientDataType.{ } import com.advancedtelematic.libtuf.data.TufCodecs.* import com.advancedtelematic.libtuf.data.TufDataType.{RoleType, SignedPayload} -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.syntax.* import java.time.Instant @@ -35,47 +38,49 @@ class RemoteSessionsRoutesSpec with Generators with ProvisionedDevicesRequests { - testWithRepo("can set a remote session for a device") { implicit ns => + testWithRepo("can set remote commands for a device") { implicit ns => val deviceId = DeviceId.generate() - val session = RemoteSessionsPayload( - SshSessionProperties("someapiversion", Map.empty, Vector.empty, Vector.empty), - "someapiversion" - ) - - val body = RemoteSessionRequest(session, 0) + val body = + RemoteCommandRequest( + RemoteCommandsPayload( + Map(RestartService -> CommandParameters(List("aktualizr"))), + "v1alpha" + ) + ) - Post(apiUri(s"admin/remote-sessions"), body).namespaced ~> routes ~> check { - status shouldBe StatusCodes.OK - val signedRole = responseAs[SignedPayload[RemoteSessionsRole]].signed - signedRole.remote_sessions shouldBe session + Post(apiUri(s"admin/remote-commands"), body).namespaced ~> routes ~> check { + status shouldBe StatusCodes.Accepted } - Get( - apiUri(s"device/${deviceId.show}/remote-sessions.json"), - body - ).namespaced ~> routes ~> check { + Get(apiUri(s"device/${deviceId.show}/remote-sessions.json")).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK val signedRole = responseAs[SignedPayload[RemoteSessionsRole]].signed - signedRole.remote_sessions shouldBe session + val payload = signedRole.remote_commands.value + + payload shouldBe body.remoteCommands } - Get(apiUri(s"admin/remote-sessions.json"), body).namespaced ~> routes ~> check { + Get(apiUri(s"admin/remote-sessions.json")).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK val signedRole = responseAs[SignedPayload[RemoteSessionsRole]].signed - signedRole.remote_sessions shouldBe session + val payload = signedRole.remote_commands.value + + payload shouldBe body.remoteCommands } } - testWithRepo("updating with wrong previousVersion returns conflict") { implicit ns => + testWithRepo("can set a remote session for a device") { implicit ns => + val deviceId = DeviceId.generate() + val session = RemoteSessionsPayload( SshSessionProperties("someapiversion", Map.empty, Vector.empty, Vector.empty), "someapiversion" ) - val body = RemoteSessionRequest(session, 0) + val body = RemoteSessionRequest(session) Post(apiUri(s"admin/remote-sessions"), body).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK @@ -83,13 +88,21 @@ class RemoteSessionsRoutesSpec signedRole.remote_sessions shouldBe session } - val body2 = RemoteSessionRequest(session, 3) + Get( + apiUri(s"device/${deviceId.show}/remote-sessions.json"), + body + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + + val signedRole = responseAs[SignedPayload[RemoteSessionsRole]].signed + signedRole.remote_sessions shouldBe session + } - Post(apiUri(s"admin/remote-sessions"), body2).namespaced ~> routes ~> check { - status shouldBe StatusCodes.Conflict + Get(apiUri(s"admin/remote-sessions.json"), body).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK - val error = responseAs[ErrorRepresentation] - error.code.code shouldBe "invalid_version_bump" + val signedRole = responseAs[SignedPayload[RemoteSessionsRole]].signed + signedRole.remote_sessions shouldBe session } } @@ -103,7 +116,7 @@ class RemoteSessionsRoutesSpec Post( apiUri(s"admin/remote-sessions"), - RemoteSessionRequest(beforeSession, 0) + RemoteSessionRequest(beforeSession) ).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK } @@ -126,7 +139,7 @@ class RemoteSessionsRoutesSpec "someapiversion" ) val remoteSessionsRole = - RemoteSessionsRole(session, Instant.now().plus(365, ChronoUnit.DAYS), 2) + RemoteSessionsRole(session, None, Instant.now().plus(365, ChronoUnit.DAYS), 2) val signature = TufCrypto.signPayload(keyPair.privkey, remoteSessionsRole.asJson).toClient(keyPair.pubkey.id) val signedPayload = @@ -144,4 +157,63 @@ class RemoteSessionsRoutesSpec } } + testWithRepo("accepts an offline signed remote commands payload") { implicit ns => + val deviceId = DeviceId.generate() + + val body = + RemoteCommandRequest( + RemoteCommandsPayload( + Map(RestartService -> CommandParameters(List("aktualizr"))), + "v1alpha" + ) + ) + + Post(apiUri(s"admin/remote-commands"), body).namespaced ~> routes ~> check { + status shouldBe StatusCodes.Accepted + } + + val keyId = Get(apiUri(s"device/${deviceId.show}/root.json")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val rootRole = responseAs[SignedPayload[RootRole]].signed + rootRole.roles.get(RoleType.REMOTE_SESSIONS).value.keyids.loneElement + } + + val keyPair = keyserverClient.fetchKeypairByKeyId(keyId).value + + val currentRole = Get(apiUri(s"admin/remote-sessions.json")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[SignedPayload[RemoteSessionsRole]].signed + } + + val payload02 = RemoteCommandsPayload( + Map(Reboot -> CommandParameters(List.empty)), + "v1alpha" + ) + + val remoteSessionsRole = RemoteSessionsRole( + currentRole.remote_sessions, + Some(payload02), + Instant.now().plus(365, ChronoUnit.DAYS), + currentRole.version + 1 + ) + + val signature = + TufCrypto.signPayload(keyPair.privkey, remoteSessionsRole.asJson).toClient(keyPair.pubkey.id) + + val signedPayload = + SignedPayload(List(signature), remoteSessionsRole, remoteSessionsRole.asJson) + + Post(apiUri(s"admin/remote-sessions.json"), signedPayload).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + } + + Get(apiUri(s"device/${deviceId.show}/remote-sessions.json")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val signedPayload = responseAs[SignedPayload[RemoteSessionsRole]].signed + signedPayload.version shouldBe currentRole.version + 1 + + signedPayload.remote_commands.value.allowed_commands shouldBe payload02.allowed_commands + } + } + } diff --git a/src/test/scala/com/advancedtelematic/director/http/ScheduledUpdatesSpec.scala b/src/test/scala/com/advancedtelematic/director/http/ScheduledUpdatesSpec.scala deleted file mode 100644 index 1853bc85..00000000 --- a/src/test/scala/com/advancedtelematic/director/http/ScheduledUpdatesSpec.scala +++ /dev/null @@ -1,178 +0,0 @@ -package com.advancedtelematic.director.http - -import akka.http.scaladsl.model.StatusCodes -import cats.syntax.show.* -import com.advancedtelematic.director.data.AdminDataType.MultiTargetUpdate -import com.advancedtelematic.director.data.ClientDataType.CreateScheduledUpdateRequest -import com.advancedtelematic.director.data.Codecs.* -import com.advancedtelematic.director.data.DataType.ScheduledUpdateId.* -import com.advancedtelematic.director.data.DataType.{ScheduledUpdate, ScheduledUpdateId} -import com.advancedtelematic.director.data.GeneratorOps.* -import com.advancedtelematic.director.data.Generators.GenTargetUpdateRequest -import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.genDeviceT -import com.advancedtelematic.director.http.deviceregistry.RegistryDeviceRequests -import com.advancedtelematic.director.util.{ - DirectorSpec, - NamespacedTests, - RepositorySpec, - ResourceSpec -} -import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.data.{ErrorRepresentation, PaginationResult} -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} -import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* -import org.scalactic.source.Position -import org.scalatest.LoneElement.* -import org.scalatest.OptionValues.* - -import java.time.Instant - -trait ScheduledUpdatesResources { - self: DirectorSpec & ResourceSpec & NamespacedTests & RegistryDeviceRequests => - - def createScheduledUpdateOk(deviceId: DeviceId, hardwareId: HardwareIdentifier)( - implicit ns: Namespace, - pos: Position): ScheduledUpdateId = { - val mtu = MultiTargetUpdate(Map(hardwareId -> GenTargetUpdateRequest.generate)) - createScheduledUpdateOk(deviceId, mtu) - } - - def createScheduledUpdateOk(deviceId: DeviceId, mtu: MultiTargetUpdate)( - implicit ns: Namespace, - pos: Position): ScheduledUpdateId = { - - val deviceT = genDeviceT.generate.copy(uuid = Some(deviceId)) - - createDeviceInNamespaceOk(deviceT, ns) - - val mtuId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { - status shouldBe StatusCodes.Created - responseAs[UpdateId] - } - - val req = - CreateScheduledUpdateRequest(device = deviceId, updateId = mtuId, scheduledAt = Instant.now()) - - Post( - apiUri(s"admin/devices/${deviceId.show}/scheduled-updates"), - req - ).namespaced ~> routes ~> check { - status shouldBe StatusCodes.Created - responseAs[ScheduledUpdateId] - } - } - - def listScheduledUpdatesOK( - deviceId: DeviceId)(implicit ns: Namespace, pos: Position): PaginationResult[ScheduledUpdate] = - Get(apiUri(s"admin/devices/${deviceId.show}/scheduled-updates")).namespaced ~> routes ~> check { - status shouldBe StatusCodes.OK - responseAs[PaginationResult[ScheduledUpdate]] - } - - def cancelScheduledUpdateOK(deviceId: DeviceId, - id: ScheduledUpdateId)(implicit ns: Namespace, pos: Position): Unit = - Delete( - apiUri(s"admin/devices/${deviceId.show}/scheduled-updates/${id.show}") - ).namespaced ~> routes ~> check { - status shouldBe StatusCodes.OK - } - -} - -class ScheduledUpdatesSpec - extends DirectorSpec - with ResourceSpec - with AdminResources - with RepositorySpec - with ProvisionedDevicesRequests - with RegistryDeviceRequests - with ScheduledUpdatesResources { - - testWithRepo("creates and lists a scheduled update") { implicit ns => - val regDev = registerAdminDeviceOk() - - val id = createScheduledUpdateOk(regDev.deviceId, regDev.primary.hardwareId) - - val existing = listScheduledUpdatesOK(regDev.deviceId).values.loneElement - - existing.deviceId shouldBe regDev.deviceId - existing.id shouldBe id - existing.status shouldBe ScheduledUpdate.Status.Scheduled - } - - testWithRepo("returns error if device already has a scheduled update") { implicit ns => - val regDev = registerAdminDeviceOk() - - val deviceT = genDeviceT.generate.copy(uuid = Some(regDev.deviceId)) - createDeviceInNamespaceOk(deviceT, ns) - - val mtu = MultiTargetUpdate(Map(regDev.primary.hardwareId -> GenTargetUpdateRequest.generate)) - - val mtuId = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { - status shouldBe StatusCodes.Created - responseAs[UpdateId] - } - - val req = CreateScheduledUpdateRequest( - device = regDev.deviceId, - updateId = mtuId, - scheduledAt = Instant.now() - ) - - Post( - apiUri(s"admin/devices/${regDev.deviceId.show}/scheduled-updates"), - req - ).namespaced ~> routes ~> check { - status shouldBe StatusCodes.Created - } - - Post( - apiUri(s"admin/devices/${regDev.deviceId.show}/scheduled-updates"), - req - ).namespaced ~> routes ~> check { - status shouldBe StatusCodes.BadRequest - val error = responseAs[ErrorRepresentation] - error.code shouldBe ErrorCodes.UpdateScheduleError - - val causeCode = error.cause.value.hcursor.downN(0).keys.flatMap(_.headOption) - causeCode should contain("scheduled_update_exists") - } - } - - testWithRepo("deletes scheduled update") { implicit ns => - val regDev = registerAdminDeviceOk() - - val id = createScheduledUpdateOk(regDev.deviceId, regDev.primary.hardwareId) - - Delete( - apiUri(s"admin/devices/${regDev.deviceId.show}/scheduled-updates/${id.show}") - ).namespaced ~> routes ~> check { - status shouldBe StatusCodes.OK - } - - val existing = listScheduledUpdatesOK(regDev.deviceId) - - existing.values.loneElement.status shouldBe ScheduledUpdate.Status.Cancelled - } - - testWithRepo("returns an error if device does not have compatible ECUs cannot be updated") { - implicit ns => - val regDev = registerAdminDeviceOk() - val mtuId = createMtuOk() - val req = CreateScheduledUpdateRequest( - device = regDev.deviceId, - updateId = mtuId, - scheduledAt = Instant.now() - ) - - Post( - apiUri(s"admin/devices/${regDev.deviceId.show}/scheduled-updates"), - req - ).namespaced ~> routes ~> check { - status shouldBe StatusCodes.BadRequest - responseAs[ErrorRepresentation].code shouldBe ErrorCodes.UpdateScheduleError - } - } - -} diff --git a/src/test/scala/com/advancedtelematic/director/http/MultiTargetUpdatesResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/TargetUpdateSpecsResourceSpec.scala similarity index 84% rename from src/test/scala/com/advancedtelematic/director/http/MultiTargetUpdatesResourceSpec.scala rename to src/test/scala/com/advancedtelematic/director/http/TargetUpdateSpecsResourceSpec.scala index 51e50b9e..a5d2bce8 100644 --- a/src/test/scala/com/advancedtelematic/director/http/MultiTargetUpdatesResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/TargetUpdateSpecsResourceSpec.scala @@ -1,22 +1,22 @@ package com.advancedtelematic.director.http -import akka.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes import cats.syntax.show.* -import com.advancedtelematic.director.data.AdminDataType.{MultiTargetUpdate, TargetUpdateRequest} +import com.advancedtelematic.director.data.AdminDataType.{TargetUpdateSpec, TargetUpdateRequest} import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.DataType.TargetSpecId import com.advancedtelematic.director.data.GeneratorOps.GenSample import com.advancedtelematic.director.data.Generators import com.advancedtelematic.director.util.{DefaultPatience, DirectorSpec, ResourceSpec} import com.advancedtelematic.libats.codecs.CirceCodecs.* import com.advancedtelematic.libats.data.ErrorCodes.MissingEntity import com.advancedtelematic.libats.data.ErrorRepresentation -import com.advancedtelematic.libats.messaging_datatype.DataType.UpdateId import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.Json import org.scalatest.OptionValues.* -class MultiTargetUpdatesResourceSpec +class TargetUpdateSpecsResourceSpec extends DirectorSpec with Generators with DefaultPatience @@ -24,7 +24,7 @@ class MultiTargetUpdatesResourceSpec with AdminResources { test("fetching non-existent target info returns 404") { - val id = UpdateId.generate() + val id = TargetSpecId.generate() Get(apiUri(s"multi_target_updates/${id.uuid.toString}")) ~> routes ~> check { status shouldBe StatusCodes.NotFound @@ -50,7 +50,7 @@ class MultiTargetUpdatesResourceSpec Get(apiUri(s"multi_target_updates/${mtu.show}")).namespaced ~> routes ~> check { status shouldBe StatusCodes.OK - responseAs[MultiTargetUpdate] + responseAs[TargetUpdateSpec] } } @@ -59,7 +59,7 @@ class MultiTargetUpdatesResourceSpec } testWithNamespace("does not accept empty mtu") { implicit ns => - val mtu = MultiTargetUpdate(Map.empty) + val mtu = TargetUpdateSpec(Map.empty) Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { status shouldBe StatusCodes.BadRequest @@ -72,11 +72,11 @@ class MultiTargetUpdatesResourceSpec val toUpdate = GenTargetUpdate.generate.copy(userDefinedCustom = Some(userDefinedCustom)) val toUpdateReq = TargetUpdateRequest(None, toUpdate) val hwId = GenHardwareIdentifier.generate - val mtu = MultiTargetUpdate(Map(hwId -> toUpdateReq)) + val mtu = TargetUpdateSpec(Map(hwId -> toUpdateReq)) val id = Post(apiUri("multi_target_updates"), mtu).namespaced ~> routes ~> check { status shouldBe StatusCodes.Created - responseAs[UpdateId] + responseAs[TargetSpecId] } Get(apiUri(s"multi_target_updates/${id.show}")).namespaced ~> routes ~> check { diff --git a/src/test/scala/com/advancedtelematic/director/http/UpdateResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/UpdateResourceSpec.scala new file mode 100644 index 00000000..6b656d88 --- /dev/null +++ b/src/test/scala/com/advancedtelematic/director/http/UpdateResourceSpec.scala @@ -0,0 +1,822 @@ +package com.advancedtelematic.director.http + +import org.apache.pekko.http.scaladsl.model.StatusCodes +import cats.syntax.option.* +import cats.syntax.show.* +import com.advancedtelematic.director.data.AdminDataType.{TargetUpdateRequest, TargetUpdateSpec} +import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.DataType.{TargetItemCustom, Update, UpdateId} +import com.advancedtelematic.director.data.GeneratorOps.* +import com.advancedtelematic.director.data.Generators.* +import com.advancedtelematic.director.db.{ + DbDeviceRoleRepositorySupport, + HardwareUpdateRepositorySupport, + RepoNamespaceRepositorySupport +} +import com.advancedtelematic.director.deviceregistry.daemon.{ + DeviceEventListener, + DeviceUpdateEventListener +} +import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.genDeviceT +import com.advancedtelematic.director.http.deviceregistry.RegistryDeviceRequests +import com.advancedtelematic.director.util.{ + DeviceManifestSpec, + DirectorSpec, + RepositorySpec, + ResourceSpec +} +import com.advancedtelematic.libats.data.ErrorRepresentation +import com.advancedtelematic.libats.messaging.test.MockMessageBus +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, Event, EventType} +import com.advancedtelematic.libats.messaging_datatype.Messages.{ + DeviceEventMessage, + DeviceUpdateAssigned, + DeviceUpdateCanceled, + DeviceUpdateCompleted, + DeviceUpdateEvent +} +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* +import io.circe.Json +import io.circe.syntax.* +import org.scalatest.EitherValues.* +import org.scalatest.LoneElement.* +import org.scalatest.OptionValues.* +import org.scalatest.concurrent.Eventually + +import java.time.Instant +import java.time.temporal.ChronoUnit + +class UpdateResourceSpec + extends DirectorSpec + with ResourceSpec + with RepoNamespaceRepositorySupport + with DbDeviceRoleRepositorySupport + with HardwareUpdateRepositorySupport + with AdminResources + with ProvisionedDevicesRequests + with DeviceManifestSpec + with AssignmentResources + with RepositorySpec + with RegistryDeviceRequests + with UpdatesResources + with Eventually { + + override implicit val msgPub: MockMessageBus = new MockMessageBus() + + testWithRepo("fetching updates for non-existent device returns empty") { implicit ns => + val deviceId = DeviceId.generate() + + val updates = listUpdatesOK(deviceId) + updates.values shouldBe empty + } + + testWithRepo("can create an update for a device") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetRequest), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(device.deviceId) + result.notAffected shouldBe empty + } + + val updates = listUpdatesOK(device.deviceId) + val update = updates.values.loneElement + + update.packages should contain(device.primary.hardwareId -> targetUpdate.target) + update.status shouldBe Update.Status.Assigned + } + + testWithRepo("device can see the update in targets.json") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetRequest), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + } + + val targets = getTargetsOk(device.deviceId) + targets.signed.targets should not be empty + + val targetItemCustom = targets.signed.targets.headOption.value._2.customParsed[TargetItemCustom] + targetItemCustom.get.ecuIdentifiers.keys.head shouldBe device.ecus.keys.head + } + + testWithRepo("update moves to Seen when devices sees the update in targets.json") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetRequest), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + } + + val targets = getTargetsOk(device.deviceId) + targets.signed.targets should not be empty + + val updates = listUpdatesOK(device.deviceId) + val update = updates.values.loneElement + + update.status shouldBe Update.Status.Seen + } + + + testWithRepo("update is marked as completed when device reports successful installation") { + implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdateRequest.generate + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetUpdate), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(device.deviceId) + } + + val updates = listUpdatesOK(device.deviceId) + val updateId = updates.values.loneElement.updateId + + getTargetsOk(device.deviceId) + + val manifest = buildPrimaryManifest(device.primary, device.primaryKey, targetUpdate.to) + putManifestOk(device.deviceId, manifest) + + val completedUpdates = listUpdatesOK(device.deviceId) + completedUpdates.values should have size 1 + val completedUpdate = completedUpdates.values.head + completedUpdate.status shouldBe Update.Status.Completed + + val msg = msgPub + .findReceived[DeviceUpdateEvent] { (msg: DeviceUpdateEvent) => + msg.deviceUuid == device.deviceId && msg.isInstanceOf[DeviceUpdateCompleted] + } + .map(_.asInstanceOf[DeviceUpdateCompleted]) + .value + msg.correlationId shouldBe updateId.toCorrelationId + } + + testWithRepo("can get update details by ID") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdateRequest.generate + val createRequest = + CreateUpdateRequest(targets = Map(hardwareId -> targetUpdate), devices = Seq(device.deviceId)) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(device.deviceId) + } + + val updates = listUpdatesOK(device.deviceId) + val updateId = updates.values.loneElement.updateId + + val updateDetail = getUpdateDetailOK(device.deviceId, updateId) + updateDetail.updateId shouldBe updateId + updateDetail.status shouldBe Update.Status.Assigned + updateDetail.packages should contain(device.primary.hardwareId -> targetUpdate.to.target) + updateDetail.ecuResults shouldBe empty + } + + testWithRepo("update details include results when update is completed") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdateRequest.generate + val createRequest = + CreateUpdateRequest(targets = Map(hardwareId -> targetUpdate), devices = Seq(device.deviceId)) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + } + + val updates = listUpdatesOK(device.deviceId) + val updateId = updates.values.loneElement.updateId + + getTargetsOk(device.deviceId) + + val installationReport = GenInstallReport( + device.primary.ecuSerial, + success = true, + correlationId = updateId.toCorrelationId.some + ).generate + + val manifest = buildPrimaryManifest( + device.primary, + device.primaryKey, + targetUpdate.to, + installationReport.some + ) + putManifestOk(device.deviceId, manifest) + + // Process all messages + msgPub.findReceivedAll((_: DeviceUpdateEvent) => true).foreach { msg => + new DeviceUpdateEventListener(msgPub).apply(msg).futureValue + } + + val updateDetail = getUpdateDetailOK(device.deviceId, updateId) + updateDetail.updateId shouldBe updateId + updateDetail.status shouldBe Update.Status.Completed + updateDetail.completedAt shouldNot be(empty) + + val resultForPrimaryEcu = updateDetail.ecuResults.get(device.primary.ecuSerial).value + resultForPrimaryEcu.success shouldBe true + resultForPrimaryEcu.hardwareId shouldBe hardwareId + resultForPrimaryEcu.id shouldBe targetUpdate.to.target + + val ecuReport = resultForPrimaryEcu.result.value + ecuReport.success shouldBe true + ecuReport.description shouldBe defined + ecuReport.description.value shouldBe installationReport.result.description.value + ecuReport.resultCode shouldBe installationReport.result.code + } + + testWithRepo("update details include results for partially completed updates") { implicit ns => + val device = registerAdminDeviceWithSecondariesOk() + val primaryHwId = device.primary.hardwareId + val secondaryHwId = device.secondaries.values.head.hardwareId + val primaryTarget = GenTargetUpdateRequest.generate + val secondaryTarget = GenTargetUpdateRequest.generate + + val createRequest = CreateUpdateRequest( + targets = Map(primaryHwId -> primaryTarget, secondaryHwId -> secondaryTarget), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + } + + val updates = listUpdatesOK(device.deviceId) + val updateId = updates.values.loneElement.updateId + + getTargetsOk(device.deviceId) + + val installationReport = GenInstallReport( + device.primary.ecuSerial, + success = true, + correlationId = updateId.toCorrelationId.some + ).generate + + val manifest = buildPrimaryManifest( + device.primary, + device.primaryKey, + primaryTarget.to, + installationReport.some + ) + putManifestOk(device.deviceId, manifest) + + // Process all messages + msgPub.findReceivedAll((_: DeviceUpdateEvent) => true).foreach { msg => + new DeviceUpdateEventListener(msgPub).apply(msg).futureValue + } + + val updateDetail = getUpdateDetailOK(device.deviceId, updateId) + updateDetail.updateId shouldBe updateId + updateDetail.status shouldBe Update.Status.PartiallyCompleted + + val resultForPrimaryEcu = updateDetail.ecuResults.get(device.primary.ecuSerial).value + resultForPrimaryEcu.success shouldBe true + resultForPrimaryEcu.hardwareId shouldBe primaryHwId + resultForPrimaryEcu.id shouldBe primaryTarget.to.target + + val ecuReport = resultForPrimaryEcu.result.value + ecuReport.success shouldBe true + ecuReport.resultCode shouldBe installationReport.result.code + ecuReport.description.value shouldBe installationReport.result.description.value + + // Secondary ECU should not have a result yet + (updateDetail.ecuResults shouldNot contain).key(device.secondaries.keys.head) + } + + testWithRepo("returns 404 when update ID doesn't exist") { implicit ns => + val device = registerAdminDeviceOk() + val nonExistentUpdateId = UpdateId.generate() + + Get( + apiUri(s"updates/devices/${device.deviceId.show}/${nonExistentUpdateId.show}") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.NotFound + } + } + + testWithRepo("update is marked as partially completed when only some ECUs report success") { + implicit ns => + val device = registerAdminDeviceWithSecondariesOk() + val primaryHwId = device.primary.hardwareId + val secondaryHwId = device.secondaries.values.head.hardwareId + val primaryTarget = GenTargetUpdateRequest.generate + val secondaryTarget = GenTargetUpdateRequest.generate + + val createRequest = CreateUpdateRequest( + targets = Map(primaryHwId -> primaryTarget, secondaryHwId -> secondaryTarget), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(device.deviceId) + } + + getTargetsOk(device.deviceId) + + val manifest = buildPrimaryManifest(device.primary, device.primaryKey, primaryTarget.to) + putManifestOk(device.deviceId, manifest) + + val partialUpdates = listUpdatesOK(device.deviceId) + partialUpdates.values should have size 1 + val partialUpdate = partialUpdates.values.head + partialUpdate.status shouldBe Update.Status.PartiallyCompleted + } + + testWithRepo("can create an update for multiple devices") { implicit ns => + val device1 = registerAdminDeviceOk() + val device2 = registerAdminDeviceOk(hardwareIdentifier = device1.primary.hardwareId.some) + val hardwareId = device1.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetRequest), + devices = Seq(device1.deviceId, device2.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + (result.affected should contain).allOf(device1.deviceId, device2.deviceId) + result.notAffected shouldBe empty + } + + val targets1 = getTargetsOk(device1.deviceId) + targets1.signed.targets should not be empty + + val targets2 = getTargetsOk(device2.deviceId) + targets2.signed.targets should not be empty + } + + testWithRepo("returns notAffected for devices with incompatible hardware") { implicit ns => + val device1 = registerAdminDeviceOk() + val device2 = registerAdminDeviceOk(Some(GenHardwareIdentifier.generate)) + val hardwareId = device1.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetRequest), + devices = Seq(device1.deviceId, device2.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain only device1.deviceId + (result.notAffected should contain).key(device2.deviceId) + } + + // Only device1 should see the update + val targets1 = getTargetsOk(device1.deviceId) + targets1.signed.targets should not be empty + + val targets2 = getTargetsOk(device2.deviceId) + targets2.signed.targets shouldBe empty + } + + testWithRepo( + "when creating an update for a device, if the device is not affected, no TargetUpdateSpec is created" + ) { implicit ns => + val device = registerAdminDeviceOk() + + val differentHardwareId = GenHardwareIdentifier.generate + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + + val req = CreateDeviceUpdateRequest( + targets = Map(differentHardwareId -> targetRequest), + scheduledFor = None + ) + + Post(apiUri(s"updates/devices/${device.deviceId.show}"), req).namespaced ~> routes ~> check { + status shouldBe StatusCodes.BadRequest + val error = responseAs[ErrorRepresentation] + error.code shouldBe ErrorCodes.DeviceCannotBeUpdated + + val deviceError = error.cause.value.as[Map[String, ErrorRepresentation]].value.values.head + deviceError.code shouldBe ErrorCodes.DeviceNoCompatibleHardware + } + + val updates = listUpdatesOK(device.deviceId) + updates.values shouldBe empty + + val targetSpecIds = hardwareUpdateRepository.findAll(ns).futureValue + targetSpecIds shouldBe empty + } + + testWithRepo("does not accept empty targets") { implicit ns => + val device = registerAdminDeviceOk() + val createRequest = CreateUpdateRequest(targets = Map.empty, devices = Seq(device.deviceId)) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.BadRequest + responseAs[ErrorRepresentation].code shouldBe ErrorCodes.InvalidMtu + } + } + + testWithRepo("accepts user defined json") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val userDefinedCustom = Json.obj("some" -> Json.fromString("val")) + val toUpdate = GenTargetUpdate.generate.copy(userDefinedCustom = Some(userDefinedCustom)) + val toUpdateReq = TargetUpdateRequest(None, toUpdate) + val createRequest = + CreateUpdateRequest(targets = Map(hardwareId -> toUpdateReq), devices = Seq(device.deviceId)) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(device.deviceId) + } + + // Device fetches targets.json + val targets = getTargetsOk(device.deviceId) + targets.signed.targets should not be empty + + // Verify the target contains the user-defined custom JSON + val targetItemCustom = targets.signed.targets.headOption.value._2.customParsed[TargetItemCustom] + targetItemCustom.get.userDefinedCustom.value shouldBe userDefinedCustom + } + + testWithRepo("can cancel an update for a device") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetRequest), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(device.deviceId) + } + + val updates = listUpdatesOK(device.deviceId) + val update = updates.values.loneElement + update.status shouldBe Update.Status.Assigned + val updateId = update.updateId + + cancelUpdateOK(device.deviceId, updateId) + + val cancelledUpdates = listUpdatesOK(device.deviceId) + cancelledUpdates.values should have size 1 + cancelledUpdates.values.head.status shouldBe Update.Status.Cancelled + + val msg = msgPub + .findReceived[DeviceUpdateEvent] { (msg: DeviceUpdateEvent) => + msg.deviceUuid == device.deviceId && msg.correlationId == updateId.toCorrelationId && + msg.isInstanceOf[DeviceUpdateCanceled] + } + + msg shouldNot be(empty) + } + + testWithRepo("can cancel a scheduled update") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + + createUpdateOk( + device.deviceId, + TargetUpdateSpec(Map(hardwareId -> targetRequest)), + Instant.now.minusSeconds(60).some + ) + + val updates = listUpdatesOK(device.deviceId) + val update = updates.values.loneElement + update.status shouldBe Update.Status.Scheduled + val updateId = update.updateId + + cancelUpdateOK(device.deviceId, updateId) + + val cancelledUpdates = listUpdatesOK(device.deviceId) + cancelledUpdates.values should have size 1 + cancelledUpdates.values.head.status shouldBe Update.Status.Cancelled + + val msg = msgPub + .findReceived[DeviceUpdateEvent] { (msg: DeviceUpdateEvent) => + msg.deviceUuid == device.deviceId && msg.correlationId == updateId.toCorrelationId && + msg.isInstanceOf[DeviceUpdateCanceled] + } + + msg shouldNot be(empty) + } + + testWithRepo("cannot cancel an update that is not in Assigned status") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdateReq = GenTargetUpdateRequest.generate + val createRequest = + CreateUpdateRequest( + targets = Map(hardwareId -> targetUpdateReq), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + } + + val updates = listUpdatesOK(device.deviceId) + updates.values should have size 1 + val updateId = updates.values.head.updateId + + val manifest = buildPrimaryManifest(device.primary, device.primaryKey, targetUpdateReq.to) + putManifestOk(device.deviceId, manifest) + + val completedUpdates = listUpdatesOK(device.deviceId) + completedUpdates.values.head.status shouldBe Update.Status.Completed + + Patch( + apiUri(s"updates/devices/${device.deviceId.show}/${updateId.show}") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.Conflict + val error = responseAs[ErrorRepresentation] + error.code shouldBe ErrorCodes.UpdateCannotBeCancelled + } + } + + testWithRepo("can cancel an update when it is InFlight if force header is provided") { + implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + val createRequest = CreateUpdateRequest( + targets = Map(hardwareId -> targetRequest), + devices = Seq(device.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(device.deviceId) + } + + val updates = listUpdatesOK(device.deviceId) + val update = updates.values.loneElement + update.status shouldBe Update.Status.Assigned + val updateId = update.updateId + + getTargetsOk(device.deviceId) + + Patch( + apiUri(s"updates/devices/${device.deviceId.show}/${updateId.show}") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.Conflict + val error = responseAs[ErrorRepresentation] + error.code.code shouldBe "update_cannot_be_cancelled" + } + + Patch(apiUri(s"updates/devices/${device.deviceId.show}/${updateId.show}")) + .addHeader(new ForceHeader(true)) + .namespaced ~> routes ~> check { + status shouldBe StatusCodes.NoContent + } + + val cancelledUpdates = listUpdatesOK(device.deviceId) + cancelledUpdates.values should have size 1 + cancelledUpdates.values.head.status shouldBe Update.Status.Cancelled + + val msg = msgPub + .findReceived[DeviceUpdateEvent] { (msg: DeviceUpdateEvent) => + msg.deviceUuid == device.deviceId && msg.correlationId == updateId.toCorrelationId && + msg.isInstanceOf[DeviceUpdateCanceled] + } + + msg shouldNot be(empty) + } + + testWithRepo("sends DeviceUpdateAssigned message when update is created with scheduledFor") { + implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + + val scheduledFor = Instant.now().plusSeconds(3600) + val mtu = TargetUpdateSpec(Map(hardwareId -> GenTargetUpdateRequest.generate)) + createUpdateOk(device.deviceId, mtu, scheduledFor.some) + + val msg = msgPub + .findReceived[DeviceUpdateEvent] { (msg: DeviceUpdateEvent) => + msg.deviceUuid == device.deviceId && msg.isInstanceOf[DeviceUpdateAssigned] + } + .map(_.asInstanceOf[DeviceUpdateAssigned]) + .value + + msg.scheduledFor should contain(scheduledFor.truncatedTo(ChronoUnit.SECONDS)) + } + + testWithRepo("creating a scheduled update sets status to Scheduled") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + + val scheduledFor = Instant.now().plusSeconds(3600) + val targetUpdateRequest = GenTargetUpdateRequest.generate + val mtu = TargetUpdateSpec(Map(hardwareId -> targetUpdateRequest)) + createUpdateOk(device.deviceId, mtu, scheduledFor.some) + + val scheduledUpdates = listUpdatesOK(device.deviceId) + val update = scheduledUpdates.values.loneElement + + update.packages should contain(device.primary.hardwareId -> targetUpdateRequest.to.target) + update.scheduledFor shouldBe Some(scheduledFor.truncatedTo(ChronoUnit.SECONDS)) + update.status shouldBe Update.Status.Scheduled + } + + testWithRepo("returns error if device already has a scheduled update") { implicit ns => + val regDev = registerAdminDeviceOk() + + val scheduledFor = Instant.now() + val mtu = TargetUpdateSpec(Map(regDev.primary.hardwareId -> GenTargetUpdateRequest.generate)) + createUpdateOk(regDev.deviceId, mtu, scheduledFor.some) + + val req = CreateDeviceUpdateRequest( + targets = Map(regDev.primary.hardwareId -> GenTargetUpdateRequest.generate), + scheduledFor = scheduledFor.some + ) + + Post(apiUri(s"updates/devices/${regDev.deviceId.show}"), req).namespaced ~> routes ~> check { + status shouldBe StatusCodes.BadRequest + val error = responseAs[ErrorRepresentation] + error.code shouldBe ErrorCodes.DeviceCannotBeUpdated + + val code = + error.cause.value.as[Map[String, ErrorRepresentation]].value.values.loneElement.code + code shouldBe ErrorCodes.DeviceHasActiveUpdate + } + } + + testWithRepo("can get update events by ID") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdateRequest.generate + val createRequest = + CreateUpdateRequest(targets = Map(hardwareId -> targetUpdate), devices = Seq(device.deviceId)) + + val deviceT = genDeviceT.generate.copy(uuid = Some(device.deviceId)) + createDeviceInNamespaceOk(deviceT, ns) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + result.affected should contain(device.deviceId) + } + + val updates = listUpdatesOK(device.deviceId) + val updateId = updates.values.loneElement.updateId + + val msg = DeviceEventMessage( + ns, + Event( + device.deviceId, + "someevent", + EventType("InstallationComplete", 0), + Instant.now.truncatedTo(ChronoUnit.SECONDS), + Instant.now.truncatedTo(ChronoUnit.SECONDS), + None, + Json.obj("correlationId" -> updateId.toCorrelationId.toString.asJson) + ) + ) + new DeviceEventListener().apply(msg).futureValue + + Get( + apiUri(s"updates/devices/${device.deviceId.show}/${updateId.show}/events") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val event = responseAs[Seq[UpdateEventResponse]].loneElement + event.receivedAt shouldBe msg.event.receivedAt + event.ecu shouldBe msg.event.ecu + event.eventType shouldBe msg.event.eventType + } + } + + testWithRepo("returns empty list when no events exist for update") { implicit ns => + val device = registerAdminDeviceOk() + val newUpdate = GenTargetUpdateRequest.generate + val mtu = TargetUpdateSpec(Map(device.primary.hardwareId -> newUpdate)) + + createUpdateOk(device.deviceId, mtu) + + val updates = listUpdatesOK(device.deviceId) + val updateId = updates.values.loneElement.updateId + + Get( + apiUri(s"updates/devices/${device.deviceId.show}/${updateId.show}/events") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + val events = responseAs[Seq[UpdateEventResponse]] + events shouldBe empty + } + } + + testWithRepo("returns 404 when getting events for non-existent update") { implicit ns => + val device = registerAdminDeviceOk() + val nonExistentUpdateId = UpdateId.generate() + + Get( + apiUri(s"updates/devices/${device.deviceId.show}/${nonExistentUpdateId.show}/events") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.NotFound + } + } + + + testWithRepo("can cancel all updates and assignments for all devices with a given update") { + implicit ns => + val device1 = registerAdminDeviceOk() + val device2 = registerAdminDeviceOk(hardwareIdentifier = device1.primary.hardwareId.some) + + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + val createRequest = CreateUpdateRequest( + targets = Map(device1.primary.hardwareId -> targetRequest), + devices = Seq(device1.deviceId, device2.deviceId) + ) + + createManyUpdates(createRequest) { + status shouldBe StatusCodes.OK + val result = responseAs[CreateUpdateResult] + (result.affected should contain).allOf(device1.deviceId, device2.deviceId) + } + + val update1 = listUpdatesOK(device1.deviceId).values.loneElement + update1.status shouldBe Update.Status.Assigned + + val update2 = listUpdatesOK(device2.deviceId).values.loneElement + update2.status shouldBe Update.Status.Assigned + + update1.updateId shouldBe update2.updateId + + cancelAllUpdatesOK(update1.updateId) + + val cancelledUpdates1 = listUpdatesOK(device1.deviceId) + cancelledUpdates1.values.loneElement.status shouldBe Update.Status.Cancelled + + val cancelledUpdates2 = listUpdatesOK(device2.deviceId) + cancelledUpdates2.values.loneElement.status shouldBe Update.Status.Cancelled + } + + testWithRepo("cancelling scheduled updates sends messages when the update is not yet assigned") { implicit ns => + val device = registerAdminDeviceOk() + val hardwareId = device.primary.hardwareId + val targetUpdate = GenTargetUpdate.generate + val targetRequest = TargetUpdateRequest(None, targetUpdate) + + createUpdateOk( + device.deviceId, + TargetUpdateSpec(Map(hardwareId -> targetRequest)), + Instant.now.minusSeconds(60).some + ) + + val updates = listUpdatesOK(device.deviceId) + val update = updates.values.loneElement + update.status shouldBe Update.Status.Scheduled + val updateId = update.updateId + + cancelAllUpdatesOK(updateId) + + val cancelledUpdates = listUpdatesOK(device.deviceId) + cancelledUpdates.values should have size 1 + cancelledUpdates.values.head.status shouldBe Update.Status.Cancelled + + val msg = msgPub + .findReceived[DeviceUpdateEvent] { (msg: DeviceUpdateEvent) => + msg.deviceUuid == device.deviceId && msg.correlationId == updateId.toCorrelationId && + msg.isInstanceOf[DeviceUpdateCanceled] + } + + msg shouldNot be(empty) + } + +} diff --git a/src/test/scala/com/advancedtelematic/director/http/UpdatesResources.scala b/src/test/scala/com/advancedtelematic/director/http/UpdatesResources.scala new file mode 100644 index 00000000..fe3f57fb --- /dev/null +++ b/src/test/scala/com/advancedtelematic/director/http/UpdatesResources.scala @@ -0,0 +1,82 @@ +package com.advancedtelematic.director.http + +import org.apache.pekko.http.scaladsl.model.StatusCodes +import cats.syntax.show.* +import com.advancedtelematic.director.data.AdminDataType.TargetUpdateSpec +import com.advancedtelematic.director.data.Codecs.* +import com.advancedtelematic.director.data.DataType.UpdateId +import com.advancedtelematic.director.data.GeneratorOps.* +import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.genDeviceT +import com.advancedtelematic.director.http.deviceregistry.RegistryDeviceRequests +import com.advancedtelematic.director.util.{DirectorSpec, NamespacedTests, ResourceSpec} +import com.advancedtelematic.libats.data.DataType.Namespace +import com.advancedtelematic.libats.data.PaginationResult +import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* +import org.scalactic.source.Position + +import java.time.Instant + +trait UpdatesResources { + self: DirectorSpec & ResourceSpec & NamespacedTests & RegistryDeviceRequests => + + def createManyUpdates[T](createRequest: CreateUpdateRequest)( + fn: => T)(implicit ns: Namespace, pos: Position): T = + Post(apiUri("updates/devices"), createRequest).namespaced ~> routes ~> check(fn) + + def createUpdateOk( + deviceId: DeviceId, + mtu: TargetUpdateSpec, + scheduledFor: Option[Instant] = None)(implicit ns: Namespace, pos: Position): UpdateId = { + + val deviceT = genDeviceT.generate.copy(uuid = Some(deviceId)) + + createDeviceInNamespaceOk(deviceT, ns) + + val req = + CreateDeviceUpdateRequest(mtu.targets, scheduledFor) + + Post(apiUri(s"updates/devices/${deviceId.show}"), req).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[UpdateId] + } + } + + def listUpdatesOK( + deviceId: DeviceId)(implicit ns: Namespace, pos: Position): PaginationResult[UpdateResponse] = + Get(apiUri(s"updates/devices/${deviceId.show}")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[PaginationResult[UpdateResponse]] + } + + def cancelUpdateOK(deviceId: DeviceId, + updateId: UpdateId)(implicit ns: Namespace, pos: Position): Unit = + Patch( + apiUri(s"updates/devices/${deviceId.show}/${updateId.show}") + ).namespaced ~> routes ~> check { + status shouldBe StatusCodes.NoContent + } + + def getUpdateDetailOK( + deviceId: DeviceId, + updateId: UpdateId)(implicit ns: Namespace, pos: Position): UpdateDetailResponse = + Get(apiUri(s"updates/devices/${deviceId.show}/${updateId.show}")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[UpdateDetailResponse] + } + + def cancelAllUpdatesOK(updateId: UpdateId)(implicit ns: Namespace, pos: Position): Unit = { + Patch(apiUri(s"updates/${updateId.show}")).namespaced ~> routes ~> check { + status shouldBe StatusCodes.NoContent + } + } + + def listAllUpdatesOK(limit: Long = 100, offset: Long = 0) + (implicit ns: Namespace, pos: Position): PaginationResult[UpdateResponse] = + Get(apiUri(s"updates?limit=${limit}&offset=${offset}")) + .namespaced ~> routes ~> check { + status shouldBe StatusCodes.OK + responseAs[PaginationResult[UpdateResponse]] + } + +} diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DeviceRegistryResourceUri.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DeviceRegistryResourceUri.scala index bd3efd5c..38edc014 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DeviceRegistryResourceUri.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DeviceRegistryResourceUri.scala @@ -1,7 +1,7 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.Uri -import akka.http.scaladsl.model.Uri.Path +import org.apache.pekko.http.scaladsl.model.Uri +import org.apache.pekko.http.scaladsl.model.Uri.Path object DeviceRegistryResourceUri { diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DeviceResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DeviceResourceSpec.scala index dfcb46c0..52b74599 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DeviceResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DeviceResourceSpec.scala @@ -8,9 +8,9 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.model.StatusCodes.* -import akka.http.scaladsl.model.Uri.Query +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes.* +import org.apache.pekko.http.scaladsl.model.Uri.Query import cats.syntax.either.* import cats.syntax.option.* import cats.syntax.show.* @@ -65,7 +65,7 @@ class DeviceResourceSpec import Device.* import com.advancedtelematic.director.deviceregistry.data.GeneratorOps.* - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* private implicit val exec: scala.concurrent.ExecutionContextExecutor = system.dispatcher @@ -968,8 +968,8 @@ class DeviceResourceSpec status shouldBe OK val paginationResult = responseAs[PaginationResult[PackageId]] paginationResult.total shouldBe allPackages.length - paginationResult.limit shouldBe limit - paginationResult.offset shouldBe offset + paginationResult.limit.toLong shouldBe limit + paginationResult.offset.toLong shouldBe offset val packages = paginationResult.values.map(canonPkg) packages.length shouldBe scala.math.min(limit, allPackages.length) packages shouldBe sorted @@ -1468,7 +1468,7 @@ class DeviceResourceSpec val expression = GroupExpression.from("tag(colour) contains ue").valueOr(throw _) val groupId = createDynamicGroupOk(expression = expression) val tagId = TagId.from("colour").valueOr(throw _) - val newTagId = TagId.from("chromatic spectrum").valueOr(throw _) + val newTagId = TagId.from("chromatic-spectrum").valueOr(throw _) val csvRows = Seq(Seq(deviceT.deviceId.underlying, "Blue")) diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DynamicGroupsResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DynamicGroupsResourceSpec.scala index 3768b1db..1fe235cc 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DynamicGroupsResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/DynamicGroupsResourceSpec.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.StatusCodes.* +import org.apache.pekko.http.scaladsl.model.StatusCodes.* import cats.syntax.either.* import cats.syntax.show.* import com.advancedtelematic.director.db.DeleteDeviceDBIO @@ -11,7 +11,7 @@ import com.advancedtelematic.director.deviceregistry.data.{GroupExpression, Grou import com.advancedtelematic.director.util.{DirectorSpec, ResourceSpec} import com.advancedtelematic.libats.data.{ErrorCodes, ErrorRepresentation, PaginationResult} import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import org.scalatest.concurrent.Eventually import org.scalatest.time.SpanSugar.* import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.* diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/GroupRequests.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/GroupRequests.scala index d295a4f6..52a51e4c 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/GroupRequests.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/GroupRequests.scala @@ -8,9 +8,9 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.StatusCodes.* -import akka.http.scaladsl.model.Uri.Query -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpRequest, Multipart} +import org.apache.pekko.http.scaladsl.model.StatusCodes.* +import org.apache.pekko.http.scaladsl.model.Uri.Query +import org.apache.pekko.http.scaladsl.model.{ContentTypes, HttpEntity, HttpRequest, Multipart} import cats.syntax.show.* import com.advancedtelematic.director.deviceregistry.data.Device.DeviceOemId import com.advancedtelematic.director.deviceregistry.data.Group.GroupId @@ -22,7 +22,7 @@ import com.advancedtelematic.director.deviceregistry.data.{GroupExpression, Grou import com.advancedtelematic.director.util.ResourceSpec import com.advancedtelematic.libats.data.PaginationResult import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.Json import org.scalatest.Assertion diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/GroupsResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/GroupsResourceSpec.scala index 9ac76dda..16964858 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/GroupsResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/GroupsResourceSpec.scala @@ -8,9 +8,10 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.model.StatusCodes.* -import akka.http.scaladsl.model.Uri.Query +import com.advancedtelematic.libats.data.PaginationResult.* +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.StatusCodes.* +import org.apache.pekko.http.scaladsl.model.Uri.Query import cats.implicits.toShow import com.advancedtelematic.director.deviceregistry.GroupMembership import com.advancedtelematic.director.deviceregistry.data.Codecs.* @@ -52,7 +53,7 @@ class GroupsResourceSpec with ResourceSpec with RegistryDeviceRequests with GroupRequests { - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* private val limit = 30 @@ -363,7 +364,7 @@ class GroupsResourceSpec status shouldEqual Created val groupId = responseAs[GroupId] val uuidsInGroup = new GroupMembership() - .listDevices(groupId, Some(0L), Some(deviceTs.size.toLong)) + .listDevices(groupId, 0L.toOffset, deviceTs.size.toLimit) .futureValue .values uuidsInGroup should contain allElementsOf uuidsCreated @@ -381,7 +382,7 @@ class GroupsResourceSpec status shouldEqual Created val groupId = responseAs[GroupId] val uuidsInGroup = new GroupMembership() - .listDevices(groupId, Some(0L), Some(deviceTs.size.toLong)) + .listDevices(groupId, 0L.toOffset, deviceTs.size.toLimit) .futureValue .values uuidsInGroup should contain allElementsOf uuidsCreated @@ -396,7 +397,7 @@ class GroupsResourceSpec status shouldEqual Created val groupId = responseAs[GroupId] val uuidsInGroup = new GroupMembership() - .listDevices(groupId, Some(0L), Some(deviceTs.size.toLong)) + .listDevices(groupId, 0L.toOffset, deviceTs.size.toLimit) .futureValue .values uuidsInGroup shouldBe empty diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/InstallationReportSpec.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/InstallationReportSpec.scala index 5bd4036c..36641bfa 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/InstallationReportSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/InstallationReportSpec.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.StatusCodes.* +import org.apache.pekko.http.scaladsl.model.StatusCodes.* import com.advancedtelematic.director.db.DeleteDeviceDBIO import com.advancedtelematic.director.db.deviceregistry.EcuReplacementRepository import com.advancedtelematic.director.deviceregistry.daemon.DeviceUpdateEventListener @@ -19,8 +19,11 @@ import com.advancedtelematic.libats.messaging_datatype.MessageCodecs.{ deviceUpdateCompletedCodec, ecuReplacementCodec } -import com.advancedtelematic.libats.messaging_datatype.Messages.{DeviceUpdateCompleted, EcuReplaced} -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.advancedtelematic.libats.messaging_datatype.Messages.{ + DeviceUpdateCompleted, + EcuReplaced +} +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.Json import org.scalacheck.Gen import org.scalatest.EitherValues.* diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PackageListsResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PackageListsResourceSpec.scala index da05ba09..3f72dd85 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PackageListsResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PackageListsResourceSpec.scala @@ -1,7 +1,7 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.HttpRequest -import akka.http.scaladsl.model.StatusCodes.{Created, NoContent, NotFound, OK} +import org.apache.pekko.http.scaladsl.model.HttpRequest +import org.apache.pekko.http.scaladsl.model.StatusCodes.{Created, NoContent, NotFound, OK} import cats.syntax.show.* import com.advancedtelematic.director.deviceregistry.data.Codecs.{ packageListItemCodec, @@ -17,7 +17,7 @@ import com.advancedtelematic.director.deviceregistry.data.PackageId import com.advancedtelematic.director.http.deviceregistry.DeviceRegistryResourceUri.uri import com.advancedtelematic.libats.data.{ErrorCodes, ErrorRepresentation} import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import org.scalacheck.{Arbitrary, Gen} import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.* import com.advancedtelematic.director.deviceregistry.data.PackageIdGenerators.* diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsRequests.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsRequests.scala index ce6e0eac..5bede5cb 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsRequests.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsRequests.scala @@ -8,7 +8,7 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.{HttpRequest, StatusCodes} +import org.apache.pekko.http.scaladsl.model.{HttpRequest, StatusCodes} import com.advancedtelematic.director.deviceregistry.data.Codecs.* import com.advancedtelematic.director.deviceregistry.data.CredentialsType.CredentialsType import com.advancedtelematic.director.deviceregistry.data.DataType.DeviceT @@ -23,7 +23,7 @@ trait PublicCredentialsRequests { self: ResourceSpec => import StatusCodes.* import com.advancedtelematic.director.deviceregistry.data.Device.* - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* private val credentialsApi = "devices" diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsResourceSpec.scala index 1161fc1d..3a9466fd 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/PublicCredentialsResourceSpec.scala @@ -8,7 +8,7 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.StatusCodes.* +import org.apache.pekko.http.scaladsl.model.StatusCodes.* import com.advancedtelematic.director.deviceregistry.data.DataType.DeviceT import com.advancedtelematic.director.deviceregistry.data.DeviceGenerators.* import com.advancedtelematic.director.deviceregistry.data.{CredentialsType, Device} @@ -25,7 +25,7 @@ class PublicCredentialsResourceSpec with PublicCredentialsRequests { import Device.* - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* val genCredentialsType: Gen[CredentialsType.CredentialsType] = Gen.oneOf(CredentialsType.values.toSeq) diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/RegistryDeviceRequests.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/RegistryDeviceRequests.scala index 4e112c4a..8d62e91f 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/RegistryDeviceRequests.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/RegistryDeviceRequests.scala @@ -8,9 +8,9 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.* -import akka.http.scaladsl.model.Uri.Query -import akka.http.scaladsl.server.Route +import org.apache.pekko.http.scaladsl.model.* +import org.apache.pekko.http.scaladsl.model.Uri.Query +import org.apache.pekko.http.scaladsl.server.Route import cats.instances.int.* import cats.instances.string.* import cats.syntax.option.* @@ -35,14 +35,14 @@ import com.advancedtelematic.libats.data.DataType.{CorrelationId, Namespace} import com.advancedtelematic.libats.http.HttpOps.* import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libtuf.data.TufDataType.HardwareIdentifier -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.Json import org.scalatest.matchers.should.Matchers import java.time.{Instant, OffsetDateTime} // Default patience required here so the RouteTestTimeout implicit defined in DefaultPatience has priority over the -// one defined by akka-testkit +// one defined by pekko-testkit trait RegistryDeviceRequests { self: DefaultPatience & ResourceSpec & Matchers => import StatusCodes.* @@ -239,8 +239,8 @@ trait RegistryDeviceRequests { self: DefaultPatience & ResourceSpec & Matchers = Get(DeviceRegistryResourceUri.uri(api, deviceUuid.show, "events").withQuery(query)) } - def getEventsV2(deviceUuid: DeviceId, updateId: Option[CorrelationId] = None): HttpRequest = { - val query = Query(updateId.map("updateId" -> _.toString).toMap) + def getEventsV2(deviceUuid: DeviceId, TargetSpecId: Option[CorrelationId] = None): HttpRequest = { + val query = Query(TargetSpecId.map("TargetSpecId" -> _.toString).toMap) Get(DeviceRegistryResourceUri.uriV2(api, deviceUuid.show, "events").withQuery(query)) } @@ -284,6 +284,10 @@ trait RegistryDeviceRequests { self: DefaultPatience & ResourceSpec & Matchers = require(tags.map(_.length == headers.length).reduce(_ && _)) val csv = (headers +: tags).map(_.mkString(";")).mkString("\n") + + println(csv) + + val multipartForm = Multipart.FormData( Multipart.FormData.BodyPart.Strict( "custom-device-fields", diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/SearchResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/SearchResourceSpec.scala index 03a97163..8796fa82 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/SearchResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/SearchResourceSpec.scala @@ -1,8 +1,8 @@ package com.advancedtelematic.director.http.deviceregistry -import akka.http.scaladsl.model.HttpRequest -import akka.http.scaladsl.model.StatusCodes.* -import akka.http.scaladsl.model.Uri.Query +import org.apache.pekko.http.scaladsl.model.HttpRequest +import org.apache.pekko.http.scaladsl.model.StatusCodes.* +import org.apache.pekko.http.scaladsl.model.Uri.Query import cats.syntax.option.* import com.advancedtelematic.director.data.Generators.GenHardwareIdentifier import com.advancedtelematic.director.db.deviceregistry.TaggedDeviceRepository @@ -21,7 +21,7 @@ import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.libats.data.{ErrorCodes, ErrorRepresentation, PaginationResult} import com.advancedtelematic.libats.http.HttpOps.HttpRequestOps import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId -import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* +import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* import io.circe.Json trait SearchRequests { @@ -129,18 +129,9 @@ class SearchResourceSpec DeviceRegistryResourceUri.uri("devices"), DevicesQuery(Some(deviceOemIds), None) ) ~> routes ~> check { - status shouldBe NotFound - val errResponse = responseAs[ErrorRepresentation] - errResponse.code shouldBe Codes.MissingDevice - errResponse.cause shouldBe defined - val errMap = errResponse.cause - .getOrElse(Json.fromString("{}")) - .as[Map[String, String]] - .getOrElse(Map.empty[String, String]) - errMap.keys should contain("missingDeviceUuids") - errMap.keys should contain("missingOemIds") - errMap("missingOemIds") should not be empty - errMap("missingDeviceUuids") shouldBe empty + status shouldBe OK + val resp = responseAs[List[Device]] + resp.map(_.deviceId) shouldNot contain(DeviceOemId("not-real-deviceId")) } } } @@ -160,18 +151,9 @@ class SearchResourceSpec DeviceRegistryResourceUri.uri("devices"), DevicesQuery(None, Some(deviceUuids)) ) ~> routes ~> check { - status shouldBe NotFound - val errResponse = responseAs[ErrorRepresentation] - errResponse.code shouldBe Codes.MissingDevice - errResponse.cause shouldBe defined - val errMap = errResponse.cause - .getOrElse(Json.fromString("{}")) - .as[Map[String, String]] - .getOrElse(Map.empty[String, String]) - errMap.keys should contain("missingDeviceUuids") - errMap.keys should contain("missingOemIds") - errMap("missingOemIds") shouldBe empty - errMap("missingDeviceUuids") should not be empty + status shouldBe OK + val resp = responseAs[List[Device]].map(_.deviceId) + resp.length shouldBe devices.length } } } @@ -192,19 +174,10 @@ class SearchResourceSpec DeviceRegistryResourceUri.uri("devices"), DevicesQuery(Some(deviceOemIds), Some(deviceUuids)) ) ~> routes ~> check { - status shouldBe NotFound - val errResponse = responseAs[ErrorRepresentation] - errResponse.code shouldBe Codes.MissingDevice - errResponse.cause shouldBe defined - - val errMap = errResponse.cause - .getOrElse(Json.fromString("{}")) - .as[Map[String, String]] - .getOrElse(Map.empty[String, String]) - errMap.keys should contain("missingDeviceUuids") - errMap.keys should contain("missingOemIds") - errMap("missingOemIds") should not be empty - errMap("missingDeviceUuids") should not be empty + status shouldBe OK + val resp = responseAs[List[Device]].map(_.deviceId) + resp.length shouldBe devices.length + resp shouldNot contain(DeviceOemId("not-real-deviceId")) } } } @@ -499,4 +472,43 @@ class SearchResourceSpec } } + test("can search by device tag") { + val ns = Namespace("search_tag_device") + + val count = 3 + val deviceTs = genConflictFreeDeviceTs(count).sample.get + val ids = deviceTs.map { + createDeviceInNamespaceOk(_, ns) + } + + val first = deviceTs.head + val firstId = ids.head + val second = deviceTs(1) + + val csvRows = Seq( + Seq(first.deviceId.underlying, "Germany", "Premium"), + Seq(second.deviceId.underlying, "China", "Deluxe") + ) + + postDeviceTags(csvRows, ns = ns) ~> routes ~> check { + status shouldBe NoContent + } + + val query = Query("tags" -> "market=Germany,trim=pt/not-premium", "sortBy" -> "activatedAt") + + Get(DeviceRegistryResourceUri.uri(api).withQuery(query)).withNs(ns) ~> routes ~> check { + status shouldBe OK + val result = responseAs[PaginationResult[Device]] + + result.total shouldBe 1 + result.values.length shouldBe 1 + + val firstIndex = result.values.indexWhere(_.uuid == firstId) + result.values(firstIndex).attributes shouldEqual Map( + TagId("market") -> "Germany", + TagId("trim") -> "Premium" + ) + } + } + } diff --git a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/SystemInfoResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/SystemInfoResourceSpec.scala index 6490fc4f..f06e6101 100644 --- a/src/test/scala/com/advancedtelematic/director/http/deviceregistry/SystemInfoResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/http/deviceregistry/SystemInfoResourceSpec.scala @@ -40,8 +40,8 @@ class SystemInfoResourceSpec with ResourcePropSpec with RegistryDeviceRequests { - import akka.http.scaladsl.model.StatusCodes.* - import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* + import org.apache.pekko.http.scaladsl.model.StatusCodes.* + import com.github.pjfanning.pekkohttpcirce.FailFastCirceSupport.* test("GET /system_info request fails on non-existent device") { forAll { (uuid: DeviceId, json: Json) => @@ -421,7 +421,7 @@ class SystemInfoResourceSpec } test("system config can be uploaded") { - import akka.http.scaladsl.unmarshalling.Unmarshaller.* + import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshaller.* val deviceUuid = createDeviceOk(genDeviceT.generate.copy(deviceId = DeviceOemId("abcd-1234"))) val config = """ @@ -448,7 +448,7 @@ class SystemInfoResourceSpec } test("system config without 'secondary_preinstall_wait_sec' can be uploaded") { - import akka.http.scaladsl.unmarshalling.Unmarshaller.* + import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshaller.* val deviceUuid = createDeviceOk(genDeviceT.generate.copy(deviceId = DeviceOemId("abcd-1234-legacy"))) @@ -475,7 +475,7 @@ class SystemInfoResourceSpec } test("system config TOML parsing error handling") { - import akka.http.scaladsl.unmarshalling.Unmarshaller.* + import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshaller.* val deviceUuid = createDeviceOk(genDeviceT.generate.copy(deviceId = DeviceOemId("abcd-1234-error"))) diff --git a/src/test/scala/com/advancedtelematic/director/manifest/ManifestCompilerSpec.scala b/src/test/scala/com/advancedtelematic/director/manifest/ManifestCompilerSpec.scala index d20fbdac..dbec7870 100644 --- a/src/test/scala/com/advancedtelematic/director/manifest/ManifestCompilerSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/manifest/ManifestCompilerSpec.scala @@ -1,25 +1,17 @@ package com.advancedtelematic.director.manifest +import cats.implicits.catsSyntaxOptionId import com.advancedtelematic.director.data.AdminDataType.TargetUpdate import com.advancedtelematic.director.data.Codecs.* -import com.advancedtelematic.director.data.DataType.{ScheduledUpdate, ScheduledUpdateId} -import com.advancedtelematic.director.data.DbDataType.{ - Assignment, - DeviceKnownState, - EcuTarget, - EcuTargetId -} -import com.advancedtelematic.director.data.DeviceRequest.{ - DeviceManifest, - EcuManifest, - MissingInstallationReport -} +import com.advancedtelematic.director.data.DataType.{TargetSpecId, Update, UpdateId} +import com.advancedtelematic.director.data.DbDataType.{Assignment, DeviceKnownState, EcuTarget, EcuTargetId} +import com.advancedtelematic.director.data.DeviceRequest.{DeviceManifest, EcuManifest, MissingInstallationReport} import com.advancedtelematic.director.data.GeneratorOps.* import com.advancedtelematic.director.data.Generators.* import com.advancedtelematic.director.data.UptaneDataType.* import com.advancedtelematic.director.util.DirectorSpec import com.advancedtelematic.libats.data.DataType.Namespace -import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, UpdateId} +import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId import com.advancedtelematic.libtuf.data.TufDataType.SignedPayload import io.circe.syntax.* import org.scalatest.LoneElement.* @@ -286,18 +278,22 @@ class ManifestCompilerSpec extends DirectorSpec { targetUpdate.userDefinedCustom ) - val su = ScheduledUpdate( + val id = UpdateId.generate() + + val su = Update( ns, - ScheduledUpdateId.generate(), + id, deviceId, - UpdateId.generate(), - Instant.now(), - ScheduledUpdate.Status.Assigned + id.toCorrelationId, + TargetSpecId.generate(), + Instant.now, + Instant.now().some, + Update.Status.Assigned ) val ecuTargets = Map(ecuTarget.id -> ecuTarget, scheduledEcuTarget.id -> scheduledEcuTarget) - val ecuTargetsByHardwareId = Map(su.updateId -> List(scheduledEcuTarget.id)) + val ecuTargetsByHardwareId = Map(su.targetSpecId -> List(scheduledEcuTarget.id)) val currentStatus = DeviceKnownState( deviceId, @@ -313,7 +309,7 @@ class ManifestCompilerSpec extends DirectorSpec { val resultStatus = ManifestCompiler(ns, manifest).apply(currentStatus).get.knownState - resultStatus.scheduledUpdates.loneElement.status shouldBe ScheduledUpdate.Status.Completed + resultStatus.updates.loneElement.status shouldBe Update.Status.Completed } test( @@ -348,13 +344,17 @@ class ManifestCompilerSpec extends DirectorSpec { targetUpdate1.userDefinedCustom ) - val su = ScheduledUpdate( + val id = UpdateId.generate() + + val su = Update( ns, - ScheduledUpdateId.generate(), + id, deviceId, - UpdateId.generate(), + id.toCorrelationId, + TargetSpecId.generate(), Instant.now(), - ScheduledUpdate.Status.Assigned + Instant.now().some, + Update.Status.Assigned ) val ecuTargets = Map( @@ -364,7 +364,7 @@ class ManifestCompilerSpec extends DirectorSpec { ) val ecuTargetsByHardwareId = - Map(su.updateId -> List(scheduledEcuTarget.id, scheduledEcuTarget1.id)) + Map(su.targetSpecId -> List(scheduledEcuTarget.id, scheduledEcuTarget1.id)) val currentStatus = DeviceKnownState( deviceId, @@ -380,7 +380,7 @@ class ManifestCompilerSpec extends DirectorSpec { val resultStatus = ManifestCompiler(ns, manifest).apply(currentStatus).get.knownState - resultStatus.scheduledUpdates.loneElement.status shouldBe ScheduledUpdate.Status.PartiallyCompleted + resultStatus.updates.loneElement.status shouldBe Update.Status.PartiallyCompleted } test( @@ -403,18 +403,22 @@ class ManifestCompilerSpec extends DirectorSpec { targetUpdate.userDefinedCustom ) - val su = ScheduledUpdate( + val id = UpdateId.generate() + + val su = Update( ns, - ScheduledUpdateId.generate(), + id, deviceId, - UpdateId.generate(), + id.toCorrelationId, + TargetSpecId.generate(), Instant.now(), - ScheduledUpdate.Status.Assigned + Instant.now().some, + Update.Status.Assigned ) val ecuTargets = Map(ecuTarget.id -> ecuTarget, scheduledEcuTarget.id -> scheduledEcuTarget) - val ecuTargetsByHardwareId = Map(su.updateId -> List(scheduledEcuTarget.id)) + val ecuTargetsByHardwareId = Map(su.targetSpecId -> List(scheduledEcuTarget.id)) val currentStatus = DeviceKnownState( deviceId, @@ -430,7 +434,7 @@ class ManifestCompilerSpec extends DirectorSpec { val resultStatus = ManifestCompiler(ns, manifest).apply(currentStatus).get.knownState - resultStatus.scheduledUpdates.loneElement.status shouldBe ScheduledUpdate.Status.Completed + resultStatus.updates.loneElement.status shouldBe Update.Status.Completed } test("Ecu.installed_target for device gets updated with new target id if target was not known") { diff --git a/src/test/scala/com/advancedtelematic/director/util/DefaultPatience.scala b/src/test/scala/com/advancedtelematic/director/util/DefaultPatience.scala index 41d5385d..142bf34c 100644 --- a/src/test/scala/com/advancedtelematic/director/util/DefaultPatience.scala +++ b/src/test/scala/com/advancedtelematic/director/util/DefaultPatience.scala @@ -1,6 +1,6 @@ package com.advancedtelematic.director.util -import akka.http.scaladsl.testkit.RouteTestTimeout +import org.apache.pekko.http.scaladsl.testkit.RouteTestTimeout import org.scalatest.concurrent.PatienceConfiguration import org.scalatest.time.{Millis, Seconds, Span} diff --git a/src/test/scala/com/advancedtelematic/director/util/DirectorSpec.scala b/src/test/scala/com/advancedtelematic/director/util/DirectorSpec.scala index 5ec43732..064712f8 100644 --- a/src/test/scala/com/advancedtelematic/director/util/DirectorSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/util/DirectorSpec.scala @@ -1,7 +1,7 @@ package com.advancedtelematic.director.util import java.security.Security -import akka.http.scaladsl.model.{headers, HttpRequest} +import org.apache.pekko.http.scaladsl.model.{headers, HttpRequest} import com.advancedtelematic.director.data.GeneratorOps._ import com.advancedtelematic.director.http.AdminResources import com.advancedtelematic.libats.data.DataType.Namespace diff --git a/src/test/scala/com/advancedtelematic/director/util/ResourceSpec.scala b/src/test/scala/com/advancedtelematic/director/util/ResourceSpec.scala index 3ff92044..54b38ae8 100644 --- a/src/test/scala/com/advancedtelematic/director/util/ResourceSpec.scala +++ b/src/test/scala/com/advancedtelematic/director/util/ResourceSpec.scala @@ -1,7 +1,7 @@ package com.advancedtelematic.director.util -import akka.http.scaladsl.server.* -import akka.http.scaladsl.testkit.ScalatestRouteTest +import org.apache.pekko.http.scaladsl.server.* +import org.apache.pekko.http.scaladsl.testkit.ScalatestRouteTest import com.advancedtelematic.director.client.FakeKeyserverClient import com.advancedtelematic.director.http.DirectorRoutes import com.advancedtelematic.libats.data.DataType.Namespace @@ -14,14 +14,7 @@ import com.advancedtelematic.director.data.AdminDataType.TargetUpdate import com.advancedtelematic.director.data.UptaneDataType.* import com.advancedtelematic.director.data.DbDataType.Ecu import com.advancedtelematic.director.data.DeviceRequest -import com.advancedtelematic.director.data.DeviceRequest.{ - DeviceManifest, - EcuManifest, - InstallationReport, - InstallationReportEntity, - MissingInstallationReport -} -import com.advancedtelematic.libats.data.EcuIdentifier +import com.advancedtelematic.director.data.DeviceRequest.{DeviceManifest, EcuManifest, InstallationReport, InstallationReportEntity, MissingInstallationReport} import com.advancedtelematic.director.data.Codecs.* import com.advancedtelematic.director.data.UptaneDataType.Image import com.advancedtelematic.director.db.deviceregistry.DeviceRepository @@ -29,7 +22,7 @@ import com.advancedtelematic.director.deviceregistry.AllowUUIDPath import com.advancedtelematic.director.http.deviceregistry.DeviceRegistryRoutes import com.advancedtelematic.libats.http.NamespaceDirectives import com.advancedtelematic.libats.messaging.test.MockMessageBus -import com.advancedtelematic.libats.messaging_datatype.DataType.DeviceId +import com.advancedtelematic.libats.messaging_datatype.DataType.{DeviceId, EcuIdentifier} import org.scalatest.concurrent.ScalaFutures import org.scalatest.matchers.should.Matchers