From 6fa5962d0189b3b0e3214dbbbd848cb63522c0db Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 21 Sep 2025 11:07:20 +0000 Subject: [PATCH 1/5] Initial plan From 015c6d5f07367611ef1648993c2489ef0779691b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 21 Sep 2025 11:19:18 +0000 Subject: [PATCH 2/5] Convert MerkleProof timestamp from Long to java.time.Instant Co-authored-by: mitchelllisle <18128531+mitchelllisle@users.noreply.github.com> --- .../org/mitchelllisle/analysers/MerkleTree.scala | 5 +++-- src/test/scala/MerkleTreeTest.scala | 14 +++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala b/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala index c6ec42e..809684c 100644 --- a/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala +++ b/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala @@ -3,6 +3,7 @@ package org.mitchelllisle.analysers import org.apache.spark.sql.{DataFrame, functions => F} import org.apache.spark.sql.types._ import java.security.MessageDigest +import java.time.Instant import scala.annotation.tailrec /** MerkleTreeAnalyser provides cryptographic proof capabilities for data retention and deletion verification. @@ -14,7 +15,7 @@ object MerkleTree { rootHash: String, recordCount: Long, leafHashes: Seq[String], - timestamp: Long = System.currentTimeMillis() + timestamp: Instant = Instant.now() ) case class DeletionProof( @@ -98,7 +99,7 @@ object MerkleTree { rootHash = rootHash, recordCount = recordCount, leafHashes = leafHashes, - timestamp = System.currentTimeMillis() + timestamp = Instant.now() ) } diff --git a/src/test/scala/MerkleTreeTest.scala b/src/test/scala/MerkleTreeTest.scala index 33dcb36..c45453f 100644 --- a/src/test/scala/MerkleTreeTest.scala +++ b/src/test/scala/MerkleTreeTest.scala @@ -1,6 +1,7 @@ import org.apache.spark.sql.DataFrame import org.mitchelllisle.analysers.MerkleTree import org.scalatest.flatspec.AnyFlatSpec +import java.time.Instant class MerkleTreeTest extends AnyFlatSpec with SparkFunSuite { import spark.implicits._ @@ -21,7 +22,8 @@ class MerkleTreeTest extends AnyFlatSpec with SparkFunSuite { assert(proof.rootHash.nonEmpty) assert(proof.recordCount == 4) assert(proof.leafHashes.length == 4) - assert(proof.timestamp > 0) + assert(proof.timestamp.isInstanceOf[Instant]) + assert(proof.timestamp.isBefore(Instant.now().plusSeconds(1))) } "apply" should "produce same result as createMerkleProof" in { @@ -183,4 +185,14 @@ class MerkleTreeTest extends AnyFlatSpec with SparkFunSuite { assert(hashes1.sameElements(hashes2)) } + + "timestamp" should "use proper Instant type and be recent" in { + val beforeTime = Instant.now().minusSeconds(1) + val proof = MerkleTree.createMerkleProof(testData, columns, idColumn) + val afterTime = Instant.now().plusSeconds(1) + + assert(proof.timestamp.isInstanceOf[Instant]) + assert(proof.timestamp.isAfter(beforeTime)) + assert(proof.timestamp.isBefore(afterTime)) + } } \ No newline at end of file From f89063f59fcbe6c52b013c267144b486e8e71b48 Mon Sep 17 00:00:00 2001 From: Mitchell Lisle Date: Sun, 21 Sep 2025 21:23:35 +1000 Subject: [PATCH 3/5] updates --- .github/workflows/build-test.yaml | 2 +- Makefile | 2 +- build.sbt | 2 +- project/plugins.sbt | 1 + .../mitchelllisle/analysers/MerkleTree.scala | 33 +++++++++---------- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 72b2c5b..809b80c 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -30,7 +30,7 @@ jobs: sudo apt-get install -y sbt - name: Run tests - run: sbt -J--add-opens=java.base/sun.nio.ch=ALL-UNNAMED clean test + run: make test - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 diff --git a/Makefile b/Makefile index e5eda1b..7d5be7c 100644 --- a/Makefile +++ b/Makefile @@ -13,5 +13,5 @@ format: sbt scalafmt test: - sbt -J--add-opens=java.base/sun.nio.ch=ALL-UNNAMED test + sbt -J--add-opens=java.base/sun.nio.ch=ALL-UNNAMED coverage test coverageReport @make clean diff --git a/build.sbt b/build.sbt index da02e45..df6a9db 100644 --- a/build.sbt +++ b/build.sbt @@ -24,7 +24,7 @@ libraryDependencies ++= Seq( "io.circe" %% "circe-yaml" % "1.15.0", "io.circe" %% "circe-core" % circeVersion, "io.circe" %% "circe-parser" % circeVersion, - "com.swoop" %% "spark-alchemy" % "1.2.1" + "com.swoop" %% "spark-alchemy" % "1.2.1", ) dependencyOverrides += "org.scala-lang.modules" %% "scala-xml" % "2.4.0" diff --git a/project/plugins.sbt b/project/plugins.sbt index e69de29..d86b6fb 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.3.1") \ No newline at end of file diff --git a/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala b/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala index 809684c..4d0ac5d 100644 --- a/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala +++ b/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala @@ -1,30 +1,27 @@ package org.mitchelllisle.analysers import org.apache.spark.sql.{DataFrame, functions => F} -import org.apache.spark.sql.types._ import java.security.MessageDigest -import java.time.Instant import scala.annotation.tailrec +case class MerkleProof( + rootHash: String, + recordCount: Long, + leafHashes: Seq[String], + timestamp: Long = System.currentTimeMillis() + ) + +case class DeletionProof( + beforeProof: MerkleProof, + afterProof: MerkleProof, + deletedRecordHashes: Seq[String], + merklePathProofs: Seq[String] + ) + /** MerkleTreeAnalyser provides cryptographic proof capabilities for data retention and deletion verification. * This complements the KHyperLogLogAnalyser by adding tamper-evident audit trails. */ object MerkleTree { - - case class MerkleProof( - rootHash: String, - recordCount: Long, - leafHashes: Seq[String], - timestamp: Instant = Instant.now() - ) - - case class DeletionProof( - beforeProof: MerkleProof, - afterProof: MerkleProof, - deletedRecordHashes: Seq[String], - merklePathProofs: Seq[String] - ) - /** Main entry point for creating a Merkle proof. This is the standard way to use MerkleTree. * * @param data The DataFrame to create proof for @@ -99,7 +96,7 @@ object MerkleTree { rootHash = rootHash, recordCount = recordCount, leafHashes = leafHashes, - timestamp = Instant.now() + timestamp = System.currentTimeMillis() ) } From 6f6977996be4d42a428eeb2ed6df3508f1d20058 Mon Sep 17 00:00:00 2001 From: Mitchell Lisle Date: Sun, 21 Sep 2025 21:23:46 +1000 Subject: [PATCH 4/5] updte version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index df6a9db..a1a86ef 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -ThisBuild / version := "1.5.0" +ThisBuild / version := "1.6.0" ThisBuild / scalaVersion := "2.12.20" From 348b28279f57484db64c61547d1c72589779f540 Mon Sep 17 00:00:00 2001 From: Mitchell Lisle Date: Sun, 21 Sep 2025 21:29:23 +1000 Subject: [PATCH 5/5] fix type --- src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala b/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala index 4d0ac5d..05f02f0 100644 --- a/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala +++ b/src/main/scala/org/mitchelllisle/analysers/MerkleTree.scala @@ -2,13 +2,14 @@ package org.mitchelllisle.analysers import org.apache.spark.sql.{DataFrame, functions => F} import java.security.MessageDigest +import java.time.Instant import scala.annotation.tailrec case class MerkleProof( rootHash: String, recordCount: Long, leafHashes: Seq[String], - timestamp: Long = System.currentTimeMillis() + timestamp: Instant = Instant.now() ) case class DeletionProof( @@ -96,7 +97,7 @@ object MerkleTree { rootHash = rootHash, recordCount = recordCount, leafHashes = leafHashes, - timestamp = System.currentTimeMillis() + timestamp = Instant.now() ) }