Merge pull request #257 from diffix/edon/refactor

Refactor context members and names

Author: edongashi

src/OpenDiffix.CLI/Program.fs

@@ -129,8 +129,8 @@ let dryRun query filePath anonParams =
 
 let runQuery query filePath anonParams =
   use dataProvider = new SQLite.DataProvider(filePath) :> IDataProvider
-  let context = EvaluationContext.make anonParams dataProvider
-  QueryEngine.run context query
+  let queryContext = QueryContext.make anonParams dataProvider
+  QueryEngine.run queryContext query
 
 let quoteString (string: string) =
   "\"" + string.Replace("\"", "\"\"") + "\""

src/OpenDiffix.Core.Tests/Analyzer.Tests.fs

@@ -18,7 +18,7 @@ let testTable: Table =
   }
 
 let dataProvider = dummyDataProvider [ testTable ]
-let context = EvaluationContext.make AnonymizationParams.Default dataProvider
+let queryContext = QueryContext.make AnonymizationParams.Default dataProvider
 
 let defaultQuery =
   {
@@ -32,10 +32,13 @@ let defaultQuery =
   }
 
 let testParsedQuery queryString expected =
-  queryString |> Parser.parse |> Analyzer.analyze context |> should equal expected
+  queryString
+  |> Parser.parse
+  |> Analyzer.analyze queryContext
+  |> should equal expected
 
 let testQueryError queryString =
-  (fun () -> queryString |> Parser.parse |> Analyzer.analyze context |> ignore)
+  (fun () -> queryString |> Parser.parse |> Analyzer.analyze queryContext |> ignore)
   |> shouldFail
 
 [<Fact>]
@@ -277,14 +280,19 @@ type Tests(db: DBFixture) =
       NoiseSD = 0.
     }
 
-  let context = EvaluationContext.make anonParams db.DataProvider
+  let queryContext = QueryContext.make anonParams db.DataProvider
 
   let idColumn = ColumnReference(4, IntegerType)
   let companyColumn = ColumnReference(2, StringType)
   let aidColumns = [ companyColumn; idColumn ] |> ListExpr
 
   let analyzeQuery query =
-    let query, _ = query |> Parser.parse |> Analyzer.analyze context |> Analyzer.anonymize context
+    let query, _ =
+      query
+      |> Parser.parse
+      |> Analyzer.analyze queryContext
+      |> Analyzer.anonymize queryContext
+
     query
 
   let ensureQueryFails query error =
@@ -296,7 +304,13 @@ type Tests(db: DBFixture) =
 
   let assertSqlSeed query (seedMaterial: string) =
     let expectedSeed = seedMaterial |> System.Text.Encoding.UTF8.GetBytes |> Hash.bytes
-    let _query, executionContext = query |> Parser.parse |> Analyzer.analyze context |> Analyzer.anonymize context
+
+    let _query, executionContext =
+      query
+      |> Parser.parse
+      |> Analyzer.analyze queryContext
+      |> Analyzer.anonymize queryContext
+
     executionContext.NoiseLayers.BucketSeed |> should equal expectedSeed
 
   [<Fact>]

src/OpenDiffix.Core.Tests/Anonymizer.Tests.fs

@@ -25,7 +25,7 @@ let strColumn = ColumnReference(1, StringType)
 let companyColumn = ColumnReference(2, StringType)
 let allAidColumns = ListExpr [ aidColumn; companyColumn ]
 
-let context =
+let executionContext =
   { ExecutionContext.Default with
       QueryContext =
         { QueryContext.Default with
@@ -45,7 +45,7 @@ let anonymizedAggregationContext =
   let threshold = { Lower = 2; Upper = 2 }
 
   let anonParams =
-    { context.QueryContext.AnonymizationParams with
+    { executionContext.AnonymizationParams with
         OutlierCount = threshold
         TopCount = threshold
     }
@@ -54,7 +54,8 @@ let anonymizedAggregationContext =
       QueryContext = { QueryContext.Default with AnonymizationParams = anonParams }
   }
 
-let evaluateAggregator fn args = evaluateAggregator context fn args
+let evaluateAggregator fn args =
+  evaluateAggregator executionContext fn args
 
 let mergeAids = AggregateFunction(MergeAids, AggregateOptions.Default)
 let distinctDiffixCount = AggregateFunction(DiffixCount, { AggregateOptions.Default with Distinct = true })
@@ -273,22 +274,22 @@ let ``allows null-values for some of the AID rows`` () =
   let allAidColumns = ListExpr [ aid1; aid2 ]
 
   rows
-  |> TestHelpers.evaluateAggregator context diffixCount [ allAidColumns; value ]
+  |> TestHelpers.evaluateAggregator executionContext diffixCount [ allAidColumns; value ]
   |> should equal (Integer 5L)
 
   rows
-  |> TestHelpers.evaluateAggregator context distinctDiffixCount [ allAidColumns; value ]
+  |> TestHelpers.evaluateAggregator executionContext distinctDiffixCount [ allAidColumns; value ]
   |> should equal (Integer 5L)
 
   // The aggregate result should not be affected by the order of the AIDs
   let allAidsFlipped = ListExpr [ aid2; aid1 ]
 
   rows
-  |> TestHelpers.evaluateAggregator context diffixCount [ allAidsFlipped; value ]
+  |> TestHelpers.evaluateAggregator executionContext diffixCount [ allAidsFlipped; value ]
   |> should equal (Integer 5L)
 
   rows
-  |> TestHelpers.evaluateAggregator context distinctDiffixCount [ allAidsFlipped; value ]
+  |> TestHelpers.evaluateAggregator executionContext distinctDiffixCount [ allAidsFlipped; value ]
   |> should equal (Integer 5L)
 
 [<Fact>]
@@ -313,7 +314,7 @@ let ``account for values where AID-value is null`` () =
   let allAidColumns = ListExpr [ aid1; aid2 ]
 
   rows
-  |> TestHelpers.evaluateAggregator context diffixCount [ allAidColumns; value ]
+  |> TestHelpers.evaluateAggregator executionContext diffixCount [ allAidColumns; value ]
   |> should equal (Integer 8L)
 
 [<Fact>]
@@ -325,11 +326,11 @@ let ``low count accepts rows with shared contribution`` () =
   let highUserRows = [ [| aidList [ "Paul"; "Cristian"; "Felix"; "Edon" ] |] ]
 
   lowUserRows
-  |> TestHelpers.evaluateAggregator context diffixLowCount [ aidsExpression ]
+  |> TestHelpers.evaluateAggregator executionContext diffixLowCount [ aidsExpression ]
   |> should equal (Boolean true)
 
   highUserRows
-  |> TestHelpers.evaluateAggregator context diffixLowCount [ aidsExpression ]
+  |> TestHelpers.evaluateAggregator executionContext diffixLowCount [ aidsExpression ]
   |> should equal (Boolean false)
 
 [<Fact>]
@@ -357,7 +358,7 @@ let ``count accepts rows with shared contribution`` () =
   // Total:                     = 7.0
 
   rows
-  |> TestHelpers.evaluateAggregator context diffixCount [ aidsExpression ]
+  |> TestHelpers.evaluateAggregator executionContext diffixCount [ aidsExpression ]
   |> should equal (Integer 5L)
 
 [<Fact>]
@@ -385,5 +386,5 @@ let ``count distinct accepts rows with shared contribution`` () =
     ]
 
   rows
-  |> TestHelpers.evaluateAggregator context distinctDiffixCount [ aidsExpression; dataColumn ]
+  |> TestHelpers.evaluateAggregator executionContext distinctDiffixCount [ aidsExpression; dataColumn ]
   |> should equal (Integer 5L)

src/OpenDiffix.Core.Tests/Executor.Tests.fs

@@ -26,13 +26,13 @@ type Tests(db: DBFixture) =
   let countDistinct expression =
     FunctionExpr(AggregateFunction(Count, { Distinct = true; OrderBy = [] }), [ expression ])
 
-  let context =
+  let executionContext =
     { ExecutionContext.Default with
         QueryContext = { QueryContext.Default with DataProvider = db.DataProvider }
     }
 
   let execute plan =
-    plan |> Executor.execute context |> Seq.toList
+    plan |> Executor.execute executionContext |> Seq.toList
 
   [<Fact>]
   let ``execute scan`` () =

src/OpenDiffix.Core.Tests/Normalizer.Tests.fs

@@ -16,10 +16,13 @@ let testTable =
   }
 
 let dataProvider = dummyDataProvider [ testTable ]
-let context = EvaluationContext.make AnonymizationParams.Default dataProvider
+let queryContext = QueryContext.make AnonymizationParams.Default dataProvider
 
 let queryPlan statement =
-  statement |> Parser.parse |> Analyzer.analyze context |> Normalizer.normalize
+  statement
+  |> Parser.parse
+  |> Analyzer.analyze queryContext
+  |> Normalizer.normalize
 
 let equivalentQueries expectedQuery testQuery =
   let testPlan = queryPlan testQuery

src/OpenDiffix.Core.Tests/QueryEngine.Tests.fs

@@ -22,8 +22,8 @@ type Tests(db: DBFixture) =
     }
 
   let runQueryWithCustomAnonParams anonymizationParams query =
-    let context = EvaluationContext.make anonymizationParams db.DataProvider
-    run context query
+    let queryContext = QueryContext.make anonymizationParams db.DataProvider
+    run queryContext query
 
   let runQuery = runQueryWithCustomAnonParams anonParams
 

src/OpenDiffix.Core.Tests/QueryValidator.Tests.fs

@@ -16,14 +16,14 @@ let testTable: Table =
 
 
 let dataProvider = dummyDataProvider [ testTable ]
-let context = EvaluationContext.make AnonymizationParams.Default dataProvider
+let queryContext = QueryContext.make AnonymizationParams.Default dataProvider
 
 let aidColIndex = Table.findColumn testTable "int_col" |> fst
 
 let analyzeQuery queryString =
   queryString
   |> Parser.parse
-  |> Analyzer.analyze context
+  |> Analyzer.analyze queryContext
   |> QueryValidator.validateQuery
 
 let ensureFailParsedQuery queryString (errorFragment: string) =

src/OpenDiffix.Core/Aggregator.fs

@@ -124,11 +124,11 @@ type private DiffixCount(minCount) =
       | [ aidInstances; _ ] -> updateAidMaps aidInstances 1L
       | _ -> invalidArgs args
 
-    member this.Final context =
+    member this.Final executionContext =
       if isNull state then
         Integer minCount
       else
-        match Anonymizer.count context state with
+        match Anonymizer.count executionContext state with
         | Null -> Integer minCount
         | Integer value -> Integer(max value minCount)
         | value -> value
@@ -161,8 +161,8 @@ type private DiffixCountDistinct(minCount) =
         )
       | _ -> invalidArgs args
 
-    member this.Final context =
-      match Anonymizer.countDistinct context aidsCount aidsPerValue with
+    member this.Final executionContext =
+      match Anonymizer.countDistinct executionContext aidsCount aidsPerValue with
       | Null -> Integer minCount
       | Integer value -> Integer(max value minCount)
       | value -> value
@@ -188,11 +188,11 @@ type private DiffixLowCount() =
 
       | _ -> invalidArgs args
 
-    member this.Final context =
+    member this.Final executionContext =
       if isNull state then
         Boolean true
       else
-        Anonymizer.isLowCount context state |> Boolean
+        Anonymizer.isLowCount executionContext state |> Boolean
 
 type private MergeAids() =
   let state = HashSet<Value>()
@@ -213,12 +213,12 @@ type private MergeAids() =
 
 type T = IAggregator
 
-let create executionContext globalBucket fn : T =
+let create (executionContext: ExecutionContext) globalBucket fn : T =
   let minDiffixCount =
     if globalBucket then
       0L
     else
-      int64 executionContext.QueryContext.AnonymizationParams.Suppression.LowThreshold
+      int64 executionContext.AnonymizationParams.Suppression.LowThreshold
 
   match fn with
   | AggregateFunction (Count, { Distinct = false }) -> Count() :> T

src/OpenDiffix.Core/Analyzer.fs

@@ -369,9 +369,9 @@ let private computeNoiseLayers anonParams query =
 // Public API
 // ----------------------------------------------------------------
 
-let analyze context (parseTree: ParserTypes.SelectQuery) : Query =
-  let schema = context.DataProvider.GetSchema()
-  let anonParams = context.AnonymizationParams
+let analyze queryContext (parseTree: ParserTypes.SelectQuery) : Query =
+  let schema = queryContext.DataProvider.GetSchema()
+  let anonParams = queryContext.AnonymizationParams
   let query = mapQuery schema anonParams false parseTree
   query
 

src/OpenDiffix.Core/Anonymizer.fs

@@ -56,7 +56,7 @@ let private generateNoise salt stepName stdDev noiseLayers =
 let isLowCount (executionContext: ExecutionContext) (aidSets: HashSet<AidHash> seq) =
   aidSets
   |> Seq.map (fun aidSet ->
-    let anonParams = executionContext.QueryContext.AnonymizationParams
+    let anonParams = executionContext.AnonymizationParams
 
     if aidSet.Count < anonParams.Suppression.LowThreshold then
       true
@@ -101,7 +101,7 @@ let inline private aidFlattening
   (unaccountedFor: int64)
   (aidContributions: (AidHash * ^Contribution) list)
   : AidCount option =
-  let anonParams = executionContext.QueryContext.AnonymizationParams
+  let anonParams = executionContext.AnonymizationParams
 
   if aidContributions.Length < anonParams.OutlierCount.Lower + anonParams.TopCount.Lower then
     None
@@ -139,7 +139,7 @@ let inline private aidFlattening
     let flattenedAvg = flattenedSum / float aidContributions.Length
 
     let noiseScale = max flattenedAvg (0.5 * topGroupAverage)
-    let noiseSD = executionContext.QueryContext.AnonymizationParams.NoiseSD * noiseScale
+    let noiseSD = executionContext.AnonymizationParams.NoiseSD * noiseScale
 
     let noise =
       [ executionContext.NoiseLayers.BucketSeed; aidContributions |> Seq.map fst |> seedFromAidSet ]

src/OpenDiffix.Core/CommonTypes.fs

@@ -184,4 +184,6 @@ type ExecutionContext =
     QueryContext: QueryContext
     NoiseLayers: NoiseLayers
   }
+  member this.AnonymizationParams = this.QueryContext.AnonymizationParams
+  member this.DataProvider = this.QueryContext.DataProvider
   static member Default = { QueryContext = QueryContext.Default; NoiseLayers = { BucketSeed = 0UL } }