[sql-44] firewalldb: add migration code for privacy mapper from kvdb to SQL

firewalldb/sql_migration.go

@@ -67,6 +67,11 @@ func (e *kvEntry) namespacedKey() string {
 	return ns
 }
 
+// privacyPairs is a type alias for a map that holds the privacy pairs, where
+// the outer key is the group ID, and the value is a map of real to pseudo
+// values.
+type privacyPairs = map[int64]map[string]string
+
 // MigrateFirewallDBToSQL runs the migration of the firwalldb stores from the
 // bbolt database to a SQL database. The migration is done in a single
 // transaction to ensure that all rows in the stores are migrated or none at
@@ -87,10 +92,14 @@ func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 		return err
 	}
 
+	err = migratePrivacyMapperDBToSQL(ctx, kvStore, sqlTx)
+	if err != nil {
+		return err
+	}
+
 	log.Infof("The rules DB has been migrated from KV to SQL.")
 
-	// TODO(viktor): Add migration for the privacy mapper and the action
-	// stores.
+	// TODO(viktor): Add migration for the action stores.
 
 	return nil
 }
@@ -490,3 +499,299 @@ func verifyBktKeys(bkt *bbolt.Bucket, errorOnKeyValues bool,
 		return fmt.Errorf("unexpected key found: %s", key)
 	})
 }
+
+func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB,
+	sqlTx SQLQueries) error {
+
+	log.Infof("Starting migration of the privacy mapper store to SQL")
+
+	// 1) Collect all privacy pairs from the KV store.
+	privPairs, err := collectPrivacyPairs(ctx, kvStore, sqlTx)
+	if err != nil {
+		return fmt.Errorf("error migrating privacy mapper store: %w",
+			err)
+	}
+
+	// 2) Insert all collected privacy pairs into the SQL database.
+	err = insertPrivacyPairs(ctx, sqlTx, privPairs)
+	if err != nil {
+		return fmt.Errorf("insertion of privacy pairs failed: %w", err)
+	}
+
+	// 3) Validate that all inserted privacy pairs match the original values
+	// in the KV store. Note that this is done after all values have been
+	// inserted, to ensure that the migration doesn't overwrite any values
+	// after they were inserted.
+	err = validatePrivacyPairsMigration(ctx, sqlTx, privPairs)
+	if err != nil {
+		return fmt.Errorf("migration validation of privacy pairs "+
+			"failed: %w", err)
+	}
+
+	log.Infof("Migration of the privacy mapper stores to SQL completed. "+
+		"Total number of rows migrated: %d", len(privPairs))
+	return nil
+}
+
+// collectPrivacyPairs collects all privacy pairs from the KV store and
+// returns them as the privacyPairs type alias.
+func collectPrivacyPairs(ctx context.Context, kvStore *bbolt.DB,
+	sqlTx SQLQueries) (privacyPairs, error) {
+
+	groupPairs := make(privacyPairs)
+
+	return groupPairs, kvStore.View(func(kvTx *bbolt.Tx) error {
+		bkt := kvTx.Bucket(privacyBucketKey)
+		if bkt == nil {
+			// If we haven't generated any privacy bucket yet,
+			// we can skip the migration, as there are no privacy
+			// pairs to migrate.
+			return nil
+		}
+
+		return bkt.ForEach(func(groupId, v []byte) error {
+			if v != nil {
+				return fmt.Errorf("expected only buckets "+
+					"under %s bkt, but found value %s",
+					privacyBucketKey, v)
+			}
+
+			gBkt := bkt.Bucket(groupId)
+			if gBkt == nil {
+				return fmt.Errorf("group bkt for group id "+
+					"%s not found", groupId)
+			}
+
+			groupSqlId, err := sqlTx.GetSessionIDByAlias(
+				ctx, groupId,
+			)
+			if errors.Is(err, sql.ErrNoRows) {
+				return fmt.Errorf("session with group id %x "+
+					"not found in sql db", groupId)
+			} else if err != nil {
+				return err
+			}
+
+			groupRealToPseudoPairs, err := collectGroupPairs(gBkt)
+			if err != nil {
+				return fmt.Errorf("processing group bkt "+
+					"for group id %s (sqlID %d) failed: %w",
+					groupId, groupSqlId, err)
+			}
+
+			groupPairs[groupSqlId] = groupRealToPseudoPairs
+
+			return nil
+		})
+	})
+}
+
+// collectGroupPairs collects all privacy pairs for a specific session group,
+// i.e. the group buckets under the privacy mapper bucket in the KV store.
+// The function returns them as a map, where the key is the real value, and
+// the value for the key is the pseudo values.
+// It also checks that the pairs are consistent, i.e. that for each real value
+// there is a corresponding pseudo value, and vice versa. If the pairs are
+// inconsistent, it returns an error indicating the mismatch.
+func collectGroupPairs(bkt *bbolt.Bucket) (map[string]string, error) {
+	var (
+		realToPseudoRes map[string]string
+		pseudoToRealRes map[string]string
+		err             error
+		missMatchErr    = errors.New("privacy mapper pairs mismatch")
+	)
+
+	if realBkt := bkt.Bucket(realToPseudoKey); realBkt != nil {
+		realToPseudoRes, err = collectPairs(realBkt)
+		if err != nil {
+			return nil, fmt.Errorf("fetching real to pseudo pairs "+
+				"failed: %w", err)
+		}
+	} else {
+		return nil, fmt.Errorf("%s bucket not found", realToPseudoKey)
+	}
+
+	if pseudoBkt := bkt.Bucket(pseudoToRealKey); pseudoBkt != nil {
+		pseudoToRealRes, err = collectPairs(pseudoBkt)
+		if err != nil {
+			return nil, fmt.Errorf("fetching pseudo to real pairs "+
+				"failed: %w", err)
+		}
+	} else {
+		return nil, fmt.Errorf("%s bucket not found", pseudoToRealKey)
+	}
+
+	if len(realToPseudoRes) != len(pseudoToRealRes) {
+		return nil, missMatchErr
+	}
+
+	for realVal, pseudoVal := range realToPseudoRes {
+		if rv, ok := pseudoToRealRes[pseudoVal]; !ok || rv != realVal {
+			return nil, missMatchErr
+		}
+	}
+
+	return realToPseudoRes, nil
+}
+
+// collectPairs collects all privacy pairs from a specific realToPseudoKey or
+// pseudoToRealKey bucket in the KV store. It returns a map where the key is
+// the real value or pseudo value, and the value is the corresponding pseudo
+// value or real value, respectively (depending on if the realToPseudo or
+// pseudoToReal bucket is passed to the function).
+func collectPairs(pairsBucket *bbolt.Bucket) (map[string]string, error) {
+	pairsRes := make(map[string]string)
+
+	return pairsRes, pairsBucket.ForEach(func(k, v []byte) error {
+		if v == nil {
+			return fmt.Errorf("expected only key-values under "+
+				"pairs bucket, but found bucket %s", k)
+		}
+
+		if len(v) == 0 {
+			return fmt.Errorf("empty value stored for privacy "+
+				"pairs key %s", k)
+		}
+
+		pairsRes[string(k)] = string(v)
+
+		return nil
+	})
+}
+
+// insertPrivacyPairs inserts the collected privacy pairs into the SQL database.
+func insertPrivacyPairs(ctx context.Context, sqlTx SQLQueries,
+	pairs privacyPairs) error {
+
+	for groupId, groupPairs := range pairs {
+		err := insertGroupPairs(ctx, sqlTx, groupPairs, groupId)
+		if err != nil {
+			return fmt.Errorf("inserting group pairs for group "+
+				"id %d failed: %w", groupId, err)
+		}
+	}
+
+	return nil
+}
+
+// insertGroupPairs inserts the privacy pairs for a specific group into
+// the SQL database. It checks for duplicates before inserting, and returns
+// an error if a duplicate pair is found. The function takes a map of real
+// to pseudo values, where the key is the real value and the value is the
+// corresponding pseudo value.
+func insertGroupPairs(ctx context.Context, sqlTx SQLQueries,
+	pairs map[string]string, groupID int64) error {
+
+	for realVal, pseudoVal := range pairs {
+		_, err := sqlTx.GetPseudoForReal(
+			ctx, sqlc.GetPseudoForRealParams{
+				GroupID: groupID,
+				RealVal: realVal,
+			},
+		)
+		if err == nil {
+			return fmt.Errorf("duplicate privacy pair %s:%s: %w",
+				realVal, pseudoVal, ErrDuplicatePseudoValue)
+		} else if !errors.Is(err, sql.ErrNoRows) {
+			return err
+		}
+
+		_, err = sqlTx.GetRealForPseudo(
+			ctx, sqlc.GetRealForPseudoParams{
+				GroupID:   groupID,
+				PseudoVal: pseudoVal,
+			},
+		)
+		if err == nil {
+			return fmt.Errorf("duplicate privacy pair %s:%s: %w",
+				realVal, pseudoVal, ErrDuplicatePseudoValue)
+		} else if !errors.Is(err, sql.ErrNoRows) {
+			return err
+		}
+
+		err = sqlTx.InsertPrivacyPair(
+			ctx, sqlc.InsertPrivacyPairParams{
+				GroupID:   groupID,
+				RealVal:   realVal,
+				PseudoVal: pseudoVal,
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("inserting privacy pair %s:%s "+
+				"failed: %w", realVal, pseudoVal, err)
+		}
+	}
+
+	return nil
+}
+
+// validatePrivacyPairsMigration validates that the migrated privacy pairs
+// match the original values in the KV store.
+func validatePrivacyPairsMigration(ctx context.Context, sqlTx SQLQueries,
+	pairs privacyPairs) error {
+
+	for groupId, groupPairs := range pairs {
+		err := validateGroupPairsMigration(
+			ctx, sqlTx, groupPairs, groupId,
+		)
+		if err != nil {
+			return fmt.Errorf("migration validation of privacy "+
+				"pairs for group %d failed: %w", groupId, err)
+		}
+	}
+
+	return nil
+}
+
+// validateGroupPairsMigration validates that the migrated privacy pairs for
+// a specific group match the original values in the KV store. It checks that
+// for each real value, the pseudo value in the SQL database matches the
+// original pseudo value, and vice versa. If any mismatch is found, it returns
+// an error indicating the mismatch.
+func validateGroupPairsMigration(ctx context.Context, sqlTx SQLQueries,
+	pairs map[string]string, groupID int64) error {
+
+	for realVal, pseudoVal := range pairs {
+		resPseudoVal, err := sqlTx.GetPseudoForReal(
+			ctx, sqlc.GetPseudoForRealParams{
+				GroupID: groupID,
+				RealVal: realVal,
+			},
+		)
+		if errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("migrated privacy pair %s:%s not "+
+				"found for real value", realVal, pseudoVal)
+		}
+		if err != nil {
+			return err
+		}
+
+		if resPseudoVal != pseudoVal {
+			return fmt.Errorf("pseudo value in db %s, does not "+
+				"match original value %s, for real value %s",
+				resPseudoVal, pseudoVal, realVal)
+		}
+
+		resRealVal, err := sqlTx.GetRealForPseudo(
+			ctx, sqlc.GetRealForPseudoParams{
+				GroupID:   groupID,
+				PseudoVal: pseudoVal,
+			},
+		)
+		if errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("migrated privacy pair %s:%s not "+
+				"found for pseudo value", realVal, pseudoVal)
+		}
+		if err != nil {
+			return err
+		}
+
+		if resRealVal != realVal {
+			return fmt.Errorf("real value in db %s, does not "+
+				"match original value %s, for pseudo value %s",
+				resRealVal, realVal, pseudoVal)
+		}
+	}
+
+	return nil
+}