Recursive sql templates (just experimental, little tested).

Author: poser55

README.md

@@ -261,3 +261,4 @@ Further Ideas
 * Extension ideas
     - Do more unification of Datatype handling. E.g. oracle treats DATE different from Postgres (so at the moment
       we need to adapt it in the JSON/ DbRecord). Refer e.g. to DbExporterBasicTests#datatypesTest(). 
+* Oracle testcontainer does not run on Arm osx. To try things out: https://dbfiddle.uk/Wh4zpMKJ

src/main/java/org/oser/tools/jdbc/DbExporter.java

@@ -25,6 +25,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -479,4 +480,28 @@ public Map<String, FieldExporter> getTypeFieldExporters() {
     public void setOrderResults(boolean orderResults) {
         this.orderResults = orderResults;
     }
+
+
+    /**
+     * Get only the cache entries that are excluded by the stopTablesExcluded
+     */
+    public Cache<String, List<Fk>> getFilteredFkCache() {
+        ConcurrentMap<String, List<Fk>> map = fkCache.asMap();
+        List<Map.Entry<String, List<Fk>>> removedKeys = map.entrySet().stream().filter(e -> !stopTablesExcluded.contains(e.getKey())).toList();
+
+        Cache<String, List<Fk>> fkCacheSubset1 = Caffeine.newBuilder()
+                .maximumSize(1000).build();
+        removedKeys.stream().map(e -> filterEntry(e, stopTablesExcluded)).forEach(e -> fkCacheSubset1.put(e.key(), e.fks()));
+        Cache<String, List<Fk>> fkCacheSubset = fkCacheSubset1;
+        return fkCacheSubset;
+    }
+
+    private static KeyAndFkList filterEntry(Map.Entry<String, List<Fk>> e, Set<String> stopTablesExcluded) {
+        List<Fk> list = e.getValue().stream().filter(fk -> !(stopTablesExcluded.contains(fk.getPktable()) || stopTablesExcluded.contains(fk.getFktable()))).toList();
+        return new KeyAndFkList(e.getKey(), list);
+    }
+
+    record KeyAndFkList(String key, List<Fk> fks) {
+    }
+
 }

src/main/java/org/oser/tools/jdbc/Fk.java

@@ -30,8 +30,8 @@
  * <p>
  * In JDBC <em>one</em> fk constraint between table 1 and table 2 has <em>two</em> representations: the link from
  * table 1 to table 2 and vice versa. One of the 2 constraints is <em>inverted</em>, refer to the field <code>inverted</code>.
- *  <a>
- *  Supports virtual foreign keys (that do not exist in the db).
+ * <a>
+ * Supports virtual foreign keys (that do not exist in the db).
  */
 @Getter
 public class Fk {
@@ -84,6 +84,25 @@ public Fk(String pktable, String[] pkcolumnArray, String fktable, String[] fkcol
         this.inverted = inverted;
     }
 
+    // convenience accessors: they check the inverted flag to either return the 1st or the 2nd entry
+    // todo can we simplify code with these methods?
+
+    public String getFirstTable() {
+        return inverted ? pktable : fktable;
+    }
+
+    public String getSecondTable() {
+        return inverted ? fktable : pktable;
+    }
+
+    public String[] getFirstColumn() {
+        return inverted ? pkcolumn : fkcolumn;
+    }
+
+    public String[] getSecondColumn() {
+        return inverted ? fkcolumn : pkcolumn;
+    }
+
 
     /**
      * Refer to {@link #getFksOfTable(Connection, String)}
@@ -340,7 +359,30 @@ public static void addVirtualForeignKeyAsString(Connection dbConnection, FkCache
         }
     }
 
-    /** To parse String FKs */
+    /**
+     * Same as #addOneVirtualForeignKeyAsString but only fills the cache (without a connection)
+     */
+    public static void addOneVirtualForeignKeyAsString(FkCacheAccessor importerOrExporter, String asString) {
+        FkMatchedFields fkMatchedFields = new FkMatchedFields(asString).parse();
+        String table1 = fkMatchedFields.getTable1();
+        String fields1AsString = fkMatchedFields.getFields1AsString();
+        String table2 = fkMatchedFields.getTable2();
+        String fields2AsString = fkMatchedFields.getFields2AsString();
+
+        List<Fk> table1fks = importerOrExporter.getFkCache().getIfPresent(table1);
+        table1fks = table1fks == null ? new ArrayList<>() : table1fks;
+        table1fks.add(new Fk(table1, fields1AsString.split(","), table2, fields2AsString.split(","), "1", table1 + fields1AsString.split(",")[0], false));
+        importerOrExporter.getFkCache().put(table1, table1fks);
+
+        List<Fk> table2fks = importerOrExporter.getFkCache().getIfPresent(table2);
+        table2fks = table2fks == null ? new ArrayList<>() : table2fks;
+        table2fks.add(new Fk(table1, fields1AsString.split(","), table2, fields2AsString.split(","), "1", table1 + fields1AsString.split(",")[0], true));
+        importerOrExporter.getFkCache().put(table2, table2fks);
+    }
+
+    /**
+     * o parse String FKs
+     */
     @Getter
     static class FkMatchedFields {
         private String asString;

src/main/java/org/oser/tools/jdbc/JdbcHelpers.java

@@ -86,6 +86,7 @@ public static class Pair<A,B> {
      * @param treated init treated to Set with all entries
      * @param exceptionWithCycles whether we should throw an exception if there are cycles
      * @return the sorted list of entries
+     * @throws IllegalStateException in case of cycles if exceptionWithCycles is true
      * */
     public static <T> List<T> topologicalSort(Map<T, Set<T>> dependencyGraph, Set<T> treated, boolean exceptionWithCycles) {
         List<T> orderedTables = new ArrayList<>();

src/main/java/org/oser/tools/jdbc/Loggers.java

@@ -15,25 +15,37 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-/** Global logging convenience abstraction (to globally enable certain details in logs, such as all select statements or all insert/update statements).
- *  Just uses Slf4j. In case logback is configured, sets the log levels in logback (users of other log implementations need to set their log levels themselves). */
+/**
+ * Global logging convenience abstraction (to globally enable certain details in logs, such as all select statements or all insert/update statements).
+ * Just uses Slf4j. In case logback is configured, sets the log levels in logback (users of other log implementations need to set their log levels themselves).
+ */
 public enum Loggers {
 
-        /** SQL select statements */
-        SELECT,
-        /** SQL update and insert statements */
-        CHANGE,
-        /** SQL delete statements */
-        DELETE,
-        WARNING,
-        INFO,
-        /** meta Logger to mean all db operations */
-        DB_OPERATIONS,
-        /** meta Logger means all Loggers */
-        ALL;
-
-
-    static final Set<Loggers> CONCRETE_LOGGERS = EnumSet.of(Loggers.SELECT, Loggers.CHANGE, Loggers.DELETE, Loggers.WARNING,  Loggers.INFO);
+    /**
+     * SQL select statements
+     */
+    SELECT,
+    /**
+     * SQL update and insert statements
+     */
+    CHANGE,
+    /**
+     * SQL delete statements
+     */
+    DELETE,
+    WARNING,
+    INFO,
+    /**
+     * meta Logger to mean all db operations
+     */
+    DB_OPERATIONS,
+    /**
+     * meta Logger means all Loggers
+     */
+    ALL;
+
+
+    static final Set<Loggers> CONCRETE_LOGGERS = EnumSet.of(Loggers.SELECT, Loggers.CHANGE, Loggers.DELETE, Loggers.WARNING, Loggers.INFO);
     static final Set<Loggers> CONCRETE_DB_OPERATIONS = EnumSet.of(Loggers.SELECT, Loggers.CHANGE, Loggers.DELETE);
     static final Set<Loggers> ALL_LOGGERS = CONCRETE_LOGGERS;
 
@@ -45,12 +57,16 @@ public enum Loggers {
 
     static boolean missingLoggerSignalled = false;
 
-    /** Convenience method to enable what you would like to see in the logs */
+    /**
+     * Convenience method to enable what you would like to see in the logs
+     */
     public static void enableLoggers(Set<Loggers> loggers) {
         setLoggerLevel(loggers, Level.INFO);
     }
 
-    /** Convenience method to enable what you would like to see in the logs */
+    /**
+     * Convenience method to enable what you would like to see in the logs
+     */
     public static void enableLoggers(Loggers... loggers) {
         setLoggerLevel(new HashSet(Arrays.asList(loggers)), Level.INFO);
     }
@@ -86,17 +102,19 @@ static void setLoggerLevel(Set<Loggers> loggers, Level newLevel) {
         disableDefaultLogs();
     }
 
-    /** Convenience method to only show warning logs */
+    /**
+     * Convenience method to only show warning logs
+     */
     public static void disableDefaultLogs() {
         setLoggerLevel(CONCRETE_LOGGERS, Level.WARN);
     }
 
 
-    public static Set<Loggers> stringListToLoggerSet(List<String> logs){
+    public static Set<Loggers> stringListToLoggerSet(List<String> logs) {
         return logs.stream().map(String::toUpperCase).map(n -> optionalGetLogger(n)).flatMap(Optional::stream).collect(Collectors.toSet());
     }
 
-    private static Optional<Loggers> optionalGetLogger(String logger){
+    private static Optional<Loggers> optionalGetLogger(String logger) {
         try {
             return Optional.of(Loggers.valueOf(logger));
         } catch (IllegalArgumentException e) {

src/main/java/org/oser/tools/jdbc/cli/ExecuteDbScriptFiles.java

@@ -29,7 +29,11 @@ public static void executeDbScriptFiles(String sqlDirectory, Connection connecti
                     .map(Path::toString).sorted().collect(toList());
 
             for (String fileName : sqlFiles) {
-                executeSqlFile(connection, fileName, placeholders);
+                try {
+                    executeSqlFile(connection, fileName, placeholders);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
             }
         }
     }