Fix delta external table schema diff, script syntax, and cleanup (#164)

* Fix delta external table schema diff, script syntax, and cleanup

Three fixes for delta external tables:

1. Schema perpetual diff: The bulk loader populates the schema for all
   external tables from the cluster, but delta tables auto-infer their
   schema from the delta log — so YAML configs intentionally omit it.
   This caused a perpetual diff on every run. Fix: before comparing,
   clear the cluster-side schema for delta tables when the YAML doesn't
   specify one. If the YAML provides a custom schema, keep it for
   proper comparison.

2. Missing closing paren: CreateDeltaScript generated an unclosed
   parenthesis around the connection string, producing invalid KQL
   that fails Kusto syntax validation and cannot be applied.

3. Remove unnecessary dataFormat requirement: Delta tables don't use
   dataformat= in the Kusto command syntax. Removed the misleading
   validation that required it.

4. Only emit with() properties when set: Delta script no longer emits
   empty folder='', docString='', fileExtension='' when not specified
   in YAML. The with() block is omitted entirely if no properties are
   set.

Fixes github/data#11973

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Address comments

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: ashleyvansp

KustoSchemaTools/Changes/DatabaseChanges.cs

@@ -113,6 +113,22 @@ public static List<IChange> GenerateChanges(Database oldState, Database newState
             result.AddRange(mvChanges);
             result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.ContinuousExports, nameof(newState.ContinuousExports), log));
             result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.Functions, nameof(newState.Functions), log));
+
+            // For delta external tables, the schema is auto-inferred from the delta log.
+            // If the YAML doesn't specify a schema, clear the cluster-side schema so it
+            // doesn't cause a perpetual diff. If the YAML does specify a schema, keep the
+            // cluster-side schema for proper comparison.
+            foreach (var et in newState.ExternalTables)
+            {
+                if (et.Value.Kind?.ToLower() == "delta"
+                    && et.Value.Schema?.Any() != true
+                    && oldState.ExternalTables.TryGetValue(et.Key, out var oldExternalTable)
+                    && oldExternalTable.Kind?.ToLower() == "delta")
+                {
+                    oldExternalTable.Schema = null;
+                }
+            }
+
             result.AddRange(GenerateScriptCompareChanges(oldState, newState, db => db.ExternalTables, nameof(newState.ExternalTables), log));
 
             if (newState.EntityGroups.Any())

KustoSchemaTools/Model/ExternalTable.cs

@@ -137,7 +137,7 @@ private string CreateSqlScript(string name)
 
         private string CreateDeltaScript(string name)
         {
-            if (string.IsNullOrWhiteSpace(DataFormat)) throw new ArgumentException("DataFormat can't be empty");
+            if (string.IsNullOrWhiteSpace(ConnectionString)) throw new ArgumentException("ConnectionString can't be empty");
 
             var sb = new StringBuilder();
             sb.AppendLine($".create-or-alter external table {name}");
@@ -146,11 +146,18 @@ private string CreateDeltaScript(string name)
                 sb.AppendLine($"({string.Join(", ", Schema.Select(c => $"{c.Key.BracketIfIdentifier()}:{c.Value}"))})");
             }
             sb.AppendLine("kind=delta");            
-            sb.AppendLine($"(h@'{ConnectionString}'");
+            sb.AppendLine($"(h@'{ConnectionString}')");
 
+            var withProps = new List<string>();
+            if (!string.IsNullOrEmpty(Folder)) withProps.Add($"folder='{Folder}'");
+            if (!string.IsNullOrEmpty(DocString)) withProps.Add($"docString='{DocString}'");
             var ext = string.IsNullOrWhiteSpace(FileExtensions) ? "" : FileExtensions.StartsWith(".") ? FileExtensions : "." + FileExtensions;
+            if (!string.IsNullOrEmpty(ext)) withProps.Add($"fileExtension='{ext}'");
 
-            sb.AppendLine($"with(folder='{Folder}', docString='{DocString}', fileExtension='{ext}') ");
+            if (withProps.Any())
+            {
+                sb.AppendLine($"with({string.Join(", ", withProps)})");
+            }
 
             return sb.ToString();
         }