Skip to content

feat: add schemaVersion to systemMetadata #16778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chakru-r merged 1 commit into from
Mar 25, 2026
Merged

feat: add schemaVersion to systemMetadata #16778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,11 @@
"type" : [ "null", "string" ],
"doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.",
"default" : null
}, {
"name" : "schemaVersion",
"type" : [ "null", "long" ],
"doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.",
"default" : null
}, {
"name" : "aspectCreated",
"type" : [ "null", {
Expand Down
5 changes: 5 additions & 0 deletions metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,11 @@
"type" : [ "null", "string" ],
"doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.",
"default" : null
}, {
"name" : "schemaVersion",
"type" : [ "null", "long" ],
"doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.",
"default" : null
}, {
"name" : "aspectCreated",
"type" : [ "null", {
Expand Down
8 changes: 7 additions & 1 deletion metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import com.linkedin.common.AuditStamp
*/
@Aspect = {
"name": "systemMetadata"
}
}
record SystemMetadata {
/**
* The timestamp the metadata was observed at
Expand Down Expand Up @@ -51,6 +51,12 @@ record SystemMetadata {
*/
version: optional string

/**
* Schema version of the aspect data model. Used to determine aspects that need migrations.
* Defaults to 1 when not present as a baseline. Incremented when an aspect is modified.
*/
schemaVersion: optional long

/**
* When the aspect was initially created and who created it, detected by version 0 -> 1 change
*/
Expand Down
25 changes: 22 additions & 3 deletions metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,11 @@
"type" : "string",
"doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.",
"optional" : true
}, {
"name" : "schemaVersion",
"type" : "long",
"doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.",
"optional" : true
}, {
"name" : "aspectCreated",
"type" : {
Expand Down Expand Up @@ -454,7 +459,13 @@
"optional" : true
} ]
},
"doc" : "Captures information about who created/last modified/deleted this chart and when"
"doc" : "Captures information about who created/last modified/deleted this chart and when",
"Searchable" : {
"/lastModified/time" : {
"fieldName" : "lastModifiedAt",
"fieldType" : "DATETIME"
}
}
}, {
"name" : "chartUrl",
"type" : "com.linkedin.common.Url",
Expand Down Expand Up @@ -1484,6 +1495,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -1644,6 +1656,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -1721,6 +1734,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -2165,6 +2179,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -2214,7 +2229,7 @@
"type" : "typeref",
"name" : "SchemaFieldPath",
"namespace" : "com.linkedin.dataset",
"doc" : "Schema field path. TODO: Add formal documentation on normalization rules.",
"doc" : "Schema field path.\n\nFor formal documentation on normalization rules, see docs/advanced/field-path-spec-v2.md\nor https://github.com/datahub-project/datahub/blob/master/docs/advanced/field-path-spec-v2.md",
"ref" : "string"
}, {
"type" : "record",
Expand Down Expand Up @@ -3151,7 +3166,8 @@
"Searchable" : {
"boostScore" : 0.1,
"fieldName" : "fieldDescriptions",
"fieldType" : "TEXT"
"fieldType" : "TEXT",
"sanitizeRichText" : true
}
}, {
"name" : "label",
Expand Down Expand Up @@ -3503,6 +3519,7 @@
"boostScore" : 0.1,
"fieldName" : "editedFieldDescriptions",
"fieldType" : "TEXT",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -3701,6 +3718,7 @@
"Searchable" : {
"boostScore" : 10.0,
"enableAutocomplete" : true,
"fieldName" : "id",
"fieldNameAliases" : [ "_entityName" ],
"fieldType" : "WORD_GRAM",
"searchLabel" : "entityName",
Expand Down Expand Up @@ -4035,6 +4053,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down
33 changes: 29 additions & 4 deletions metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,13 @@
"optional" : true
} ]
},
"doc" : "Captures information about who created/last modified/deleted this chart and when"
"doc" : "Captures information about who created/last modified/deleted this chart and when",
"Searchable" : {
"/lastModified/time" : {
"fieldName" : "lastModifiedAt",
"fieldType" : "DATETIME"
}
}
}, {
"name" : "chartUrl",
"type" : "com.linkedin.common.Url",
Expand Down Expand Up @@ -349,7 +355,8 @@
"optional" : true,
"Searchable" : {
"fieldName" : "editedDescription",
"fieldType" : "TEXT"
"fieldType" : "TEXT",
"sanitizeRichText" : true
}
} ],
"Aspect" : {
Expand Down Expand Up @@ -1503,6 +1510,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -1651,6 +1659,7 @@
"Searchable" : {
"fieldName" : "editedDescription",
"fieldType" : "TEXT",
"sanitizeRichText" : true,
"searchTier" : 2
}
} ],
Expand Down Expand Up @@ -1683,6 +1692,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -1760,6 +1770,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -2133,6 +2144,7 @@
"Searchable" : {
"fieldName" : "editedDescription",
"fieldType" : "TEXT",
"sanitizeRichText" : true,
"searchTier" : 2
}
} ],
Expand All @@ -2153,6 +2165,7 @@
"Searchable" : {
"fieldName" : "editedDescription",
"fieldType" : "TEXT",
"sanitizeRichText" : true,
"searchTier" : 2
}
} ],
Expand Down Expand Up @@ -2389,6 +2402,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -2465,6 +2479,7 @@
"Searchable" : {
"fieldName" : "editedDescription",
"fieldType" : "TEXT",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand All @@ -2484,7 +2499,7 @@
"type" : "typeref",
"name" : "SchemaFieldPath",
"namespace" : "com.linkedin.dataset",
"doc" : "Schema field path. TODO: Add formal documentation on normalization rules.",
"doc" : "Schema field path.\n\nFor formal documentation on normalization rules, see docs/advanced/field-path-spec-v2.md\nor https://github.com/datahub-project/datahub/blob/master/docs/advanced/field-path-spec-v2.md",
"ref" : "string"
}, {
"type" : "record",
Expand Down Expand Up @@ -3534,7 +3549,8 @@
"Searchable" : {
"boostScore" : 0.1,
"fieldName" : "fieldDescriptions",
"fieldType" : "TEXT"
"fieldType" : "TEXT",
"sanitizeRichText" : true
}
}, {
"name" : "label",
Expand Down Expand Up @@ -3886,6 +3902,7 @@
"boostScore" : 0.1,
"fieldName" : "editedFieldDescriptions",
"fieldType" : "TEXT",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -4105,6 +4122,7 @@
"Searchable" : {
"boostScore" : 10.0,
"enableAutocomplete" : true,
"fieldName" : "id",
"fieldNameAliases" : [ "_entityName" ],
"fieldType" : "WORD_GRAM",
"searchLabel" : "entityName",
Expand Down Expand Up @@ -4186,6 +4204,7 @@
"Searchable" : {
"fieldType" : "TEXT",
"hasValuesFieldName" : "hasDescription",
"sanitizeRichText" : true,
"searchTier" : 2
}
}, {
Expand Down Expand Up @@ -5146,6 +5165,7 @@
"Searchable" : {
"boostScore" : 10.0,
"enableAutocomplete" : true,
"fieldName" : "id",
"fieldNameAliases" : [ "_entityName" ],
"fieldType" : "WORD_GRAM",
"searchLabel" : "entityName",
Expand Down Expand Up @@ -6967,6 +6987,11 @@
"type" : "string",
"doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.",
"optional" : true
}, {
"name" : "schemaVersion",
"type" : "long",
"doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.",
"optional" : true
}, {
"name" : "aspectCreated",
"type" : "com.linkedin.common.AuditStamp",
Expand Down
5 changes: 5 additions & 0 deletions metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,11 @@
"type" : "string",
"doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.",
"optional" : true
}, {
"name" : "schemaVersion",
"type" : "long",
"doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.",
"optional" : true
}, {
"name" : "aspectCreated",
"type" : "com.linkedin.common.AuditStamp",
Expand Down
5 changes: 5 additions & 0 deletions ...ervice/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@
"type" : "string",
"doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.",
"optional" : true
}, {
"name" : "schemaVersion",
"type" : "long",
"doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.",
"optional" : true
}, {
"name" : "aspectCreated",
"type" : "com.linkedin.common.AuditStamp",
Expand Down
Loading
Loading