diff --git a/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc b/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc index 06a994625c5b1b..4e4d5d8cc24bbc 100644 --- a/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc +++ b/metadata-integration/java/datahub-client/src/main/resources/MetadataChangeProposal.avsc @@ -172,6 +172,11 @@ "type" : [ "null", "string" ], "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", "default" : null + }, { + "name" : "schemaVersion", + "type" : [ "null", "long" ], + "doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.", + "default" : null }, { "name" : "aspectCreated", "type" : [ "null", { diff --git a/metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc b/metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc index 06a994625c5b1b..4e4d5d8cc24bbc 100644 --- a/metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc +++ b/metadata-integration/java/datahub-event/src/main/resources/MetadataChangeProposal.avsc @@ -172,6 +172,11 @@ "type" : [ "null", "string" ], "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", "default" : null + }, { + "name" : "schemaVersion", + "type" : [ "null", "long" ], + "doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.", + "default" : null }, { "name" : "aspectCreated", "type" : [ "null", { diff --git a/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl index 7973174b995a3e..31283f3310f316 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl @@ -7,7 +7,7 @@ import com.linkedin.common.AuditStamp */ @Aspect = { "name": "systemMetadata" - } +} record SystemMetadata { /** * The timestamp the metadata was observed at @@ -51,6 +51,12 @@ record SystemMetadata { */ version: optional string + /** + * Schema version of the aspect data model. Used to determine aspects that need migrations. + * Defaults to 1 when not present as a baseline. Incremented when an aspect is modified. + */ + schemaVersion: optional long + /** * When the aspect was initially created and who created it, detected by version 0 -> 1 change */ diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index 9ea2aa5b44229e..d60cf3c9b016d2 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -106,6 +106,11 @@ "type" : "string", "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", "optional" : true + }, { + "name" : "schemaVersion", + "type" : "long", + "doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.", + "optional" : true }, { "name" : "aspectCreated", "type" : { @@ -454,7 +459,13 @@ "optional" : true } ] }, - "doc" : "Captures information about who created/last modified/deleted this chart and when" + "doc" : "Captures information about who created/last modified/deleted this chart and when", + "Searchable" : { + "/lastModified/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } }, { "name" : "chartUrl", "type" : "com.linkedin.common.Url", @@ -1484,6 +1495,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1644,6 +1656,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1721,6 +1734,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -2165,6 +2179,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -2214,7 +2229,7 @@ "type" : "typeref", "name" : "SchemaFieldPath", "namespace" : "com.linkedin.dataset", - "doc" : "Schema field path. TODO: Add formal documentation on normalization rules.", + "doc" : "Schema field path.\n\nFor formal documentation on normalization rules, see docs/advanced/field-path-spec-v2.md\nor https://github.com/datahub-project/datahub/blob/master/docs/advanced/field-path-spec-v2.md", "ref" : "string" }, { "type" : "record", @@ -3151,7 +3166,8 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "fieldDescriptions", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "sanitizeRichText" : true } }, { "name" : "label", @@ -3503,6 +3519,7 @@ "boostScore" : 0.1, "fieldName" : "editedFieldDescriptions", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -3701,6 +3718,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, + "fieldName" : "id", "fieldNameAliases" : [ "_entityName" ], "fieldType" : "WORD_GRAM", "searchLabel" : "entityName", @@ -4035,6 +4053,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index c4d2bb18ebcc2d..7881b8ffcf71bd 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -173,7 +173,13 @@ "optional" : true } ] }, - "doc" : "Captures information about who created/last modified/deleted this chart and when" + "doc" : "Captures information about who created/last modified/deleted this chart and when", + "Searchable" : { + "/lastModified/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } }, { "name" : "chartUrl", "type" : "com.linkedin.common.Url", @@ -349,7 +355,8 @@ "optional" : true, "Searchable" : { "fieldName" : "editedDescription", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "sanitizeRichText" : true } } ], "Aspect" : { @@ -1503,6 +1510,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1651,6 +1659,7 @@ "Searchable" : { "fieldName" : "editedDescription", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } } ], @@ -1683,6 +1692,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1760,6 +1770,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -2133,6 +2144,7 @@ "Searchable" : { "fieldName" : "editedDescription", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } } ], @@ -2153,6 +2165,7 @@ "Searchable" : { "fieldName" : "editedDescription", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } } ], @@ -2389,6 +2402,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -2465,6 +2479,7 @@ "Searchable" : { "fieldName" : "editedDescription", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -2484,7 +2499,7 @@ "type" : "typeref", "name" : "SchemaFieldPath", "namespace" : "com.linkedin.dataset", - "doc" : "Schema field path. TODO: Add formal documentation on normalization rules.", + "doc" : "Schema field path.\n\nFor formal documentation on normalization rules, see docs/advanced/field-path-spec-v2.md\nor https://github.com/datahub-project/datahub/blob/master/docs/advanced/field-path-spec-v2.md", "ref" : "string" }, { "type" : "record", @@ -3534,7 +3549,8 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "fieldDescriptions", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "sanitizeRichText" : true } }, { "name" : "label", @@ -3886,6 +3902,7 @@ "boostScore" : 0.1, "fieldName" : "editedFieldDescriptions", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -4105,6 +4122,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, + "fieldName" : "id", "fieldNameAliases" : [ "_entityName" ], "fieldType" : "WORD_GRAM", "searchLabel" : "entityName", @@ -4186,6 +4204,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -5146,6 +5165,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, + "fieldName" : "id", "fieldNameAliases" : [ "_entityName" ], "fieldType" : "WORD_GRAM", "searchLabel" : "entityName", @@ -6967,6 +6987,11 @@ "type" : "string", "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", "optional" : true + }, { + "name" : "schemaVersion", + "type" : "long", + "doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.", + "optional" : true }, { "name" : "aspectCreated", "type" : "com.linkedin.common.AuditStamp", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json index 36347a6646b83d..9051f891b99db5 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesV2.snapshot.json @@ -151,6 +151,11 @@ "type" : "string", "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", "optional" : true + }, { + "name" : "schemaVersion", + "type" : "long", + "doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.", + "optional" : true }, { "name" : "aspectCreated", "type" : "com.linkedin.common.AuditStamp", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json index ce1be464d7d6b3..96f5df62ce1a72 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entitiesVersionedV2.snapshot.json @@ -160,6 +160,11 @@ "type" : "string", "doc" : "Aspect version\n Initial implementation will use the aspect version's number, however stored as\n a string in the case where a different aspect versioning scheme is later adopted.", "optional" : true + }, { + "name" : "schemaVersion", + "type" : "long", + "doc" : "Schema version of the aspect data model. Used to determine aspects that need migrations.\nDefaults to 1 when not present as a baseline. Incremented when an aspect is modified.", + "optional" : true }, { "name" : "aspectCreated", "type" : "com.linkedin.common.AuditStamp", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json index 0bd4a5dff8da92..6db253ecd5677d 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json @@ -173,7 +173,13 @@ "optional" : true } ] }, - "doc" : "Captures information about who created/last modified/deleted this chart and when" + "doc" : "Captures information about who created/last modified/deleted this chart and when", + "Searchable" : { + "/lastModified/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } }, { "name" : "chartUrl", "type" : "com.linkedin.common.Url", @@ -1203,6 +1209,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1363,6 +1370,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1440,6 +1448,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1884,6 +1893,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1933,7 +1943,7 @@ "type" : "typeref", "name" : "SchemaFieldPath", "namespace" : "com.linkedin.dataset", - "doc" : "Schema field path. TODO: Add formal documentation on normalization rules.", + "doc" : "Schema field path.\n\nFor formal documentation on normalization rules, see docs/advanced/field-path-spec-v2.md\nor https://github.com/datahub-project/datahub/blob/master/docs/advanced/field-path-spec-v2.md", "ref" : "string" }, { "type" : "record", @@ -2861,7 +2871,8 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "fieldDescriptions", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "sanitizeRichText" : true } }, { "name" : "label", @@ -3213,6 +3224,7 @@ "boostScore" : 0.1, "fieldName" : "editedFieldDescriptions", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -3411,6 +3423,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, + "fieldName" : "id", "fieldNameAliases" : [ "_entityName" ], "fieldType" : "WORD_GRAM", "searchLabel" : "entityName", @@ -3745,6 +3758,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json index 75d9b636325566..45f4e5d201cfdb 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json @@ -173,7 +173,13 @@ "optional" : true } ] }, - "doc" : "Captures information about who created/last modified/deleted this chart and when" + "doc" : "Captures information about who created/last modified/deleted this chart and when", + "Searchable" : { + "/lastModified/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } }, { "name" : "chartUrl", "type" : "com.linkedin.common.Url", @@ -1203,6 +1209,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1363,6 +1370,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1440,6 +1448,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1884,6 +1893,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1933,7 +1943,7 @@ "type" : "typeref", "name" : "SchemaFieldPath", "namespace" : "com.linkedin.dataset", - "doc" : "Schema field path. TODO: Add formal documentation on normalization rules.", + "doc" : "Schema field path.\n\nFor formal documentation on normalization rules, see docs/advanced/field-path-spec-v2.md\nor https://github.com/datahub-project/datahub/blob/master/docs/advanced/field-path-spec-v2.md", "ref" : "string" }, { "type" : "record", @@ -2855,7 +2865,8 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "fieldDescriptions", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "sanitizeRichText" : true } }, { "name" : "label", @@ -3207,6 +3218,7 @@ "boostScore" : 0.1, "fieldName" : "editedFieldDescriptions", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -3405,6 +3417,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, + "fieldName" : "id", "fieldNameAliases" : [ "_entityName" ], "fieldType" : "WORD_GRAM", "searchLabel" : "entityName", @@ -3739,6 +3752,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index 803058e324a8f1..c2cddc70c5a2b2 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -173,7 +173,13 @@ "optional" : true } ] }, - "doc" : "Captures information about who created/last modified/deleted this chart and when" + "doc" : "Captures information about who created/last modified/deleted this chart and when", + "Searchable" : { + "/lastModified/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } }, { "name" : "chartUrl", "type" : "com.linkedin.common.Url", @@ -349,7 +355,8 @@ "optional" : true, "Searchable" : { "fieldName" : "editedDescription", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "sanitizeRichText" : true } } ], "Aspect" : { @@ -1503,6 +1510,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1651,6 +1659,7 @@ "Searchable" : { "fieldName" : "editedDescription", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } } ], @@ -1683,6 +1692,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -1760,6 +1770,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -2133,6 +2144,7 @@ "Searchable" : { "fieldName" : "editedDescription", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } } ], @@ -2153,6 +2165,7 @@ "Searchable" : { "fieldName" : "editedDescription", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } } ], @@ -2389,6 +2402,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -2465,6 +2479,7 @@ "Searchable" : { "fieldName" : "editedDescription", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -2484,7 +2499,7 @@ "type" : "typeref", "name" : "SchemaFieldPath", "namespace" : "com.linkedin.dataset", - "doc" : "Schema field path. TODO: Add formal documentation on normalization rules.", + "doc" : "Schema field path.\n\nFor formal documentation on normalization rules, see docs/advanced/field-path-spec-v2.md\nor https://github.com/datahub-project/datahub/blob/master/docs/advanced/field-path-spec-v2.md", "ref" : "string" }, { "type" : "record", @@ -3528,7 +3543,8 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "fieldDescriptions", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "sanitizeRichText" : true } }, { "name" : "label", @@ -3880,6 +3896,7 @@ "boostScore" : 0.1, "fieldName" : "editedFieldDescriptions", "fieldType" : "TEXT", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -4099,6 +4116,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, + "fieldName" : "id", "fieldNameAliases" : [ "_entityName" ], "fieldType" : "WORD_GRAM", "searchLabel" : "entityName", @@ -4180,6 +4198,7 @@ "Searchable" : { "fieldType" : "TEXT", "hasValuesFieldName" : "hasDescription", + "sanitizeRichText" : true, "searchTier" : 2 } }, { @@ -5140,6 +5159,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, + "fieldName" : "id", "fieldNameAliases" : [ "_entityName" ], "fieldType" : "WORD_GRAM", "searchLabel" : "entityName",