SALTO-582: custom obj translation reference fields and validation rules

package.json

@@ -7,7 +7,7 @@
       "packages/*"
     ],
     "nohoist": [
-      "salto-vscode/**"
+      "vscode/**"
     ]
   },
   "devDependencies": {

packages/adapter-api/src/elements.ts

@@ -298,6 +298,7 @@ export class InstanceElement extends Element {
    * @return {InstanceElement} the cloned instance
    */
   clone(): InstanceElement {
-    return new InstanceElement(this.elemID.name, this.type, _.cloneDeep(this.value), this.path)
+    return new InstanceElement(this.elemID.name, this.type, _.cloneDeep(this.value), this.path,
+      _.cloneDeep(this.annotations))
   }
 }

packages/salesforce-adapter/src/adapter.ts

@@ -56,6 +56,7 @@ import topicsForObjectsFilter from './filters/topics_for_objects'
 import globalValueSetFilter from './filters/global_value_sets'
 import instanceReferences from './filters/instance_references'
 import valueSetFilter from './filters/value_set'
+import customObjectTranslationFilter from './filters/custom_object_translation'
 import {
   FilterCreator, Filter, filtersRunner,
 } from './filter'
@@ -84,6 +85,8 @@ export const DEFAULT_FILTERS = [
   topicsForObjectsFilter,
   valueSetFilter,
   globalValueSetFilter,
+  // customObjectTranslationFilter depends on customObjectsFilter
+  customObjectTranslationFilter,
   // The following filters should remain last in order to make sure they fix all elements
   convertListsFilter,
   convertTypeFilter,

packages/salesforce-adapter/src/constants.ts

@@ -233,6 +233,7 @@ export const TOPICS_FOR_OBJECTS_METADATA_TYPE = 'TopicsForObjects'
 export const PROFILE_METADATA_TYPE = 'Profile'
 export const WORKFLOW_METADATA_TYPE = 'Workflow'
 export const ASSIGNMENT_RULES_METADATA_TYPE = 'AssignmentRules'
+export const VALIDATION_RULES_METADATA_TYPE = 'ValidationRule'
 export const LEAD_CONVERT_SETTINGS_METADATA_TYPE = 'LeadConvertSettings'
 export const QUICK_ACTION_METADATA_TYPE = 'QuickAction'
 export const CUSTOM_TAB_METADATA_TYPE = 'CustomTab'

packages/salesforce-adapter/src/filters/custom_object_translation.ts

@@ -0,0 +1,87 @@
+/*
+*                      Copyright 2020 Salto Labs Ltd.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+import wu from 'wu'
+import {
+  Element, ElemID, ReferenceExpression, Field, ObjectType,
+} from '@salto-io/adapter-api'
+import { findInstances, findElements } from '@salto-io/adapter-utils'
+import { collections } from '@salto-io/lowerdash'
+import _ from 'lodash'
+import { logger } from '@salto-io/logging'
+import { apiName } from '../transformers/transformer'
+import { FilterWith } from '../filter'
+import { instanceShortName, instanceParent, customObjectToMetadataTypeInstances } from './utils'
+import { SALESFORCE, CUSTOM_OBJECT_TRANSLATION_METADATA_TYPE, VALIDATION_RULES_METADATA_TYPE } from '../constants'
+
+const log = logger(module)
+
+const { makeArray } = collections.array
+
+const FIELDS = 'fields'
+const NAME = 'name'
+const VALIDATION_RULES = 'validationRules'
+
+/**
+* This filter change CustomObjectTranslation logical references to fields and validation rules to
+* Salto referenes
+*/
+const filterCreator = (): FilterWith<'onFetch'> => ({
+  onFetch: async (elements: Element[]) => {
+    const allCustomObjectFields = (elemID: ElemID): Iterable<Field> =>
+      wu(findElements(elements, elemID))
+        .map(elem => Object.values((elem as ObjectType).fields))
+        .flatten()
+
+    const customToRule = customObjectToMetadataTypeInstances(
+      elements, VALIDATION_RULES_METADATA_TYPE
+    )
+
+    wu(findInstances(elements, new ElemID(SALESFORCE, CUSTOM_OBJECT_TRANSLATION_METADATA_TYPE)))
+      .forEach(customTranslation => {
+        const customObjectElemId = instanceParent(customTranslation)
+        if (_.isUndefined(customObjectElemId)) {
+          log.warn('failed to find custom object for custom translation %s',
+            apiName(customTranslation))
+          return
+        }
+
+        // Change fields to reference
+        makeArray(customTranslation.value[FIELDS]).forEach(field => {
+          const customField = wu(allCustomObjectFields(customObjectElemId))
+            .find(f => apiName(f, true) === field[NAME])
+          if (customField) {
+            field[NAME] = new ReferenceExpression(customField.elemID)
+          } else {
+            log.warn('failed to find field %s in %s', field[NAME], customObjectElemId.getFullName())
+          }
+        })
+
+        // Change validation rules to refs
+        const objRules = customToRule[customObjectElemId.getFullName()]
+        makeArray(customTranslation.value[VALIDATION_RULES]).forEach(rule => {
+          const ruleInstance = objRules?.find(r => instanceShortName(r) === rule[NAME])
+          if (ruleInstance) {
+            rule[NAME] = new ReferenceExpression(ruleInstance.elemID)
+          } else {
+            log.warn('failed to validation rule %s for %s', rule[NAME],
+              customObjectElemId.getFullName())
+          }
+        })
+      })
+  },
+})
+
+export default filterCreator

packages/salesforce-adapter/src/filters/custom_objects.ts

@@ -35,6 +35,7 @@ import {
   FORMULA, LEAD_CONVERT_SETTINGS_METADATA_TYPE, ASSIGNMENT_RULES_METADATA_TYPE,
   WORKFLOW_METADATA_TYPE, QUICK_ACTION_METADATA_TYPE, CUSTOM_TAB_METADATA_TYPE,
   DUPLICATE_RULE_METADATA_TYPE, CUSTOM_OBJECT_TRANSLATION_METADATA_TYPE,
+  VALIDATION_RULES_METADATA_TYPE,
 } from '../constants'
 import { FilterCreator } from '../filter'
 import {
@@ -43,7 +44,8 @@ import {
 } from '../transformers/transformer'
 import {
   id, addApiName, addMetadataType, addLabel, hasNamespace, getNamespace, boolValue,
-  buildAnnotationsObjectType, generateApiNameToCustomObject, addObjectParentReference,
+  buildAnnotationsObjectType, generateApiNameToCustomObject, addObjectParentReference, apiNameParts,
+  customObjectApiName,
 } from './utils'
 import { convertList } from './convert_lists'
 import { WORKFLOW_FIELD_TO_TYPE } from './workflow'
@@ -68,7 +70,7 @@ export const NESTED_INSTANCE_VALUE_NAME = {
 
 export const NESTED_INSTANCE_TYPE_NAME = {
   WEB_LINK: 'WebLink',
-  VALIDATION_RULE: 'ValidationRule',
+  VALIDATION_RULE: VALIDATION_RULES_METADATA_TYPE,
   BUSINESS_PROCESS: 'BusinessProcess',
   RECORD_TYPE: 'RecordType',
   LIST_VIEW: 'ListView',
@@ -484,9 +486,6 @@ const hasCustomObjectParent = (instance: InstanceElement): boolean =>
   dependentMetadataTypes.has(metadataType(instance))
 
 const fixDependentInstancesPathAndSetParent = (elements: Element[]): void => {
-  const apiNameParts = (instance: InstanceElement): string[] =>
-    apiName(instance).split(/\.|-/g)
-
   const setDependingInstancePath = (instance: InstanceElement, customObject: ObjectType):
     void => {
     if (customObject.path) {
@@ -502,7 +501,7 @@ const fixDependentInstancesPathAndSetParent = (elements: Element[]): void => {
   const apiNameToCustomObject = generateApiNameToCustomObject(elements)
 
   const getDependentCustomObj = (instance: InstanceElement): ObjectType | undefined => {
-    const customObject = apiNameToCustomObject.get(apiNameParts(instance)[0])
+    const customObject = apiNameToCustomObject.get(customObjectApiName(instance))
     if (_.isUndefined(customObject)
       && metadataType(instance) === LEAD_CONVERT_SETTINGS_METADATA_TYPE) {
       return apiNameToCustomObject.get('Lead')

packages/salesforce-adapter/src/filters/utils.ts

@@ -13,7 +13,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-import _ from 'lodash'
+import _, { Dictionary } from 'lodash'
 import { logger } from '@salto-io/logging'
 import {
   Element, Field, isObjectType, ObjectType, InstanceElement, isInstanceElement, isField,
@@ -121,6 +121,18 @@ export const buildAnnotationsObjectType = (annotationTypes: TypeMap): ObjectType
 export const generateApiNameToCustomObject = (elements: Element[]): Map<string, ObjectType> =>
   new Map(getCustomObjects(elements).map(obj => [apiName(obj), obj]))
 
+export const apiNameParts = (instance: InstanceElement): string[] =>
+  apiName(instance).split(/\.|-/g)
+
+export const customObjectApiName = (instance: InstanceElement): string =>
+  apiNameParts(instance)[0]
+
+export const instanceShortName = (instance: InstanceElement): string =>
+  apiName(instance).slice(customObjectApiName(instance).length + 1)
+
+export const instanceParent = (instance: InstanceElement): ElemID | undefined =>
+  instance.annotations[INSTANCE_ANNOTATIONS.PARENT]?.elemId
+
 export const addObjectParentReference = (instance: InstanceElement,
   { elemID: objectID }: ObjectType): void => {
   const instanceDeps = makeArray(instance.annotations[INSTANCE_ANNOTATIONS.PARENT])
@@ -133,3 +145,8 @@ export const addObjectParentReference = (instance: InstanceElement,
 
 export const fullApiName = (parent: string, child: string): string =>
   ([parent, child].join(API_NAME_SEPERATOR))
+
+export const customObjectToMetadataTypeInstances = (elements: Element[], type: string):
+Dictionary<InstanceElement[]> => _(getInstancesOfMetadataType(elements, type))
+  .groupBy(instance => instanceParent(instance)?.getFullName())
+  .value() as Dictionary<InstanceElement[]>

packages/salesforce-adapter/test/filters/custom_object_translation.test.ts

@@ -0,0 +1,151 @@
+/*
+*                      Copyright 2020 Salto Labs Ltd.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+import {
+  ObjectType, InstanceElement, Field, BuiltinTypes, ElemID, ReferenceExpression,
+  INSTANCE_ANNOTATIONS,
+} from '@salto-io/adapter-api'
+import filterCreator from '../../src/filters/custom_object_translation'
+import {
+  SALESFORCE, CUSTOM_OBJECT_TRANSLATION_METADATA_TYPE, METADATA_TYPE, CUSTOM_OBJECT, API_NAME,
+  VALIDATION_RULES_METADATA_TYPE,
+  INSTANCE_FULL_NAME_FIELD,
+} from '../../src/constants'
+import { FilterWith } from '../../src/filter'
+
+describe('custom object translation filter', () => {
+  const customObjName = 'MockCustomObject'
+  const customFieldName = 'MockField'
+  const customObjElemID = new ElemID(SALESFORCE, customObjName)
+  const customObjectAnno = new ObjectType(
+    {
+      elemID: customObjElemID,
+      annotations: {
+        [METADATA_TYPE]: CUSTOM_OBJECT,
+        [API_NAME]: customObjName,
+      },
+    }
+  )
+  const customObjectField = new ObjectType(
+    {
+      elemID: customObjElemID,
+      fields: {
+        [customFieldName]: new Field(customObjElemID, customFieldName, BuiltinTypes.STRING,
+          { [API_NAME]: `${customObjName}.${customFieldName}` }),
+      },
+    }
+  )
+  const customObjectAdditionalField = new ObjectType(
+    {
+      elemID: customObjElemID,
+      fields: {
+        additional: new Field(customObjElemID, 'additional', BuiltinTypes.STRING,
+          { [API_NAME]: `${customObjName}.additional` }),
+      },
+    }
+  )
+  const validationRuleName = 'Last_price_must_for_recently_sold'
+  const validationRuleType = new ObjectType({
+    elemID: new ElemID(SALESFORCE, VALIDATION_RULES_METADATA_TYPE),
+    annotations: { [METADATA_TYPE]: VALIDATION_RULES_METADATA_TYPE },
+  })
+  const validationRuleInstance = new InstanceElement(
+    `${customObjName}_${validationRuleName}`,
+    validationRuleType,
+    { [INSTANCE_FULL_NAME_FIELD]: `${customObjName}.${validationRuleName}` },
+    undefined,
+    { [INSTANCE_ANNOTATIONS.PARENT]: new ReferenceExpression(customObjElemID) }
+  )
+  const fakeValidationRuleInstance = new InstanceElement(
+    `${customObjName}_BLA`,
+    validationRuleType,
+    { [INSTANCE_FULL_NAME_FIELD]: `${customObjName}.BLA` }
+  )
+  const objTranslationType = new ObjectType({
+    elemID: new ElemID(SALESFORCE, CUSTOM_OBJECT_TRANSLATION_METADATA_TYPE),
+  })
+  const objTranslationInstance = new InstanceElement(
+    `${customObjName}-en_US`,
+    objTranslationType,
+    {
+      [INSTANCE_FULL_NAME_FIELD]: `${customObjName}-en_US`,
+      fields: [{ name: customFieldName }, { name: 'not-exists' }],
+      validationRules: [{ name: validationRuleName }, { name: 'not-exists' }],
+    },
+    undefined,
+    { [INSTANCE_ANNOTATIONS.PARENT]: new ReferenceExpression(customObjElemID) }
+  )
+  const objTranslationNoCustomObjInstance = new InstanceElement(
+    'BLA-en_US',
+    objTranslationType,
+    {
+      [INSTANCE_FULL_NAME_FIELD]: 'BLA.en_US',
+      // Use here also single element and not a list
+      fields: { name: customFieldName },
+      validationRules: { name: validationRuleName },
+    },
+  )
+
+  describe('on fetch', () => {
+    let postFilter: InstanceElement
+    let postFilterNoObj: InstanceElement
+
+    beforeAll(async () => {
+      const filter = filterCreator() as FilterWith<'onFetch'>
+      const testElements = [
+        objTranslationInstance.clone(),
+        objTranslationNoCustomObjInstance.clone(),
+        customObjectAnno.clone(),
+        customObjectField.clone(),
+        customObjectAdditionalField.clone(),
+        objTranslationType.clone(),
+        validationRuleType.clone(),
+        validationRuleInstance.clone(),
+        fakeValidationRuleInstance.clone(),
+      ]
+      await filter.onFetch(testElements)
+      postFilter = testElements[0] as InstanceElement
+      postFilterNoObj = testElements[1] as InstanceElement
+    })
+
+    describe('fields reference', () => {
+      it('should transform fields to reference', async () => {
+        expect(postFilter.value.fields[0].name)
+          .toEqual(new ReferenceExpression(customObjectField.fields[customFieldName].elemID))
+      })
+      it('should keep name as is if no referenced field was found', async () => {
+        expect(postFilter.value.fields[1].name).toBe('not-exists')
+      })
+
+      it('should keep name as is if no custom object', async () => {
+        expect(postFilterNoObj.value.fields.name).toBe(customFieldName)
+      })
+    })
+
+    describe('validation rules reference', () => {
+      it('should transform validation rules to reference', async () => {
+        expect(postFilter.value.validationRules[0].name)
+          .toEqual(new ReferenceExpression(validationRuleInstance.elemID))
+      })
+      it('should keep name as is if no referenced validation rules was found', async () => {
+        expect(postFilter.value.validationRules[1].name).toBe('not-exists')
+      })
+
+      it('should keep name as is if no custom object', async () => {
+        expect(postFilterNoObj.value.validationRules.name).toBe(validationRuleName)
+      })
+    })
+  })
+})