Skip to content
This repository was archived by the owner on Mar 13, 2025. It is now read-only.

skill & skillsProvider removal #126

Open
wants to merge 5 commits into
base: feature/removing_skill_model
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -41,13 +41,10 @@ Configuration for the application is at config/default.js and config/production.
- ES_HOST: Elasticsearch host
- ES.DOCUMENTS: Elasticsearch index, type and id mapping for resources.
- ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline`
- SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline`
- USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline`
- ATTRIBUTE_GROUP_ENRICH_POLICYNAME: The enrich policy for attribute group. Default is `attributegroup-policy`
- SKILL_PROVIDER_ENRICH_POLICYNAME: The enrich policy for skill provider. Default is `skillprovider-policy`
- ROLE_ENRICH_POLICYNAME: The enrich policy for role. Default is `role-policy`
- ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy`
- SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy`
- ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy`
- ELASTICCLOUD_ID: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this
- ELASTICCLOUD_USERNAME: The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
@@ -105,3 +102,8 @@ Make sure all config values are right, and you can run on local successfully, th
5. When you are running the application for the first time, It will take some time initially to download the image and install the dependencies

You can also head into `docker-pgsql-es` folder and run `docker-compose up -d` to have docker instances of pgsql and elasticsearch to use with the api

## Testing

- Run `npm run test` to execute unit tests
- Run `npm run test:cov` to execute unit tests and generate coverage report.
4 changes: 4 additions & 0 deletions app.js
Original file line number Diff line number Diff line change
@@ -112,3 +112,7 @@ app.use('*', (req, res) => {
logger.info(`Express server listening on port ${app.get('port')}`)
})
})()

if (process.env.NODE_ENV === 'test') {
module.exports = app
}
12 changes: 1 addition & 11 deletions config/default.js
Original file line number Diff line number Diff line change
@@ -32,6 +32,7 @@ module.exports = {
BUSAPI_URL: process.env.BUSAPI_URL || 'https://api.topcoder-dev.com/v5',

TOPCODER_GROUP_API: process.env.TOPCODER_GROUP_API || 'https://api.topcoder-dev.com/v5/groups',
TOPCODER_SKILL_API: process.env.TOPCODER_SKILL_API || 'https://api.topcoder-dev.com/v5/skills',

KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC || 'common.error.reporting',
KAFKA_MESSAGE_ORIGINATOR: process.env.KAFKA_MESSAGE_ORIGINATOR || 'u-bahn-api',
@@ -101,17 +102,6 @@ module.exports = {
type: '_doc',
enrichPolicyName: process.env.ROLE_ENRICH_POLICYNAME || 'role-policy'
},
skill: {
index: process.env.SKILL_INDEX || 'skill',
type: '_doc',
enrichPolicyName: process.env.SKILL_ENRICH_POLICYNAME || 'skill-policy'
},
skillprovider: {
index: process.env.SKILL_PROVIDER_INDEX || 'skill_provider',
type: '_doc',
pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline',
enrichPolicyName: process.env.SKILL_PROVIDER_ENRICH_POLICYNAME || 'skillprovider-policy'
},
user: {
index: process.env.USER_INDEX || 'user',
type: '_doc',
8 changes: 8 additions & 0 deletions config/test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
module.exports = {
AUTH0_URL: 'https://topcoder-dev.auth0.com/oauth/token',
AUTH0_AUDIENCE: 'https://m2m.topcoder-dev.com/',
TOKEN_CACHE_TIME: 6000,
AUTH0_CLIENT_ID: 'client_id',
AUTH0_CLIENT_SECRET: 'secret',
AUTH0_PROXY_SERVER_URL: 'proxy_url'
}
995 changes: 179 additions & 816 deletions docs/UBahn_API.postman_collection.json

Large diffs are not rendered by default.

14 changes: 7 additions & 7 deletions docs/UBahn_ENV.postman_environment.json
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
{
"id": "1f07c3cc-5af9-4dc1-b038-129aaf1fac6e",
"id": "00b2830d-9838-44e4-a15e-37980c7aba23",
"name": "UBahn_ENV",
"values": [
{
"key": "HOST",
"value": "http://127.0.0.1:3002/api/1.0",
"value": "http://127.0.0.1:3001/api/1.0",
"enabled": true
},
{
"key": "token",
"value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIiwiQ29waWxvdCIsIkFkbWluIl0sImlzcyI6Imh0dHBzOi8vYXBpLnRvcGNvZGVyLmNvbSIsImhhbmRsZSI6InRjLUFkbWluIiwiZXhwIjoxNjg1NTcxNDYwLCJ1c2VySWQiOiIyMzE2Njc2OCIsImlhdCI6MTU4NTU3MDg2MCwiZW1haWwiOiJ0Yy1BZG1pbkBnbWFpbC5jb20iLCJqdGkiOiIwZjFlZjFkMy0yYjMzLTQ5MDAtYmI0My00OGYyMjg1Zjk2MzAifQ.D_TtClF4xkuSPSWoUYvkWigUWVFhH5UuF7Eci4S1_xg",
"value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIiwiQ29waWxvdCIsIkFkbWluIl0sImlzcyI6Imh0dHBzOi8vYXBpLnRvcGNvZGVyLmNvbSIsImhhbmRsZSI6InRjLUFkbWluIiwiZXhwIjoxNjg1NTcxNDYwLCJ1c2VySWQiOiIyMzE2Njc2OCIsImlhdCI6MTY4NTU3MTQ2MCwiZW1haWwiOiJ0Yy1BZG1pbkBnbWFpbC5jb20iLCJqdGkiOiIwZjFlZjFkMy0yYjMzLTQ5MDAtYmI0My00OGYyMjg1Zjk2MzAifQ.PeJDhtvFY5Io-ZNfiiWQZtAesp-rYS-1TCSsu3sncpE",
"enabled": true
},
{
"key": "userId",
"value": "55f6fa1c-fc38-4b74-83ad-278babd7efd2",
"value": "ce348067-e73f-49d7-af72-fcf11a6c88bf",
"enabled": true
},
{
@@ -24,7 +24,7 @@
},
{
"key": "skillsProviderId",
"value": "7637ae1a-3b7c-44eb-a5ed-10ea02f1885d",
"value": "6a21394e-1278-4835-9e4d-cb4ff151fcd3",
"enabled": true
},
{
@@ -39,6 +39,6 @@
}
],
"_postman_variable_scope": "environment",
"_postman_exported_at": "2020-08-02T07:47:16.803Z",
"_postman_exported_using": "Postman/7.29.1"
"_postman_exported_at": "2021-10-02T14:50:38.084Z",
"_postman_exported_using": "Postman/8.12.1"
}
15 changes: 0 additions & 15 deletions docs/swagger.yaml
Original file line number Diff line number Diff line change
@@ -520,11 +520,6 @@ paths:
required: true
type: "string"
format: "UUID"
- name: "skillName"
in: "query"
description: "Filter by skill name (through skill id)"
required: false
type: "string"
responses:
"200":
description: "OK - the request was successful"
@@ -579,11 +574,6 @@ paths:
required: true
type: "string"
format: "UUID"
- name: "skillName"
in: "query"
description: "Filter by skill name (through skill id)"
required: false
type: "string"
responses:
"200":
description: "Success response"
@@ -4565,11 +4555,6 @@ definitions:
description: "The organization id"
type: "string"
format: "UUID"
skills:
type: "array"
items:
type: "string"
description: "The skill name to filter users with. Specify multiple times to provide multiple values"
Unauthorized:
type: "object"
properties:
2,152 changes: 2,143 additions & 9 deletions package-lock.json

Large diffs are not rendered by default.

21 changes: 20 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -11,7 +11,9 @@
"migrations": "node scripts/db/migrations.js",
"insert-data": "node scripts/db/insert-data.js",
"migrate-qldb-to-pg": "node scripts/db/migrateQldbToPg.js",
"import-s3-data": "node scripts/db/importS3ToQldb.js"
"import-s3-data": "node scripts/db/importS3ToQldb.js",
"test": "mocha test/unit/*.test.js --timeout 30000 --require test/prepare.js --exit",
"test:cov": "nyc --reporter=html --reporter=text npm run test"
},
"repository": {
"type": "git",
@@ -47,8 +49,25 @@
"winston": "^3.2.1"
},
"devDependencies": {
"chai": "^4.3.4",
"chai-as-promised": "^7.1.1",
"chai-http": "^4.3.0",
"mocha": "^9.1.2",
"nyc": "^15.1.0",
"sinon": "^11.1.2",
"standard": "^14.3.0"
},
"standard": {
"env": [
"mocha"
]
},
"nyc": {
"exclude": [
"src/common/logger.js",
"test/unit/**"
]
},
"engines": {
"node": "12.x"
}
40 changes: 0 additions & 40 deletions scripts/constants.js
Original file line number Diff line number Diff line change
@@ -6,22 +6,6 @@
const config = require('config')

const topResources = {
skillprovider: {
index: config.get('ES.DOCUMENTS.skillprovider.index'),
type: config.get('ES.DOCUMENTS.skillprovider.type'),
enrich: {
policyName: config.get('ES.DOCUMENTS.skillprovider.enrichPolicyName'),
matchField: 'id',
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
},
pipeline: {
id: config.get('ES.DOCUMENTS.skillprovider.pipelineId'),
field: 'skillProviderId',
targetField: 'skillprovider',
maxMatches: '1'
}
},

role: {
index: config.get('ES.DOCUMENTS.role.index'),
type: config.get('ES.DOCUMENTS.role.type'),
@@ -58,21 +42,6 @@ const topResources = {
}
},

skill: {
index: config.get('ES.DOCUMENTS.skill.index'),
type: config.get('ES.DOCUMENTS.skill.type'),
enrich: {
policyName: config.get('ES.DOCUMENTS.skill.enrichPolicyName'),
matchField: 'id',
enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider']
},
ingest: {
pipeline: {
id: config.get('ES.DOCUMENTS.skillprovider.pipelineId')
}
}
},

attribute: {
index: config.get('ES.DOCUMENTS.attribute.index'),
type: config.get('ES.DOCUMENTS.attribute.type'),
@@ -126,13 +95,6 @@ const topResources = {
field: '_ingest._value.roleId',
targetField: '_ingest._value.role',
maxMatches: '1'
},
{
referenceField: config.get('ES.DOCUMENTS.userskill.userField'),
enrichPolicyName: 'skill-policy',
field: '_ingest._value.skillId',
targetField: '_ingest._value.skill',
maxMatches: '1'
}
]
}
@@ -178,9 +140,7 @@ const organizationResources = {
const modelToESIndexMapping = {
User: 'user',
Role: 'role',
SkillsProvider: 'skillprovider',
Organization: 'organization',
Skill: 'skill',
UsersRole: 'userrole',
UsersSkill: 'userskill',
Achievement: 'achievement',
9 changes: 9 additions & 0 deletions scripts/db/data/OrganizationSkillsProvider.json
Original file line number Diff line number Diff line change
@@ -7,5 +7,14 @@
"updatedBy": "tc-Copilot",
"organizationId": "36ed815b-3da1-49f1-a043-aaed0a4e81ad",
"skillProviderId": "7637ae1a-3b7c-44eb-a5ed-10ea02f1885d"
},
{
"id": "6a21394e-1278-4835-9e4d-cb4ff151fcd3",
"created": "2020-05-05T11:01:31.334Z",
"updated": "2020-05-05T11:02:10.574Z",
"createdBy": "tc-Copilot",
"updatedBy": "tc-Copilot",
"organizationId": "6a21394e-1278-4835-9e4d-cb4ff151fcd3",
"skillProviderId": "26fb37b1-5f9f-4727-baa9-f3c87de84ab1"
}
]
46 changes: 0 additions & 46 deletions scripts/db/data/Skill.json

This file was deleted.

10 changes: 0 additions & 10 deletions scripts/db/data/SkillsProvider.json

This file was deleted.

3 changes: 0 additions & 3 deletions scripts/db/dropAll.js
Original file line number Diff line number Diff line change
@@ -20,9 +20,6 @@ async function main () {
await client.ingest.deletePipeline({
id: topResources.user.pipeline.id
})
await client.ingest.deletePipeline({
id: topResources.skillprovider.pipeline.id
})
await client.ingest.deletePipeline({
id: topResources.attributegroup.pipeline.id
})
14 changes: 0 additions & 14 deletions scripts/db/dumpDbToEs.js
Original file line number Diff line number Diff line change
@@ -15,11 +15,9 @@ const models = sequelize.models

// Declares the ordering of the resource data insertion, to ensure that enrichment happens correctly
const RESOURCES_IN_ORDER = [
'skillprovider',
'role',
'achievementprovider',
'attributegroup',
'skill',
'attribute',
'organization',
'organizationskillprovider',
@@ -54,18 +52,6 @@ async function cleanupES (keys) {
}
}

try {
await client.ingest.deletePipeline({
id: topResources.skillprovider.pipeline.id
})
} catch (e) {
if (e.meta && e.meta.body.error.type === RESOURCE_NOT_FOUND) {
// Ignore
} else {
throw e
}
}

try {
await client.ingest.deletePipeline({
id: topResources.attributegroup.pipeline.id
4 changes: 2 additions & 2 deletions scripts/db/insert-data.js
Original file line number Diff line number Diff line change
@@ -4,8 +4,8 @@ const logger = require('../../src/common/logger')
const models = sequelize.models

const dataKeys = ['User', 'Organization', 'AchievementsProvider', 'Achievement',
'AttributeGroup', 'Attribute', 'ExternalProfile', 'SkillsProvider',
'OrganizationSkillsProvider', 'Role', 'Skill', 'UserAttribute', 'UsersRole', 'UsersSkill']
'AttributeGroup', 'Attribute', 'ExternalProfile',
'OrganizationSkillsProvider', 'Role', 'UserAttribute', 'UsersRole', 'UsersSkill']

/**
* import seed data
4 changes: 2 additions & 2 deletions scripts/db/migrateQldbToPg.js
Original file line number Diff line number Diff line change
@@ -9,8 +9,8 @@ const logger = require('../../src/common/logger')
const models = sequelize.models

const dataKeys = ['User', 'Organization', 'AchievementsProvider', 'Achievement',
'AttributeGroup', 'Attribute', 'ExternalProfile', 'SkillsProvider',
'OrganizationSkillsProvider', 'Role', 'Skill', 'UserAttribute', 'UsersRole', 'UsersSkill']
'AttributeGroup', 'Attribute', 'ExternalProfile',
'OrganizationSkillsProvider', 'Role', 'UserAttribute', 'UsersRole', 'UsersSkill']

/**
* Query all records from db.
1 change: 0 additions & 1 deletion scripts/db/migrations.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
const sequelize = require('../../src/models/index')
const path = require('path')
const Umzug = require('umzug')
const { createDb } = require('../../src/common/db-helper')

function getUmzug () {
return new Umzug({
92 changes: 92 additions & 0 deletions scripts/db/migrations/15_remove-skills-skillsProvider.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
const { DataTypes } = require('sequelize')

module.exports = {
up: async (query) => {
await query.removeConstraint('UsersSkills', 'UsersSkills_skillId_fkey')
await query.removeConstraint('OrganizationSkillsProviders', 'OrganizationSkillsProviders_skillProviderId_fkey')
await query.dropTable('Skills')
await query.dropTable('SkillsProviders')
},
down: async (query) => {
await query.createTable('Skills', {
id: {
primaryKey: true,
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4
},
createdBy: {
type: DataTypes.STRING
},
updatedBy: {
type: DataTypes.STRING
},
name: {
type: DataTypes.STRING
},
externalId: {
type: DataTypes.STRING
},
uri: {
type: DataTypes.STRING
},
created: {
type: DataTypes.DATE,
allowNull: false
},
updated: {
type: DataTypes.DATE
}
})
await query.createTable('SkillsProviders', {
id: {
primaryKey: true,
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4
},
createdBy: {
type: DataTypes.STRING
},
updatedBy: {
type: DataTypes.STRING
},
name: {
type: DataTypes.STRING
},
created: {
type: DataTypes.DATE,
allowNull: false
},
updated: {
type: DataTypes.DATE
}
})
await query.addColumn('Skills', 'skillProviderId', {
type: DataTypes.UUID,
references: {
model: 'SkillsProviders',
key: 'id'
},
onUpdate: 'CASCADE'
})
await query.addConstraint('OrganizationSkillsProviders', {
fields: ['skillProviderId'],
type: 'foreign key',
name: 'OrganizationSkillsProviders_skillProviderId_fkey',
references: {
table: 'SkillsProviders',
field: 'id'
},
onUpdate: 'CASCADE'
})
await query.addConstraint('UsersSkills', {
fields: ['skillId'],
type: 'foreign key',
name: 'UsersSkills_skillId_fkey',
references: {
table: 'Skills',
field: 'id'
},
onUpdate: 'CASCADE'
})
}
}
164 changes: 11 additions & 153 deletions src/common/es-helper.js
Original file line number Diff line number Diff line change
@@ -43,14 +43,8 @@ const USER_ORGANIZATION = {

const USER_FILTER_TO_MODEL = {
skill: {
name: 'skill',
isAttribute: false,
model: require('../models/Skill'),
queryField: 'name',
esDocumentQuery: 'skills.skillId.keyword',
values: []
esDocumentQuery: 'skills.skillId.keyword'
},
get skills () { return this.skill },
achievement: {
name: 'achievement',
isAttribute: false,
@@ -114,26 +108,6 @@ const RESOURCE_FILTER = {
queryField: 'name'
}
},
skill: {
externalId: {
resource: 'skill',
queryField: 'externalId'
},
skillProviderId: {
resource: 'skill',
queryField: 'skillProviderId'
},
name: {
resource: 'skill',
queryField: 'name'
}
},
skillprovider: {
name: {
resource: 'skillprovider',
queryField: 'name'
}
},
achievementprovider: {
name: {
resource: 'achievementprovider',
@@ -167,12 +141,6 @@ const RESOURCE_FILTER = {
}
},
// sub-resources
userskill: {
skillName: {
resource: 'skill',
queryField: 'name'
}
},
externalprofile: {
organizationName: {
resource: 'organization',
@@ -214,16 +182,6 @@ const FILTER_CHAIN = {
user: {
idField: 'id'
},
skillprovider: {
filterNext: 'skill',
queryField: 'skillProviderId',
idField: 'id'
},
skill: {
filterNext: 'userskill',
queryField: 'skillId',
idField: 'skillProviderId'
},
attribute: {
filterNext: 'userattribute',
queryField: 'attributeId',
@@ -365,7 +323,7 @@ async function updateOrg (organizationId, body, seqNo, primaryTerm) {
*/
async function processCreate (resource, entity) {
if (_.includes(_.keys(TopResources), resource)) {
// process the top resources such as user, skill...
// process the top resources such as user, attribute...
helper.validProperties(entity, ['id'])
await esClient.index({
index: TopResources[resource].index,
@@ -505,7 +463,7 @@ async function processUpdate (resource, entity) {
*/
async function processDelete (resource, entity) {
if (_.includes(_.keys(TopResources), resource)) {
// process the top resources such as user, skill...
// process the top resources such as user, attribute...
helper.validProperties(entity, ['id'])
await esClient.delete({
index: TopResources[resource].index,
@@ -585,13 +543,16 @@ async function getAttributeId (organizationId, attributeName) {
const attributeIdLookupResults = await dBHelper.find(
sequelize.models.Attribute,
{
'$AttributeGroup.organizationId$': organizationId,
name: attributeName
},
[{
model: sequelize.models.AttributeGroup,
as: 'AttributeGroup',
attributes: []
attributes: [],
where: {
organizationId
},
required: true
}]
)

@@ -877,59 +838,8 @@ function hasNonAlphaNumeric (text) {
return !regex.test(text)
}

/**
* Get skillIds matching the search keyword
*
* @param keyword the search keyword
* @returns array of skillIds
*/
async function searchSkills (keyword, skillProviderIds) {
const queryDoc = DOCUMENTS.skill
keyword = escapeRegex(keyword)
const query = hasNonAlphaNumeric(keyword) ? `\\*${keyword}\\*` : `*${keyword}*`

const keywordSearchClause = {
query_string: {
default_field: 'name',
minimum_should_match: '100%',
query
}
}

const searchClause = {
query: {}
}

if (skillProviderIds == null) {
searchClause.query = keywordSearchClause
searchClause._source = 'id'
} else {
searchClause.query = {
bool: {
filter: [{
terms: {
[`${RESOURCE_FILTER.skill.skillProviderId.queryField}.keyword`]: skillProviderIds
}
}],
must: [keywordSearchClause]
}
}
}

const esQuery = {
index: queryDoc.index,
type: queryDoc.type,
body: searchClause
}

logger.debug(`ES query for searching skills: ${JSON.stringify(esQuery, null, 2)}`)
const { body: results } = await esClient.search(esQuery)

return results.hits.hits.map(hit => hit._source)
}

async function setUserSearchClausesToEsQuery (boolClause, keyword) {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Sande3p I don't see any update in the code where setUserSearchClausesToEsQuery is used. I mean if we have removed searching skills in this method's implementation, we must be doing that where this function is being called, right? Am I missing something here?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, I don't think we need to do anything in searchUsers, the keyword be used to match user records. since we remove skill model, we just remove the matching skill name and reserve other matchings.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What I meant is searchUsers method would not support searching by skills name now, if we don't add some other way to provide filtering for user based on skills names. I want to be sure that we are not removing any feature from the calling code. We need to remove skills model but we should not remove any feature from the calling code.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

const skillIds = (await searchSkills(keyword)).map(skill => skill.id)
const skillIds = (await helper.getSkillsByName(keyword)).map(skill => skill.id)
boolClause.should.push({
query_string: {
fields: ['firstName', 'lastName', 'handle'],
@@ -1093,30 +1003,8 @@ function buildEsQueryToGetAttributeValues (attributeId, attributeValue, size) {
return esQuery
}

function buildEsQueryToGetSkillProviderIds (organizationId) {
const queryDoc = DOCUMENTS.organization

const esQuery = {
index: queryDoc.index,
type: queryDoc.type,
body: {
size: 1000,
query: {
term: {
'id.keyword': {
value: organizationId
}
}
}
}
}

return esQuery
}

async function resolveUserFilterFromDb (filter, { handle }, organizationId) {
const DBHelper = require('../models/index').DBHelper

const DBHelper = require('../common/db-helper')
if (filter.isAttribute) {
const esQueryClause = {
bool: {
@@ -1303,7 +1191,7 @@ async function resolveResFilter (filter, initialRes) {
*/
function applySubResFilters (results, preResFilterResults, ownResFilters, perPage) {
let count = 0
const filtered = results.filter(item => {
const filtered = _.filter(results, item => {
for (const filter of preResFilterResults) {
if (item[filter.queryField] !== filter.value) {
return false
@@ -1635,35 +1523,6 @@ async function searchUsers (authUser, filter, params) {
}
}

/**
* Search for skills matching the given keyword and are part of the given organization
* @param {Object} param0 the organizationId and keyword
*/
async function searchSkillsInOrganization ({ organizationId, keyword }) {
if (!organizationId) {
throw Error('Cannot search for skills without organization info')
}
const esQueryToGetSkillProviders = buildEsQueryToGetSkillProviderIds(organizationId)
logger.debug(`ES query to get skill provider ids: ${JSON.stringify(esQueryToGetSkillProviders, null, 2)}`)

const { body: esResultOfQueryToGetSkillProviders } = await esClient.search(esQueryToGetSkillProviders)
logger.debug(`ES result: ${JSON.stringify(esResultOfQueryToGetSkillProviders, null, 2)}`)

const skillProviderIds = _.flatten(esResultOfQueryToGetSkillProviders.hits.hits.map(hit => hit._source.skillProviders == null ? [] : hit._source.skillProviders.map(sp => sp.skillProviderId)))
logger.debug(`Organization ${organizationId} yielded skillProviderIds: ${JSON.stringify(skillProviderIds, null, 2)}`)

const skills = await searchSkills(keyword, skillProviderIds)

return {
result: skills.map(skill => ({
name: skill.name,
skillId: skill.id,
skillProviderId: skill.skillProviderId
// skillProviderName: 'TODO'
}))
}
}

/**
* Searches for matching values for the given attribute value, under the given attribute id
* @param {Object} param0 The attribute id and the attribute value properties
@@ -1730,7 +1589,6 @@ module.exports = {
searchElasticSearch,
getFromElasticSearch,
searchUsers,
searchSkillsInOrganization,
searchAttributeValues,
searchAchievementValues
}
21 changes: 21 additions & 0 deletions src/common/helper.js
Original file line number Diff line number Diff line change
@@ -63,6 +63,26 @@ async function getUserGroup (memberId) {
return groups
}

/**
* Returns the skills in topcoder skills
* @param {String} name The skill name
*/
async function getSkillsByName (name) {
const url = config.TOPCODER_SKILL_API
const token = await getTopcoderM2Mtoken()
const params = { name, page: 1 }

logger.debug(`request GET ${url} with params: ${JSON.stringify(params)}`)
let skills = []
let skillsRes = await axios.get(url, { headers: { Authorization: `Bearer ${token}` }, params })
while (skillsRes.data.length > 0) {
skills = _.concat(skills, _.map(skillsRes.data, g => _.pick(g, 'id', 'name', 'taxonomyId')))
params.page = params.page + 1
skillsRes = await axios.get(url, { headers: { Authorization: `Bearer ${token}` }, params })
}
return skills
}

/**
* Checks if the source matches the term.
*
@@ -226,6 +246,7 @@ module.exports = {
validProperties,
getAuthUser,
getUserGroup,
getSkillsByName,
permissionCheck,
checkIfExists,
injectSearchMeta,
1 change: 0 additions & 1 deletion src/common/service-helper.js
Original file line number Diff line number Diff line change
@@ -27,7 +27,6 @@ _.forOwn(config.ES.DOCUMENTS, (value, key) => {
// map model name to bus message resource if different
const MODEL_TO_RESOURCE = {
UsersSkill: 'userskill',
SkillsProvider: 'skillprovider',
AchievementsProvider: 'achievementprovider',
UserAttribute: 'userattribute',
UsersRole: 'userrole',
39 changes: 0 additions & 39 deletions src/consts.js
Original file line number Diff line number Diff line change
@@ -17,8 +17,6 @@ function validProperties (payload, keys) {
}
}



/**
* roles that used in service, all roles must match topcoder roles
* Admin and Administrator are both admin user
@@ -42,7 +40,6 @@ const AllAuthenticatedUsers = [
UserRoles.ubahn
]


/**
* all admin user
*/
@@ -106,35 +103,6 @@ const TopResources = {
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
}
},
skill: {
index: config.get('ES.DOCUMENTS.skill.index'),
type: config.get('ES.DOCUMENTS.skill.type'),
enrich: {
policyName: config.get('ES.DOCUMENTS.skill.enrichPolicyName'),
matchField: 'id',
enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider']
},
ingest: {
pipeline: {
id: config.get('ES.DOCUMENTS.skillprovider.pipelineId')
}
}
},
skillprovider: {
index: config.get('ES.DOCUMENTS.skillprovider.index'),
type: config.get('ES.DOCUMENTS.skillprovider.type'),
enrich: {
policyName: config.get('ES.DOCUMENTS.skillprovider.enrichPolicyName'),
matchField: 'id',
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
},
pipeline: {
id: config.get('ES.DOCUMENTS.skillprovider.pipelineId'),
field: 'skillProviderId',
targetField: 'skillprovider',
maxMatches: '1'
}
},
user: {
index: config.get('ES.DOCUMENTS.user.index'),
type: config.get('ES.DOCUMENTS.user.type'),
@@ -166,13 +134,6 @@ const TopResources = {
field: '_ingest._value.roleId',
targetField: '_ingest._value.role',
maxMatches: '1'
},
{
referenceField: config.get('ES.DOCUMENTS.userskill.userField'),
enrichPolicyName: config.get('ES.DOCUMENTS.skill.enrichPolicyName'),
field: '_ingest._value.skillId',
targetField: '_ingest._value.skill',
maxMatches: '1'
}
]
}
4 changes: 3 additions & 1 deletion src/models/OrganizationSkillsProvider.js
Original file line number Diff line number Diff line change
@@ -10,6 +10,9 @@ module.exports = (sequelize) => {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4
},
skillProviderId: {
type: DataTypes.UUID
},
createdBy: {
type: DataTypes.STRING
},
@@ -23,7 +26,6 @@ module.exports = (sequelize) => {
createdAt: 'created'
})
OrganizationSkillsProvider.associate = (models) => {
OrganizationSkillsProvider.belongsTo(models.SkillsProvider, { foreignKey: 'skillProviderId', type: DataTypes.UUID })
OrganizationSkillsProvider.belongsTo(models.Organization, { foreignKey: 'organizationId', type: DataTypes.UUID })
}
return OrganizationSkillsProvider
36 changes: 0 additions & 36 deletions src/models/Skill.js

This file was deleted.

30 changes: 0 additions & 30 deletions src/models/SkillsProvider.js

This file was deleted.

4 changes: 3 additions & 1 deletion src/models/UsersSkill.js
Original file line number Diff line number Diff line change
@@ -7,6 +7,9 @@ module.exports = (sequelize) => {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4
},
skillId: {
type: DataTypes.UUID
},
createdBy: {
type: DataTypes.STRING
},
@@ -30,7 +33,6 @@ module.exports = (sequelize) => {
})
UsersSkill.associate = (models) => {
UsersSkill.belongsTo(models.User, { foreignKey: 'userId', type: DataTypes.UUID })
UsersSkill.belongsTo(models.Skill, { foreignKey: 'skillId', type: DataTypes.UUID })
}
return UsersSkill
}
2 changes: 0 additions & 2 deletions src/modules/organizationSkillsProvider/service.js
Original file line number Diff line number Diff line change
@@ -12,7 +12,6 @@ const dbHelper = require('../../common/db-helper')
const serviceHelper = require('../../common/service-helper')
const sequelize = require('../../models/index')

const SkillsProvider = sequelize.models.SkillsProvider
const Organization = sequelize.models.Organization
const OrganizationSkillsProvider = sequelize.models.OrganizationSkillsProvider
const resource = serviceHelper.getResource('OrganizationSkillsProvider')
@@ -26,7 +25,6 @@ const uniqueFields = [['organizationId', 'skillProviderId']]
*/
async function create (entity, auth) {
await dbHelper.get(Organization, entity.organizationId)
await dbHelper.get(SkillsProvider, entity.skillProviderId)
await dbHelper.makeSureUnique(OrganizationSkillsProvider, entity, uniqueFields)

let newEntity
19 changes: 14 additions & 5 deletions src/modules/search/service.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
const axios = require('axios')
const config = require('config')
const _ = require('lodash')
const querystring = require('querystring')
const NodeCache = require('node-cache')
const joi = require('@hapi/joi')
const orgSkillsProviderService = require('../organizationSkillsProvider/service')
const esHelper = require('../../common/es-helper')
const helper = require('../../common/helper')

// cache the emsi token
const tokenCache = new NodeCache()
@@ -60,17 +61,25 @@ async function getEmsiObject (path, params) {
* @returns {Object} the Object with skills
*/
async function getSkills (query, auth) {
let result
const skillProviderIds = await orgSkillsProviderService.search({ organizationId: query.organizationId }, auth)

if (skillProviderIds.result.length === 1 && skillProviderIds.result[0].skillProviderId === config.EMSI_SKILLPROVIDER_ID) {
result = await getEmsiObject('/skills', { q: query.keyword })
const result = await getEmsiObject('/skills', { q: query.keyword })
return { result: formatEmsiSkills(result) }
}

result = await esHelper.searchSkillsInOrganization(query)
const skills = await helper.getSkillsByName(query.keyword)
// filter by skill providerIds
const providerIds = _.map(skillProviderIds.result, 'skillProviderId')
const filteredSkills = _.filter(skills, skill => _.includes(providerIds, skill.taxonomyId))

return result
return {
result: _.map(filteredSkills, skill => ({
name: skill.name,
skillId: skill.id,
skillProviderId: skill.taxonomyId
}))
}
}

getSkills.schema = {
19 changes: 2 additions & 17 deletions src/modules/usersSkill/service.js
Original file line number Diff line number Diff line change
@@ -11,7 +11,6 @@ const dbHelper = require('../../common/db-helper')
const serviceHelper = require('../../common/service-helper')
const sequelize = require('../../models/index')

const Skill = sequelize.models.Skill
const User = sequelize.models.User
const UsersSkill = sequelize.models.UsersSkill
const resource = serviceHelper.getResource('UsersSkill')
@@ -24,7 +23,6 @@ const uniqueFields = [['userId', 'skillId']]
* @return {Promise} the created device
*/
async function create (entity, auth) {
await dbHelper.get(Skill, entity.skillId)
await dbHelper.get(User, entity.userId)
await dbHelper.makeSureUnique(UsersSkill, entity, uniqueFields)

@@ -64,9 +62,6 @@ create.schema = {
* @return {Promise} the updated device
*/
async function patch (id, entity, auth, params) {
if (entity.skillId) {
await dbHelper.get(Skill, entity.skillId)
}
if (entity.userId) {
await dbHelper.get(User, entity.userId)
}
@@ -146,16 +141,7 @@ async function search (query, auth) {
return esResult
}

// add query for associations
if (query.skillName) {
query['$Skill.name$'] = query.skillName
delete query.skillName
}
const items = await dbHelper.find(UsersSkill, query, auth, [{
model: Skill,
as: 'Skill',
attributes: []
}])
const items = await dbHelper.find(UsersSkill, query, auth)

return { fromDb: true, result: items, total: items.length }
}
@@ -164,8 +150,7 @@ search.schema = {
query: {
page: joi.id(),
perPage: joi.pageSize(),
userId: joi.string().required(),
skillName: joi.string()
userId: joi.string().required()
},
auth: joi.object()
}
5 changes: 5 additions & 0 deletions test/prepare.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
/*
* Prepare for tests.
*/
process.env.NODE_ENV = 'test'
require('../src/bootstrap')
131 changes: 131 additions & 0 deletions test/unit/UserSkillsEndpoint.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
const _ = require('lodash')
const config = require('config')
const chai = require('chai')
const chaiHttp = require('chai-http')
const chaiAsPromised = require('chai-as-promised')
const service = require('../../src/modules/usersSkill/service')
const sinon = require('sinon')
const app = require('../../app')
const commonData = require('./common/commonData')

chai.use(chaiHttp)
chai.use(chaiAsPromised)
const should = chai.should()
const expect = chai.expect

describe('Unit tests for /users/{userId}/skills', () => {
afterEach(() => {
sinon.restore()
})
describe('Test GET', () => {
beforeEach(() => {
sinon.stub(service, 'search').resolves({
page: 1,
perPage: 20,
total: 1,
result: [{
id: 1,
userId: 1,
skillId: 1
}]
})
})
for (const tokenName of ['admin', 'administrator', 'topcoderUser', 'copilot', 'ubahn']) {
it(`Call GET API with ${tokenName} token successfully`, async () => {
const response = await chai.request(app).get(`/${config.API_VERSION}/users/test_user/skills`)
.set('Authorization', 'Bearer ' + commonData[`${tokenName}Token`])
should.equal(response.status, 200)
should.equal(response.headers['x-page'], '1')
should.equal(response.headers['x-per-page'], '20')
should.equal(response.headers['x-total'], '1')
should.equal(response.headers['x-total-pages'], '1')
should.exist(response.headers.link)
should.equal(response.body.length, 1)
})
}
it('Call GET API with other token', async () => {
const response = await chai.request(app).get(`/${config.API_VERSION}/users/test_user/skills`)
.set('Authorization', `Bearer ${commonData.unKnownToken}`)
should.equal(response.status, 403)
})
it('Call GET API without token', async () => {
const response = await chai.request(app).get(`/${config.API_VERSION}/users/test_user/skills`)
should.equal(response.status, 403)
})
})
describe('Test HEAD', () => {
beforeEach(() => {
sinon.stub(service, 'search').resolves({
page: 1,
perPage: 20,
total: 1,
result: [{
id: 1,
userId: 1,
skillId: 1
}]
})
})
for (const tokenName of ['admin', 'administrator', 'topcoderUser', 'copilot', 'ubahn']) {
it(`Call HEAD API with ${tokenName} token successfully`, async () => {
const response = await chai.request(app).head(`/${config.API_VERSION}/users/test_user/skills`)
.set('Authorization', 'Bearer ' + commonData[`${tokenName}Token`])
should.equal(response.status, 200)
should.equal(response.headers['x-page'], '1')
should.equal(response.headers['x-per-page'], '20')
should.equal(response.headers['x-total'], '1')
should.equal(response.headers['x-total-pages'], '1')
should.exist(response.headers.link)
should.equal(_.isEmpty(response.body), true)
})
}
it('Call HEAD API with other token', async () => {
const response = await chai.request(app).head(`/${config.API_VERSION}/users/test_user/skills`)
.set('Authorization', `Bearer ${commonData.unKnownToken}`)
should.equal(response.status, 403)
})
it('Call HEAD API without token', async () => {
const response = await chai.request(app).head(`/${config.API_VERSION}/users/test_user/skills`)
should.equal(response.status, 403)
})
})
describe('Test POST', () => {
const userSkill = {
userId: 'string',
skillId: 'string',
metricValue: 'string',
certifierId: 'string',
certifiedDate: '2021-10-11T10:59:12.816Z',
created: '2021-10-11T10:59:12.816Z',
updated: '2021-10-11T10:59:12.817Z',
createdBy: 'string',
updatedBy: 'string'
}
let stubCreate
beforeEach(() => {
stubCreate = sinon.stub(service, 'create').resolves(userSkill)
})
for (const tokenName of ['admin', 'administrator', 'topcoderUser', 'copilot', 'ubahn']) {
it(`Call POST API with ${tokenName} token successfully`, async () => {
const response = await chai.request(app).post(`/${config.API_VERSION}/users/test_user/skills`)
.set('Authorization', 'Bearer ' + commonData[`${tokenName}Token`])
.send({ skillId: 'testSkill' })
should.equal(response.status, 200)
expect(response.body).to.deep.eq(userSkill)
should.equal(stubCreate.calledOnce, true)
expect(stubCreate.getCall(0).args[0]).to.deep.eq({ userId: 'test_user', skillId: 'testSkill' })
})
}
it('Call POST API with other token', async () => {
const response = await chai.request(app).post(`/${config.API_VERSION}/users/test_user/skills`)
.set('Authorization', `Bearer ${commonData.unKnownToken}`)
.send({ skillId: 'testSkill' })
should.equal(response.status, 403)
})
it('Call POST API without token', async () => {
const response = await chai.request(app).post(`/${config.API_VERSION}/users/test_user/skills`)
.send({ skillId: 'testSkill' })
should.equal(response.status, 403)
})
})
})
110 changes: 110 additions & 0 deletions test/unit/UserSkillsService.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/* eslint-disable no-unused-expressions */

const _ = require('lodash')
const chai = require('chai')
const chaiAsPromised = require('chai-as-promised')
const sinon = require('sinon')
const sequelize = require('../../src/models/index')
const service = require('../../src/modules/usersSkill/service')
const helper = require('../../src/common/helper')
const serviceHelper = require('../../src/common/service-helper')
const dbHelper = require('../../src/common/db-helper')
const errors = require('../../src/common/errors')

chai.use(chaiAsPromised)
const expect = chai.expect

describe('user skills service test', () => {
beforeEach(() => {
sinon.stub(sequelize, 'transaction').callsFake(f => f())
})
afterEach(() => {
sinon.restore()
})

describe('Create user skills', () => {
it('Create user skills successfully', async () => {
const entity = { userId: 'userId', skillId: 'skillId' }
const newEntity = _.assign({ createdBy: 'test' }, entity)
sinon.stub(dbHelper, 'get').resolves({})
sinon.stub(dbHelper, 'makeSureUnique').resolves({})
const stubDBCreate = sinon.stub(dbHelper, 'create').resolves({ toJSON: () => newEntity })
const stubEsInsert = sinon.stub(serviceHelper, 'createRecordInEs').resolves({})
const result = await service.create(entity, {})
expect(result).to.deep.eql(newEntity)
expect(stubDBCreate.calledOnce).to.be.true
expect(stubEsInsert.calledOnce).to.be.true
expect(stubDBCreate.getCall(0).args[1]).to.deep.eq(entity)
expect(stubEsInsert.getCall(0).args[1]).to.deep.eq(newEntity)
})
it('Throw error when require parameters absence', async () => {
await expect(service.create({ userId: 'userId' })).to.be.rejectedWith('"entity.skillId" is required')
await expect(service.create({ skillId: 'skillId' })).to.be.rejectedWith('"entity.userId" is required')
})
it('Throw error when user id does not exist', async () => {
const error = errors.newEntityNotFoundError('cannot find user where id:userId')
sinon.stub(dbHelper, 'get').rejects(error)
await expect(service.create({ userId: 'userId', skillId: 'skillId' })).to.eventually.rejectedWith(error)
})
it('Throw error when a record with same user id and skill id exist', async () => {
const error = errors.newConflictError('usersSkill already exists with userId:userId,skillId:skillId')
sinon.stub(dbHelper, 'get').resolves({})
sinon.stub(dbHelper, 'makeSureUnique').rejects(error)
await expect(service.create({ userId: 'userId', skillId: 'skillId' })).to.eventually.rejectedWith(error)
})
it('Throw error and does not publish error event when insert into db error', async () => {
const error = new Error('db error')
sinon.stub(dbHelper, 'get').resolves({})
sinon.stub(dbHelper, 'makeSureUnique').resolves({})
const stubPublishError = sinon.stub(helper, 'publishError')
sinon.stub(dbHelper, 'create').rejects(error)
await expect(service.create({ userId: 'userId', skillId: 'skillId' })).to.eventually.rejectedWith(error)
expect(stubPublishError.called).to.be.false
})
it('Throw error and publish error event when insert into es error', async () => {
const error = new Error('es error')
const entity = { userId: 'userId', skillId: 'skillId' }
const newEntity = _.assign({ createdBy: 'test' }, entity)
sinon.stub(dbHelper, 'get').resolves({})
sinon.stub(dbHelper, 'makeSureUnique').resolves({})
const stubPublishError = sinon.stub(helper, 'publishError').resolves({})
const stubDBCreate = sinon.stub(dbHelper, 'create').resolves({ toJSON: () => newEntity })
sinon.stub(serviceHelper, 'createRecordInEs').rejects(error)
await expect(service.create({ userId: 'userId', skillId: 'skillId' })).to.eventually.rejectedWith(error)
expect(stubPublishError.called).to.be.true
expect(stubDBCreate.called).to.be.true
})
})
describe('Search user skills', () => {
it('Throw error when require parameters absence', async () => {
await expect(service.search({ })).to.be.rejectedWith(errors.BadRequestError)
})
it('Throw error when user id does not exist', async () => {
const error = errors.newEntityNotFoundError('cannot find user where id:userId')
sinon.stub(dbHelper, 'get').rejects(error)
await expect(service.search({ userId: 'userId' })).to.eventually.rejectedWith(error)
})
it('Search user skills from es successfully', async () => {
const entity = { userId: 'userId', skillId: 'skillId' }
const newEntity = _.assign({ createdBy: 'test' }, entity)
sinon.stub(dbHelper, 'get').resolves({})
const searchEs = sinon.stub(serviceHelper, 'searchRecordInEs').resolves([newEntity])
const searchDb = sinon.stub(dbHelper, 'find')
const result = await service.search({ userId: 'userId' }, {})
expect(result).to.deep.eql([newEntity])
expect(searchEs.calledOnce).to.be.true
expect(searchDb.called).to.be.false
})
it('Search user skills from db when es throw errors', async () => {
const entity = { userId: 'userId', skillId: 'skillId' }
const newEntity = _.assign({ createdBy: 'test' }, entity)
sinon.stub(dbHelper, 'get').resolves({})
const searchEs = sinon.stub(serviceHelper, 'searchRecordInEs').resolves(null)
const searchDb = sinon.stub(dbHelper, 'find').resolves([newEntity])
const result = await service.search({ userId: 'userId' }, {})
expect(result).to.deep.eql({ fromDb: true, result: [newEntity], total: 1 })
expect(searchEs.calledOnce).to.be.true
expect(searchDb.called).to.be.true
})
})
})
15 changes: 15 additions & 0 deletions test/unit/common/commonData.js