Skip to content

Commit 079c35d

Browse files
author
James Cori
committedNov 3, 2020
Merge branch 'develop'
2 parents 0f7856b + ed2993b commit 079c35d

8 files changed

+747
-490
lines changed
 

‎Verification.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
- You need to run command `npm run sync-es` before you run `Challenges/get challenge` and `Challenges/search challenge` test case.
88

99
## DynamoDB Verification
10-
Run command `npm run view-data <ModelName>` to view table data, ModelName can be `Challenge`, `ChallengeType`, `AuditLog`, `Phase`, `TimelineTemplate`, `Attachment` or `ChallengeTypeTimelineTemplate`
10+
Run command `npm run view-data <ModelName>` to view table data, ModelName can be `Challenge`, `ChallengeType`, `AuditLog`, `Phase`, `TimelineTemplate`, `Attachment` or `ChallengeTimelineTemplate`
1111

1212
## S3 Verification
1313

‎docs/swagger.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,13 @@ paths:
135135
required: false
136136
type: string
137137
format: UUID
138+
- name: search
139+
in: query
140+
description: >-
141+
Filter by name, description and tags fields,
142+
case-insensitive, partial matches are allowed.
143+
required: false
144+
type: string
138145
- name: name
139146
in: query
140147
description: 'Filter by name, case-insensitive, partial matches are allowed.'

‎docs/topcoder-challenge-api.postman_collection.json

Lines changed: 592 additions & 416 deletions
Large diffs are not rendered by default.

‎docs/topcoder-challenge-api.postman_environment.json

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"id": "e2000d9a-93af-4fb0-94e5-2c633c539422",
2+
"id": "92c2786b-c4d0-4842-b41c-67d9faa9d5aa",
33
"name": "topcoder-challenge-api",
44
"values": [
55
{
@@ -24,7 +24,7 @@
2424
},
2525
{
2626
"key": "admin_token",
27-
"value": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczovL2FwaS50b3Bjb2Rlci1kZXYuY29tIiwiaWF0IjoxNTgyNjcyMTM1LCJleHAiOjE4OTgyMDQ5MzUsImF1ZCI6Ind3dy5leGFtcGxlLmNvbSIsInN1YiI6Impyb2NrZXRAZXhhbXBsZS5jb20iLCJyb2xlcyI6WyJUb3Bjb2RlciBVc2VyIiwiQ29ubmVjdCBTdXBwb3J0IiwiYWRtaW5pc3RyYXRvciIsIkNvbm5lY3QgTWFuYWdlciIsIkNvbm5lY3QgQWRtaW4iLCJjb3BpbG90IiwiQ29ubmVjdCBDb3BpbG90IE1hbmFnZXIiXSwiaGFuZGxlIjoiVG9ueUoiLCJ1c2VySWQiOiI4NTQ3ODk5IiwiZW1haWwiOiJ0amVmdHMrZml4QHRvcGNvZGVyLmNvbSJ9.FjJlEoQa6YXsgR0ioIjc7g36C6sVa_TdUcXIqacRwC4",
27+
"value": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Ik5VSkZORGd4UlRVME5EWTBOVVkzTlRkR05qTXlRamxETmpOQk5UYzVRVUV3UlRFeU56TTJRUSJ9.eyJpc3MiOiJodHRwczovL3RvcGNvZGVyLWRldi5hdXRoMC5jb20vIiwic3ViIjoiakdJZjJwZDNmNDRCMWpxdk9haTMwQklLVFphbllCZlVAY2xpZW50cyIsImF1ZCI6Imh0dHBzOi8vbTJtLnRvcGNvZGVyLWRldi5jb20vIiwiaWF0IjoxNjAzMDM3NjUxLCJleHAiOjE2MDMxMjQwNTEsImF6cCI6ImpHSWYycGQzZjQ0QjFqcXZPYWkzMEJJS1RaYW5ZQmZVIiwic2NvcGUiOiJ1cGRhdGU6dXNlcl9wcm9maWxlcyB3cml0ZTp1c2VyX3Byb2ZpbGVzIGNyZWF0ZTpjb25uZWN0X3Byb2plY3QgYWxsOmNoYWxsZW5nZXMgcmVhZDpjaGFsbGVuZ2VzIHdyaXRlOmNoYWxsZW5nZXMgYWxsOmdyb3VwcyB3cml0ZTpncm91cHMgcmVhZDpncm91cHMgdXBkYXRlOnN1Ym1pc3Npb24gcmVhZDpzdWJtaXNzaW9uIGRlbGV0ZTpzdWJtaXNzaW9uIGNyZWF0ZTpzdWJtaXNzaW9uIGFsbDpzdWJtaXNzaW9uIHJlYWQ6cHJvamVjdCBhbGw6Y29ubmVjdF9wcm9qZWN0IHJlYWQ6YnVzX3RvcGljcyB3cml0ZTpidXNfYXBpIHJlYWQ6ZW1haWxfdGVtcGxhdGVzIHJlYWQ6dXNlcl9wcm9maWxlcyByZWFkOnJvbGVzIHJlYWQ6cHJvamVjdC11c2VyIHJlYWQ6cHJvamVjdC1wZXJtaXNzaW9uIHJlYWQ6cmVzb3VyY2VzIHdyaXRlOnJlc291cmNlcyBkZWxldGU6cmVzb3VyY2VzIHVwZGF0ZTpyZXNvdXJjZXMgYWxsOnJlc291cmNlcyByZWFkOnRlcm1zIGFsbDp0ZXJtcyBhbGw6cHJvamVjdHMgcmVhZDpwcm9qZWN0cyIsImd0eSI6ImNsaWVudC1jcmVkZW50aWFscyJ9.Mnv7w3fgwuJg8SrrY1cg1CKYOfNUZD2rQCdYW7oTtuhGvew9_7WRw_WW8DokcFT0ID8c7iIzR_75Nxl2wHT-v6paZ5c6Z3VDduiA1ZPgqx0hUqqxYgK9ts-Ut-MBnUhcZN684grBDD8YNLMEOqNccl9C84du_43r3HYsiJvbx-P0366O2XFHLv-UomK5SUKiIzVvF4wmUDKCHMrkg8r4q9iAH4tQ_jEi3_UHGOUFSnnfaD5a3ZatqbSKroI7W9DextifLvlFypckXPRXC658gDPpqxfxix4OYpBvS5yvYWfiB5erzcXehcXc5yzcM5xpAp_BsVJrAdwa2x-7rvMykQ",
2828
"enabled": true
2929
},
3030
{
@@ -279,6 +279,6 @@
279279
}
280280
],
281281
"_postman_variable_scope": "environment",
282-
"_postman_exported_at": "2020-03-23T08:00:10.994Z",
283-
"_postman_exported_using": "Postman/7.13.0"
284-
}
282+
"_postman_exported_at": "2020-10-20T04:56:48.763Z",
283+
"_postman_exported_using": "Postman/7.34.0"
284+
}

‎src/init-db.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ const initDB = async () => {
1616
for (const challenge of challenges) {
1717
await challenge.delete()
1818
}
19-
const typeTimelineTemplates = await helper.scan('ChallengeTypeTimelineTemplate')
20-
for (const typeTT of typeTimelineTemplates) {
21-
await typeTT.delete()
19+
const challengeTimelineTemplates = await helper.scan('ChallengeTimelineTemplate')
20+
for (const challengeTT of challengeTimelineTemplates) {
21+
await challengeTT.delete()
2222
}
2323
const types = await helper.scan('ChallengeType')
2424
for (const type of types) {

‎src/init-es.js

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,22 @@ const initES = async () => {
3131
const body = { mappings: {} }
3232
body.mappings[config.get('ES.ES_TYPE')] = {
3333
properties: {
34-
id: { type: 'keyword' }
34+
id: { type: 'keyword' },
35+
name: {
36+
type: 'keyword',
37+
normalizer: 'custom_sort_normalizer'
38+
},
39+
prizeSets: {
40+
properties: {
41+
type: { type: 'text' },
42+
prizes: {
43+
properties: {
44+
type: { type: 'text' },
45+
value: { type: 'float' }
46+
}
47+
}
48+
}
49+
}
3550
},
3651
dynamic_templates: [{
3752
metadata: {
@@ -42,6 +57,17 @@ const initES = async () => {
4257
}
4358
}]
4459
}
60+
body.settings = {
61+
analysis: {
62+
normalizer: {
63+
custom_sort_normalizer: {
64+
type: 'custom',
65+
char_filter: [],
66+
filter: ['lowercase', 'asciifolding']
67+
}
68+
}
69+
}
70+
}
4571

4672
await client.indices.create({
4773
index: config.ES.ES_INDEX,

‎src/scripts/update-es-mappings.js

Lines changed: 70 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,17 @@ function createIndex (indexName) {
1818
name: {
1919
type: 'keyword',
2020
normalizer: 'custom_sort_normalizer'
21+
},
22+
prizeSets: {
23+
properties: {
24+
type: { type: 'text' },
25+
prizes: {
26+
properties: {
27+
type: { type: 'text' },
28+
value: { type: 'float' }
29+
}
30+
}
31+
}
2132
}
2233
},
2334
dynamic_templates: [{
@@ -50,59 +61,73 @@ function createIndex (indexName) {
5061
async function updateMappings () {
5162
const tempReindexing = config.get('ES.TEMP_REINDEXING')
5263
let indexName = config.get('ES.ES_INDEX')
53-
let newIndexName = `${indexName}_tmp_dont_use_for_querying`
54-
55-
if (tempReindexing) {
56-
try {
57-
logger.info(`Attemp to remove temporary index ${newIndexName}`)
58-
await esClient.indices.delete({
59-
index: newIndexName
60-
})
61-
await sleep(500)
62-
} catch (e) {
63-
logger.info(`Index ${newIndexName} does not exist`)
64-
}
65-
}
64+
let backupIndex = 'challenge_tmp_dont_use_for_querying_backup'
65+
let newIndexName = 'challenge_tmp_dont_use_for_querying'
6666

67-
await createIndex(newIndexName)
68-
await sleep(500)
69-
logger.info(`Reindexing from ${indexName} to ${newIndexName}`)
70-
await esClient.reindex({
71-
body: {
72-
source: { index: indexName },
73-
dest: { index: newIndexName }
74-
},
75-
waitForCompletion: true
76-
})
67+
// if (tempReindexing) {
68+
// try {
69+
// logger.info(`Attemp to remove temporary index ${newIndexName}`)
70+
// await esClient.indices.delete({
71+
// index: newIndexName
72+
// })
73+
// await sleep(500)
74+
// } catch (e) {
75+
// logger.info(`Index ${newIndexName} does not exist`)
76+
// }
77+
// }
7778

78-
if (tempReindexing) {
79-
return
80-
}
79+
// await createIndex(backupIndex)
80+
// await sleep(500)
81+
// logger.info(`Reindexing from ${indexName} to ${backupIndex}`)
82+
// await esClient.reindex({
83+
// body: {
84+
// source: { index: indexName },
85+
// dest: { index: backupIndex }
86+
// },
87+
// waitForCompletion: true
88+
// })
8189

82-
logger.warn(`Deleting ${indexName}. If script crashes after this point data may be lost and a recreation of index will be required.`)
90+
// await createIndex(newIndexName)
91+
// await sleep(500)
92+
// logger.info(`Reindexing from ${indexName} to ${newIndexName}`)
93+
// await esClient.reindex({
94+
// body: {
95+
// source: { index: indexName },
96+
// dest: { index: newIndexName }
97+
// },
98+
// waitForCompletion: true
99+
// })
83100

84-
await esClient.indices.delete({
85-
index: indexName
86-
})
101+
// if (tempReindexing) {
102+
// return
103+
// }
87104

88-
logger.info(`Copying data back into ${indexName}`)
105+
// logger.warn(`Deleting ${indexName}. If script crashes after this point data may be lost and a recreation of index will be required.`)
89106

90-
// This should be replaced with cloneIndex after migration to 7.4+
91-
await createIndex(indexName)
92-
await sleep(500)
93-
await esClient.reindex({
94-
body: {
95-
source: { index: newIndexName },
96-
dest: { index: indexName }
97-
},
98-
waitForCompletion: true
99-
})
107+
// await esClient.indices.delete({
108+
// index: indexName
109+
// })
100110

101-
logger.info(`Removing ${newIndexName} index`)
111+
// indexName = 'challenge' // overridding source so it's not deleted.
102112

103-
await esClient.indices.delete({
104-
index: newIndexName
105-
})
113+
// logger.info(`Copying data back into ${indexName}`)
114+
115+
// // This should be replaced with cloneIndex after migration to 7.4+
116+
// await createIndex(indexName)
117+
// await sleep(500)
118+
// await esClient.reindex({
119+
// body: {
120+
// source: { index: 'challenge_tmp_dont_use_for_querying' },
121+
// dest: { index: 'challenge' }
122+
// },
123+
// waitForCompletion: true
124+
// })
125+
126+
// logger.info(`Removing ${newIndexName} index`)
127+
128+
// await esClient.indices.delete({
129+
// index: newIndexName
130+
// })
106131
}
107132

108133
updateMappings()

‎src/services/ChallengeService.js

Lines changed: 42 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ async function searchChallenges (currentUser, criteria) {
110110
const page = criteria.page || 1
111111
const perPage = criteria.perPage || 20
112112
const boolQuery = []
113+
let sortByScore = false
113114
const matchPhraseKeys = [
114115
'id',
115116
'timelineTemplateId',
@@ -202,11 +203,46 @@ async function searchChallenges (currentUser, criteria) {
202203
})
203204
}
204205

205-
if (criteria.name) {
206-
boolQuery.push({ wildcard: { 'name': `*${criteria.name}*` } })
206+
if (criteria.search) {
207+
boolQuery.push({
208+
bool: {
209+
should: [
210+
{ match_phrase_prefix: { 'name': criteria.search } },
211+
{ match_phrase_prefix: { 'description': criteria.search } },
212+
{ match_phrase_prefix: { 'tags': criteria.search } }
213+
]
214+
}
215+
})
216+
} else {
217+
if (criteria.name) {
218+
// boolQuery.push({ wildcard: { 'name': `*${criteria.name}*` } })
219+
boolQuery.push({ match_phrase_prefix: { 'name': criteria.name } })
220+
}
221+
222+
if (criteria.description) {
223+
boolQuery.push({ match_phrase_prefix: { 'description': criteria.description } })
224+
}
225+
}
226+
227+
// 'search', 'name', 'description' fields should be sorted by function score unless sortBy param provided.
228+
if (!criteria.sortBy && (
229+
criteria.search ||
230+
criteria.name ||
231+
criteria.description
232+
)) {
233+
sortByScore = true
234+
}
235+
236+
if (criteria.tag) {
237+
boolQuery.push({ match_phrase: { tags: criteria.tag } })
207238
}
208-
if (criteria.description) {
209-
boolQuery.push({ match: { 'description': criteria.description } })
239+
240+
if (criteria.tags) {
241+
boolQuery.push({
242+
bool: {
243+
[criteria.includeAllTags ? 'must' : 'should']: _.map(criteria.tags, t => ({ match_phrase: { tags: t } }))
244+
}
245+
})
210246
}
211247

212248
if (criteria.forumId) {
@@ -221,9 +257,6 @@ async function searchChallenges (currentUser, criteria) {
221257
if (criteria.directProjectId) {
222258
boolQuery.push({ match_phrase: { 'legacy.directProjectId': criteria.directProjectId } })
223259
}
224-
if (criteria.tag) {
225-
boolQuery.push({ match_phrase: { tags: criteria.tag } })
226-
}
227260
if (criteria.currentPhaseName) {
228261
boolQuery.push({ match_phrase: { 'currentPhaseNames': criteria.currentPhaseName } })
229262
}
@@ -277,23 +310,12 @@ async function searchChallenges (currentUser, criteria) {
277310
}
278311

279312
let sortByProp = criteria.sortBy ? criteria.sortBy : 'created'
280-
// If property to sort is text, then use its sub-field 'keyword' for sorting
281-
sortByProp = _.includes(constants.challengeTextSortField, sortByProp) ? sortByProp + '.keyword' : sortByProp
282313
const sortOrderProp = criteria.sortOrder ? criteria.sortOrder : 'desc'
283314

284315
const mustQuery = []
285316

286317
const groupsQuery = []
287318

288-
// logger.debug(`Tags: ${criteria.tags}`)
289-
if (criteria.tags) {
290-
boolQuery.push({
291-
bool: {
292-
[criteria.includeAllTags ? 'must' : 'should']: _.map(criteria.tags, t => ({ match_phrase: { tags: t } }))
293-
}
294-
})
295-
}
296-
297319
if (criteria.events) {
298320
boolQuery.push({
299321
bool: {
@@ -491,7 +513,7 @@ async function searchChallenges (currentUser, criteria) {
491513
from: (page - 1) * perPage, // Es Index starts from 0
492514
body: {
493515
query: finalQuery,
494-
sort: [{ [sortByProp]: { 'order': sortOrderProp, 'missing': '_last', 'unmapped_type': 'String' } }]
516+
sort: [sortByScore ? { '_score': { 'order': 'desc' } } : { [sortByProp]: { 'order': sortOrderProp, 'missing': '_last', 'unmapped_type': 'String' } }]
495517
}
496518
}
497519

@@ -573,6 +595,7 @@ searchChallenges.schema = {
573595
type: Joi.string(),
574596
track: Joi.string(),
575597
name: Joi.string(),
598+
search: Joi.string(),
576599
description: Joi.string(),
577600
timelineTemplateId: Joi.string(), // Joi.optionalId(),
578601
reviewType: Joi.string(),

0 commit comments

Comments
 (0)
Please sign in to comment.