Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 637b9ae

Browse files
rakibansaryrakibansary
committedFeb 21, 2023
refactor: upgrade to opensearch & build fix
Signed-off-by: Rakib Ansary <[email protected]>
1 parent 134814e commit 637b9ae

File tree

13 files changed

+218
-265
lines changed

13 files changed

+218
-265
lines changed
 

‎.circleci/config.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ install_deploysuite: &install_deploysuite
1818
cp ./../buildscript/buildenv.sh .
1919
cp ./../buildscript/awsconfiguration.sh .
2020
restore_cache_settings_for_build: &restore_cache_settings_for_build
21-
key: docker-node-modules-{{ checksum "package-lock.json" }}
21+
key: docker-node-modules-{{ checksum "yarn.lock" }}
2222

2323
save_cache_settings: &save_cache_settings
24-
key: docker-node-modules-{{ checksum "package-lock.json" }}
24+
key: docker-node-modules-{{ checksum "yarn.lock" }}
2525
paths:
2626
- node_modules
2727

‎README.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ The following parameters can be set in config files or in env variables:
5858
- ES.HOST: Elasticsearch host
5959
- ES.API_VERSION: Elasticsearch API version
6060
- ES.ES_INDEX: Elasticsearch index name
61-
- ES.ES_TYPE: Elasticsearch index type
6261
- ES.ES_REFRESH: Elasticsearch refresh method. Default to string `true`(i.e. refresh immediately)
6362
- FILE_UPLOAD_SIZE_LIMIT: the file upload size limit in bytes
6463
- RESOURCES_API_URL: TC resources API base URL

‎config/default.js

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ module.exports = {
4343
HOST: process.env.ES_HOST || "localhost:9200",
4444
API_VERSION: process.env.ES_API_VERSION || "6.8",
4545
ES_INDEX: process.env.ES_INDEX || "challenge",
46-
ES_TYPE: process.env.ES_TYPE || "_doc", // ES 6.x accepts only 1 Type per index and it's mandatory to define it
4746
ES_REFRESH: process.env.ES_REFRESH || "true",
4847
TEMP_REINDEXING: process.env.TEMP_REINDEXING || true, // if true, it won't delete the existing index when reindexing data
4948
},

‎docker/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Use the base image with Node.js
2-
FROM node:14.21.2-bullseye
2+
FROM node:18.14.1-alpine3.17
33

44
# Copy the current directory into the Docker image
55
COPY . /challenge-api

‎docs/prod.env

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ S3_API_VERSION=
1414
ES_HOST=
1515
ES_API_VERSION=
1616
ES_INDEX=
17-
ES_TYPE=
1817
ES_REFRESH=true
1918
RESOURCES_API_URL=
2019
GROUPS_API_URL=

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
"prettier": "^2.8.1"
4040
},
4141
"dependencies": {
42+
"@opensearch-project/opensearch": "^2.2.0",
4243
"@topcoder-framework/domain-challenge": "^0.4.3",
4344
"@topcoder-framework/lib-common": "^0.4.3",
4445
"aws-sdk": "^2.1145.0",
@@ -50,7 +51,6 @@
5051
"cors": "^2.7.1",
5152
"dotenv": "^8.2.0",
5253
"dynamoose": "^1.11.1",
53-
"elasticsearch": "^16.1.1",
5454
"express": "^4.15.4",
5555
"express-fileupload": "^1.1.6",
5656
"express-interceptor": "^1.2.0",

‎src/common/helper.js

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,13 @@ const config = require("config");
1313
const axios = require("axios");
1414
const axiosRetry = require("axios-retry");
1515
const busApi = require("topcoder-bus-api-wrapper");
16-
const elasticsearch = require("elasticsearch");
1716
const NodeCache = require("node-cache");
1817
const HttpStatus = require("http-status-codes");
1918
const xss = require("xss");
2019
const logger = require("./logger");
2120

21+
const { Client: ESClient } = require("@opensearch-project/opensearch");
22+
2223
const projectHelper = require("./project-helper");
2324
const m2mHelper = require("./m2m-helper");
2425

@@ -850,23 +851,14 @@ function getESClient() {
850851
return esClient;
851852
}
852853
const esHost = config.get("ES.HOST");
853-
// AWS ES configuration is different from other providers
854-
if (/.*amazonaws.*/.test(esHost)) {
855-
esClient = elasticsearch.Client({
856-
apiVersion: config.get("ES.API_VERSION"),
857-
hosts: esHost,
858-
connectionClass: require("http-aws-es"), // eslint-disable-line global-require
859-
amazonES: {
860-
region: config.get("AMAZON.AWS_REGION"),
861-
credentials: new AWS.EnvironmentCredentials("AWS"),
862-
},
863-
});
864-
} else {
865-
esClient = new elasticsearch.Client({
866-
apiVersion: config.get("ES.API_VERSION"),
867-
hosts: esHost,
868-
});
869-
}
854+
855+
esClient = new ESClient({
856+
node: esHost,
857+
ssl: {
858+
rejectUnauthorized: false,
859+
},
860+
});
861+
870862
return esClient;
871863
}
872864

‎src/init-es.js

Lines changed: 54 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -7,81 +7,85 @@
77
* node src/init-es force
88
*/
99
const config = require("config");
10-
// const logger = require('./common/logger')
10+
const logger = require("./common/logger");
1111
const helper = require("./common/helper");
1212

1313
const client = helper.getESClient();
1414

1515
const initES = async () => {
1616
if (process.argv.length === 3 && process.argv[2] === "force") {
17-
console.log(`Delete index ${config.ES.ES_INDEX} if any.`);
17+
logger.info(`Delete index ${config.ES.ES_INDEX} if any.`);
1818
try {
1919
await client.indices.delete({ index: config.ES.ES_INDEX });
2020
} catch (err) {
2121
// ignore
2222
}
2323
}
2424

25-
const exists = await client.indices.exists({ index: config.ES.ES_INDEX });
25+
const { body: exists } = await client.indices.exists({ index: config.ES.ES_INDEX });
2626
if (exists) {
27-
console.log(`The index ${config.ES.ES_INDEX} exists.`);
27+
logger.info(`The index ${config.ES.ES_INDEX} exists.`);
2828
} else {
29-
console.log(`The index ${config.ES.ES_INDEX} will be created.`);
29+
logger.info(`The index ${config.ES.ES_INDEX} will be created.`);
3030

31-
const body = { mappings: {} };
32-
body.mappings[config.get("ES.ES_TYPE")] = {
33-
properties: {
34-
id: { type: "keyword" },
35-
name: {
36-
type: "keyword",
37-
fields: {
38-
text: {
39-
type: "text",
31+
const body = {
32+
mappings: {
33+
// properties: {
34+
properties: {
35+
id: { type: "keyword" },
36+
name: {
37+
type: "keyword",
38+
fields: {
39+
text: {
40+
type: "text",
41+
},
4042
},
43+
normalizer: "custom_sort_normalizer",
4144
},
42-
normalizer: "custom_sort_normalizer",
43-
},
44-
status: {
45-
type: 'keyword',
46-
fields: {
47-
text: {
48-
type: 'text'
49-
}
45+
status: {
46+
type: "keyword",
47+
fields: {
48+
text: {
49+
type: "text",
50+
},
51+
},
52+
normalizer: "custom_sort_normalizer",
5053
},
51-
normalizer: 'custom_sort_normalizer'
52-
},
53-
type: {
54-
type: 'keyword',
55-
fields: {
56-
text: {
57-
type: 'text'
58-
}
54+
type: {
55+
type: "keyword",
56+
fields: {
57+
text: {
58+
type: "text",
59+
},
60+
},
61+
normalizer: "custom_sort_normalizer",
5962
},
60-
normalizer: 'custom_sort_normalizer'
61-
},
62-
prizeSets: {
63-
properties: {
64-
type: { type: "text" },
65-
prizes: {
66-
properties: {
67-
type: { type: "text" },
68-
value: { type: "float" },
63+
prizeSets: {
64+
properties: {
65+
type: { type: "text" },
66+
prizes: {
67+
properties: {
68+
type: { type: "text" },
69+
value: { type: "float" },
70+
},
6971
},
7072
},
7173
},
7274
},
73-
},
74-
dynamic_templates: [
75-
{
76-
metadata: {
77-
path_match: "metadata.*",
78-
mapping: {
79-
type: "text",
75+
dynamic_templates: [
76+
{
77+
metadata: {
78+
path_match: "metadata.*",
79+
mapping: {
80+
type: "text",
81+
},
8082
},
8183
},
82-
},
83-
],
84+
],
85+
//}
86+
},
8487
};
88+
8589
body.settings = {
8690
analysis: {
8791
normalizer: {
@@ -103,11 +107,10 @@ const initES = async () => {
103107

104108
initES()
105109
.then(() => {
106-
console.log("Done!");
110+
logger.info("Done!");
107111
process.exit();
108112
})
109113
.catch((e) => {
110-
// logger.logFullError(e);
111-
e.printStackTrace();
114+
logger.logFullError(e);
112115
process.exit();
113116
});

‎src/scripts/sync-es.js

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ async function migrateRecords() {
1616
for (const challenge of result) {
1717
await esClient.update({
1818
index: config.get("ES.ES_INDEX"),
19-
type: config.get("ES.ES_TYPE"),
2019
id: challenge.id,
2120
body: { doc: challenge, doc_as_upsert: true },
2221
});

‎src/scripts/view-es-data.js

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ const esClient = helper.getESClient();
1111
async function showESData() {
1212
const result = await esClient.search({
1313
index: config.get("ES.ES_INDEX"),
14-
type: config.get("ES.ES_TYPE"),
1514
});
1615
return result;
1716
}

‎src/services/ChallengeService.js

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,6 @@ async function searchChallenges(currentUser, criteria) {
794794

795795
const esQuery = {
796796
index: config.get("ES.ES_INDEX"),
797-
type: config.get("ES.ES_TYPE"),
798797
size: perPage,
799798
from: (page - 1) * perPage, // Es Index starts from 0
800799
body: {
@@ -817,7 +816,7 @@ async function searchChallenges(currentUser, criteria) {
817816
// Search with constructed query
818817
let docs;
819818
try {
820-
docs = await esClient.search(esQuery);
819+
docs = (await esClient.search(esQuery)).body;
821820
} catch (e) {
822821
// Catch error when the ES is fresh and has no data
823822
logger.error(`Query Error from ES ${JSON.stringify(e, null, 4)}`);
@@ -1139,7 +1138,6 @@ async function createChallenge(currentUser, challenge, userToken) {
11391138
// Create in ES
11401139
await esClient.create({
11411140
index: config.get("ES.ES_INDEX"),
1142-
type: config.get("ES.ES_TYPE"),
11431141
refresh: config.get("ES.ES_REFRESH"),
11441142
id: ret.id,
11451143
body: ret,
@@ -1304,15 +1302,15 @@ async function getChallenge(currentUser, id, checkIfExists) {
13041302
let challenge;
13051303
// logger.warn(JSON.stringify({
13061304
// index: config.get('ES.ES_INDEX'),
1307-
// type: config.get('ES.ES_TYPE'),
13081305
// _id: id
13091306
// }))
13101307
try {
1311-
challenge = await esClient.getSource({
1312-
index: config.get("ES.ES_INDEX"),
1313-
type: config.get("ES.ES_TYPE"),
1314-
id,
1315-
});
1308+
challenge = (
1309+
await esClient.getSource({
1310+
index: config.get("ES.ES_INDEX"),
1311+
id,
1312+
})
1313+
).body;
13161314
} catch (e) {
13171315
if (e.statusCode === HttpStatus.NOT_FOUND) {
13181316
throw new errors.NotFoundError(`Challenge of id ${id} is not found.`);
@@ -2195,7 +2193,6 @@ async function update(currentUser, challengeId, data, isFull) {
21952193
// Update ES
21962194
await esClient.update({
21972195
index: config.get("ES.ES_INDEX"),
2198-
type: config.get("ES.ES_TYPE"),
21992196
refresh: config.get("ES.ES_REFRESH"),
22002197
id: challengeId,
22012198
body: {
@@ -2701,7 +2698,6 @@ async function deleteChallenge(currentUser, challengeId) {
27012698
// delete ES document
27022699
await esClient.delete({
27032700
index: config.get("ES.ES_INDEX"),
2704-
type: config.get("ES.ES_TYPE"),
27052701
refresh: config.get("ES.ES_REFRESH"),
27062702
id: challengeId,
27072703
});

0 commit comments

Comments
 (0)
Please sign in to comment.