Skip to content
This repository was archived by the owner on Jan 23, 2025. It is now read-only.

Commit 521860c

Browse files
skyhitskyhit
authored
job_cleanup_improve2 (#66)
1 parent 7ee7b32 commit 521860c

File tree

76 files changed

+1198
-3220
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+1198
-3220
lines changed

‎.deploy/ecs_deploy.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ ELASTIC_SEARCH_URL=$(eval "echo \$${ENV}_ELASTIC_SEARCH_URL")
2323
TC_JWT_KEY=$(eval "echo \$${ENV}_TC_JWT_KEY")
2424
REDISSON_JOB_SINGLE_SERVER_ADDRESS=$(eval "echo \$${ENV}_REDISSON_JOB_SINGLE_SERVER_ADDRESS")
2525
LOG_LEVEL=$(eval "echo \$${ENV}_LOG_LEVEL")
26-
CHALLENGES_INDEX_NAME=$(eval "echo \$${ENV}_CHALLENGES_INDEX_NAME")
2726
#APP_NAME
2827
OLTP_USER=$(eval "echo \$${ENV}_OLTP_USER")
2928
OLTP_PW=$(eval "echo \$${ENV}_OLTP_PW")
@@ -85,7 +84,7 @@ push_ecr_image() {
8584
make_task_def(){
8685
echo "Creating ECS task definition..."
8786
task_template=`cat ecs_task_template.json`
88-
task_def=$(printf "$task_template" $AWS_ACCOUNT_ID $AWS_ECS_SERVICE $AWS_REGION "$AUTH_DOMAIN" $AWS_SIGNING_ENABLED $CHALLENGES_INDEX_NAME $ELASTIC_SEARCH_URL $OLTP_PW "$OLTP_URL" $OLTP_USER $TC_JWT_KEY $REDISSON_JOB_SINGLE_SERVER_ADDRESS $LOG_LEVEL $TAG $AWS_ECS_SERVICE $AWS_ACCOUNT_ID $AWS_ECS_SERVICE)
87+
task_def=$(printf "$task_template" $AWS_ACCOUNT_ID $AWS_ECS_SERVICE $AWS_REGION "$AUTH_DOMAIN" $AWS_SIGNING_ENABLED $ELASTIC_SEARCH_URL $OLTP_PW "$OLTP_URL" $OLTP_USER $TC_JWT_KEY $REDISSON_JOB_SINGLE_SERVER_ADDRESS $LOG_LEVEL $TAG $AWS_ECS_SERVICE $AWS_ACCOUNT_ID $AWS_ECS_SERVICE)
8988
echo $task_def > task_def.json
9089
echo "ECS task definition is created : "
9190
echo $task_def

‎.deploy/ecs_task_template.json

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,6 @@
3232
"name": "AWS_SIGNING_ENABLED",
3333
"value": "%s"
3434
},
35-
{
36-
"name": "CHALLENGES_INDEX_NAME",
37-
"value": "%s"
38-
},
3935
{
4036
"name": "ELASTIC_SEARCH_URL",
4137
"value": "%s"

‎docs/Job Cleanup and Improvement.postman_collection.json

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
{
2+
"info": {
3+
"_postman_id": "d1083801-f6dd-4220-a6c6-55ddab035754",
4+
"name": "Job Cleanup and Improvement",
5+
"schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json"
6+
},
7+
"item": [
8+
{
9+
"name": "View challenges in challenges listing index",
10+
"request": {
11+
"method": "GET",
12+
"header": [],
13+
"body": {
14+
"mode": "raw",
15+
"raw": ""
16+
},
17+
"url": "http://cockpit.cloud.topcoder.com:9200/challengeslisting/challenges/30005520"
18+
},
19+
"response": []
20+
},
21+
{
22+
"name": "View challenges in challenges details index",
23+
"request": {
24+
"method": "GET",
25+
"header": [],
26+
"body": {
27+
"mode": "raw",
28+
"raw": ""
29+
},
30+
"url": "http://cockpit.cloud.topcoder.com:9200/challengesdetail/challenges/30005520"
31+
},
32+
"response": []
33+
},
34+
{
35+
"name": "View marachon match in challenges listing index",
36+
"request": {
37+
"method": "GET",
38+
"header": [],
39+
"body": {
40+
"mode": "raw",
41+
"raw": ""
42+
},
43+
"url": "http://cockpit.cloud.topcoder.com:9200/challengeslisting/challenges/13675"
44+
},
45+
"response": []
46+
},
47+
{
48+
"name": "View marathon match in challenges details index",
49+
"request": {
50+
"method": "GET",
51+
"header": [],
52+
"body": {
53+
"mode": "raw",
54+
"raw": ""
55+
},
56+
"url": "http://cockpit.cloud.topcoder.com:9200/challengeslisting/challenges/13675"
57+
},
58+
"response": []
59+
},
60+
{
61+
"name": "View marathon match",
62+
"request": {
63+
"method": "GET",
64+
"header": [],
65+
"body": {
66+
"mode": "raw",
67+
"raw": ""
68+
},
69+
"url": "http://cockpit.cloud.topcoder.com:9200/mmatches/mmatches/_search"
70+
},
71+
"response": []
72+
},
73+
{
74+
"name": "View single round match",
75+
"request": {
76+
"method": "GET",
77+
"header": [],
78+
"body": {
79+
"mode": "raw",
80+
"raw": ""
81+
},
82+
"url": "http://cockpit.cloud.topcoder.com:9200/srms/srms/_search"
83+
},
84+
"response": []
85+
}
86+
],
87+
"event": [
88+
{
89+
"listen": "prerequest",
90+
"script": {
91+
"id": "d61f2275-4803-4bac-9787-1bfd6c2540e8",
92+
"type": "text/javascript",
93+
"exec": [
94+
""
95+
]
96+
}
97+
},
98+
{
99+
"listen": "test",
100+
"script": {
101+
"id": "40db868c-a50a-4b47-a1cd-73f959da9145",
102+
"type": "text/javascript",
103+
"exec": [
104+
""
105+
]
106+
}
107+
}
108+
]
109+
}

‎docs/Verification_Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement version 1.0.docx

716 KB
Binary file not shown.

‎local/run.bat

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ REM Please provide the AWS keys if you use AWS ElasticSearch and AWS_SIGNING_ENA
55
REM set AWS_ACCESS_KEY=your key
66
REM set AWS_SECRET_KEY=your secret
77

8-
java -jar ../target/elasticsearch-feeder-service-1.0.0.jar server ../src/main/resources/elasticsearch-feeder-service.yaml
8+
java -jar -Duser.timezone=America/New_York ../target/elasticsearch-feeder-service-1.0.0.jar server ../src/main/resources/elasticsearch-feeder-service.yaml

‎local/run.sh

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,4 @@ export TC_JWT_KEY="secret"
55
# export AWS_ACCESS_KEY=your key
66
# export AWS_SECRET_KEY=your secret
77

8-
export REDISSON_JOB_CLUSTER_ENABLED=false
9-
export REDISSON_JOB_FORCE_INITIAL_LOAD=true
10-
118
java -jar -Duser.timezone=America/New_York ../target/elasticsearch-feeder-service-*.jar server ../src/main/resources/elasticsearch-feeder-service.yaml

‎src/main/java/com/appirio/service/challengefeeder/ChallengeFeederServiceApplication.java

Lines changed: 51 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
import com.appirio.service.challengefeeder.resources.HealthCheckResource;
99
import com.appirio.service.challengefeeder.util.JestClientUtils;
1010
import com.appirio.service.resourcefactory.ChallengeFeederFactory;
11-
import com.appirio.service.resourcefactory.MmFeederResourceFactory;
1211
import com.appirio.service.resourcefactory.MarathonMatchFeederFactory;
12+
import com.appirio.service.resourcefactory.MmFeederResourceFactory;
1313
import com.appirio.service.resourcefactory.SRMFeederFactory;
1414
import com.appirio.service.supply.resources.SupplyDatasourceFactory;
1515

@@ -70,8 +70,13 @@
7070
* <li>Add job for LoadChangedChallengeDetailJob</li>
7171
* </ul>
7272
* </p>
73+
*
74+
* <p>
75+
* Version 1.8 - Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement v1.0
76+
* - remove the useless resources, jobs and logging of configuration values
77+
* </p>
7378
* @author TCSCODER
74-
* @version 1.7
79+
* @version 1.8
7580
*/
7681
public class ChallengeFeederServiceApplication extends BaseApplication<ChallengeFeederServiceConfiguration> {
7782
/**
@@ -107,19 +112,47 @@ protected void logServiceSpecificConfigs(ChallengeFeederServiceConfiguration con
107112
logger.info("\t\tAWS region : " + config.getJestClientConfiguration().getAwsRegion());
108113
logger.info("\t\tAWS service : " + config.getJestClientConfiguration().getAwsService());
109114

110-
logger.info("\tRedissonConfiguration ");
111-
logger.info("\t\tChallenges index: " + config.getRedissonConfiguration().getChallengesIndex());
112-
logger.info("\t\tChallenges type: " + config.getRedissonConfiguration().getChallengesType());
113-
logger.info("\t\tSingle server address: " + config.getRedissonConfiguration().getSingleServerAddress());
114-
logger.info("\t\tLast run timestamp prefix for job LoadChangedChallengesJob: " + config.getRedissonConfiguration().getLoadChangedChallengesJobLastRunTimestampPrefix());
115-
logger.info("\t\tLast run timestamp prefix for job MarathonMatchesJob: " + config.getRedissonConfiguration().getMarathonMatchesJobLastRunTimestampPrefix());
116-
logger.info("\t\tLast run timestamp prefix for job SingleRoundMatchesJob: " + config.getRedissonConfiguration().getSingleRoundMatchesJobLastRunTimestampPrefix());
117-
logger.info("\t\tCluster enabled: " + config.getRedissonConfiguration().isClusterEnabled());
118-
logger.info("\t\tLoadChangedChallengesJob Locker key name: " + config.getRedissonConfiguration().getLoadChangedChallengesJobLockerKeyName());
119-
logger.info("\t\tMarathonMatchesJob Locker key name: " + config.getRedissonConfiguration().getMarathonMatchesJobLockerKeyName());
120-
logger.info("\t\tSingleRoundMatchesJob Locker key name: " + config.getRedissonConfiguration().getSingleRoundMatchesJobLockerKeyName());
121-
logger.info("\t\tLock watchdog timeout: " + config.getRedissonConfiguration().getLockWatchdogTimeout());
122-
logger.info("\t\tNode addresses: " + config.getRedissonConfiguration().getNodeAddresses());
115+
logger.info("\tJobsConfiguration");
116+
logger.info("\t\tRedissonConfiguration");
117+
logger.info("\t\t\tSingle server address: " + config.getJobsConfiguration().getRedissonConfiguration().getSingleServerAddress());
118+
logger.info("\t\t\tCluster enabled: " + config.getJobsConfiguration().getRedissonConfiguration().isClusterEnabled());
119+
logger.info("\t\t\tLock watchdog timeout: " + config.getJobsConfiguration().getRedissonConfiguration().getLockWatchdogTimeout());
120+
logger.info("\t\t\tNode addresses: " + config.getJobsConfiguration().getRedissonConfiguration().getNodeAddresses());
121+
122+
123+
logger.info("\tJobs configuration");
124+
logger.info("\t\t\tRedisson configuration");
125+
logger.info("\t\t\t\t\tLock watchdog timeout : " + config.getJobsConfiguration().getRedissonConfiguration().getLockWatchdogTimeout());
126+
logger.info("\t\t\t\t\tSingle server address : " + config.getJobsConfiguration().getRedissonConfiguration().getSingleServerAddress());
127+
logger.info("\t\t\t\t\tCluster enabled : " + config.getJobsConfiguration().getRedissonConfiguration().isClusterEnabled());
128+
logger.info("\t\t\tNode addresses: " + config.getJobsConfiguration().getRedissonConfiguration().getNodeAddresses());
129+
logger.info("\t\t\t\t\tNode addresses");
130+
logger.info("\t\t\tLoad changed challenges listing job");
131+
logger.info("\t\t\t\t\tIndex name");
132+
logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getLoadChangedChallengesListingJob().getBatchUpdateSize());
133+
logger.info("\t\t\tLoad changed challenges detail job");
134+
logger.info("\t\t\t\t\tIndex name");
135+
logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getLoadChangedChallengesDetailJob().getBatchUpdateSize());
136+
logger.info("\t\t\tLegacymm to challenge listing job");
137+
logger.info("\t\t\t\t\tIndex name");
138+
logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getLegacyMMToChallengeListingJob().getBatchUpdateSize());
139+
logger.info("\t\t\t\t\tMarathon matches days to subtract : " + config.getJobsConfiguration().getLegacyMMToChallengeListingJob().getMarathonMatchesDaysToSubtract());
140+
logger.info("\t\t\t\t\tMarathon matches forum url : " + config.getJobsConfiguration().getLegacyMMToChallengeListingJob().getMarathonMatchesForumUrl());
141+
logger.info("\t\t\tLoad changedmm challenge detail job");
142+
logger.info("\t\t\t\t\tIndex name");
143+
logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getLoadChangedMMChallengeDetailJob().getBatchUpdateSize());
144+
logger.info("\t\t\t\t\tMarathon matches days to subtract : " + config.getJobsConfiguration().getLoadChangedMMChallengeDetailJob().getMarathonMatchesDaysToSubtract());
145+
logger.info("\t\t\t\t\tMarathon matches forum url : " + config.getJobsConfiguration().getLoadChangedMMChallengeDetailJob().getMarathonMatchesForumUrl());
146+
logger.info("\t\t\tMarathon matches job");
147+
logger.info("\t\t\t\t\tIndex name");
148+
logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getMarathonMatchesJob().getBatchUpdateSize());
149+
logger.info("\t\t\t\t\tMarathon matches days to subtract : " + config.getJobsConfiguration().getMarathonMatchesJob().getMarathonMatchesDaysToSubtract());
150+
logger.info("\t\t\t\t\tMarathon matches forum url : " + config.getJobsConfiguration().getMarathonMatchesJob().getMarathonMatchesForumUrl());
151+
logger.info("\t\t\tSingle round matches job");
152+
logger.info("\t\t\t\t\tIndex name");
153+
logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getSingleRoundMatchesJob().getBatchUpdateSize());
154+
logger.info("\t\t\t\t\tSingle round matches days to subtract : " + config.getJobsConfiguration().getSingleRoundMatchesJob().getSingleRoundMatchesDaysToSubtract());
155+
123156

124157
logger.info("\tJobs ");
125158
logger.info("\t\tJobs: " + config.getJobs());
@@ -149,9 +182,9 @@ protected void registerResources(ChallengeFeederServiceConfiguration config, Env
149182
JestClient jestClient = JestClientUtils.get(config.getJestClientConfiguration());
150183

151184
// Register resources here
185+
env.jersey().register(new HealthCheckResource());
152186
env.jersey().register(new ChallengeFeederFactory(jestClient).getResourceInstance());
153187
env.jersey().register(new MmFeederResourceFactory(jestClient).getResourceInstance());
154-
env.jersey().register(new HealthCheckResource());
155188
env.jersey().register(new MarathonMatchFeederFactory(jestClient).getResourceInstance());
156189
env.jersey().register(new SRMFeederFactory(jestClient).getResourceInstance());
157190
logger.info("Services registered");
@@ -182,7 +215,7 @@ public void initialize(Bootstrap<ChallengeFeederServiceConfiguration> bootstrap)
182215
bootstrap.setConfigurationSourceProvider(
183216
new SubstitutingSourceProvider(bootstrap.getConfigurationSourceProvider(), new EnvironmentVariableSubstitutor(false)));
184217

185-
bootstrap.addBundle(new JobsBundle(new LoadChangedChallengesListingJob(), new LegacyMMToChallengeListingJob(),
186-
new MarathonMatchesJob(), new SingleRoundMatchesJob(), new LoadChangedChallengesDetailJob(), new LoadChangedMMChallengeDetailJob()));
218+
bootstrap.addBundle(new JobsBundle(new LoadChangedChallengesListingJob(), new LegacyMMToChallengeListingJob(), new MarathonMatchesJob(),
219+
new SingleRoundMatchesJob(), new LoadChangedChallengesDetailJob(), new LoadChangedMMChallengeDetailJob()));
187220
}
188221
}

‎src/main/java/com/appirio/service/challengefeeder/ChallengeFeederServiceConfiguration.java

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@
1111
import com.appirio.service.challengefeeder.config.ChallengeConfiguration;
1212
import com.appirio.service.challengefeeder.config.CommonConfiguration;
1313
import com.appirio.service.challengefeeder.config.JestClientConfiguration;
14-
import com.appirio.service.challengefeeder.config.RedissonConfiguration;
14+
import com.appirio.service.challengefeeder.config.JobsConfiguration;
1515
import com.appirio.service.supply.resources.SupplyDatasourceFactory;
1616
import com.fasterxml.jackson.annotation.JsonProperty;
1717

1818
import de.spinscale.dropwizard.jobs.JobConfiguration;
19+
import lombok.Getter;
1920

2021
import javax.validation.Valid;
2122
import javax.validation.constraints.NotNull;
@@ -34,8 +35,13 @@
3435
*
3536
* Version 1.3 - Topcoder ElasticSearch Feeder Service - Way To Populate Challenge-Listing Index
3637
* - add commonConfiguration
38+
*
39+
* Version 1.4 - Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement v1.0
40+
* - refactor the job configuration
41+
*
42+
*
3743
* @author TCSCODER
38-
* @version 1.3
44+
* @version 1.4
3945
*/
4046
public class ChallengeFeederServiceConfiguration extends BaseAppConfiguration implements JobConfiguration {
4147

@@ -62,10 +68,11 @@ public class ChallengeFeederServiceConfiguration extends BaseAppConfiguration im
6268
private Map<String , String> jobs;
6369

6470
/**
65-
* Represents the redissonConfiguration field
71+
* Represents the jobsConfiguration field
6672
*/
67-
@JsonProperty("redissonConfiguration")
68-
private RedissonConfiguration redissonConfiguration;
73+
@JsonProperty("jobsConfiguration")
74+
@Getter
75+
private JobsConfiguration jobsConfiguration;
6976

7077
/**
7178
* The challengeConfiguration
@@ -110,14 +117,6 @@ public Map<String, String> getJobs() {
110117
return jobs;
111118
}
112119

113-
/**
114-
* Get redissonConfiguration
115-
* @return the redissonConfiguration
116-
*/
117-
public RedissonConfiguration getRedissonConfiguration() {
118-
return this.redissonConfiguration;
119-
}
120-
121120
/**
122121
* Get challenge configuration
123122
*

‎src/main/java/com/appirio/service/challengefeeder/config/BaseJobConfiguration.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package com.appirio.service.challengefeeder.config;
2+
3+
import org.hibernate.validator.constraints.NotEmpty;
4+
5+
import com.fasterxml.jackson.annotation.JsonProperty;
6+
7+
import lombok.Getter;
8+
import lombok.Setter;
9+
10+
/**
11+
* Represents the BaseJobConfiguration
12+
*
13+
* It's added in Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement v1.0
14+
*
15+
* @author TCCoder
16+
* @version 1.0
17+
*
18+
*/
19+
public class BaseJobConfiguration {
20+
/**
21+
* The index field
22+
*/
23+
@JsonProperty
24+
@NotEmpty
25+
@Getter
26+
@Setter
27+
private String indexName;
28+
29+
/**
30+
* The type name field
31+
*/
32+
@JsonProperty
33+
@NotEmpty
34+
@Getter
35+
@Setter
36+
private String typeName;
37+
38+
/**
39+
* Represents the batch update size field.
40+
*/
41+
@JsonProperty
42+
@Getter
43+
@Setter
44+
private int batchUpdateSize;
45+
46+
}

0 commit comments

Comments
 (0)