topcoder-archive · skyhit · Jun 11, 2018 · Jun 10, 2018 · Jun 11, 2018 · Jun 11, 2018
diff --git a/.deploy/ecs_deploy.sh b/.deploy/ecs_deploy.sh
@@ -23,7 +23,6 @@ ELASTIC_SEARCH_URL=$(eval "echo \$${ENV}_ELASTIC_SEARCH_URL")
 TC_JWT_KEY=$(eval "echo \$${ENV}_TC_JWT_KEY")
 REDISSON_JOB_SINGLE_SERVER_ADDRESS=$(eval "echo \$${ENV}_REDISSON_JOB_SINGLE_SERVER_ADDRESS")
 LOG_LEVEL=$(eval "echo \$${ENV}_LOG_LEVEL")
-CHALLENGES_INDEX_NAME=$(eval "echo \$${ENV}_CHALLENGES_INDEX_NAME")
 #APP_NAME
 OLTP_USER=$(eval "echo \$${ENV}_OLTP_USER")
 OLTP_PW=$(eval "echo \$${ENV}_OLTP_PW")
@@ -85,7 +84,7 @@ push_ecr_image() {
 make_task_def(){
   echo "Creating ECS task definition..."  
   task_template=`cat ecs_task_template.json`
-  task_def=$(printf "$task_template" $AWS_ACCOUNT_ID $AWS_ECS_SERVICE $AWS_REGION "$AUTH_DOMAIN" $AWS_SIGNING_ENABLED $CHALLENGES_INDEX_NAME $ELASTIC_SEARCH_URL $OLTP_PW "$OLTP_URL" $OLTP_USER $TC_JWT_KEY $REDISSON_JOB_SINGLE_SERVER_ADDRESS $LOG_LEVEL $TAG $AWS_ECS_SERVICE $AWS_ACCOUNT_ID $AWS_ECS_SERVICE)
+  task_def=$(printf "$task_template" $AWS_ACCOUNT_ID $AWS_ECS_SERVICE $AWS_REGION "$AUTH_DOMAIN" $AWS_SIGNING_ENABLED $ELASTIC_SEARCH_URL $OLTP_PW "$OLTP_URL" $OLTP_USER $TC_JWT_KEY $REDISSON_JOB_SINGLE_SERVER_ADDRESS $LOG_LEVEL $TAG $AWS_ECS_SERVICE $AWS_ACCOUNT_ID $AWS_ECS_SERVICE)
   echo $task_def > task_def.json
   echo "ECS task definition is created : "
   echo $task_def

diff --git a/.deploy/ecs_task_template.json b/.deploy/ecs_task_template.json
@@ -32,10 +32,6 @@
             "name": "AWS_SIGNING_ENABLED",
             "value": "%s"
           },
-          {
-            "name": "CHALLENGES_INDEX_NAME",
-            "value": "%s"
-          },
           {
             "name": "ELASTIC_SEARCH_URL",
             "value": "%s"

diff --git a/docs/Job Cleanup and Improvement.postman_collection.json b/docs/Job Cleanup and Improvement.postman_collection.json
@@ -0,0 +1,109 @@
+{
+	"info": {
+		"_postman_id": "d1083801-f6dd-4220-a6c6-55ddab035754",
+		"name": "Job Cleanup and Improvement",
+		"schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json"
+	},
+	"item": [
+		{
+			"name": "View challenges in challenges listing index",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": ""
+				},
+				"url": "http://cockpit.cloud.topcoder.com:9200/challengeslisting/challenges/30005520"
+			},
+			"response": []
+		},
+		{
+			"name": "View challenges in challenges details index",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": ""
+				},
+				"url": "http://cockpit.cloud.topcoder.com:9200/challengesdetail/challenges/30005520"
+			},
+			"response": []
+		},
+		{
+			"name": "View marachon match in challenges listing index",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": ""
+				},
+				"url": "http://cockpit.cloud.topcoder.com:9200/challengeslisting/challenges/13675"
+			},
+			"response": []
+		},
+		{
+			"name": "View marathon match in challenges details index",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": ""
+				},
+				"url": "http://cockpit.cloud.topcoder.com:9200/challengeslisting/challenges/13675"
+			},
+			"response": []
+		},
+		{
+			"name": "View marathon match",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": ""
+				},
+				"url": "http://cockpit.cloud.topcoder.com:9200/mmatches/mmatches/_search"
+			},
+			"response": []
+		},
+		{
+			"name": "View single round match",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"body": {
+					"mode": "raw",
+					"raw": ""
+				},
+				"url": "http://cockpit.cloud.topcoder.com:9200/srms/srms/_search"
+			},
+			"response": []
+		}
+	],
+	"event": [
+		{
+			"listen": "prerequest",
+			"script": {
+				"id": "d61f2275-4803-4bac-9787-1bfd6c2540e8",
+				"type": "text/javascript",
+				"exec": [
+					""
+				]
+			}
+		},
+		{
+			"listen": "test",
+			"script": {
+				"id": "40db868c-a50a-4b47-a1cd-73f959da9145",
+				"type": "text/javascript",
+				"exec": [
+					""
+				]
+			}
+		}
+	]
+}
diff --git a/...ion_Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement version 1.0.docx b/...ion_Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement version 1.0.docx
diff --git a/local/run.bat b/local/run.bat
@@ -5,4 +5,4 @@ REM Please provide the AWS keys if you use AWS ElasticSearch and AWS_SIGNING_ENA
 REM set AWS_ACCESS_KEY=your key
 REM set AWS_SECRET_KEY=your secret
 
-java -jar ../target/elasticsearch-feeder-service-1.0.0.jar server ../src/main/resources/elasticsearch-feeder-service.yaml
+java -jar -Duser.timezone=America/New_York ../target/elasticsearch-feeder-service-1.0.0.jar server ../src/main/resources/elasticsearch-feeder-service.yaml
diff --git a/local/run.sh b/local/run.sh
@@ -5,7 +5,4 @@ export TC_JWT_KEY="secret"
 # export AWS_ACCESS_KEY=your key
 # export AWS_SECRET_KEY=your secret
 
-export REDISSON_JOB_CLUSTER_ENABLED=false
-export REDISSON_JOB_FORCE_INITIAL_LOAD=true
-
 java -jar -Duser.timezone=America/New_York ../target/elasticsearch-feeder-service-*.jar server ../src/main/resources/elasticsearch-feeder-service.yaml
diff --git a/src/main/java/com/appirio/service/challengefeeder/ChallengeFeederServiceApplication.java b/src/main/java/com/appirio/service/challengefeeder/ChallengeFeederServiceApplication.java
@@ -8,8 +8,8 @@
 import com.appirio.service.challengefeeder.resources.HealthCheckResource;
 import com.appirio.service.challengefeeder.util.JestClientUtils;
 import com.appirio.service.resourcefactory.ChallengeFeederFactory;
-import com.appirio.service.resourcefactory.MmFeederResourceFactory;
 import com.appirio.service.resourcefactory.MarathonMatchFeederFactory;
+import com.appirio.service.resourcefactory.MmFeederResourceFactory;
 import com.appirio.service.resourcefactory.SRMFeederFactory;
 import com.appirio.service.supply.resources.SupplyDatasourceFactory;
 
@@ -70,8 +70,13 @@
  *      <li>Add job for LoadChangedChallengeDetailJob</li>
  * </ul>
  * </p>
+ * 
+ * <p>
+ * Version 1.8 - Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement v1.0
+ * - remove the useless resources, jobs and logging of configuration values
+ * </p>
  * @author TCSCODER
- * @version 1.7
+ * @version 1.8 
  */
 public class ChallengeFeederServiceApplication extends BaseApplication<ChallengeFeederServiceConfiguration> {
     /**
@@ -107,19 +112,47 @@ protected void logServiceSpecificConfigs(ChallengeFeederServiceConfiguration con
         logger.info("\t\tAWS region : " + config.getJestClientConfiguration().getAwsRegion());
         logger.info("\t\tAWS service : " + config.getJestClientConfiguration().getAwsService());
 
-        logger.info("\tRedissonConfiguration ");
-        logger.info("\t\tChallenges index: " + config.getRedissonConfiguration().getChallengesIndex());
-        logger.info("\t\tChallenges type: " + config.getRedissonConfiguration().getChallengesType());
-        logger.info("\t\tSingle server address: " + config.getRedissonConfiguration().getSingleServerAddress());
-        logger.info("\t\tLast run timestamp prefix for job LoadChangedChallengesJob: " + config.getRedissonConfiguration().getLoadChangedChallengesJobLastRunTimestampPrefix());
-        logger.info("\t\tLast run timestamp prefix for job MarathonMatchesJob: " + config.getRedissonConfiguration().getMarathonMatchesJobLastRunTimestampPrefix());
-        logger.info("\t\tLast run timestamp prefix for job SingleRoundMatchesJob: " + config.getRedissonConfiguration().getSingleRoundMatchesJobLastRunTimestampPrefix());
-        logger.info("\t\tCluster enabled: " + config.getRedissonConfiguration().isClusterEnabled());
-        logger.info("\t\tLoadChangedChallengesJob Locker key name: " + config.getRedissonConfiguration().getLoadChangedChallengesJobLockerKeyName());
-        logger.info("\t\tMarathonMatchesJob Locker key name: " + config.getRedissonConfiguration().getMarathonMatchesJobLockerKeyName());
-        logger.info("\t\tSingleRoundMatchesJob Locker key name: " + config.getRedissonConfiguration().getSingleRoundMatchesJobLockerKeyName());
-        logger.info("\t\tLock watchdog timeout: " + config.getRedissonConfiguration().getLockWatchdogTimeout());
-        logger.info("\t\tNode addresses: " + config.getRedissonConfiguration().getNodeAddresses());
+        logger.info("\tJobsConfiguration");
+        logger.info("\t\tRedissonConfiguration");
+        logger.info("\t\t\tSingle server address: " + config.getJobsConfiguration().getRedissonConfiguration().getSingleServerAddress());
+        logger.info("\t\t\tCluster enabled: " + config.getJobsConfiguration().getRedissonConfiguration().isClusterEnabled());
+        logger.info("\t\t\tLock watchdog timeout: " + config.getJobsConfiguration().getRedissonConfiguration().getLockWatchdogTimeout());
+        logger.info("\t\t\tNode addresses: " + config.getJobsConfiguration().getRedissonConfiguration().getNodeAddresses());
+
+
+        logger.info("\tJobs configuration");
+        logger.info("\t\t\tRedisson configuration");
+        logger.info("\t\t\t\t\tLock watchdog timeout : " + config.getJobsConfiguration().getRedissonConfiguration().getLockWatchdogTimeout());
+        logger.info("\t\t\t\t\tSingle server address : " + config.getJobsConfiguration().getRedissonConfiguration().getSingleServerAddress());
+        logger.info("\t\t\t\t\tCluster enabled : " + config.getJobsConfiguration().getRedissonConfiguration().isClusterEnabled());
+        logger.info("\t\t\tNode addresses: " + config.getJobsConfiguration().getRedissonConfiguration().getNodeAddresses());
+        logger.info("\t\t\t\t\tNode addresses");
+        logger.info("\t\t\tLoad changed challenges listing job");
+        logger.info("\t\t\t\t\tIndex name");
+        logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getLoadChangedChallengesListingJob().getBatchUpdateSize());
+        logger.info("\t\t\tLoad changed challenges detail job");
+        logger.info("\t\t\t\t\tIndex name");
+        logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getLoadChangedChallengesDetailJob().getBatchUpdateSize());
+        logger.info("\t\t\tLegacymm to challenge listing job");
+        logger.info("\t\t\t\t\tIndex name");
+        logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getLegacyMMToChallengeListingJob().getBatchUpdateSize());
+        logger.info("\t\t\t\t\tMarathon matches days to subtract : " + config.getJobsConfiguration().getLegacyMMToChallengeListingJob().getMarathonMatchesDaysToSubtract());
+        logger.info("\t\t\t\t\tMarathon matches forum url : " + config.getJobsConfiguration().getLegacyMMToChallengeListingJob().getMarathonMatchesForumUrl());
+        logger.info("\t\t\tLoad changedmm challenge detail job");
+        logger.info("\t\t\t\t\tIndex name");
+        logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getLoadChangedMMChallengeDetailJob().getBatchUpdateSize());
+        logger.info("\t\t\t\t\tMarathon matches days to subtract : " + config.getJobsConfiguration().getLoadChangedMMChallengeDetailJob().getMarathonMatchesDaysToSubtract());
+        logger.info("\t\t\t\t\tMarathon matches forum url : " + config.getJobsConfiguration().getLoadChangedMMChallengeDetailJob().getMarathonMatchesForumUrl());
+        logger.info("\t\t\tMarathon matches job");
+        logger.info("\t\t\t\t\tIndex name");
+        logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getMarathonMatchesJob().getBatchUpdateSize());
+        logger.info("\t\t\t\t\tMarathon matches days to subtract : " + config.getJobsConfiguration().getMarathonMatchesJob().getMarathonMatchesDaysToSubtract());
+        logger.info("\t\t\t\t\tMarathon matches forum url : " + config.getJobsConfiguration().getMarathonMatchesJob().getMarathonMatchesForumUrl());
+        logger.info("\t\t\tSingle round matches job");
+        logger.info("\t\t\t\t\tIndex name");
+        logger.info("\t\t\t\t\tBatch size : " + config.getJobsConfiguration().getSingleRoundMatchesJob().getBatchUpdateSize());
+        logger.info("\t\t\t\t\tSingle round matches days to subtract : " + config.getJobsConfiguration().getSingleRoundMatchesJob().getSingleRoundMatchesDaysToSubtract());
+
 
         logger.info("\tJobs ");
         logger.info("\t\tJobs: " + config.getJobs());
@@ -149,9 +182,9 @@ protected void registerResources(ChallengeFeederServiceConfiguration config, Env
         JestClient jestClient = JestClientUtils.get(config.getJestClientConfiguration());
 
         // Register resources here
+        env.jersey().register(new HealthCheckResource());
         env.jersey().register(new ChallengeFeederFactory(jestClient).getResourceInstance());
         env.jersey().register(new MmFeederResourceFactory(jestClient).getResourceInstance());
-        env.jersey().register(new HealthCheckResource());
         env.jersey().register(new MarathonMatchFeederFactory(jestClient).getResourceInstance());
         env.jersey().register(new SRMFeederFactory(jestClient).getResourceInstance());
         logger.info("Services registered");
@@ -182,7 +215,7 @@ public void initialize(Bootstrap<ChallengeFeederServiceConfiguration> bootstrap)
         bootstrap.setConfigurationSourceProvider(
                 new SubstitutingSourceProvider(bootstrap.getConfigurationSourceProvider(), new EnvironmentVariableSubstitutor(false)));
 
-        bootstrap.addBundle(new JobsBundle(new LoadChangedChallengesListingJob(), new LegacyMMToChallengeListingJob(),
-                new MarathonMatchesJob(), new SingleRoundMatchesJob(), new LoadChangedChallengesDetailJob(), new LoadChangedMMChallengeDetailJob()));
+        bootstrap.addBundle(new JobsBundle(new LoadChangedChallengesListingJob(), new LegacyMMToChallengeListingJob(), new MarathonMatchesJob(),
+                new SingleRoundMatchesJob(), new LoadChangedChallengesDetailJob(), new LoadChangedMMChallengeDetailJob()));
     }
 }
diff --git a/src/main/java/com/appirio/service/challengefeeder/ChallengeFeederServiceConfiguration.java b/src/main/java/com/appirio/service/challengefeeder/ChallengeFeederServiceConfiguration.java
@@ -11,11 +11,12 @@
 import com.appirio.service.challengefeeder.config.ChallengeConfiguration;
 import com.appirio.service.challengefeeder.config.CommonConfiguration;
 import com.appirio.service.challengefeeder.config.JestClientConfiguration;
-import com.appirio.service.challengefeeder.config.RedissonConfiguration;
+import com.appirio.service.challengefeeder.config.JobsConfiguration;
 import com.appirio.service.supply.resources.SupplyDatasourceFactory;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import de.spinscale.dropwizard.jobs.JobConfiguration;
+import lombok.Getter;
 
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
@@ -34,8 +35,13 @@
  *
  * Version 1.3 - Topcoder ElasticSearch Feeder Service - Way To Populate Challenge-Listing Index
  * - add commonConfiguration
+ * 
+ * Version 1.4 - Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement v1.0
+ * - refactor the job configuration
+ * 
+ * 
  * @author TCSCODER
- * @version 1.3
+ * @version 1.4 
  */
 public class ChallengeFeederServiceConfiguration extends BaseAppConfiguration implements JobConfiguration {
 
@@ -62,10 +68,11 @@ public class ChallengeFeederServiceConfiguration extends BaseAppConfiguration im
     private Map<String , String> jobs;
 
     /**
-     * Represents the redissonConfiguration field
+     * Represents the jobsConfiguration field
      */
-    @JsonProperty("redissonConfiguration")
-    private RedissonConfiguration redissonConfiguration;
+    @JsonProperty("jobsConfiguration")
+    @Getter
+    private JobsConfiguration jobsConfiguration;
 
     /**
      * The challengeConfiguration
@@ -110,14 +117,6 @@ public Map<String, String> getJobs() {
         return jobs;
     }
 
-    /**
-     * Get redissonConfiguration
-     * @return the redissonConfiguration
-     */
-    public RedissonConfiguration getRedissonConfiguration() {
-        return this.redissonConfiguration;
-    }
-
     /**
      * Get challenge configuration
      *

diff --git a/src/main/java/com/appirio/service/challengefeeder/config/BaseJobConfiguration.java b/src/main/java/com/appirio/service/challengefeeder/config/BaseJobConfiguration.java
@@ -0,0 +1,46 @@
+package com.appirio.service.challengefeeder.config;
+
+import org.hibernate.validator.constraints.NotEmpty;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * Represents the BaseJobConfiguration
+ * 
+ * It's added in Topcoder Elasticsearch Feeder Service - Jobs Cleanup And Improvement v1.0
+ * 
+ * @author TCCoder
+ * @version 1.0
+ *
+ */
+public class BaseJobConfiguration {
+    /**
+     * The index field
+     */
+    @JsonProperty
+    @NotEmpty
+    @Getter
+    @Setter
+    private String indexName;
+
+    /**
+     * The type name field
+     */
+    @JsonProperty
+    @NotEmpty
+    @Getter
+    @Setter
+    private String typeName;
+
+    /**
+     * Represents the batch update size field.
+     */
+    @JsonProperty
+    @Getter
+    @Setter
+    private int batchUpdateSize;
+
+}