topcoder-platform · Gunasekar-K · Mar 6, 2025 · May 5, 2023 · Sep 11, 2023 · Sep 12, 2023
diff --git a/master_deploy.sh b/master_deploy.sh
@@ -18,6 +18,7 @@ SHARED_PROPERTY_FILENAME=""
 TAG=""
 SEC_LIST=""
 SECPS_LIST=""
+ARG_SECPS_LIST=""
 #COUNTER_LIMIT=12
 
 if [ -z "$COUNTER_LIMIT" ]; then
@@ -399,6 +400,29 @@ ECS_template_create_register() {
             IFS=$o  
         done
     fi
+    if [ -z $ARG_SECPS_LIST ];
+    then
+        log "No ps file provided"
+    else
+        Buffer_seclist=$(echo $ARG_SECPS_LIST | sed 's/,/ /g')
+        for listname in $Buffer_seclist;
+        do
+            local o=$IFS
+            IFS=$(echo -en "\n\b")
+            k=$listname
+            echo $k
+            aws ssm get-parameters-by-path --path $k --query "Parameters[*].{Name:Name}" > paramnames.json
+            ###paramnames=$(cat paramnames.json | jq -r .[].Name | rev | cut -d / -f 1 | rev)
+            for s in $(cat paramnames.json | jq -r .[].Name )
+            do
+                varname=$(echo $s | rev | cut -d / -f 1 | rev)
+                varvalue="arn:aws:ssm:$AWS_REGION:$AWS_ACCOUNT_ID:parameter$s"
+                psenvaddition "$varname" "$varvalue"
+                #echo "$varname" "$varvalue"
+            done
+            IFS=$o  
+        done
+    fi    
     log "Environment has updated"
 
     # Log Configuration
@@ -812,30 +836,50 @@ configure_Lambda_template()
     if [ "$AWS_LAMBDA_DEPLOY_TYPE" == "SLS" ]
     then
         mkdir -p /home/circleci/project/config
-        Buffer_seclist=$(echo $SEC_LIST | sed 's/,/ /g')
-	    #envvars=$( cat $listname.json | jq  -c ' .app_var ')
+       if [ -z $SEC_LIST ];
+       then
+            log "No ps path provided"
+       else
+            Buffer_seclist=$(echo $SEC_LIST | sed 's/,/ /g')
+            #envvars=$( cat $listname.json | jq  -c ' .app_var ')
+
+            for listname in $Buffer_seclist;
+            do
+                local o=$IFS
+                IFS=$(echo -en "\n\b")
+                envvars=$( cat $listname.json | jq  -c ' . ')	     
+                echo "$envvars" > /home/circleci/project/config/$AWS_LAMBDA_STAGE.json
+                sed -i 's/\\n/\\\\n/g' /home/circleci/project/config/$AWS_LAMBDA_STAGE.json
+                IFS=$o 
+        done
+       fi
+
+       if [ -z $ARG_SECPS_LIST ];
+       then
+            log "No ps path provided"
+       else
+            Buffer_seclist=$(echo $ARG_SECPS_LIST | sed 's/,/ /g')
+            for listname in $Buffer_seclist;
+            do
+                local o=$IFS
+                IFS=$(echo -en "\n\b")
+                k=$listname
+                echo $k
+                aws ssm get-parameters-by-path --with-decryption --path $k --query "Parameters[*].{Name:Name, Value:Value}" >fetched_parameters.json
+                cat fetched_parameters.json | jq  -r ' . |= (map({ (.Name): .Value }) | add)' | sed -e "s~$k/~~" >paramwithvalue.json
+                envvars=$( cat paramwithvalue.json | jq  -c ' . ')	     
+                echo "$envvars" > /home/circleci/project/config/$AWS_LAMBDA_STAGE.json
+                sed -i 's/\\n/\\\\n/g' /home/circleci/project/config/$AWS_LAMBDA_STAGE.json
+                ###paramnames=$(cat paramnames.json | jq -r .[].Name | rev | cut -d / -f 1 | rev)
+                IFS=$o  
+            done
+        fi 
+
 
-        for listname in $Buffer_seclist;
-        do
-            o=$IFS
-            IFS=$(echo -en "\n\b")
-            envvars=$( cat $listname.json | jq  -c ' . ')	     
-            echo "$envvars" > /home/circleci/project/config/$AWS_LAMBDA_STAGE.json
-            sed -i 's/\\n/\\\\n/g' /home/circleci/project/config/$AWS_LAMBDA_STAGE.json
-
-            #yq r $listname.json  >$listname.yml
-            #a=serverless.yml
-            #b="$listname.json"
-            #python -c "import sys; from ruamel.yaml import YAML; yaml = YAML(); cfg = yaml.load(open('$a','r')); cfg_env = yaml.load(open('$b','r')); cfg['Resources']['tcdevhandler']['Properties']['Environment']['Variables']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml', 'w'))"
-            #python -c "import sys; from ruamel.yaml import YAML; yaml = YAML(); cfg = yaml.load(open('$a','r')); cfg_env = yaml.load(open('$b','r')); cfg['provider']['environment']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml', 'w'))"
-            #python -c "import sys , json , ruamel.yaml , cStringIO; jsondata = cStringIO.StringIO(); yaml = ruamel.yaml.YAML(); yaml.explicit_start = True; data = json.load(open('$b','r'), object_pairs_hook=ruamel.yaml.comments.CommentedMap) ; ruamel.yaml.scalarstring.walk_tree(data) ; yaml.dump(data, jsondata); cfg = yaml.load(open('$a','r')); cfg_env = yaml.load(jsondata.getvalue()); cfg['Resources']['tcdevhandler']['Properties']['Environment']['Variables']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml', 'w'))"
-            #python -c "import sys , json , ruamel.yaml , cStringIO; jsondata = cStringIO.StringIO(); yaml = ruamel.yaml.YAML(); yaml.explicit_start = True; data = json.load(open('$b','r'), object_pairs_hook=ruamel.yaml.comments.CommentedMap) ; ruamel.yaml.scalarstring.walk_tree(data) ; yaml.dump(data, jsondata); cfg = yaml.load(open('$a','r')); cfg_env = yaml.load(jsondata.getvalue()); cfg['provider']['environment']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml', 'w'))"
-            #python -c "import sys , json , ruamel.yaml ; from io import BytesIO as StringIO ; jsondata = StringIO(); yaml = ruamel.yaml.YAML(); yaml.explicit_start = True; data = json.load(open('$b','r'), object_pairs_hook=ruamel.yaml.comments.CommentedMap) ; ruamel.yaml.scalarstring.walk_tree(data) ; yaml.dump(data, jsondata); cfg = yaml.load(open('$a','r')); cfg_env= yaml.load(jsondata.getvalue()); cfg['provider']['environment']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml','w'))"
-            #python -c "import sys , json , ruamel.yaml ; from io import BytesIO as StringIO ; jsondata = StringIO(); yaml = ruamel.yaml.YAML(); data = json.load(open('$b','r')) ; yaml.dump(data, jsondata); cfg = yaml.load(open('$a','r')); cfg_env= yaml.load(jsondata.getvalue()); cfg['provider']['environment']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml','w'))"
-            #mv -f appeneded.yaml serverless.yml 
-       done
-       IFS=$o 
     fi
+
+
+
 }
 
 deploy_lambda_package()
@@ -857,7 +901,7 @@ deploy_lambda_package()
 # Input Collection and validation
 input_parsing_validation()
 {
-    while getopts .d:h:i:e:l:t:v:s:p:g:c:m:. OPTION
+    while getopts .d:h:i:e:l:j:t:v:s:p:g:c:m:. OPTION
     do
         case $OPTION in
             d)
@@ -876,6 +920,9 @@ input_parsing_validation()
             l)
                 SECPS_LIST=$OPTARG
                 ;;
+            j)
+                ARG_SECPS_LIST=$OPTARG
+                ;;                
             t)
                 TAG=$OPTARG
                 ;;

diff --git a/psvar-processor.sh b/psvar-processor.sh
@@ -0,0 +1,213 @@
+#!/bin/bash
+set -eo pipefail
+usage()
+{
+cat << EOF
+usage: $0 options
+
+This script needs to be executed with below options.
+
+OPTIONS:
+ -e environment
+ -t type appenv,appconf and appjson
+ -p parameter store path without final slash
+ -l parameter store list  without final slash
+
+EOF
+}
+
+create_env_file_format()
+{
+    file_name=$1
+    fetch_path=$2
+    echo $fetch_path
+    echo $file_name
+    aws ssm get-parameters-by-path --with-decryption --path $fetch_path --query "Parameters[*].{Name:Name, Value:Value}" >fetched_parameters.json
+    cat fetched_parameters.json | jq -r '.[] | "export " + .Name + "=\"" + .Value + "\""  '  | sed -e "s~$fetch_path/~~" >${file_name}_env
+    rm -rf fetched_parameters.json
+}
+
+create_conf_file_format()
+{
+    file_name=$1
+    fetch_path=$2
+    aws ssm get-parameters-by-path --with-decryption --path $fetch_path --query "Parameters[*].{Name:Name, Value:Value}" >fetched_parameters.json
+    cat fetched_parameters.json | jq -r '.[] | .Name + "=\"" + .Value + "\""  '  | sed -e "s~$fetch_path/~~" >${file_name}.conf
+    rm -rf fetched_parameters.json    
+}
+
+create_json_file_format()
+{
+    file_name=$1
+    fetch_path=$2
+    echo $fetch_path
+    echo $file_name
+    echo "aws ssm get-parameters-by-path --with-decryption --path $fetch_path --query \"Parameters[*].{Name:Name, Value:Value}\""
+    aws ssm get-parameters-by-path --with-decryption --path $fetch_path --query "Parameters[*].{Name:Name, Value:Value}" >fetched_parameters.json
+    cat fetched_parameters.json | jq  -r ' . |= (map({ (.Name): .Value }) | add)' | sed -e "s~$fetch_path/~~" >${file_name}.json
+#    rm -rf fetched_parameters.json    
+}
+
+create_jsonso_file_format()
+{
+    file_name=$1
+    fetch_path=$2
+    echo $fetch_path
+    echo $file_name
+    echo "aws ssm get-parameters --with-decryption --name $fetch_path | jq '.Parameters | .[] | .Value' | jq '.|fromjson'"
+    aws ssm get-parameters --with-decryption --name $fetch_path | jq '.Parameters | .[] | .Value' | jq '.|fromjson' >${file_name}.json
+#    rm -rf fetched_parameters.json    
+}
+
+fetching_specific_path()
+{   
+    type_to_fetch=$1
+    PS_PATH=${PS_PATH%/}
+    fname=${PS_PATH##*/}
+    fpath=$PS_PATH
+    echo $fpath
+    echo $PS_PATH 
+    if [ "$type_to_fetch" == "appenv" ]
+    then
+        create_env_file_format $fname $fpath
+    fi
+    if [ "$type_to_fetch" == "appconf" ]
+    then
+        create_conf_file_format $fname $fpath
+    fi
+    if [ "$type_to_fetch" == "appjson" ]
+    then
+        create_json_file_format $fname $fpath
+    fi 
+    if [ "$type_to_fetch" == "appjsonso" ]
+    then
+        create_jsonso_file_format $fname $fpath
+    fi        
+}
+
+fetching_multiple_path()
+{
+    type_to_fetch=$1
+    Buffer_seclist=$(echo $PS_PATH_LIST | sed 's/,/ /g' )
+    for listname in $Buffer_seclist;
+    do
+        listname=${listname%/}
+        fname=${listname##*/}
+        fpath=$listname
+        if [ "$type_to_fetch" == "appenv" ]
+        then
+            create_env_file_format $fname $fpath
+        fi
+        if [ "$type_to_fetch" == "appconf" ]
+        then
+            create_conf_file_format $fname $fpath
+        fi
+        if [ "$type_to_fetch" == "appjson" ]
+        then
+            create_json_file_format $fname $fpath
+        fi 
+        if [ "$type_to_fetch" == "appjsonso" ]
+        then
+            create_jsonso_file_format $fname $fpath
+        fi                        
+    done
+}
+
+
+while getopts .t:e:p:l:. OPTION
+do
+     case $OPTION in
+         e)
+             ENV=$OPTARG
+             ;;
+         t)
+             APP_TYPE=$OPTARG
+             ;;
+         p)
+             PS_PATH=$OPTARG
+             ;;
+         l)
+             PS_PATH_LIST=$OPTARG
+             ;;             
+         ?)
+             log "additional param required"
+             usage
+             exit
+             ;;
+     esac
+done
+
+ENV_CONFIG=`echo "$ENV" | tr '[:upper:]' '[:lower:]'`
+APP_TYPE_LOWERCASE=`echo "$APP_TYPE" | tr '[:upper:]' '[:lower:]'`
+
+echo "APP_TYPE: $APP_TYPE_LOWERCASE"
+echo "PS_PATH: $PS_PATH"
+echo "PS_PATH_LIST: $PS_PATH_LIST"
+
+if [ "$APP_TYPE_LOWERCASE" == "appenv" ]
+then
+        echo "env configuration"
+        if [ -z $PS_PATH ];
+        then
+            echo "Info: no ps path"
+        else                    
+            fetching_specific_path $APP_TYPE_LOWERCASE
+        fi
+        if [ -z $PS_PATH_LIST ];
+        then
+            echo "Info: no path list provided. So skipping pathlist"
+        else        
+            fetching_multiple_path  $APP_TYPE_LOWERCASE
+        fi        
+fi
+
+if [ "$APP_TYPE_LOWERCASE" == "appconf" ]
+then
+        echo "conf file configuration"
+        if [ -z $PS_PATH ];
+        then
+            echo "Info: no ps path"
+        else
+            fetching_specific_path $APP_TYPE_LOWERCASE
+        fi
+        if [ -z $PS_PATH_LIST ];
+        then
+            echo "Info: no path list provided. So skipping pathlist"
+        else        
+            fetching_multiple_path $APP_TYPE_LOWERCASE
+        fi          
+fi
+
+if [ "$APP_TYPE_LOWERCASE" == "appjson" ]
+then
+        echo "json file configuration"
+        if [ -z $PS_PATH ];
+        then
+            echo "Info: no ps path"
+        else        
+            fetching_specific_path $APP_TYPE_LOWERCASE
+        fi
+        if [ -z $PS_PATH_LIST ];
+        then
+            echo "Info: no path list provided. So skipping pathlist"
+        else        
+            fetching_multiple_path $APP_TYPE_LOWERCASE
+        fi          
+fi
+
+if [ "$APP_TYPE_LOWERCASE" == "appjsonso" ]
+then
+        echo "json file configuration"
+        if [ -z $PS_PATH ];
+        then
+            echo "Info: no ps path"
+        else        
+            fetching_specific_path $APP_TYPE_LOWERCASE
+        fi
+        if [ -z $PS_PATH_LIST ];
+        then
+            echo "Info: no path list provided. So skipping pathlist"
+        else        
+            fetching_multiple_path $APP_TYPE_LOWERCASE
+        fi          
+fi
diff --git a/uploadjson-ps.sh b/uploadjson-ps.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eo pipefail
+UPLOAD_FILENAME=$1
+PARAMETER_PATH=$2
+
+cat $UPLOAD_FILENAME  | jq  -r ' . ' | jq --arg PARAMETER_PATH $PARAMETER_PATH ' . | to_entries[] | { "Name": ($PARAMETER_PATH+"/"+.key) , "Value": .value, "Type" : "SecureString" } ' | jq -s . >upload_object.json
+o=$IFS
+IFS=$(echo -en "\n\b")
+
+for s in $(cat upload_object.json | jq -c .[] )
+do
+    echo $s>cli-input.json
+    aws ssm put-parameter --cli-input-json file://cli-input.json
+done
+IFS=$o  
+
+[ -f upload_object.json ] && rm -f upload_object.json
+[ -f cli-input.json ] && rm -f cli-input.json
diff --git a/uploadjsonso-ps.sh b/uploadjsonso-ps.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -eo pipefail
+UPLOAD_FILENAME=$1
+PARAMETER_PATH=$2
+
+aws ssm put-parameter \
+    --name $PARAMETER_PATH \
+    --type SecureString \
+    --value file://$UPLOAD_FILENAME