diff --git a/master_deploy.sh b/master_deploy.sh index 98ec043..3fe6b59 100755 --- a/master_deploy.sh +++ b/master_deploy.sh @@ -18,6 +18,7 @@ SHARED_PROPERTY_FILENAME="" TAG="" SEC_LIST="" SECPS_LIST="" +ARG_SECPS_LIST="" #COUNTER_LIMIT=12 if [ -z "$COUNTER_LIMIT" ]; then @@ -399,6 +400,29 @@ ECS_template_create_register() { IFS=$o done fi + if [ -z $ARG_SECPS_LIST ]; + then + log "No ps file provided" + else + Buffer_seclist=$(echo $ARG_SECPS_LIST | sed 's/,/ /g') + for listname in $Buffer_seclist; + do + local o=$IFS + IFS=$(echo -en "\n\b") + k=$listname + echo $k + aws ssm get-parameters-by-path --path $k --query "Parameters[*].{Name:Name}" > paramnames.json + ###paramnames=$(cat paramnames.json | jq -r .[].Name | rev | cut -d / -f 1 | rev) + for s in $(cat paramnames.json | jq -r .[].Name ) + do + varname=$(echo $s | rev | cut -d / -f 1 | rev) + varvalue="arn:aws:ssm:$AWS_REGION:$AWS_ACCOUNT_ID:parameter$s" + psenvaddition "$varname" "$varvalue" + #echo "$varname" "$varvalue" + done + IFS=$o + done + fi log "Environment has updated" # Log Configuration @@ -812,30 +836,50 @@ configure_Lambda_template() if [ "$AWS_LAMBDA_DEPLOY_TYPE" == "SLS" ] then mkdir -p /home/circleci/project/config - Buffer_seclist=$(echo $SEC_LIST | sed 's/,/ /g') - #envvars=$( cat $listname.json | jq -c ' .app_var ') + if [ -z $SEC_LIST ]; + then + log "No ps path provided" + else + Buffer_seclist=$(echo $SEC_LIST | sed 's/,/ /g') + #envvars=$( cat $listname.json | jq -c ' .app_var ') + + for listname in $Buffer_seclist; + do + local o=$IFS + IFS=$(echo -en "\n\b") + envvars=$( cat $listname.json | jq -c ' . ') + echo "$envvars" > /home/circleci/project/config/$AWS_LAMBDA_STAGE.json + sed -i 's/\\n/\\\\n/g' /home/circleci/project/config/$AWS_LAMBDA_STAGE.json + IFS=$o + done + fi + + if [ -z $ARG_SECPS_LIST ]; + then + log "No ps path provided" + else + Buffer_seclist=$(echo $ARG_SECPS_LIST | sed 's/,/ /g') + for listname in $Buffer_seclist; + do + local o=$IFS + IFS=$(echo -en "\n\b") + k=$listname + echo $k + aws ssm get-parameters-by-path --with-decryption --path $k --query "Parameters[*].{Name:Name, Value:Value}" >fetched_parameters.json + cat fetched_parameters.json | jq -r ' . |= (map({ (.Name): .Value }) | add)' | sed -e "s~$k/~~" >paramwithvalue.json + envvars=$( cat paramwithvalue.json | jq -c ' . ') + echo "$envvars" > /home/circleci/project/config/$AWS_LAMBDA_STAGE.json + sed -i 's/\\n/\\\\n/g' /home/circleci/project/config/$AWS_LAMBDA_STAGE.json + ###paramnames=$(cat paramnames.json | jq -r .[].Name | rev | cut -d / -f 1 | rev) + IFS=$o + done + fi + - for listname in $Buffer_seclist; - do - o=$IFS - IFS=$(echo -en "\n\b") - envvars=$( cat $listname.json | jq -c ' . ') - echo "$envvars" > /home/circleci/project/config/$AWS_LAMBDA_STAGE.json - sed -i 's/\\n/\\\\n/g' /home/circleci/project/config/$AWS_LAMBDA_STAGE.json - - #yq r $listname.json >$listname.yml - #a=serverless.yml - #b="$listname.json" - #python -c "import sys; from ruamel.yaml import YAML; yaml = YAML(); cfg = yaml.load(open('$a','r')); cfg_env = yaml.load(open('$b','r')); cfg['Resources']['tcdevhandler']['Properties']['Environment']['Variables']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml', 'w'))" - #python -c "import sys; from ruamel.yaml import YAML; yaml = YAML(); cfg = yaml.load(open('$a','r')); cfg_env = yaml.load(open('$b','r')); cfg['provider']['environment']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml', 'w'))" - #python -c "import sys , json , ruamel.yaml , cStringIO; jsondata = cStringIO.StringIO(); yaml = ruamel.yaml.YAML(); yaml.explicit_start = True; data = json.load(open('$b','r'), object_pairs_hook=ruamel.yaml.comments.CommentedMap) ; ruamel.yaml.scalarstring.walk_tree(data) ; yaml.dump(data, jsondata); cfg = yaml.load(open('$a','r')); cfg_env = yaml.load(jsondata.getvalue()); cfg['Resources']['tcdevhandler']['Properties']['Environment']['Variables']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml', 'w'))" - #python -c "import sys , json , ruamel.yaml , cStringIO; jsondata = cStringIO.StringIO(); yaml = ruamel.yaml.YAML(); yaml.explicit_start = True; data = json.load(open('$b','r'), object_pairs_hook=ruamel.yaml.comments.CommentedMap) ; ruamel.yaml.scalarstring.walk_tree(data) ; yaml.dump(data, jsondata); cfg = yaml.load(open('$a','r')); cfg_env = yaml.load(jsondata.getvalue()); cfg['provider']['environment']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml', 'w'))" - #python -c "import sys , json , ruamel.yaml ; from io import BytesIO as StringIO ; jsondata = StringIO(); yaml = ruamel.yaml.YAML(); yaml.explicit_start = True; data = json.load(open('$b','r'), object_pairs_hook=ruamel.yaml.comments.CommentedMap) ; ruamel.yaml.scalarstring.walk_tree(data) ; yaml.dump(data, jsondata); cfg = yaml.load(open('$a','r')); cfg_env= yaml.load(jsondata.getvalue()); cfg['provider']['environment']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml','w'))" - #python -c "import sys , json , ruamel.yaml ; from io import BytesIO as StringIO ; jsondata = StringIO(); yaml = ruamel.yaml.YAML(); data = json.load(open('$b','r')) ; yaml.dump(data, jsondata); cfg = yaml.load(open('$a','r')); cfg_env= yaml.load(jsondata.getvalue()); cfg['provider']['environment']=cfg_env['app_var'] ; yaml.dump(cfg, open('appeneded.yaml','w'))" - #mv -f appeneded.yaml serverless.yml - done - IFS=$o fi + + + } deploy_lambda_package() @@ -857,7 +901,7 @@ deploy_lambda_package() # Input Collection and validation input_parsing_validation() { - while getopts .d:h:i:e:l:t:v:s:p:g:c:m:. OPTION + while getopts .d:h:i:e:l:j:t:v:s:p:g:c:m:. OPTION do case $OPTION in d) @@ -876,6 +920,9 @@ input_parsing_validation() l) SECPS_LIST=$OPTARG ;; + j) + ARG_SECPS_LIST=$OPTARG + ;; t) TAG=$OPTARG ;; diff --git a/psvar-processor.sh b/psvar-processor.sh new file mode 100755 index 0000000..10c0dd7 --- /dev/null +++ b/psvar-processor.sh @@ -0,0 +1,213 @@ +#!/bin/bash +set -eo pipefail +usage() +{ +cat << EOF +usage: $0 options + +This script needs to be executed with below options. + +OPTIONS: + -e environment + -t type appenv,appconf and appjson + -p parameter store path without final slash + -l parameter store list without final slash + +EOF +} + +create_env_file_format() +{ + file_name=$1 + fetch_path=$2 + echo $fetch_path + echo $file_name + aws ssm get-parameters-by-path --with-decryption --path $fetch_path --query "Parameters[*].{Name:Name, Value:Value}" >fetched_parameters.json + cat fetched_parameters.json | jq -r '.[] | "export " + .Name + "=\"" + .Value + "\"" ' | sed -e "s~$fetch_path/~~" >${file_name}_env + rm -rf fetched_parameters.json +} + +create_conf_file_format() +{ + file_name=$1 + fetch_path=$2 + aws ssm get-parameters-by-path --with-decryption --path $fetch_path --query "Parameters[*].{Name:Name, Value:Value}" >fetched_parameters.json + cat fetched_parameters.json | jq -r '.[] | .Name + "=\"" + .Value + "\"" ' | sed -e "s~$fetch_path/~~" >${file_name}.conf + rm -rf fetched_parameters.json +} + +create_json_file_format() +{ + file_name=$1 + fetch_path=$2 + echo $fetch_path + echo $file_name + echo "aws ssm get-parameters-by-path --with-decryption --path $fetch_path --query \"Parameters[*].{Name:Name, Value:Value}\"" + aws ssm get-parameters-by-path --with-decryption --path $fetch_path --query "Parameters[*].{Name:Name, Value:Value}" >fetched_parameters.json + cat fetched_parameters.json | jq -r ' . |= (map({ (.Name): .Value }) | add)' | sed -e "s~$fetch_path/~~" >${file_name}.json +# rm -rf fetched_parameters.json +} + +create_jsonso_file_format() +{ + file_name=$1 + fetch_path=$2 + echo $fetch_path + echo $file_name + echo "aws ssm get-parameters --with-decryption --name $fetch_path | jq '.Parameters | .[] | .Value' | jq '.|fromjson'" + aws ssm get-parameters --with-decryption --name $fetch_path | jq '.Parameters | .[] | .Value' | jq '.|fromjson' >${file_name}.json +# rm -rf fetched_parameters.json +} + +fetching_specific_path() +{ + type_to_fetch=$1 + PS_PATH=${PS_PATH%/} + fname=${PS_PATH##*/} + fpath=$PS_PATH + echo $fpath + echo $PS_PATH + if [ "$type_to_fetch" == "appenv" ] + then + create_env_file_format $fname $fpath + fi + if [ "$type_to_fetch" == "appconf" ] + then + create_conf_file_format $fname $fpath + fi + if [ "$type_to_fetch" == "appjson" ] + then + create_json_file_format $fname $fpath + fi + if [ "$type_to_fetch" == "appjsonso" ] + then + create_jsonso_file_format $fname $fpath + fi +} + +fetching_multiple_path() +{ + type_to_fetch=$1 + Buffer_seclist=$(echo $PS_PATH_LIST | sed 's/,/ /g' ) + for listname in $Buffer_seclist; + do + listname=${listname%/} + fname=${listname##*/} + fpath=$listname + if [ "$type_to_fetch" == "appenv" ] + then + create_env_file_format $fname $fpath + fi + if [ "$type_to_fetch" == "appconf" ] + then + create_conf_file_format $fname $fpath + fi + if [ "$type_to_fetch" == "appjson" ] + then + create_json_file_format $fname $fpath + fi + if [ "$type_to_fetch" == "appjsonso" ] + then + create_jsonso_file_format $fname $fpath + fi + done +} + + +while getopts .t:e:p:l:. OPTION +do + case $OPTION in + e) + ENV=$OPTARG + ;; + t) + APP_TYPE=$OPTARG + ;; + p) + PS_PATH=$OPTARG + ;; + l) + PS_PATH_LIST=$OPTARG + ;; + ?) + log "additional param required" + usage + exit + ;; + esac +done + +ENV_CONFIG=`echo "$ENV" | tr '[:upper:]' '[:lower:]'` +APP_TYPE_LOWERCASE=`echo "$APP_TYPE" | tr '[:upper:]' '[:lower:]'` + +echo "APP_TYPE: $APP_TYPE_LOWERCASE" +echo "PS_PATH: $PS_PATH" +echo "PS_PATH_LIST: $PS_PATH_LIST" + +if [ "$APP_TYPE_LOWERCASE" == "appenv" ] +then + echo "env configuration" + if [ -z $PS_PATH ]; + then + echo "Info: no ps path" + else + fetching_specific_path $APP_TYPE_LOWERCASE + fi + if [ -z $PS_PATH_LIST ]; + then + echo "Info: no path list provided. So skipping pathlist" + else + fetching_multiple_path $APP_TYPE_LOWERCASE + fi +fi + +if [ "$APP_TYPE_LOWERCASE" == "appconf" ] +then + echo "conf file configuration" + if [ -z $PS_PATH ]; + then + echo "Info: no ps path" + else + fetching_specific_path $APP_TYPE_LOWERCASE + fi + if [ -z $PS_PATH_LIST ]; + then + echo "Info: no path list provided. So skipping pathlist" + else + fetching_multiple_path $APP_TYPE_LOWERCASE + fi +fi + +if [ "$APP_TYPE_LOWERCASE" == "appjson" ] +then + echo "json file configuration" + if [ -z $PS_PATH ]; + then + echo "Info: no ps path" + else + fetching_specific_path $APP_TYPE_LOWERCASE + fi + if [ -z $PS_PATH_LIST ]; + then + echo "Info: no path list provided. So skipping pathlist" + else + fetching_multiple_path $APP_TYPE_LOWERCASE + fi +fi + +if [ "$APP_TYPE_LOWERCASE" == "appjsonso" ] +then + echo "json file configuration" + if [ -z $PS_PATH ]; + then + echo "Info: no ps path" + else + fetching_specific_path $APP_TYPE_LOWERCASE + fi + if [ -z $PS_PATH_LIST ]; + then + echo "Info: no path list provided. So skipping pathlist" + else + fetching_multiple_path $APP_TYPE_LOWERCASE + fi +fi diff --git a/uploadjson-ps.sh b/uploadjson-ps.sh new file mode 100755 index 0000000..daa1212 --- /dev/null +++ b/uploadjson-ps.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -eo pipefail +UPLOAD_FILENAME=$1 +PARAMETER_PATH=$2 + +cat $UPLOAD_FILENAME | jq -r ' . ' | jq --arg PARAMETER_PATH $PARAMETER_PATH ' . | to_entries[] | { "Name": ($PARAMETER_PATH+"/"+.key) , "Value": .value, "Type" : "SecureString" } ' | jq -s . >upload_object.json +o=$IFS +IFS=$(echo -en "\n\b") + +for s in $(cat upload_object.json | jq -c .[] ) +do + echo $s>cli-input.json + aws ssm put-parameter --cli-input-json file://cli-input.json +done +IFS=$o + +[ -f upload_object.json ] && rm -f upload_object.json +[ -f cli-input.json ] && rm -f cli-input.json diff --git a/uploadjsonso-ps.sh b/uploadjsonso-ps.sh new file mode 100755 index 0000000..3c7bdb0 --- /dev/null +++ b/uploadjsonso-ps.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -eo pipefail +UPLOAD_FILENAME=$1 +PARAMETER_PATH=$2 + +aws ssm put-parameter \ + --name $PARAMETER_PATH \ + --type SecureString \ + --value file://$UPLOAD_FILENAME \ No newline at end of file