Skip to content

Commit 5d16e04

Browse files
TotktonadaTotktonada
committed
travis-ci: deploy RPM / Deb packages
Now php-tarantool packages are deployed into two kinds of repositories: packagecloud.io and to S3 based ones. The download.tarantool.org now points to the latter, but we keep packagecloud.io for a while to don't break users, which use it directly (we don't recommend it though). See [1] for more info about deprecation of our packagecloud.io repositories. The deployment scripts allows to provide two configurations: production and staging. The former is for deployment from the master branch and from a git tag. The latter is for deployments from a developer branch. It is useful when something should be verified in a specific environment using a built package or when the deployment process itself was modified and should be tested. The difference between production and staging deployment process is how duplicate package versions are handled. It give an error for production deployment, but does not for staging. The new package is discarded in the case for packagecloud.io, but it replaces the old package in S3 repositories. Read comments in the deployment scripts for more details. [1]: tarantool/tarantool#4947 Fixes #117
1 parent 2d506f7 commit 5d16e04

10 files changed

+440
-2
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,6 @@ debian/php5-tarantool.substvars
4646
debian/php5-tarantool/
4747
build
4848
.*.sw[a-z]
49+
50+
# Unencrypted private GPG keys for deployment.
51+
.travis/*.asc

‎.travis.yml

+43-2
Original file line numberDiff line numberDiff line change
@@ -129,20 +129,61 @@ python:
129129

130130
script:
131131
- |
132+
# Make shell strictier.
133+
#
134+
# - Exit with a failure on a first failed command.
135+
# - Print each executed commmand.
136+
set -ex
137+
132138
if [ -n "${TARANTOOL_VERSION}" ]; then
133139
./test.sh
134140
elif [ -n "${OS}" ] && [ -n "${DIST}" ]; then
135141
git clone --depth 1 https://github.com/packpack/packpack.git
136142
./packpack/packpack
137143
if [ "${OS}" = "el" ]; then
138-
export OS=centos
144+
DOCKER_IMAGE="centos:${DIST}"
145+
else
146+
DOCKER_IMAGE="${OS}:${DIST}"
139147
fi
140148
docker run \
141149
--volume "$(realpath .):/tarantool-php" \
142150
--workdir /tarantool-php \
143151
--rm \
144-
"${OS}:${DIST}" \
152+
"${DOCKER_IMAGE}" \
145153
./test.pkg.sh
146154
else
147155
exit 1
148156
fi
157+
158+
# Deploy
159+
# ------
160+
161+
# Skip deployment when it is not expected.
162+
if [ -z "${OS}" ] || [ -z "${DIST}" ]; then
163+
echo "Skip deployment: it is pure testing job w/o any RPM / Deb artefacts"
164+
exit 0
165+
fi
166+
if [ "${TRAVIS_REPO_SLUG}" != "tarantool/tarantool-php" ]; then
167+
echo "Skip deployment: it is a fork, not the base repository"
168+
exit 0
169+
fi
170+
if [ "${TRAVIS_EVENT_TYPE}" != "push" ]; then
171+
echo "Skip deployment: event is not 'push', but ${TRAVIS_EVENT_TYPE}"
172+
exit 0
173+
fi
174+
175+
# Choose destination to push packages.
176+
if [ "${TRAVIS_BRANCH}" == "master" ] || [ -n "${TRAVIS_TAG}" ]; then
177+
echo "Set production deployment parameters"
178+
configuration=production
179+
else
180+
echo "Set staging deployment parameters"
181+
configuration=staging
182+
fi
183+
184+
# Deploy to packagecloud repositories.
185+
./.travis/deploy_packagecloud.sh ${configuration}
186+
187+
# Deploy to S3 based repositories.
188+
./.travis/deploy_s3_dependencies.sh
189+
./.travis/deploy_s3.sh ${configuration}

‎.travis/deploy_packagecloud.sh

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#!/bin/sh
2+
3+
# Deploy to packagecloud repositories
4+
# -----------------------------------
5+
#
6+
# `deploy_packagecloud.sh` is equivalent to
7+
# `deploy_packagecloud.sh staging`.
8+
#
9+
# `deploy_packagecloud.sh staging` requires the following
10+
# environment variables:
11+
#
12+
# - OS
13+
# - DIST
14+
# - DEPLOY_STAGING_PACKAGECLOUD_USER
15+
# - DEPLOY_STAGING_PACKAGECLOUD_TOKEN
16+
#
17+
# `deploy_packagecloud.sh production` requires the following
18+
# environment variables:
19+
#
20+
# - OS
21+
# - DIST
22+
# - DEPLOY_PRODUCTION_PACKAGECLOUD_USER
23+
# - DEPLOY_PRODUCTION_PACKAGECLOUD_TOKEN
24+
#
25+
# If one of those variables is not set or empty, then deployment
26+
# will be skipped.
27+
28+
# Make shell strictier.
29+
#
30+
# - Exit with a failure on a first failed command.
31+
# - Exit with a failure on an attempt to use an unset variable.
32+
# - Print each executed commmand.
33+
#
34+
# Note: The script expects that Travis-CI will filter sensitive
35+
# information (such as a token): 'Display value in build log'
36+
# toogle should be OFF for to keep a value secure.
37+
set -eux
38+
39+
configuration=${1:-staging}
40+
41+
# Choose credentials.
42+
if [ ${configuration} = staging ]; then
43+
DEPLOY_PACKAGECLOUD_USER="${DEPLOY_STAGING_PACKAGECLOUD_USER:-}"
44+
DEPLOY_PACKAGECLOUD_TOKEN="${DEPLOY_STAGING_PACKAGECLOUD_TOKEN:-}"
45+
elif [ ${configuration} = production ]; then
46+
DEPLOY_PACKAGECLOUD_USER="${DEPLOY_PRODUCTION_PACKAGECLOUD_USER:-}"
47+
DEPLOY_PACKAGECLOUD_TOKEN="${DEPLOY_PRODUCTION_PACKAGECLOUD_TOKEN:-}"
48+
else
49+
echo "Unknown configuration: ${configuration}"
50+
exit 1
51+
fi
52+
53+
# Skip deployment if some variables are not set or empty.
54+
if [ -z "${OS:-}" ] || [ -z "${DIST:-}" ] || \
55+
[ -z "${DEPLOY_PACKAGECLOUD_USER}" ] || \
56+
[ -z "${DEPLOY_PACKAGECLOUD_TOKEN}" ]; then
57+
echo "Skip deployment: some of necessary environment"
58+
echo "variables are not set or empty"
59+
exit 0
60+
fi
61+
62+
# Verify that packpack is cloned into the current directory.
63+
packagecloud_tool=./packpack/tools/packagecloud
64+
if [ ! -f "${packagecloud_tool}" ]; then
65+
echo "Could not find ${packagecloud_tool}"
66+
exit 1
67+
fi
68+
69+
# Staging repository: keep older packages in case of a
70+
# version clash.
71+
#
72+
# It would be better to replace old ones, but there is no
73+
# such option in the packagecloud tool we use. It may be
74+
# important if we'll have some manual or automatic testing
75+
# upward a staging repository. But at least CI will not fail
76+
# because a package is already exists.
77+
push_args=""
78+
if [ "${configuration}" = staging ]; then
79+
push_args="${push_args} --ignore-duplicates"
80+
fi
81+
82+
# Setup environment variables for the packagecloud tool.
83+
export PACKAGECLOUD_TOKEN="${DEPLOY_PACKAGECLOUD_TOKEN}"
84+
85+
# We have tarantool repositories on packagecloud.io up to
86+
# 2_4. The next ones present only in the S3 based storage.
87+
for repo in 1_6 1_7 1_9 1_10 2x 2_2 2_3 2_4; do
88+
# FIXME: Enable *.ddeb when packagecloud.io will support it.
89+
for file in build/*.rpm build/*.deb build/*.dsc; do
90+
extension=${file##*.}
91+
92+
# Skip non-matched globs: say, build/*.rpm on Debian.
93+
basename="$(basename "${file}" ".${extension}")"
94+
[ "${basename}" = "*" ] && continue
95+
96+
# Push all source files listed in .dsc file together with
97+
# the file.
98+
#
99+
# FIXME: It seems logical to move this logic to the
100+
# packagecloud tool we use.
101+
files="${file}"
102+
if [ "${extension}" = "dsc" ]; then
103+
parse_dsc_file='{
104+
if ($0 == "Files:") {
105+
FILES_SECTION = 1;
106+
} else if (FILES_SECTION != 0) {
107+
print "build/"$3;
108+
}
109+
}'
110+
files="${files} $(awk "${parse_dsc_file}" "${file}")"
111+
fi
112+
113+
user=${DEPLOY_PACKAGECLOUD_USER}
114+
115+
# Retry failed attempts to upload a package.
116+
#
117+
# packagecloud.io sometimes replieds with 502 Bad Gateway
118+
# for attempts to push, so retrying is important here.
119+
#
120+
# FIXME: This way we don't differentiate network errors
121+
# and all other ones. It would be much better to retry
122+
# from inside the packagecloud tool (requests library
123+
# supports it).
124+
for i in $(seq 1 5); do
125+
# FIXME: The tool fetches distributions.json each
126+
# time. It can cache the data somewhere and reuse
127+
# during some time period until expiration.
128+
${packagecloud_tool} push ${push_args} ${user}/${repo} \
129+
${extension} ${OS} ${DIST} ${files} && break
130+
done
131+
done
132+
done

‎.travis/deploy_production_s3_gpg_private_key.asc.enc

5.39 KB
Binary file not shown.

‎.travis/deploy_s3.sh

+180
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
#!/bin/sh
2+
3+
# Deploy to S3 based repositories
4+
# -------------------------------
5+
#
6+
# `deploy_s3.sh` is equivalent to `deploy_s3.sh staging`.
7+
#
8+
# `deploy_s3.sh staging` requires the following environment
9+
# variables:
10+
#
11+
# - OS
12+
# - DIST
13+
# - DEPLOY_STAGING_S3_ENDPOINT_URL="https://..."
14+
# - DEPLOY_STAGING_S3_LIVE_DIR="s3://my_bucket/foo/bar/live"
15+
# - DEPLOY_STAGING_S3_RELEASE_DIR="s3://my_bucket/foo/bar/release"
16+
# - DEPLOY_STAGING_S3_ACCESS_KEY_ID
17+
# - DEPLOY_STAGING_S3_SECRET_ACCESS_KEY
18+
# - DEPLOY_STAGING_S3_GPG_KEY_FILE_KEY (32 bytes in hex)
19+
# - DEPLOY_STAGING_S3_GPG_KEY_FILE_IV (16 bytes in hex)
20+
#
21+
# `deploy_s3.sh production` requires the following environment
22+
# variables:
23+
#
24+
# - OS
25+
# - DIST
26+
# - DEPLOY_PRODUCTION_S3_ENDPOINT_URL="https://..."
27+
# - DEPLOY_PRODUCTION_S3_LIVE_DIR="s3://my_bucket/foo/bar/live"
28+
# - DEPLOY_PRODUCTION_S3_RELEASE_DIR="s3://my_bucket/foo/bar/release"
29+
# - DEPLOY_PRODUCTION_S3_ACCESS_KEY_ID
30+
# - DEPLOY_PRODUCTION_S3_SECRET_ACCESS_KEY
31+
# - DEPLOY_PRODUCTION_S3_GPG_KEY_FILE_KEY (32 bytes in hex)
32+
# - DEPLOY_PRODUCTION_S3_GPG_KEY_FILE_IV (16 bytes in hex)
33+
#
34+
# If one of those variables is not set or empty, then deployment
35+
# will be skipped.
36+
37+
# Make shell strictier.
38+
#
39+
# - Exit with a failure on a first failed command.
40+
# - Exit with a failure on an attempt to use an unset variable.
41+
# - Print each executed commmand.
42+
#
43+
# Note: The script expects that Travis-CI will filter sensitive
44+
# information (such as a token): 'Display value in build log'
45+
# toogle should be OFF for to keep a value secure.
46+
set -eux
47+
48+
configuration=${1:-staging}
49+
50+
# Choose URLs, directories, keys and so.
51+
if [ ${configuration} = staging ]; then
52+
DEPLOY_S3_ENDPOINT_URL="${DEPLOY_STAGING_S3_ENDPOINT_URL:-}"
53+
DEPLOY_S3_LIVE_DIR="${DEPLOY_STAGING_S3_LIVE_DIR:-}"
54+
DEPLOY_S3_RELEASE_DIR="${DEPLOY_STAGING_S3_RELEASE_DIR:-}"
55+
DEPLOY_S3_ACCESS_KEY_ID="${DEPLOY_STAGING_S3_ACCESS_KEY_ID:-}"
56+
DEPLOY_S3_SECRET_ACCESS_KEY="${DEPLOY_STAGING_S3_SECRET_ACCESS_KEY:-}"
57+
DEPLOY_S3_GPG_KEY_FILE_KEY="${DEPLOY_STAGING_S3_GPG_KEY_FILE_KEY:-}"
58+
DEPLOY_S3_GPG_KEY_FILE_IV="${DEPLOY_STAGING_S3_GPG_KEY_FILE_IV:-}"
59+
elif [ ${configuration} = production ]; then
60+
DEPLOY_S3_ENDPOINT_URL="${DEPLOY_PRODUCTION_S3_ENDPOINT_URL:-}"
61+
DEPLOY_S3_LIVE_DIR="${DEPLOY_PRODUCTION_S3_LIVE_DIR:-}"
62+
DEPLOY_S3_RELEASE_DIR="${DEPLOY_PRODUCTION_S3_RELEASE_DIR:-}"
63+
DEPLOY_S3_ACCESS_KEY_ID="${DEPLOY_PRODUCTION_S3_ACCESS_KEY_ID:-}"
64+
DEPLOY_S3_SECRET_ACCESS_KEY="${DEPLOY_PRODUCTION_S3_SECRET_ACCESS_KEY:-}"
65+
DEPLOY_S3_GPG_KEY_FILE_KEY="${DEPLOY_PRODUCTION_S3_GPG_KEY_FILE_KEY:-}"
66+
DEPLOY_S3_GPG_KEY_FILE_IV="${DEPLOY_PRODUCTION_S3_GPG_KEY_FILE_IV:-}"
67+
else
68+
echo "Unknown configuration: ${configuration}"
69+
exit 1
70+
fi
71+
72+
# Skip deployment if some variables are not set or empty.
73+
if [ -z "${OS:-}" ] || [ -z "${DIST:-}" ] || \
74+
[ -z "${DEPLOY_S3_ENDPOINT_URL}" ] || \
75+
[ -z "${DEPLOY_S3_LIVE_DIR}" ] || \
76+
[ -z "${DEPLOY_S3_RELEASE_DIR}" ] || \
77+
[ -z "${DEPLOY_S3_ACCESS_KEY_ID}" ] || \
78+
[ -z "${DEPLOY_S3_SECRET_ACCESS_KEY}" ] || \
79+
[ -z "${DEPLOY_S3_GPG_KEY_FILE_KEY}" ] || \
80+
[ -z "${DEPLOY_S3_GPG_KEY_FILE_IV}" ]; then
81+
echo "Skip deployment: some of necessary environment"
82+
echo "variables are not set or empty"
83+
exit 0
84+
fi
85+
86+
# Download the tool to deploy to an S3 based repository.
87+
ref=f84cb1aae3144f5677feacf6be31bd4f15e91c2d
88+
base_url="https://raw.githubusercontent.com/tarantool/tarantool/${ref}"
89+
curl -Ssfo update_repo.sh "${base_url}/tools/update_repo.sh"
90+
chmod a+x update_repo.sh
91+
92+
# FIXME: Upstream the patches.
93+
patch -p1 -i .travis/gh-5112-update-repo-sh-use-right-gpg-key.patch
94+
patch -p1 -i .travis/gh-5113-update-repo-sh-add-fedora-25-26.patch
95+
patch -p1 -i .travis/gh-5114-update-repo-sh-fix-unbound-var-access.patch
96+
97+
# Decrypt a GPG key.
98+
gpg_key_file=".travis/deploy_${configuration}_s3_gpg_private_key.asc"
99+
openssl aes-256-cbc -K "${DEPLOY_S3_GPG_KEY_FILE_KEY}" \
100+
-iv "${DEPLOY_S3_GPG_KEY_FILE_IV}" -in "${gpg_key_file}.enc" \
101+
-out "${gpg_key_file}" -d
102+
103+
# Import GPG key for signing repository files.
104+
gpg --import --batch "${gpg_key_file}"
105+
106+
# Extract GPG key id for signing repository files.
107+
#
108+
# This way works for both GnuPG 1 and GnuPG 2. The alternative
109+
# would be using '--import-options show-only', but it is available
110+
# only in GnuPG 2. See https://unix.stackexchange.com/a/468889
111+
mkdir -m 0700 temp-gpg-home
112+
gpg --homedir temp-gpg-home --import --batch "${gpg_key_file}"
113+
export GPG_SIGN_KEY="$(gpg --homedir temp-gpg-home --list-secret-keys \
114+
--with-colons | grep ^sec: | cut -d: -f5)"
115+
rm -rf temp-gpg-home
116+
117+
# Use SHA256 hashing algorithm for files signing.
118+
#
119+
# `apt-get update` gives a warning when InRelease file signature
120+
# is calculated with SHA1. We should configure GnuPG (which is
121+
# used by reprepro, which is used by update_repo.sh) to sign using
122+
# SHA265.
123+
#
124+
# https://askubuntu.com/a/819868
125+
mkdir -p ~/.gnupg
126+
echo 'digest-algo sha256' >> ~/.gnupg/gpg.conf
127+
128+
# Setup environment variables for the update_repo.sh tool.
129+
export AWS_S3_ENDPOINT_URL="${DEPLOY_S3_ENDPOINT_URL}"
130+
export AWS_ACCESS_KEY_ID="${DEPLOY_S3_ACCESS_KEY_ID}"
131+
export AWS_SECRET_ACCESS_KEY="${DEPLOY_S3_SECRET_ACCESS_KEY}"
132+
133+
# ${product} value may affect location of *.deb, *.rpm and related
134+
# files relative to a base repository URL. We can provide it or
135+
# miss: the script will generate correct repository metainfo
136+
# anyway.
137+
#
138+
# However providing meaningful value for this option enables
139+
# grouping of related set of packages into a subdirectory named as
140+
# ${product} (only for Deb repositories at moment of writing
141+
# this).
142+
#
143+
# It is enabled here for consistency with locations of other Deb
144+
# packages in our repositories, but in fact it is the internal
145+
# detail, which does not lead to any change in the user
146+
# experience.
147+
product=php-tarantool
148+
149+
# Setup arguments that are common for all repositories
150+
# (1.10, 2.1, ...).
151+
update_repo_args="--os=${OS} --distribution=${DIST} --product=${product}"
152+
153+
# Staging repository: rewrite a package if there is a previous one
154+
# of the same version.
155+
#
156+
# Note: It differs from a logic in deploy_packagecloud.sh.
157+
if [ "${configuration}" = staging ]; then
158+
update_repo_args="${update_repo_args} --force"
159+
fi
160+
161+
# Deploy to S3 based repositories.
162+
for repo in 1.10 2.1 2.2 2.3 2.4 2.5; do
163+
# Note: The update_repo.sh tool automatically find
164+
# *.{rpm,deb,dsc} within a passed directory, so we just
165+
# pass the directory name: 'build'.
166+
167+
# FIXME: Machine-local locking that is used in the
168+
# update_repo.sh tool is insufficient when we deploy from a
169+
# just created virtual machine.
170+
171+
# Deploy to live repository (per-push).
172+
bucket="${DEPLOY_S3_LIVE_DIR}/${repo}"
173+
./update_repo.sh ${update_repo_args} --bucket="${bucket}" build
174+
175+
# Deploy to release repository (tagged commits).
176+
if [ -n "${TRAVIS_TAG:-}" ]; then
177+
bucket="${DEPLOY_S3_RELEASE_DIR}/${repo}"
178+
./update_repo.sh ${update_repo_args} --bucket="${bucket}" build
179+
fi
180+
done

0 commit comments

Comments
 (0)