Skip to content

Commit 83c7ef3

Browse files
tkiliastkilias
authored
EXASLCT: bug fixes, refactoring and CI Job for tests (#112)
Bug Fixes: - curl calls use --silent --show-error if we need to parse the result of request - Fix output in error case of command_log_handler.py - Replace manual docker client creation in Task by DockerBaseTask where it was missing - Increase docker socket timeout - Exclude Docker test environment from cleanup if should be reused or started via cli command spawn-test-environment - Fix triggers for Google Cloud Build CI Job Refactoring: - Extract pipenv_utils.sh from exaslct start script Google Cloud Build CI Job for exaslct tests: - Add distinction between local test execution and Google Cloud Build, because Google Cloud Build runs the tests in a docker container with the host docker socket mounted in and this produce some issues with host volumes and networking - Add initial tests for exaslct init script
1 parent 9c1494d commit 83c7ef3

34 files changed

+506
-213
lines changed

‎Dockerfile

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
FROM ubuntu:18.04
2+
3+
COPY ext/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
4+
5+
RUN apt-get -y update && \
6+
apt-get -y install \
7+
locales \
8+
python3-pip \
9+
git \
10+
bash \
11+
curl && \
12+
locale-gen en_US.UTF-8 && \
13+
update-locale LC_ALL=en_US.UTF-8 && \
14+
apt-get -y clean && \
15+
apt-get -y autoremove && \
16+
ldconfig
17+
18+
RUN pip3 install virtualenv
19+
RUN python3 -m virtualenv --python=python3 venv
20+
21+
COPY . /test
22+
RUN rm /test/Pipfile.lock || true

‎exaslct

Lines changed: 8 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -8,115 +8,19 @@
88
COMMAND_LINE_ARGS=$*
99
SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
1010

11+
source pipenv_utils.sh
12+
1113
run () {
12-
PIPENV_BIN="$1"
13-
echo "Using following pipenv executable: $PIPENV_BIN"
14-
if $PIPENV_BIN --venv &> /dev/null;
15-
then
16-
echo "Using existing virtual environment"
17-
else
18-
echo "Creating new virtual environment and installing dependencies"
19-
$PIPENV_BIN install
20-
fi
21-
if [ ! -f Pipfile.lock ];
22-
then
23-
echo "Installing dependencies"
24-
$PIPENV_BIN install
25-
fi
2614
export PYTHONPATH="$SCRIPT_DIR"
2715
$PIPENV_BIN run python3 "$SCRIPT_DIR/exaslct_src/exaslct.py" $COMMAND_LINE_ARGS
2816
}
2917

30-
find_pip_bin () {
31-
if python3 -m pip list &> /dev/null
32-
then
33-
PIP_BIN="python3 -m pip "
34-
else
35-
echo "ERROR: cant find pip"
36-
exit 1
37-
fi
38-
}
39-
40-
find_and_run_via_pip () {
41-
find_pip_bin
42-
local PIPENV_LOCATION=$($PIP_BIN show pipenv | grep 'Location:' | cut -f 2 -d " ")
43-
local PIPENV_BIN_IN_LOCATION=$($PIP_BIN show -f pipenv | grep 'bin/pipenv$' | awk '{$1=$1};1')
44-
PIPENV_BIN=$(command -v "$PIPENV_LOCATION/$PIPENV_BIN_IN_LOCATION")
45-
if [ -n "$PIPENV_BIN" ];
46-
then
47-
run "$PIPENV_BIN"
48-
else
49-
echo "ERROR: pipenv seems to be installed, but I can't find in the PATH or via pip"
50-
exit 1
51-
fi
52-
}
53-
54-
request_install_to_virtual_env () {
55-
if [ "$PIP_INSTALL" = YES ]; then
56-
ANSWER=yes
57-
else
58-
echo "Do you want to install pipenv into the current virtual environment: yes/no"
59-
read ANSWER
60-
fi
61-
if [ "$ANSWER" == "yes" ];
62-
then
63-
find_pip_bin
64-
$PIP_BIN install pipenv
65-
find_and_run_via_pip
66-
run "$PIPENV_BIN"
67-
else
68-
echo "Aborting"
69-
fi
70-
}
71-
72-
request_install_to_user () {
73-
if [ "$PIP_INSTALL" = YES ]; then
74-
ANSWER=yes
75-
else
76-
echo "Do you want to install pipenv local to the user: yes/no"
77-
read ANSWER
78-
fi
79-
if [ "$ANSWER" == "yes" ];
80-
then
81-
find_pip_bin
82-
$PIP_BIN install --user pipenv
83-
PIPENV_BIN="$(python3 -m site --user-base)/bin/pipenv"
84-
run "$PIPENV_BIN"
85-
else
86-
echo "Aborting"
87-
fi
88-
}
89-
90-
request_install_and_run () {
91-
IS_IN_VIRTUAL_ENV=$(python3 -c "import sys; print(hasattr(sys, 'real_prefix'))")
92-
if [ "$IS_IN_VIRTUAL_ENV" == "True" ];
93-
then
94-
request_install_to_virtual_env
95-
else
96-
request_install_to_user
97-
fi
98-
}
99-
100-
101-
discover_pipenv_and_run() {
102-
PIPENV_BIN=$(command -v pipenv)
103-
if [ -n "$PIPENV_BIN" ];
104-
then
105-
run "$PIPENV_BIN"
106-
else
107-
find_pip_bin
108-
if $PIP_BIN show pipenv &> /dev/null;
109-
then
110-
find_and_run_via_pip
111-
else
112-
request_install_and_run
113-
fi
114-
fi
115-
}
116-
117-
if [ -n "$PIPENV_BIN" ];
18+
discover_pipenv
19+
init_pipenv "$PIPENV_BIN"
20+
if [ -n "$PIPENV_BIN" ]
11821
then
119-
run "$PIPENV_BIN"
22+
run
12023
else
121-
discover_pipenv_and_run
24+
echo "Could not find pipenv!"
25+
exit 1
12226
fi

‎exaslct_src/cli/commands/run_db_tests.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,9 +162,11 @@ def run_db_test(flavor_path: Tuple[str, ...],
162162
environment_type=EnvironmentType[environment_type],
163163
reuse_database_setup=reuse_database_setup,
164164
reuse_test_container=reuse_test_container,
165+
reuse_database=reuse_database,
166+
no_database_cleanup_after_end=reuse_database,
167+
no_test_container_cleanup_after_end=reuse_test_container,
165168
docker_db_image_name=docker_db_image_name,
166169
docker_db_image_version=docker_db_image_version,
167-
reuse_database=reuse_database,
168170
max_start_attempts=max_start_attempts,
169171
external_exasol_db_host=external_exasol_db_host,
170172
external_exasol_db_port=external_exasol_db_port,

‎exaslct_src/cli/commands/spawn_test_environment.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,12 @@ def spawn_test_environment(
4545
docker_db_image_name=docker_db_image_name,
4646
db_user="sys",
4747
db_password="exasol",
48-
bucketfs_write_password="write"
48+
bucketfs_write_password="write",
49+
no_test_container_cleanup_after_end=True,
50+
no_database_cleanup_after_end=True
4951
)
5052

5153
set_job_id(SpawnTestEnvironmentWithDockerDB.__name__)
5254
success, task = run_task(task_creator, workers, task_dependencies_dot_file)
5355
if not success:
54-
exit(1)
56+
exit(1)

‎exaslct_src/lib/command_log_handler.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ def handle_log_line(self, log_line, error: bool = False):
1414
log_line = log_line.decode("utf-8")
1515
self._log_file.write(log_line)
1616
self._complete_log.append(log_line)
17+
if error:
18+
self._error_message=log_line
1719

1820
def finish(self):
1921
if self._log_config.write_log_files_to_console==WriteLogFilesToConsole.all:
@@ -31,4 +33,4 @@ def finish(self):
3133
% (self._description,
3234
self._error_message,
3335
self._log_file_path.absolute()),
34-
self._log_file_path.absolute())
36+
self._log_file_path.absolute())

‎exaslct_src/lib/docker/docker_load_task.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ def run_task(self):
1212
self.logger.info("Try to load docker image %s from %s",
1313
self.image_info.get_source_complete_name(),image_archive_path)
1414
with image_archive_path.open("rb") as f:
15-
self.client.images.load(f)
16-
self.client.images.get(self.image_info.get_source_complete_name()).tag(
15+
self._client.images.load(f)
16+
self._client.images.get(self.image_info.get_source_complete_name()).tag(
1717
repository=self.image_info.target_repository_name,
1818
tag=self.image_info.get_target_complete_tag()
19-
)
19+
)

‎exaslct_src/lib/docker_config.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55

66
class docker_client_config(luigi.Config):
7-
timeout = luigi.IntParameter(1000, significant=False, visibility=ParameterVisibility.PRIVATE)
7+
timeout = luigi.IntParameter(100000, significant=False, visibility=ParameterVisibility.PRIVATE)
88
no_cache = luigi.BoolParameter(False)
99

1010
def get_client(self):

‎exaslct_src/lib/test_runner/abstract_spawn_test_environment.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ def _spawn_database_and_test_container(self,
8484
network_info=network_info,
8585
ip_address_index_in_subnet=1,
8686
reuse_test_container=self.reuse_test_container,
87+
no_test_container_cleanup_after_end=self.no_test_container_cleanup_after_end,
8788
attempt=attempt),
8889
DATABASE: self.create_spawn_database_task(network_info, attempt)
8990
})

‎exaslct_src/lib/test_runner/docker_db_test_environment_parameter.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@ class DockerDBTestEnvironmentParameter(Config):
66
docker_db_image_name = luigi.OptionalParameter(None)
77
docker_db_image_version = luigi.OptionalParameter(None)
88
reuse_database = luigi.BoolParameter(False, significant=False)
9+
no_database_cleanup_after_end = luigi.BoolParameter(False, significant=False)
910
database_port_forward = luigi.OptionalParameter(None, significant=False)
1011
bucketfs_port_forward = luigi.OptionalParameter(None, significant=False)

‎exaslct_src/lib/test_runner/general_spawn_test_environment_parameter.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,5 @@ class GeneralSpawnTestEnvironmentParameter(Config):
99

1010
reuse_database_setup = luigi.BoolParameter(False, significant=False)
1111
reuse_test_container = luigi.BoolParameter(False, significant=False)
12-
13-
max_start_attempts = luigi.IntParameter(2, significant=False)
12+
no_test_container_cleanup_after_end = luigi.BoolParameter(False, significant=False)
13+
max_start_attempts = luigi.IntParameter(2, significant=False)

‎exaslct_src/lib/test_runner/is_database_ready_thread.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,6 @@ def create_db_connection_command(self):
5353
def create_bucketfs_connection_command(self):
5454
username = "w"
5555
password = self.database_credentials.bucketfs_write_password
56-
cmd = f"""curl --fail '{username}:{password}@{self._database_info.host}:{self._database_info.bucketfs_port}'"""
56+
cmd = f"""curl --silent --show-error --fail '{username}:{password}@{self._database_info.host}:{self._database_info.bucketfs_port}'"""
5757
bash_cmd = f"""bash -c "{cmd}" """
58-
return bash_cmd
58+
return bash_cmd

‎exaslct_src/lib/test_runner/prepare_network_for_test_environment.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ class PrepareDockerNetworkForTestEnvironment(DockerBaseTask):
1111
test_container_name = luigi.Parameter(significant=False)
1212
db_container_name = luigi.OptionalParameter(None, significant=False)
1313
reuse = luigi.BoolParameter(False, significant=False)
14+
no_cleanup_after_end = luigi.BoolParameter(False, significant=False)
1415
attempt = luigi.IntParameter(-1)
1516

1617
def run_task(self):
@@ -75,7 +76,8 @@ def remove_container(self, container_name: str):
7576
pass
7677

7778
def cleanup_task(self):
78-
self.remove_container(self.test_container_name)
79-
if self.db_container_name is not None:
80-
self.remove_container(self.db_container_name)
81-
self.remove_network(self.network_name)
79+
if not self.no_cleanup_after_end:
80+
self.remove_container(self.test_container_name)
81+
if self.db_container_name is not None:
82+
self.remove_container(self.db_container_name)
83+
self.remove_network(self.network_name)

‎exaslct_src/lib/test_runner/spawn_test_container.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ class SpawnTestContainer(DockerBaseTask):
2626
ip_address_index_in_subnet = luigi.IntParameter(significant=False)
2727
attempt = luigi.IntParameter(1)
2828
reuse_test_container = luigi.BoolParameter(False, significant=False)
29+
no_test_container_cleanup_after_end = luigi.BoolParameter(False, significant=False)
30+
2931

3032
def __init__(self, *args, **kwargs):
3133
super().__init__(*args, **kwargs)
@@ -46,6 +48,8 @@ def run_task(self):
4648
container_info = self._try_to_reuse_test_container(ip_address, self.network_info)
4749
if container_info is None:
4850
container_info = self._create_test_container(ip_address, self.network_info)
51+
test_container = \
52+
self._client.containers.get(self.test_container_name)
4953
self._copy_tests()
5054
self.return_object(container_info)
5155

@@ -143,4 +147,5 @@ def _remove_container(self, container_name: str):
143147
pass
144148

145149
def cleanup_task(self):
146-
self._remove_container(self.test_container_name)
150+
if not self.no_test_container_cleanup_after_end:
151+
self._remove_container(self.test_container_name)

‎exaslct_src/lib/test_runner/spawn_test_database.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -240,8 +240,9 @@ def _execute_init_db(self, db_volume: Volume, volume_preperation_container: Cont
240240
"Error during preperation of docker-db volume %s got following output %s" % (db_volume.name, output))
241241

242242
def cleanup_task(self):
243-
db_volume_preperation_container_name = self._get_db_volume_preperation_container_name()
244-
self._remove_container(db_volume_preperation_container_name)
245-
self._remove_container(self.db_container_name)
246-
db_volume_name = self._get_db_volume_name()
247-
self._remove_volume(db_volume_name)
243+
if not self.no_database_cleanup_after_end:
244+
db_volume_preperation_container_name = self._get_db_volume_preperation_container_name()
245+
self._remove_container(db_volume_preperation_container_name)
246+
self._remove_container(self.db_container_name)
247+
db_volume_name = self._get_db_volume_name()
248+
self._remove_volume(db_volume_name)

‎exaslct_src/lib/test_runner/spawn_test_environment_with_docker_db.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ def create_network_task(self, attempt: int):
2929
test_container_name=self.test_container_name,
3030
network_name=self.network_name,
3131
db_container_name=self.db_container_name,
32-
reuse=self.reuse_database,
32+
reuse=self.reuse_database or self.reuse_test_container,
33+
no_cleanup_after_end=self.no_database_cleanup_after_end or self.no_test_container_cleanup_after_end,
3334
attempt=attempt
3435
)
3536

‎exaslct_src/lib/test_runner/upload_exported_container.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ def get_pattern_to_wait_for(self):
1919
return self.export_info.name + ".*extracted"
2020

2121
def get_file_to_upload(self):
22-
return "/exports/" + pathlib.Path(self.export_info.cache_file).name # TODO directory /exports is as data dependency to SpawnTestContainer
22+
file="/exports/" + pathlib.Path(self.export_info.cache_file).name
23+
return file
2324

2425
def get_upload_target(self):
2526
return "myudfs/" + self.export_info.name + ".tar.gz"

‎exaslct_src/lib/test_runner/upload_file_to_db.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,13 @@ def generate_list_command(self, upload_target: str):
107107
url = "http://w:{password}@{host}:{port}/{bucket}".format(
108108
host=self._database_info.host, port=self._database_info.bucketfs_port,
109109
bucket=bucket, password=self.bucketfs_write_password)
110-
cmd = f"curl --fail '{url}'"
110+
cmd = f"curl --silent --show-error --fail '{url}'"
111111
return cmd
112112

113113
def upload_file(self, file_to_upload: str, upload_target: str):
114114
self.logger.info("upload file %s to %s",
115115
file_to_upload, upload_target)
116+
exit_code, log_output=self.run_command("upload", "ls "+str(Path(file_to_upload).parent))
116117
command = self.generate_upload_command(file_to_upload, upload_target)
117118
exit_code, log_output = self.run_command("upload", command)
118119
if exit_code != 0:
@@ -125,7 +126,7 @@ def generate_upload_command(self, file_to_upload, upload_target):
125126
url = "http://w:{password}@{host}:{port}/{target}".format(
126127
host=self._database_info.host, port=self._database_info.bucketfs_port,
127128
target=upload_target, password=self.bucketfs_write_password)
128-
cmd = f"curl --fail -X PUT -T '{file_to_upload}' '{url}'"
129+
cmd = f"curl --silent --show-error --fail -X PUT -T '{file_to_upload}' '{url}'"
129130
return cmd
130131

131132
def run_command(self, command_type, cmd):

‎exaslct_src/test/test_docker_pull.py

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,32 +8,10 @@
88

99
class DockerPullTest(unittest.TestCase):
1010

11-
def create_registry(self):
12-
self.registry_port = utils.find_free_port()
13-
registry_container_name = self.test_environment.name.replace("/", "_") + "_registry"
14-
docker_client = docker.from_env()
15-
try:
16-
print("Start pull of registry:2")
17-
docker_client.images.pull(repository="registry", tag="2")
18-
print(f"Start container of {registry_container_name}")
19-
try:
20-
docker_client.containers.get(registry_container_name).remove(force=True)
21-
except:
22-
pass
23-
self.registry_container = docker_client.containers.run(
24-
image="registry:2", name=registry_container_name,
25-
ports={5000: self.registry_port},
26-
detach=True
27-
)
28-
print(f"Finished start container of {registry_container_name}")
29-
self.test_environment.repository_prefix = f"localhost:{self.registry_port}"
30-
finally:
31-
docker_client.close()
32-
3311
def setUp(self):
3412
print(f"SetUp {self.__class__.__name__}")
3513
self.test_environment = utils.ExaslctTestEnvironment(self)
36-
self.create_registry()
14+
self.registry_container,self.registry_host,self.registry_port=self.test_environment.create_registry()
3715
self.test_environment.clean_images()
3816
command = f"./exaslct push "
3917
self.test_environment.run_command(command, track_task_dependencies=False)
@@ -64,5 +42,7 @@ def find_all(self, search_root, search_name, path):
6442
return result
6543

6644

45+
46+
6747
if __name__ == '__main__':
6848
unittest.main()

0 commit comments

Comments
 (0)