Skip to content

Commit aef7a85

Browse files
potiukpotiuk
committed
Change default MySQL client to MariaDB (#36243)
This PR is a response to pretty catastrophic issue caused by expiring key on MySQL repository on 14th of December. Oracle does not follow the best practices for signing their packages (while all others do) and their packages and repositories are signed with a key with short expiry date. This basically puts an expiry on their repository, and anyone who releases images following best practices of installation, while keeping the repository after installing mysql libraries has to rebuild their past released images every 2 years or so. This is the last straw for our MySQL client installation problems (we had a few, especially for ARM images) and we decided that we will switch to MariaDB client libraries by default (while allowing our users to build custom images with MySQL libraries). The issue tracked in Airflow repository is: #36231 The issues in Oracle's MySQL repo: * https://bugs.mysql.com/bug.php?id=113427 * https://bugs.mysql.com/bug.php?id=113428 * https://bugs.mysql.com/bug.php?id=113432 This PR implements a number of changes: * MariaDB client is now default client used for both ARM and X86 * both pre-2023 and 2023 keys for MySQL are now added to be trusted when custom image with MySQL client is built * MySQL repository is removed after installing MySQL (to avoid repeating similar fiasco for MySQL users in 2025 * changelog added and instructions on how to build custom image with MySQL client * one of our test suites is converted to use "current" image not latest released image (that was bug in our CI). * test was added in `canary` and `release` builds in CI to also test build of custom image with MySQL client (cherry picked from commit 8a7bea2)
1 parent 67031a1 commit aef7a85

24 files changed

+430
-174
lines changed

‎.github/workflows/ci.yml

Lines changed: 150 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -395,32 +395,6 @@ jobs:
395395
- name: "Generate client codegen diff"
396396
run: ./scripts/ci/openapi/client_codegen_diff.sh
397397

398-
test-examples-of-prod-image-building:
399-
timeout-minutes: 60
400-
name: "Test examples of production image building"
401-
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
402-
needs: [build-info]
403-
if: needs.build-info.outputs.ci-image-build == 'true'
404-
steps:
405-
- name: Cleanup repo
406-
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
407-
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
408-
uses: actions/checkout@v4
409-
with:
410-
fetch-depth: 2
411-
persist-credentials: false
412-
- name: "Setup python"
413-
uses: actions/setup-python@v4
414-
with:
415-
python-version: "${{needs.build-info.outputs.default-python-version}}"
416-
cache: 'pip'
417-
cache-dependency-path: ./dev/requirements.txt
418-
- name: "Test examples of PROD image building"
419-
run: >
420-
cd ./docker_tests &&
421-
python -m pip install -r requirements.txt &&
422-
python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes
423-
424398
test-git-clone-on-windows:
425399
timeout-minutes: 5
426400
name: "Test git clone on Windows"
@@ -1721,6 +1695,63 @@ jobs:
17211695
# TODO: improve caching for that build
17221696
IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
17231697
1698+
build-prod-images-mysql-client:
1699+
timeout-minutes: 80
1700+
name: >
1701+
Build MysQL Client PROD images (main)
1702+
${{needs.build-info.outputs.all-python-versions-list-as-string}}
1703+
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
1704+
needs: [build-info, build-ci-images]
1705+
if: needs.build-info.outputs.canary-run == 'true'
1706+
env:
1707+
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
1708+
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
1709+
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
1710+
BACKEND: sqlite
1711+
VERSION_SUFFIX_FOR_PYPI: "dev0"
1712+
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
1713+
# Force more parallelism for build even on public images
1714+
PARALLELISM: 6
1715+
steps:
1716+
- name: Cleanup repo
1717+
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
1718+
if: >
1719+
needs.build-info.outputs.in-workflow-build == 'true' &&
1720+
needs.build-info.outputs.default-branch == 'main'
1721+
- uses: actions/checkout@v3
1722+
with:
1723+
ref: ${{ needs.build-info.outputs.targetCommitSha }}
1724+
persist-credentials: false
1725+
submodules: recursive
1726+
if: >
1727+
needs.build-info.outputs.in-workflow-build == 'true' &&
1728+
needs.build-info.outputs.default-branch == 'main'
1729+
- name: "Install Breeze"
1730+
uses: ./.github/actions/breeze
1731+
if: >
1732+
needs.build-info.outputs.in-workflow-build == 'true' &&
1733+
needs.build-info.outputs.default-branch == 'main'
1734+
- name: >
1735+
Build Bullseye PROD Images
1736+
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
1737+
uses: ./.github/actions/build-prod-images
1738+
if: >
1739+
needs.build-info.outputs.in-workflow-build == 'true' &&
1740+
needs.build-info.outputs.default-branch == 'main'
1741+
with:
1742+
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
1743+
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
1744+
env:
1745+
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
1746+
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
1747+
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
1748+
DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
1749+
INSTALL_MYSQL_CLIENT_TYPE: "mysql"
1750+
# Do not override the "mariadb" (original) image - just push a new mysql image
1751+
# TODO: improve caching for that build
1752+
IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
1753+
1754+
17241755
build-prod-images-release-branch:
17251756
timeout-minutes: 80
17261757
name: >
@@ -1827,6 +1858,62 @@ jobs:
18271858
# TODO: improve caching for that build
18281859
IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
18291860
1861+
build-prod-images-mysql-release-branch:
1862+
timeout-minutes: 80
1863+
name: >
1864+
Build MySQL PROD images (v2_*_test)
1865+
${{needs.build-info.outputs.all-python-versions-list-as-string}}
1866+
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
1867+
needs: [build-info, generate-constraints]
1868+
if: needs.build-info.outputs.canary-run == 'true'
1869+
env:
1870+
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
1871+
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
1872+
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
1873+
BACKEND: sqlite
1874+
VERSION_SUFFIX_FOR_PYPI: "dev0"
1875+
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
1876+
# Force more parallelism for build even on public images
1877+
PARALLELISM: 6
1878+
steps:
1879+
- name: Cleanup repo
1880+
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
1881+
if: >
1882+
needs.build-info.outputs.in-workflow-build == 'true' &&
1883+
needs.build-info.outputs.default-branch != 'main'
1884+
- uses: actions/checkout@v3
1885+
with:
1886+
ref: ${{ needs.build-info.outputs.targetCommitSha }}
1887+
persist-credentials: false
1888+
submodules: recursive
1889+
if: >
1890+
needs.build-info.outputs.in-workflow-build == 'true' &&
1891+
needs.build-info.outputs.default-branch != 'main'
1892+
- name: "Install Breeze"
1893+
uses: ./.github/actions/breeze
1894+
if: >
1895+
needs.build-info.outputs.in-workflow-build == 'true' &&
1896+
needs.build-info.outputs.default-branch != 'main'
1897+
- name: >
1898+
Build Bullseye PROD Images
1899+
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
1900+
uses: ./.github/actions/build-prod-images
1901+
if: >
1902+
needs.build-info.outputs.in-workflow-build == 'true' &&
1903+
needs.build-info.outputs.default-branch != 'main'
1904+
with:
1905+
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
1906+
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
1907+
env:
1908+
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
1909+
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
1910+
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
1911+
DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
1912+
INSTALL_MYSQL_CLIENT_TYPE: "mysql"
1913+
# Do not override the "mariadb" image - just push a new mysql image
1914+
# TODO: improve caching for that build
1915+
IMAGE_TAG: "mysql-${{ github.event.pull_request.head.sha || github.sha }}"
1916+
18301917
wait-for-prod-images:
18311918
timeout-minutes: 80
18321919
name: "Wait for PROD images"
@@ -1862,6 +1949,43 @@ jobs:
18621949
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
18631950
if: needs.build-info.outputs.in-workflow-build == 'false'
18641951

1952+
test-examples-of-prod-image-building:
1953+
timeout-minutes: 60
1954+
name: "Test examples of production image building"
1955+
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
1956+
needs: [build-info, wait-for-prod-images]
1957+
if: needs.build-info.outputs.prod-image-build == 'true'
1958+
steps:
1959+
- name: Cleanup repo
1960+
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
1961+
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
1962+
uses: actions/checkout@v4
1963+
with:
1964+
fetch-depth: 2
1965+
persist-credentials: false
1966+
- name: "Install Breeze"
1967+
uses: ./.github/actions/breeze
1968+
- name: Pull PROD image "${{needs.build-info.outputs.default-python-version}}":${{ env.IMAGE_TAG }}
1969+
run: breeze prod-image pull --tag-as-latest
1970+
env:
1971+
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
1972+
- name: "Setup python"
1973+
uses: actions/setup-python@v4
1974+
with:
1975+
python-version: ${{needs.build-info.outputs.default-python-version}}
1976+
cache: 'pip'
1977+
cache-dependency-path: ./dev/requirements.txt
1978+
- name: "Test examples of PROD image building"
1979+
# yamllint disable-line rule:line-length
1980+
run: >
1981+
cd ./docker_tests &&
1982+
python -m pip install -r requirements.txt &&
1983+
TEST_IMAGE="ghcr.io/${GITHUB_REPOSITORY}/main/prod/python${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
1984+
python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes
1985+
env:
1986+
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
1987+
1988+
18651989
test-docker-compose-quick-start:
18661990
timeout-minutes: 60
18671991
name: "Verify PROD image and docker-compose quick start"

‎CI_DIAGRAMS.md

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,6 @@ sequenceDiagram
6060
Note over Tests: OpenAPI client gen
6161
and
6262
Note over Tests: React WWW tests
63-
and
64-
Note over Tests: Test examples<br>PROD image building
6563
and
6664
Note over Tests: Test git clone on Windows
6765
and
@@ -152,6 +150,11 @@ sequenceDiagram
152150
end
153151
end
154152
par
153+
opt
154+
GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
155+
Note over Tests: Test examples<br>PROD image building
156+
end
157+
and
155158
opt
156159
GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
157160
Note over Tests: Run Kubernetes <br>tests
@@ -343,8 +346,6 @@ sequenceDiagram
343346
Note over Tests: OpenAPI client gen
344347
and
345348
Note over Tests: React WWW tests
346-
and
347-
Note over Tests: Test examples<br>PROD image building
348349
and
349350
Note over Tests: Test git clone on Windows
350351
end
@@ -400,6 +401,9 @@ sequenceDiagram
400401
Artifacts ->> Tests: Download source,pypi,no-providers constraints
401402
Note over Tests: Display constraints diff
402403
Tests ->> Airflow Repo: Push constraints if changed to 'constraints-BRANCH'
404+
and
405+
GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
406+
Note over Tests: Test examples<br>PROD image building
403407
and
404408
GitHub Registry ->> Tests: Pull PROD Image<br>[COMMIT_SHA]
405409
Note over Tests: Run Kubernetes <br>tests

‎Dockerfile

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ COLOR_RESET=$'\e[0m'
232232
readonly COLOR_RESET
233233

234234
: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
235-
: "${INSTALL_MYSQL_CLIENT_TYPE:-mysql}"
235+
: "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}"
236236

237237
export_key() {
238238
local key="${1}"
@@ -271,7 +271,7 @@ install_mysql_client() {
271271
exit 1
272272
fi
273273

274-
export_key "467B942D3A79BD29" "mysql"
274+
export_key "B7B3B788A8D3785C" "mysql"
275275

276276
echo
277277
echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}"
@@ -283,6 +283,13 @@ install_mysql_client() {
283283
apt-get install --no-install-recommends -y "${packages[@]}"
284284
apt-get autoremove -yqq --purge
285285
apt-get clean && rm -rf /var/lib/apt/lists/*
286+
287+
# Remove mysql repository from sources.list.d as MySQL repos have a basic flaw that they put expiry
288+
# date on their GPG signing keys and they sign their repo with those keys. This means that after a
289+
# certain date, the GPG key becomes invalid and if you have the repository added in your sources.list
290+
# then you will not be able to install anything from any other repository. This id unlike any other
291+
# repository we have seen (for example Postgres, MariaDB, MsSQL - all have non-expiring signing keys)
292+
rm /etc/apt/sources.list.d/mysql.list
286293
}
287294

288295
install_mariadb_client() {
@@ -327,6 +334,9 @@ install_mariadb_client() {
327334
if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
328335
if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
329336
INSTALL_MYSQL_CLIENT_TYPE="mariadb"
337+
echo
338+
echo "${COLOR_YELLOW}Client forced to mariadb for ARM${COLOR_RESET}"
339+
echo
330340
fi
331341

332342
if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then
@@ -1229,7 +1239,7 @@ COPY --from=scripts install_os_dependencies.sh /scripts/docker/
12291239
RUN bash /scripts/docker/install_os_dependencies.sh dev
12301240

12311241
ARG INSTALL_MYSQL_CLIENT="true"
1232-
ARG INSTALL_MYSQL_CLIENT_TYPE="mysql"
1242+
ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb"
12331243
ARG INSTALL_MSSQL_CLIENT="true"
12341244
ARG INSTALL_POSTGRES_CLIENT="true"
12351245
ARG AIRFLOW_PIP_VERSION

‎Dockerfile.ci

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ COLOR_RESET=$'\e[0m'
192192
readonly COLOR_RESET
193193

194194
: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
195-
: "${INSTALL_MYSQL_CLIENT_TYPE:-mysql}"
195+
: "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}"
196196

197197
export_key() {
198198
local key="${1}"
@@ -231,7 +231,7 @@ install_mysql_client() {
231231
exit 1
232232
fi
233233

234-
export_key "467B942D3A79BD29" "mysql"
234+
export_key "B7B3B788A8D3785C" "mysql"
235235

236236
echo
237237
echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}"
@@ -243,6 +243,13 @@ install_mysql_client() {
243243
apt-get install --no-install-recommends -y "${packages[@]}"
244244
apt-get autoremove -yqq --purge
245245
apt-get clean && rm -rf /var/lib/apt/lists/*
246+
247+
# Remove mysql repository from sources.list.d as MySQL repos have a basic flaw that they put expiry
248+
# date on their GPG signing keys and they sign their repo with those keys. This means that after a
249+
# certain date, the GPG key becomes invalid and if you have the repository added in your sources.list
250+
# then you will not be able to install anything from any other repository. This id unlike any other
251+
# repository we have seen (for example Postgres, MariaDB, MsSQL - all have non-expiring signing keys)
252+
rm /etc/apt/sources.list.d/mysql.list
246253
}
247254

248255
install_mariadb_client() {
@@ -287,6 +294,9 @@ install_mariadb_client() {
287294
if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
288295
if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
289296
INSTALL_MYSQL_CLIENT_TYPE="mariadb"
297+
echo
298+
echo "${COLOR_YELLOW}Client forced to mariadb for ARM${COLOR_RESET}"
299+
echo
290300
fi
291301

292302
if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then

‎dev/breeze/src/airflow_breeze/commands/ci_image_commands.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
option_image_tag_for_building,
4646
option_image_tag_for_pulling,
4747
option_image_tag_for_verifying,
48+
option_install_mysql_client_type,
4849
option_install_providers_from_sources,
4950
option_platform_multiple,
5051
option_prepare_buildx_cache,
@@ -300,6 +301,7 @@ def get_exitcode(status: int) -> int:
300301
@option_eager_upgrade_additional_requirements
301302
@option_github_repository
302303
@option_github_token
304+
@option_install_mysql_client_type
303305
@option_image_tag_for_building
304306
@option_include_success_outputs
305307
@option_install_providers_from_sources
@@ -342,6 +344,7 @@ def build(
342344
github_token: str | None,
343345
image_tag: str,
344346
include_success_outputs,
347+
install_mysql_client_type: str,
345348
install_providers_from_sources: bool,
346349
parallelism: int,
347350
platform: str | None,
@@ -413,6 +416,7 @@ def run_build(ci_image_params: BuildCiParams) -> None:
413416
github_repository=github_repository,
414417
github_token=github_token,
415418
image_tag=image_tag,
419+
install_mysql_client_type=install_mysql_client_type,
416420
install_providers_from_sources=install_providers_from_sources,
417421
prepare_buildx_cache=prepare_buildx_cache,
418422
push=push,

‎dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,22 +43,23 @@
4343
{
4444
"name": "Building images in parallel",
4545
"options": [
46-
"--run-in-parallel",
46+
"--debug-resources",
47+
"--include-success-outputs",
4748
"--parallelism",
4849
"--python-versions",
50+
"--run-in-parallel",
4951
"--skip-cleanup",
50-
"--debug-resources",
51-
"--include-success-outputs",
5252
],
5353
},
5454
{
5555
"name": "Advanced build options (for power users)",
5656
"options": [
57-
"--debian-version",
58-
"--python-image",
59-
"--commit-sha",
6057
"--additional-pip-install-flags",
58+
"--commit-sha",
59+
"--debian-version",
60+
"--install-mysql-client-type",
6161
"--install-providers-from-sources",
62+
"--python-image",
6263
],
6364
},
6465
{

0 commit comments

Comments
 (0)