Skip to content

Commit f8a7e9b

Browse files
wbarnhawbarnha
authored
Transition CI/CD to GitHub Workflows (#2378)
* Create GH workflows to test code * Update tests for future Python versions
1 parent 7ac6c6e commit f8a7e9b

File tree

10 files changed

+303
-32
lines changed

10 files changed

+303
-32
lines changed

‎.github/dependabot.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
version: 2
2+
updates:
3+
# Maintain dependencies for GitHub Actions
4+
- package-ecosystem: "github-actions"
5+
directory: "/"
6+
schedule:
7+
interval: "daily"

‎.github/workflows/codeql-analysis.yml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
---
2+
# For most projects, this workflow file will not need changing; you simply need
3+
# to commit it to your repository.
4+
#
5+
# You may wish to alter this file to override the set of languages analyzed,
6+
# or to provide custom queries or build logic.
7+
#
8+
# ******** NOTE ********
9+
# We have attempted to detect the languages in your repository. Please check
10+
# the `language` matrix defined below to confirm you have the correct set of
11+
# supported CodeQL languages.
12+
#
13+
name: CodeQL
14+
on:
15+
push:
16+
branches: [master]
17+
pull_request:
18+
# The branches below must be a subset of the branches above
19+
branches: [master]
20+
schedule:
21+
- cron: 19 10 * * 6
22+
jobs:
23+
analyze:
24+
name: Analyze
25+
runs-on: ubuntu-latest
26+
permissions:
27+
actions: read
28+
contents: read
29+
security-events: write
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
language: [python]
34+
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
35+
# Learn more:
36+
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
37+
steps:
38+
- name: Checkout repository
39+
uses: actions/checkout@v3
40+
41+
# Initializes the CodeQL tools for scanning.
42+
- name: Initialize CodeQL
43+
uses: github/codeql-action/init@v2
44+
with:
45+
languages: ${{ matrix.language }}
46+
# If you wish to specify custom queries, you can do so here or in a config file.
47+
# By default, queries listed here will override any specified in a config file.
48+
# Prefix the list here with "+" to use these queries and those in the config file.
49+
# queries: ./path/to/local/query, your-org/your-repo/queries@main
50+
51+
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
52+
# If this step fails, then you should remove it and run the build manually (see below)
53+
- name: Autobuild
54+
uses: github/codeql-action/autobuild@v2
55+
56+
# ℹ️ Command-line programs to run using the OS shell.
57+
# 📚 https://git.io/JvXDl
58+
59+
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
60+
# and modify them (or add more) to build your code if your project
61+
# uses a compiled language
62+
63+
#- run: |
64+
# make bootstrap
65+
# make release
66+
- name: Perform CodeQL Analysis
67+
uses: github/codeql-action/analyze@v2

‎.github/workflows/python-package.yml

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
name: CI/CD
2+
3+
on:
4+
push:
5+
branches: ["master"]
6+
pull_request:
7+
branches: ["master"]
8+
release:
9+
types: [created]
10+
branches:
11+
- 'master'
12+
workflow_dispatch:
13+
14+
env:
15+
FORCE_COLOR: "1" # Make tools pretty.
16+
PIP_DISABLE_PIP_VERSION_CHECK: "1"
17+
PIP_NO_PYTHON_VERSION_WARNING: "1"
18+
PYTHON_LATEST: "3.11"
19+
KAFKA_LATEST: "2.6.0"
20+
21+
# For re-actors/checkout-python-sdist
22+
sdist-artifact: python-package-distributions
23+
24+
jobs:
25+
26+
build-sdist:
27+
name: 📦 Build the source distribution
28+
runs-on: ubuntu-latest
29+
steps:
30+
- name: Checkout project
31+
uses: actions/checkout@v3
32+
with:
33+
fetch-depth: 0
34+
- name: Set up Python
35+
uses: actions/setup-python@v4
36+
with:
37+
python-version: ${{ env.PYTHON_LATEST }}
38+
cache: pip
39+
- run: python -m pip install build
40+
name: Install core libraries for build and install
41+
- name: Build artifacts
42+
run: python -m build
43+
- name: Upload built artifacts for testing
44+
uses: actions/upload-artifact@v3
45+
with:
46+
name: ${{ env.sdist-artifact }}
47+
# NOTE: Exact expected file names are specified here
48+
# NOTE: as a safety measure — if anything weird ends
49+
# NOTE: up being in this dir or not all dists will be
50+
# NOTE: produced, this will fail the workflow.
51+
path: dist/${{ env.sdist-name }}
52+
retention-days: 15
53+
54+
test-python:
55+
name: Tests on ${{ matrix.python-version }}
56+
needs:
57+
- build-sdist
58+
runs-on: ubuntu-latest
59+
continue-on-error: ${{ matrix.experimental }}
60+
strategy:
61+
fail-fast: false
62+
matrix:
63+
python-version:
64+
- "3.8"
65+
- "3.9"
66+
- "3.10"
67+
- "3.11"
68+
experimental: [ false ]
69+
include:
70+
- python-version: "pypy3.9"
71+
experimental: true
72+
# - python-version: "~3.12.0-0"
73+
# experimental: true
74+
steps:
75+
- name: Checkout the source code
76+
uses: actions/checkout@v3
77+
with:
78+
fetch-depth: 0
79+
- name: Setup java
80+
uses: actions/setup-java@v3
81+
with:
82+
distribution: temurin
83+
java-version: 11
84+
- name: Set up Python
85+
uses: actions/setup-python@v4
86+
with:
87+
python-version: ${{ matrix.python-version }}
88+
cache: pip
89+
cache-dependency-path: |
90+
requirements-dev.txt
91+
- name: Check Java installation
92+
run: source travis_java_install.sh
93+
- name: Pull Kafka releases
94+
run: ./build_integration.sh
95+
env:
96+
PLATFORM: ${{ matrix.platform }}
97+
KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
98+
# TODO: Cache releases to expedite testing
99+
- name: Install dependencies
100+
run: |
101+
sudo apt install -y libsnappy-dev libzstd-dev
102+
python -m pip install --upgrade pip
103+
python -m pip install tox tox-gh-actions
104+
pip install .
105+
pip install -r requirements-dev.txt
106+
- name: Test with tox
107+
run: tox
108+
env:
109+
PLATFORM: ${{ matrix.platform }}
110+
KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
111+
112+
test-kafka:
113+
name: Tests for Kafka ${{ matrix.kafka-version }}
114+
needs:
115+
- build-sdist
116+
runs-on: ubuntu-latest
117+
strategy:
118+
fail-fast: false
119+
matrix:
120+
kafka-version:
121+
- "0.8.2.2"
122+
- "0.9.0.1"
123+
- "0.10.2.2"
124+
- "0.11.0.2"
125+
- "0.11.0.3"
126+
- "1.1.1"
127+
- "2.4.0"
128+
- "2.5.0"
129+
- "2.6.0"
130+
steps:
131+
- name: Checkout the source code
132+
uses: actions/checkout@v3
133+
with:
134+
fetch-depth: 0
135+
- name: Setup java
136+
uses: actions/setup-java@v3
137+
with:
138+
distribution: temurin
139+
java-version: 8
140+
- name: Set up Python
141+
uses: actions/setup-python@v4
142+
with:
143+
python-version: ${{ env.PYTHON_LATEST }}
144+
cache: pip
145+
cache-dependency-path: |
146+
requirements-dev.txt
147+
- name: Pull Kafka releases
148+
run: ./build_integration.sh
149+
env:
150+
# This is fast enough as long as you pull only one release at a time,
151+
# no need to worry about caching
152+
PLATFORM: ${{ matrix.platform }}
153+
KAFKA_VERSION: ${{ matrix.kafka-version }}
154+
- name: Install dependencies
155+
run: |
156+
sudo apt install -y libsnappy-dev libzstd-dev
157+
python -m pip install --upgrade pip
158+
python -m pip install tox tox-gh-actions
159+
pip install .
160+
pip install -r requirements-dev.txt
161+
- name: Test with tox
162+
run: tox
163+
env:
164+
PLATFORM: ${{ matrix.platform }}
165+
KAFKA_VERSION: ${{ matrix.kafka-version }}
166+
167+
check: # This job does nothing and is only used for the branch protection
168+
name: ✅ Ensure the required checks passing
169+
if: always()
170+
needs:
171+
- build-sdist
172+
- test-python
173+
- test-kafka
174+
runs-on: ubuntu-latest
175+
steps:
176+
- name: Decide whether the needed jobs succeeded or failed
177+
uses: re-actors/alls-green@release/v1
178+
with:
179+
jobs: ${{ toJSON(needs) }}

‎Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ test37: build-integration
2020
test27: build-integration
2121
KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py27 -- $(FLAGS)
2222

23-
# Test using py.test directly if you want to use local python. Useful for other
23+
# Test using pytest directly if you want to use local python. Useful for other
2424
# platforms that require manual installation for C libraries, ie. Windows.
2525
test-local: build-integration
26-
KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) py.test \
26+
KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \
2727
--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF $(FLAGS) kafka test
2828

2929
cov-local: build-integration
30-
KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) py.test \
30+
KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \
3131
--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka \
3232
--cov-config=.covrc --cov-report html $(FLAGS) kafka test
3333
@echo "open file://`pwd`/htmlcov/index.html"

‎README.rst

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,16 @@ Kafka Python client
77
:target: https://pypi.python.org/pypi/kafka-python
88
.. image:: https://coveralls.io/repos/dpkp/kafka-python/badge.svg?branch=master&service=github
99
:target: https://coveralls.io/github/dpkp/kafka-python?branch=master
10-
.. image:: https://travis-ci.org/dpkp/kafka-python.svg?branch=master
11-
:target: https://travis-ci.org/dpkp/kafka-python
1210
.. image:: https://img.shields.io/badge/license-Apache%202-blue.svg
1311
:target: https://github.com/dpkp/kafka-python/blob/master/LICENSE
12+
.. image:: https://img.shields.io/pypi/dw/kafka-python.svg
13+
:target: https://pypistats.org/packages/kafka-python
14+
.. image:: https://img.shields.io/pypi/v/kafka-python.svg
15+
:target: https://pypi.org/project/kafka-python
16+
.. image:: https://img.shields.io/pypi/implementation/kafka-python
17+
:target: https://github.com/dpkp/kafka-python/blob/master/setup.py
18+
19+
1420

1521
Python client for the Apache Kafka distributed stream processing system.
1622
kafka-python is designed to function much like the official java client, with a

‎requirements-dev.txt

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
coveralls==2.1.2
2-
crc32c==2.1
3-
docker-py==1.10.6
4-
flake8==3.8.3
5-
lz4==3.1.0
6-
mock==4.0.2
7-
py==1.9.0
8-
pylint==2.6.0
9-
pytest==6.0.2
10-
pytest-cov==2.10.1
11-
pytest-mock==3.3.1
12-
pytest-pylint==0.17.0
13-
python-snappy==0.5.4
14-
Sphinx==3.2.1
15-
sphinx-rtd-theme==0.5.0
16-
tox==3.20.0
17-
xxhash==2.0.0
1+
coveralls
2+
crc32c
3+
docker-py
4+
flake8
5+
lz4
6+
mock
7+
py
8+
pylint
9+
pytest
10+
pytest-cov
11+
pytest-mock
12+
pytest-pylint
13+
python-snappy
14+
Sphinx
15+
sphinx-rtd-theme
16+
tox
17+
xxhash

‎setup.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,10 @@ def run(cls):
5050
license="Apache License 2.0",
5151
description="Pure Python client for Apache Kafka",
5252
long_description=README,
53-
keywords="apache kafka",
53+
keywords=[
54+
"apache kafka",
55+
"kafka",
56+
],
5457
classifiers=[
5558
"Development Status :: 5 - Production/Stable",
5659
"Intended Audience :: Developers",
@@ -64,6 +67,10 @@ def run(cls):
6467
"Programming Language :: Python :: 3.6",
6568
"Programming Language :: Python :: 3.7",
6669
"Programming Language :: Python :: 3.8",
70+
"Programming Language :: Python :: 3.9",
71+
"Programming Language :: Python :: 3.10",
72+
"Programming Language :: Python :: 3.11",
73+
"Programming Language :: Python :: Implementation :: CPython",
6774
"Programming Language :: Python :: Implementation :: PyPy",
6875
"Topic :: Software Development :: Libraries :: Python Modules",
6976
]

‎test/test_assignors.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -655,7 +655,7 @@ def test_conflicting_previous_assignments(mocker):
655655
'execution_number,n_topics,n_consumers', [(i, randint(10, 20), randint(20, 40)) for i in range(100)]
656656
)
657657
def test_reassignment_with_random_subscriptions_and_changes(mocker, execution_number, n_topics, n_consumers):
658-
all_topics = set(['t{}'.format(i) for i in range(1, n_topics + 1)])
658+
all_topics = sorted(['t{}'.format(i) for i in range(1, n_topics + 1)])
659659
partitions = dict([(t, set(range(1, i + 1))) for i, t in enumerate(all_topics)])
660660
cluster = create_cluster(mocker, topics=all_topics, topic_partitions_lambda=lambda t: partitions[t])
661661

0 commit comments

Comments
 (0)