Skip to content

Commit ad0ed59

Browse files
authored
feat(integrations): Add integration for clickhouse-driver (#2167)
Adds an integration that automatically facilitates tracing/recording of all queries, their parameters, data, and results.
1 parent 113b461 commit ad0ed59

File tree

8 files changed

+1129
-0
lines changed

8 files changed

+1129
-0
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Test clickhouse_driver
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
- release/**
8+
9+
pull_request:
10+
11+
# Cancel in progress workflows on pull_requests.
12+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
13+
concurrency:
14+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
15+
cancel-in-progress: true
16+
17+
permissions:
18+
contents: read
19+
20+
env:
21+
BUILD_CACHE_KEY: ${{ github.sha }}
22+
CACHED_BUILD_PATHS: |
23+
${{ github.workspace }}/dist-serverless
24+
25+
jobs:
26+
test:
27+
name: clickhouse_driver, python ${{ matrix.python-version }}, ${{ matrix.os }}
28+
runs-on: ${{ matrix.os }}
29+
timeout-minutes: 30
30+
31+
strategy:
32+
fail-fast: false
33+
matrix:
34+
python-version: ["3.8","3.9","3.10","3.11"]
35+
# python3.6 reached EOL and is no longer being supported on
36+
# new versions of hosted runners on Github Actions
37+
# ubuntu-20.04 is the last version that supported python3.6
38+
# see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
39+
os: [ubuntu-20.04]
40+
41+
steps:
42+
- uses: actions/checkout@v4
43+
- uses: actions/setup-python@v4
44+
with:
45+
python-version: ${{ matrix.python-version }}
46+
47+
- uses: getsentry/action-clickhouse-in-ci@v1
48+
49+
- name: Setup Test Env
50+
run: |
51+
pip install coverage "tox>=3,<4"
52+
53+
- name: Test clickhouse_driver
54+
uses: nick-fields/retry@v2
55+
with:
56+
timeout_minutes: 15
57+
max_attempts: 2
58+
retry_wait_seconds: 5
59+
shell: bash
60+
command: |
61+
set -x # print commands that are executed
62+
coverage erase
63+
64+
# Run tests
65+
./scripts/runtox.sh "py${{ matrix.python-version }}-clickhouse_driver" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
66+
coverage combine .coverage* &&
67+
coverage xml -i
68+
69+
- uses: codecov/codecov-action@v3
70+
with:
71+
token: ${{ secrets.CODECOV_TOKEN }}
72+
files: coverage.xml
73+
74+
75+
check_required_tests:
76+
name: All clickhouse_driver tests passed or skipped
77+
needs: test
78+
# Always run this, even if a dependent job failed
79+
if: always()
80+
runs-on: ubuntu-20.04
81+
steps:
82+
- name: Check for failures
83+
if: contains(needs.test.result, 'failure')
84+
run: |
85+
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

scripts/split-tox-gh-actions/ci-yaml-test-snippet.txt

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
- uses: actions/setup-python@v4
1111
with:
1212
python-version: ${{ matrix.python-version }}
13+
{{ additional_uses }}
1314

1415
- name: Setup Test Env
1516
run: |

scripts/split-tox-gh-actions/split-tox-gh-actions.py

+13
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@
3636
"asyncpg",
3737
]
3838

39+
FRAMEWORKS_NEEDING_CLICKHOUSE = [
40+
"clickhouse_driver",
41+
]
42+
3943
MATRIX_DEFINITION = """
4044
strategy:
4145
fail-fast: false
@@ -48,6 +52,11 @@
4852
os: [ubuntu-20.04]
4953
"""
5054

55+
ADDITIONAL_USES_CLICKHOUSE = """\
56+
57+
- uses: getsentry/action-clickhouse-in-ci@v1
58+
"""
59+
5160
CHECK_NEEDS = """\
5261
needs: test
5362
"""
@@ -119,6 +128,10 @@ def write_yaml_file(
119128
f = open(TEMPLATE_FILE_SETUP_DB, "r")
120129
out += "".join(f.readlines())
121130

131+
elif template_line.strip() == "{{ additional_uses }}":
132+
if current_framework in FRAMEWORKS_NEEDING_CLICKHOUSE:
133+
out += ADDITIONAL_USES_CLICKHOUSE
134+
122135
elif template_line.strip() == "{{ check_needs }}":
123136
if py27_supported:
124137
out += CHECK_NEEDS_PY27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
from sentry_sdk import Hub
2+
from sentry_sdk.consts import OP, SPANDATA
3+
from sentry_sdk.hub import _should_send_default_pii
4+
from sentry_sdk.integrations import Integration, DidNotEnable
5+
from sentry_sdk.tracing import Span
6+
from sentry_sdk._types import TYPE_CHECKING
7+
from sentry_sdk.utils import capture_internal_exceptions
8+
9+
from typing import TypeVar
10+
11+
# Hack to get new Python features working in older versions
12+
# without introducing a hard dependency on `typing_extensions`
13+
# from: https://stackoverflow.com/a/71944042/300572
14+
if TYPE_CHECKING:
15+
from typing import ParamSpec, Callable
16+
else:
17+
# Fake ParamSpec
18+
class ParamSpec:
19+
def __init__(self, _):
20+
self.args = None
21+
self.kwargs = None
22+
23+
# Callable[anything] will return None
24+
class _Callable:
25+
def __getitem__(self, _):
26+
return None
27+
28+
# Make instances
29+
Callable = _Callable()
30+
31+
32+
try:
33+
import clickhouse_driver # type: ignore[import]
34+
35+
except ImportError:
36+
raise DidNotEnable("clickhouse-driver not installed.")
37+
38+
if clickhouse_driver.VERSION < (0, 2, 0):
39+
raise DidNotEnable("clickhouse-driver >= 0.2.0 required")
40+
41+
42+
class ClickhouseDriverIntegration(Integration):
43+
identifier = "clickhouse_driver"
44+
45+
@staticmethod
46+
def setup_once() -> None:
47+
# Every query is done using the Connection's `send_query` function
48+
clickhouse_driver.connection.Connection.send_query = _wrap_start(
49+
clickhouse_driver.connection.Connection.send_query
50+
)
51+
52+
# If the query contains parameters then the send_data function is used to send those parameters to clickhouse
53+
clickhouse_driver.client.Client.send_data = _wrap_send_data(
54+
clickhouse_driver.client.Client.send_data
55+
)
56+
57+
# Every query ends either with the Client's `receive_end_of_query` (no result expected)
58+
# or its `receive_result` (result expected)
59+
clickhouse_driver.client.Client.receive_end_of_query = _wrap_end(
60+
clickhouse_driver.client.Client.receive_end_of_query
61+
)
62+
clickhouse_driver.client.Client.receive_result = _wrap_end(
63+
clickhouse_driver.client.Client.receive_result
64+
)
65+
66+
67+
P = ParamSpec("P")
68+
T = TypeVar("T")
69+
70+
71+
def _wrap_start(f: Callable[P, T]) -> Callable[P, T]:
72+
def _inner(*args: P.args, **kwargs: P.kwargs) -> T:
73+
hub = Hub.current
74+
if hub.get_integration(ClickhouseDriverIntegration) is None:
75+
return f(*args, **kwargs)
76+
connection = args[0]
77+
query = args[1]
78+
query_id = args[2] if len(args) > 2 else kwargs.get("query_id")
79+
params = args[3] if len(args) > 3 else kwargs.get("params")
80+
81+
span = hub.start_span(op=OP.DB, description=query)
82+
83+
connection._sentry_span = span # type: ignore[attr-defined]
84+
85+
_set_db_data(span, connection)
86+
87+
span.set_data("query", query)
88+
89+
if query_id:
90+
span.set_data("db.query_id", query_id)
91+
92+
if params and _should_send_default_pii():
93+
span.set_data("db.params", params)
94+
95+
# run the original code
96+
ret = f(*args, **kwargs)
97+
98+
return ret
99+
100+
return _inner
101+
102+
103+
def _wrap_end(f: Callable[P, T]) -> Callable[P, T]:
104+
def _inner_end(*args: P.args, **kwargs: P.kwargs) -> T:
105+
res = f(*args, **kwargs)
106+
instance = args[0]
107+
span = instance.connection._sentry_span # type: ignore[attr-defined]
108+
109+
if span is not None:
110+
if res is not None and _should_send_default_pii():
111+
span.set_data("db.result", res)
112+
113+
with capture_internal_exceptions():
114+
span.hub.add_breadcrumb(
115+
message=span._data.pop("query"), category="query", data=span._data
116+
)
117+
118+
span.finish()
119+
120+
return res
121+
122+
return _inner_end
123+
124+
125+
def _wrap_send_data(f: Callable[P, T]) -> Callable[P, T]:
126+
def _inner_send_data(*args: P.args, **kwargs: P.kwargs) -> T:
127+
instance = args[0] # type: clickhouse_driver.client.Client
128+
data = args[2]
129+
span = instance.connection._sentry_span
130+
131+
_set_db_data(span, instance.connection)
132+
133+
if _should_send_default_pii():
134+
db_params = span._data.get("db.params", [])
135+
db_params.extend(data)
136+
span.set_data("db.params", db_params)
137+
138+
return f(*args, **kwargs)
139+
140+
return _inner_send_data
141+
142+
143+
def _set_db_data(
144+
span: Span, connection: clickhouse_driver.connection.Connection
145+
) -> None:
146+
span.set_data(SPANDATA.DB_SYSTEM, "clickhouse")
147+
span.set_data(SPANDATA.SERVER_ADDRESS, connection.host)
148+
span.set_data(SPANDATA.SERVER_PORT, connection.port)
149+
span.set_data(SPANDATA.DB_NAME, connection.database)
150+
span.set_data(SPANDATA.DB_USER, connection.user)

setup.py

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ def get_file_text(file_name):
5151
"bottle": ["bottle>=0.12.13"],
5252
"celery": ["celery>=3"],
5353
"chalice": ["chalice>=1.16.0"],
54+
"clickhouse-driver": ["clickhouse-driver>=0.2.0"],
5455
"django": ["django>=1.8"],
5556
"falcon": ["falcon>=1.4"],
5657
"fastapi": ["fastapi>=0.79.0"],
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import pytest
2+
3+
pytest.importorskip("clickhouse_driver")

0 commit comments

Comments
 (0)