Skip to content

Commit 343acac

Browse files
committed
Add SQLAlchemy storage backend
- add sqlastore.DatabaseStore and unittests - update API unittests for use in database tests - update README
1 parent 01fbcb3 commit 343acac

File tree

7 files changed

+298
-10
lines changed

7 files changed

+298
-10
lines changed

.github/workflows/python-app.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
- name: Install application
2727
run: |
2828
python -m pip install --upgrade pip setuptools wheel
29-
pip install -e .
29+
pip install -e .[sqla]
3030
- name: Lint with flake8 and black
3131
run: |
3232
pip install -r requirements/lint.txt

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,3 +134,4 @@ dmypy.json
134134

135135
# Project specific
136136
configurable_http_proxy/version.txt
137+
*sqlite*

README.md

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ The following items are supported:
2727
- Customizable storage backends
2828
- PID file writing
2929
- Logging
30+
- Configurable storage backend
3031

3132
The following options are not supported (yet):
3233

@@ -35,3 +36,36 @@ The following options are not supported (yet):
3536
- Change Origin: `--change-origin`
3637
- Rewrites in Location header: `--protocol-rewrite` and `--auto-rewrite`
3738
- Metrics server: `--metrics-port` and `--metrics-ip`
39+
40+
41+
## Database-backed storage backend
42+
43+
Using a SQL DBMS instead of the default in-memory store enables chp to be replicated
44+
in a High Availability scenario.
45+
46+
To use a SQL DBMS as the storage backend:
47+
48+
1. Install DBMS support
49+
50+
$ pip install configurable-http-proxy[sqla]
51+
52+
2. Set the CHP_DATABASE_URL env var to any db URL supported by SQLAlchemy.
53+
The default is "sqlite://chp.sqlite".
54+
55+
$ export CHP_DATABASE_URL="sqlite:///chp.sqlite"
56+
$ configurable-http-proxy --storage-backend configurable_http_proxy.dbstore.DatabaseStore
57+
58+
3. Optionally you may set the table name by setting the CHP_DATABASE_TABLE.
59+
The default is 'chp_routes'
60+
61+
$ export CHP_DATABASE_TABLE="chp_routes"
62+
63+
64+
Note::
65+
66+
While nodejs configurable-http-proxy supports custom storage backend, it does
67+
not currently support to use an SQL DBMS. This is a unique feature of the Python
68+
version.
69+
70+
71+

configurable_http_proxy/dbstore.py

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
import json
2+
import logging
3+
import os
4+
from datetime import datetime
5+
6+
from dataset import connect
7+
8+
from configurable_http_proxy.store import BaseStore
9+
10+
log = logging.getLogger(__name__)
11+
12+
13+
class DatabaseStore(BaseStore):
14+
"""A DBMS storage backend for configurable-http-proxy
15+
16+
This enables chp to run multiple times and serve routes from a central
17+
DBMS. It uses SQLAlchemy as the database backend.
18+
19+
Usage:
20+
Set the CHP_DATABASE_URL env var to any db URL supported by SQLAlchemy.
21+
The default is "sqlite://chp.sqlite".
22+
23+
$ export CHP_DATABASE_URL="sqlite:///chp.sqlite"
24+
$ configurable-http-proxy --storage-backend configurable_http_proxy.dbstore.DatabaseStore
25+
26+
Optionally you may set the table name by setting the CHP_DATABASE_TABLE.
27+
The default is 'chp_routes'
28+
29+
$ export CHP_DATABASE_TABLE="chp_routes"
30+
31+
See Also:
32+
* Valid URLs https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls
33+
"""
34+
35+
default_db_url = "sqlite:///chp.sqlite"
36+
default_db_table = "chp_routes"
37+
38+
def __init__(self):
39+
super().__init__()
40+
db_url = os.environ.get("CHP_DATABASE_URL", self.default_db_url)
41+
db_table = os.environ.get("CHP_DATABASE_TABLE", self.default_db_table)
42+
self.routes: TableTrie = TableTrie(db_url, table=db_table)
43+
log.info(f"Using database {db_url}")
44+
for route, data in self.get_all().items():
45+
log.info(f'Restoring {route} => {data.get("target", "<no target>")}')
46+
47+
def clean(self):
48+
# remove all information stored so far
49+
self.routes.clean()
50+
51+
def get_target(self, path: str):
52+
# return the data for the most specific matching route
53+
return self.routes.get(self.clean_path(path), trie=True)
54+
55+
def get_all(self):
56+
# return all routes as route => data
57+
return self.routes.all()
58+
59+
def add(self, path: str, data):
60+
# add a new route /path, storing data
61+
if self.get(path):
62+
self.update(path, data)
63+
else:
64+
self.routes.add(path, data)
65+
66+
def update(self, path: str, data):
67+
# update an existing route
68+
self.routes.update(self.clean_path(path), data)
69+
70+
def remove(self, path: str):
71+
# remove an existing route
72+
path = self.clean_path(path)
73+
route = self.routes.get(path)
74+
if route:
75+
self.routes.remove(path)
76+
return route
77+
78+
def get(self, path):
79+
# return the data for the exact match
80+
return self.routes.get(self.clean_path(path))
81+
82+
83+
class TableTrie:
84+
"""A URLtrie-like backed by a database
85+
86+
This stores URL-path => data mappings. On retrieving, it will try
87+
to retrieve all subpaths up to the default path.
88+
89+
Usage:
90+
91+
# create mapping
92+
routes = TableTrie('sqlite:///:memory:')
93+
routes.add('/', {'some': 'default'})
94+
routes.add('/foo/bar', {'some': 'value'})
95+
96+
# query a mapping that exists
97+
routes.get('/foo/bar/baz')
98+
=> {
99+
'prefix': '/foo/bar',
100+
'some': 'value'
101+
}
102+
103+
# query a mapping that does not exist
104+
routes.get('/fox/bax')
105+
=> {
106+
'prefix': '/',
107+
'some': 'default'
108+
}
109+
110+
How values are stored:
111+
112+
Routes are stored in the given table (defaults to 'chp_routes').
113+
The table has the following columns:
114+
115+
id: integer (primary key)
116+
key: varchar(128, unique)
117+
data: varchar
118+
119+
The data is the serialized JSON equivalent of the dictionary stored
120+
by TableTrie.add() or .update(). The rationale for storing a serialized
121+
version of the dict instead of using the sqlalchemy JSON support directly
122+
is to improve compatibility across db dialects.
123+
124+
DB backend:
125+
126+
The backend is any database supported by SQLAlchemy. To simplify
127+
implementation this uses the dataset library, which provides a very
128+
straight-forward way of working with tables created from Python dicts.
129+
"""
130+
131+
def __init__(self, url, table=None):
132+
table = table or "chp_routes"
133+
self.db = connect(url)
134+
self.table = self.db[table]
135+
self.table.create_column("path", self.db.types.string(length=128), unique=True)
136+
137+
def get(self, path, trie=False):
138+
# return the data store for path
139+
# -- if trie is False (default), will return data for the exact path
140+
# -- if trie is True, will return the data and the matching prefix
141+
try_routes = self._split_routes(path) if trie else [path]
142+
for path in try_routes:
143+
doc = self.table.find_one(path=path, order_by="id")
144+
if doc:
145+
if not trie:
146+
data = self._from_json(doc["data"])
147+
else:
148+
data = doc
149+
data["data"] = self._from_json(doc["data"])
150+
data["prefix"] = path
151+
break
152+
else:
153+
data = None
154+
return attrdict(data) if data else None
155+
156+
def add(self, path, data):
157+
# add the data for the given exact path
158+
self.table.insert({"path": path, "data": self._to_json(data)})
159+
160+
def update(self, path, data):
161+
# update the data for the given exact path
162+
doc = self.table.find_one(path=path, order_by="id")
163+
doc["data"] = self._from_json(doc["data"])
164+
doc["data"].update(data)
165+
doc["data"] = self._to_json(doc["data"])
166+
self.table.update(doc, "id")
167+
168+
def remove(self, path):
169+
# remove all matching routes for the given path, except default route
170+
for subpath in self._split_routes(path):
171+
if subpath == "/" and path != "/":
172+
continue
173+
self.table.delete(path=subpath)
174+
175+
def all(self):
176+
# return all data for all paths
177+
return {item["path"]: self._from_json(item["data"]) for item in self.table.find(order_by="id")}
178+
179+
def _to_json(self, data):
180+
# simple converter for serializable data
181+
for k, v in dict(data).items():
182+
if isinstance(v, datetime):
183+
data[k] = f"_dt_:{v.isoformat()}"
184+
elif isinstance(v, dict):
185+
data[k] = self._to_json(v)
186+
return json.dumps(data)
187+
188+
def _from_json(self, data):
189+
# simple converter from serialized data
190+
data = json.loads(data) if isinstance(data, (str, bytes)) else data
191+
for k, v in dict(data).items():
192+
if isinstance(v, str) and v.startswith("_dt_:"):
193+
data[k] = datetime.fromisoformat(v.split(":", 1)[-1])
194+
elif isinstance(v, dict):
195+
data[k] = self._from_json(v)
196+
return data
197+
198+
def _split_routes(self, path):
199+
# generator for reverse tree of routes
200+
# e.g. /path/to/document
201+
# => yields /path/to/document, /path/to, /path, /
202+
levels = path.split("/")
203+
for i, e in enumerate(levels):
204+
yield "/".join(levels[: len(levels) - i + 1])
205+
# always yield top level route
206+
yield "/"
207+
208+
def clean(self):
209+
self.table.delete()
210+
211+
212+
class attrdict(dict):
213+
# enable .attribute for dicts
214+
def __init__(self, *args, **kwargs):
215+
super().__init__(*args, **kwargs)
216+
self.__dict__ = self

configurable_http_proxy_test/test_api.py

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
import datetime
22
import json
3+
import os
34

45
from tornado.testing import AsyncHTTPTestCase
56

67
from configurable_http_proxy.configproxy import PythonProxy
78
from configurable_http_proxy_test.testutil import pytest_regex
89

910

10-
class TestAPI(AsyncHTTPTestCase):
11-
def get_app(self):
12-
self.proxy = PythonProxy({"auth_token": "secret"})
13-
self.proxy.add_route("/", {"target": "http://127.0.0.1:54321"})
14-
return self.proxy.api_app
11+
class APITestsMixin:
12+
"""
13+
Test cases for TestAPI
14+
This allows to reuse test cases for MemoryStore and DatabaseStore backends
15+
"""
1516

1617
def fetch(self, path, raise_error=True, with_auth=True, **kwargs):
1718
headers = kwargs.pop("headers", {})
@@ -144,3 +145,22 @@ def test_get_routes_with_inactive_since(self):
144145
resp = self.fetch(f"/api/routes?inactiveSince={hour_from_now.isoformat()}")
145146
reply = json.loads(resp.body)
146147
assert set(reply.keys()) == {"/", "/today", "/yesterday"}
148+
149+
150+
class TestAPI_MemoryStore(APITestsMixin, AsyncHTTPTestCase):
151+
def get_app(self):
152+
self.proxy = PythonProxy({"auth_token": "secret"})
153+
self.proxy.add_route("/", {"target": "http://127.0.0.1:54321"})
154+
return self.proxy.api_app
155+
156+
157+
class TestAPI_DatabaseStore(APITestsMixin, AsyncHTTPTestCase):
158+
def get_app(self):
159+
os.environ["CHP_DATABASE_URL"] = "sqlite:///chp_test.sqlite"
160+
self.proxy = PythonProxy(
161+
{"auth_token": "secret", "storage_backend": "configurable_http_proxy.dbstore.DatabaseStore"}
162+
)
163+
self.proxy._routes.clean()
164+
assert self.proxy._routes.get_all() == {}
165+
self.proxy.add_route("/", {"target": "http://127.0.0.1:54321"})
166+
return self.proxy.api_app

configurable_http_proxy_test/test_store.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1-
from configurable_http_proxy.store import MemoryStore
1+
import os
22

3+
from configurable_http_proxy.dbstore import DatabaseStore
4+
from configurable_http_proxy.store import MemoryStore
35

4-
class TestMemoryStore:
5-
def setup_method(self, method):
6-
self.subject = MemoryStore()
76

7+
class StoreTestMixin:
8+
# test cases for the storage
9+
# -- this allows to reuse tests for MemoryStore and DatabaseStore
810
def test_get(self):
911
self.subject.add("/myRoute", {"test": "value"})
1012
route = self.subject.get("/myRoute")
@@ -73,3 +75,15 @@ def test_has_route(self):
7375
def test_has_route_path_not_found(self):
7476
route = self.subject.get("/wut")
7577
assert route is None
78+
79+
80+
class TestMemoryStore(StoreTestMixin):
81+
def setup_method(self, method):
82+
self.subject = MemoryStore()
83+
84+
85+
class TestDataBaseStore(StoreTestMixin):
86+
def setup_method(self, method):
87+
os.environ["CHP_DATABASE_URL"] = "sqlite:///chp_test.sqlite"
88+
self.subject = DatabaseStore()
89+
self.subject.clean()

setup.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
},
2020
setup_requires=["setuptools_scm"],
2121
install_requires=open(os.path.join(BASE_PATH, "requirements", "base.txt")).readlines(),
22+
extras_require={
23+
"sqla": ["dataset"],
24+
},
2225
python_requires=">=3.6",
2326
include_package_data=True,
2427
zip_safe=False,

0 commit comments

Comments
 (0)