add SQLAlchemy storage backend

- add sqastore.DatabaseStore and unittests
- update API unittests for use in database tests
- black code formatting
- update README

Author: miraculixx

.github/workflows/python-app.yml

@@ -28,7 +28,7 @@ jobs:
         python -m pip install --upgrade pip
         pip install flake8 pytest
         pip install -r requirements.txt
-        pip install -e .
+        pip install -e .[sqla]
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names

.gitignore

@@ -135,3 +135,4 @@ configurable_http_proxy/version.txt
 /.idea/misc.xml
 /.idea/modules.xml
 /.idea/vcs.xml
+*sqlite*

README.md

@@ -27,6 +27,7 @@ The following items are supported:
 - Customizable storage backends
 - PID file writing
 - Logging
+- Configurable storage backend
 
 The following options are not supported (yet):
 
@@ -35,3 +36,24 @@ The following options are not supported (yet):
 - Change Origin: `--change-origin`
 - Rewrites in Location header: `--protocol-rewrite` and `--auto-rewrite`
 - Metrics server: `--metrics-port` and `--metrics-ip`
+
+
+## Database-backed storage backend
+
+Using a DBMS instead of the default in-memory store enables configurable-http-proxy 
+to be replicated in a High Availability scenario. 
+
+To use a DBMS as the storage backend:
+
+Set the CHP_DATABASE_URL env var to any db URL supported by SQLAlchemy.
+The default is "sqlite://chp.sqlite".
+
+        $ export CHP_DATABASE_URL="sqlite:///chp.sqlite"
+        $ configurable-http-proxy --storage-backend configurable_http_proxy.sqastore.DatabaseStore
+
+Optionally you may set the table name by setting the CHP_DATABASE_TABLE.
+The default is 'chp_routes'
+
+        $ export CHP_DATABASE_TABLE="chp_routes"
+
+ 

configurable_http_proxy/__init__.py

@@ -6,5 +6,5 @@
 log.setLevel(logging.INFO)
 
 handler = logging.StreamHandler()
-handler.setFormatter(logging.Formatter('[%(levelname)-.1s %(asctime)s %(name)s] %(message)s'))
+handler.setFormatter(logging.Formatter("[%(levelname)-.1s %(asctime)s %(name)s] %(message)s"))
 logging.root.addHandler(handler)

configurable_http_proxy/_version.py

@@ -7,7 +7,7 @@
     try:
         from pkg_resources import get_distribution
 
-        module_name = __name__.split('.', 1)[0]
+        module_name = __name__.split(".", 1)[0]
         version = get_distribution(module_name).version
     except Exception:  # noqa: S110
         pass
@@ -24,6 +24,6 @@
 if version is None:
     # When version.txt file is available - use that
     try:
-        version = open(os.path.join(os.path.dirname(__file__), 'version.txt')).read()
+        version = open(os.path.join(os.path.dirname(__file__), "version.txt")).read()
     except Exception:  # noqa: S110
         pass

configurable_http_proxy/handlers.py

@@ -275,7 +275,7 @@ def _get_proxy_request(self, url):
             fwd_values = {
                 "for": self.request.remote_ip,
                 "port": str(port),
-                "proto": 'https' if encrypted else 'http',
+                "proto": "https" if encrypted else "http",
             }
 
             for key in ["for", "port", "proto"]:

configurable_http_proxy/sqastore.py

@@ -0,0 +1,143 @@
+import logging
+import os
+from datetime import datetime
+
+from dataset import connect
+
+from configurable_http_proxy.store import BaseStore
+
+log = logging.getLogger(__name__)
+
+
+class DatabaseStore(BaseStore):
+    """A DBMS storage backend for configurable-http-proxy
+
+    This enables chp to run multiple times and serve routes from a central
+    DBMS. It uses SQLAlchemy as the database backend.
+
+    Usage:
+        Set the CHP_DATABASE_URL env var to any db URL supported by SQLAlchemy.
+        The default is "sqlite://chp.sqlite".
+
+        $ export CHP_DATABASE_URL="sqlite:///chp.sqlite"
+        $ configurable-http-proxy --storage-backend configurable_http_proxy.sqastore.DatabaseStore
+
+        Optionally you may set the table name by setting the CHP_DATABASE_TABLE.
+        The default is 'chp_routes'
+
+        $ export CHP_DATABASE_TABLE="chp_routes"
+
+    See Also:
+        * Valid URLs https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls
+    """
+
+    default_db_url = "sqlite:///chp.sqlite"
+    default_db_table = "chp_routes"
+
+    def __init__(self):
+        super().__init__()
+        db_url = os.environ.get("CHP_DATABASE_URL", self.default_db_url)
+        db_table = os.environ.get("CHP_DATABASE_TABLE", self.default_db_table)
+        self.routes: TableTrie = TableTrie(db_url, table=db_table)
+        log.info(f"Using database {db_url}")
+        for route, data in self.get_all().items():
+            log.info(f'Restoring {route} => {data.get("target", "<no target>")}')
+
+    def clean(self):
+        self.routes.clean()
+
+    def get_target(self, path: str):
+        return self.routes.get(self.clean_path(path))
+
+    def get_all(self):
+        return self.routes.all()
+
+    def add(self, path: str, data):
+        if self.get(path):
+            self.update(path, data)
+        else:
+            self.routes.add(path, data)
+
+    def update(self, path: str, data):
+        self.routes.update(self.clean_path(path), data)
+
+    def remove(self, path: str):
+        path = self.clean_path(path)
+        route = self.routes.get(path)
+        if route:
+            self.routes.remove(path)
+        return route
+
+    def get(self, path):
+        path = self.clean_path(path)
+        return self.get_all().get(path)
+
+
+class TableTrie:
+    # A databased URLTrie-alike
+    def __init__(self, url, table=None):
+        table = table or "chp_routes"
+        self.db = connect(url)
+        self.table = self.db[table]
+
+    def get(self, path):
+        for path in self._split_routes(path):
+            doc = self.table.find_one(path=path)
+            if doc:
+                data = self._from_json(doc)
+                data["prefix"] = path
+                break
+        else:
+            data = None
+        return attrdict(data) if data else None
+
+    def add(self, path, data):
+        self.table.insert(self._clean_json({"path": path, "data": data}))
+
+    def update(self, path, data):
+        doc = self.table.find_one(path=path)
+        doc["data"].update(data)
+        self.table.update(self._clean_json(doc), "id")
+
+    def remove(self, path):
+        for path in self._split_routes(path):
+            self.table.delete(path=path)
+
+    def all(self):
+        return self._from_json({item["path"]: item["data"] for item in self.table.find()})
+
+    def _clean_json(self, data):
+        for k, v in dict(data).items():
+            if isinstance(v, datetime):
+                data[k] = f"_dt_:{v.isoformat()}"
+            if isinstance(v, dict):
+                data[k] = self._clean_json(v)
+        return data
+
+    def _from_json(self, data):
+        for k, v in dict(data).items():
+            if isinstance(v, str) and v.startswith("_dt_:"):
+                data[k] = datetime.fromisoformat(v.split(":", 1)[-1])
+            if isinstance(v, dict):
+                data[k] = self._from_json(v)
+        return data
+
+    def _split_routes(self, path):
+        # generator for reverse tree of routes
+        # e.g. /path/to/document
+        # => yields /path/to/document, /path/to, /path, /
+        levels = path.split("/")
+        for i, e in enumerate(levels):
+            yield "/".join(levels[: len(levels) - i + 1])
+        # always yield top level route
+        yield "/"
+
+    def clean(self):
+        self.table.delete()
+
+
+class attrdict(dict):
+    # enable .attribute for dicts
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self

configurable_http_proxy_test/test_api.py

@@ -1,18 +1,16 @@
 import datetime
 import json
+import os
 
 from tornado.testing import AsyncHTTPTestCase
 
 from configurable_http_proxy.configproxy import PythonProxy
 from configurable_http_proxy_test.testutil import pytest_regex
 
 
-class TestAPI(AsyncHTTPTestCase):
-    def get_app(self):
-        self.proxy = PythonProxy({"auth_token": "secret"})
-        self.proxy.add_route("/", {"target": "http://127.0.0.1:54321"})
-        return self.proxy.api_app
-
+class APITestsMixin:
+    # test cases for TestAPI
+    # -- this allows to reuse these test cases in test_sqastore.TestAPI_SQAStore
     def fetch(self, path, raise_error=True, with_auth=True, **kwargs):
         headers = kwargs.pop("headers", {})
         if with_auth:
@@ -147,8 +145,17 @@ def test_get_routes_with_inactive_since(self):
 
         resp = self.fetch(f"/api/routes?inactiveSince={hour_ago.isoformat()}")
         reply = json.loads(resp.body)
-        assert set(reply.keys()) == {'/yesterday'}
+        assert set(reply.keys()) == {"/yesterday"}
 
         resp = self.fetch(f"/api/routes?inactiveSince={hour_from_now.isoformat()}")
         reply = json.loads(resp.body)
-        assert set(reply.keys()) == {'/', '/today', '/yesterday'}
+        assert set(reply.keys()) == {"/", "/today", "/yesterday"}
+
+
+class TestAPI(APITestsMixin, AsyncHTTPTestCase):
+    # actual test case
+    def get_app(self):
+        os.environ["CHP_DATABASE_URL"] = "sqlite:///chp_test.sqlite"
+        self.proxy = PythonProxy({"auth_token": "secret"})
+        self.proxy.add_route("/", {"target": "http://127.0.0.1:54321"})
+        return self.proxy.api_app

configurable_http_proxy_test/test_proxy.py

@@ -473,13 +473,16 @@ def test_receiving_headers_setcookie(self):
         headers = list(resp.headers.get_all())
         cookies = {}
         for header_name, header in headers:
-            if header_name.lower() !='set-cookie':
+            if header_name.lower() != "set-cookie":
                 continue
             key, val = header.split("=", 1)
             cookies[key] = val
         assert "key" in cookies
-        assert cookies['key'] == 'val'
+        assert cookies["key"] == "val"
         assert "combined_key" in cookies
-        assert cookies['combined_key'] == 'val; Secure=; HttpOnly=; SameSite=None; Path=/; Domain=example.com; Max-Age=999999; Expires=Fri, 01 Oct 2020 06:12:16 GMT'
+        assert (
+            cookies["combined_key"]
+            == "val; Secure=; HttpOnly=; SameSite=None; Path=/; Domain=example.com; Max-Age=999999; Expires=Fri, 01 Oct 2020 06:12:16 GMT"
+        )
         for prefix in ["Secure", "HttpOnly", "SameSite", "Path", "Domain", "Max-Age", "Expires"]:
             assert prefix + "_key" in cookies

configurable_http_proxy_test/test_sqastore.py

@@ -0,0 +1,101 @@
+import os
+
+from tornado.testing import AsyncHTTPTestCase
+
+from configurable_http_proxy.configproxy import PythonProxy
+from configurable_http_proxy.sqastore import DatabaseStore
+from configurable_http_proxy_test.test_api import APITestsMixin
+
+
+class TestDataBaseStore:
+    def setup_method(self, method):
+        os.environ["CHP_DATABASE_URL"] = "sqlite:///chp_test.sqlite"
+        self.subject = DatabaseStore()
+        self.subject.clean()
+
+    def test_get(self):
+        self.subject.add("/myRoute", {"test": "value"})
+        route = self.subject.get("/myRoute")
+        assert route == {"test": "value"}
+
+    def test_get_with_invalid_path(self):
+        route = self.subject.get("/wut")
+        assert route is None
+
+    def test_get_target(self):
+        self.subject.add("/myRoute", {"target": "http://localhost:8213"})
+        self.subject.add("/myRoute/specific", {"target": "http://localhost:8214"})
+        target = self.subject.get_target("/myRoute")
+        assert target.prefix == "/myRoute"
+        assert target.data["target"] == "http://localhost:8213"
+        target = self.subject.get_target("/myRoute/some/path")
+        assert target.prefix == "/myRoute"
+        assert target.data["target"] == "http://localhost:8213"
+        target = self.subject.get_target("/myRoute/specific/path")
+        assert target.prefix == "/myRoute/specific"
+        assert target.data["target"] == "http://localhost:8214"
+
+    def test_get_all(self):
+        self.subject.add("/myRoute", {"test": "value1"})
+        self.subject.add("/myOtherRoute", {"test": "value2"})
+
+        routes = self.subject.get_all()
+        assert len(routes) == 2
+        assert routes["/myRoute"] == {"test": "value1"}
+        assert routes["/myOtherRoute"] == {"test": "value2"}
+
+    def test_get_all_with_no_routes(self):
+        routes = self.subject.get_all()
+        assert routes == {}
+
+    def test_add(self):
+        self.subject.add("/myRoute", {"test": "value"})
+
+        route = self.subject.get("/myRoute")
+        assert route == {"test": "value"}
+
+    def test_add_overwrite(self):
+        self.subject.add("/myRoute", {"test": "value"})
+        self.subject.add("/myRoute", {"test": "updatedValue"})
+
+        route = self.subject.get("/myRoute")
+        assert route == {"test": "updatedValue"}
+
+    def test_update(self):
+        self.subject.add("/myRoute", {"version": 1, "test": "value"})
+        self.subject.update("/myRoute", {"version": 2})
+
+        route = self.subject.get("/myRoute")
+        assert route["version"] == 2
+        assert route["test"] == "value"
+
+    def test_remove(self):
+        self.subject.add("/myRoute", {"test": "value"})
+        self.subject.remove("/myRoute")
+
+        route = self.subject.get("/myRoute")
+        assert route is None
+
+    def test_remove_with_invalid_route(self):
+        # No error should occur
+        self.subject.remove("/myRoute/foo/bar")
+
+    def test_has_route(self):
+        self.subject.add("/myRoute", {"test": "value"})
+        route = self.subject.get("/myRoute")
+        assert route == {"test": "value"}
+
+    def test_has_route_path_not_found(self):
+        route = self.subject.get("/wut")
+        assert route is None
+
+
+class TestAPI_SQAStore(APITestsMixin, AsyncHTTPTestCase):
+    def get_app(self):
+        self.proxy = PythonProxy(
+            {"auth_token": "secret", "storage_backend": "configurable_http_proxy.sqastore.DatabaseStore"}
+        )
+        self.proxy._routes.clean()
+        assert self.proxy._routes.get_all() == {}
+        self.proxy.add_route("/", {"target": "http://127.0.0.1:54321"})
+        return self.proxy.api_app

setup.py

@@ -19,6 +19,9 @@
         },
         setup_requires=["setuptools_scm"],
         install_requires=open(os.path.join(BASE_PATH, "requirements.txt")).readlines(),
+        extras_require={
+            "sqla": ["dataset"],
+        },
         python_requires=">=3.6",
         include_package_data=True,
         zip_safe=False,