redis: URL support for connection parameters

Make it possible to specify the Redis connection parameters via URL
which can be supplied in the config file with key "url" (and leaving
"host", "port", "password" and "db" empty/unspecified) or via
environment variable PASSARI_WORKFLOW_REDIS_URL or REDIS_URL.

The REDIS_URL environment variable is also used by the RQ workers by
default so specifying it will also make RQ work without a config file.
The default value if no connection parameters are given is to use db 0
on localhost (which is the same as RQ uses).

Additionally provide a function "get_redis_url" so that passari-web-ui
can use it to just get the URL without creating a Redis connection.

Also tune the PostgreSQL URL handling to support similar environment
variable based configuration via PASSARI_WORKFLOW_DB_URL or DATABASE_URL
variables and fix the password escaping to use correct `quote` function
instead of `quote_plus`.

Note: The get_redis_connection used to accept a db index argument, but
not anymore.  Implementing override of the db index for the URLs would
need some extra code, because the db index can be specified in various
formats in the URL, but the argument was not used by any callers (in
this code base or in passari-web-ui) so it's not worth it to implement.

Author: suutari-ai

src/passari_workflow/db/connection.py

@@ -1,13 +1,15 @@
+import os
+
 from passari_workflow.db import DBSession
 
 from sqlalchemy import create_engine
 
-from urllib.parse import quote_plus
+from urllib.parse import quote
 
 from passari_workflow.config import CONFIG
 
 
-def get_connection_uri():
+def get_connection_uri(*, default="error"):
     """
     Get the connection URI used to connect to the database
     """
@@ -21,12 +23,15 @@ def get_connection_uri():
     if url and not name:
         return url
     elif not url and not name:
+        url = os.getenv("PASSARI_WORKFLOW_DB_URL", os.getenv("DATABASE_URL"))
+        if url or (default != "error"):
+            return url or default
         raise EnvironmentError("Either 'url' or 'name' is required for db")
     elif url and name:
         raise EnvironmentError("The db 'url' and 'name' are exclusive")
 
     if user and host and port:
-        return f"postgresql://{user}:{quote_plus(password)}@{host}:{port}/{name}"
+        return f"postgresql://{user}:{quote(password)}@{host}:{port}/{name}"
     elif user or host or port or password:
         raise EnvironmentError(
             "If 'host' is given in PostgreSQL config,"

src/passari_workflow/redis/connection.py

@@ -1,19 +1,36 @@
+import os
+from urllib.parse import quote
+
 from passari_workflow.config import CONFIG
 
 from redis import Redis
 
 
-def get_redis_connection(db=0):
+def get_redis_connection():
     """
     Get Redis connection used for the workflow, distributed locks and other
     miscellaneous tasks
     """
-    password = CONFIG["redis"].get("password", None)
-    redis = Redis(
-        host=CONFIG["redis"]["host"],
-        port=CONFIG["redis"]["port"],
-        db=db,
-        password=password if password else None
-    )
-
-    return redis
+    redis_url = get_redis_url()
+    return Redis.from_url(redis_url)
+
+
+def get_redis_url():
+    redis_config = CONFIG.get("redis", {})
+    url = redis_config.get("url")
+    host = redis_config.get("host")
+    port = redis_config.get("port")
+    db = redis_config.get("db") or 0
+    password = redis_config.get("password") or None
+
+    if not url and not host:  # Get URL from environment
+        return os.getenv(
+            "PASSARI_WORKFLOW_REDIS_URL",
+            os.getenv("REDIS_URL", "redis://localhost/0"),
+        )
+    elif url and (host or port or db or password):
+        raise EnvironmentError("The 'url' config for Redis is exclusive")
+
+    password_part = f":{quote(password)}@" if password else ""
+    port_part = f":{port}" if port else ""
+    return f"redis://{password_part}{host}{port_part}/{db}"