[WIP] Add the Ability to use Redis for Storage

.travis.yml

@@ -1,4 +1,6 @@
 language: node_js
+services:
+  - redis-server
 sudo: false
 node_js:
   - "6"

README.md

@@ -33,7 +33,7 @@ using the npm package manager:
     npm install -g configurable-http-proxy
 
 To install from the source code found in this GitHub repo:
-    
+
     git clone https://github.com/jupyterhub/configurable-http-proxy.git
     cd configurable-http-proxy
     # Use -g for global install
@@ -54,7 +54,7 @@ The configurable proxy runs two HTTP(S) servers:
 
 ### Setting a default target
 
-When you start the proxy from the command line, you can set a 
+When you start the proxy from the command line, you can set a
 default target (`--default-target` option) to be used when no
 matching route is found in the proxy table:
 
@@ -63,41 +63,49 @@ matching route is found in the proxy table:
 ### Command-line options
 
 ```
-  Usage: configurable-http-proxy [options]
+Usage: configurable-http-proxy [options]
 
   Options:
 
     -h, --help                       output usage information
     -V, --version                    output the version number
     --ip <ip-address>                Public-facing IP of the proxy
     --port <n> (defaults to 8000)    Public-facing port of the proxy
-    
+
     --ssl-key <keyfile>              SSL key to use, if any
     --ssl-cert <certfile>            SSL certificate to use, if any
     --ssl-ca <ca-file>               SSL certificate authority, if any
     --ssl-request-cert               Request SSL certs to authenticate clients
     --ssl-reject-unauthorized        Reject unauthorized SSL connections (only meaningful if --ssl-request-cert is given)
-    --ssl-protocol <ssl-protocol>    Set specific HTTPS protocol, e.g. TLSv1_2, TLSv1, etc.
+    --ssl-protocol <ssl-protocol>    Set specific SSL protocol, e.g. TLSv1.2, SSLv3
     --ssl-ciphers <ciphers>          `:`-separated ssl cipher list. Default excludes RC4
     --ssl-allow-rc4                  Allow RC4 cipher for SSL (disabled by default)
     --ssl-dhparam <dhparam-file>     SSL Diffie-Helman Parameters pem file, if any
-    
+
     --api-ip <ip>                    Inward-facing IP for API requests
     --api-port <n>                   Inward-facing port for API requests (defaults to --port=value+1)
     --api-ssl-key <keyfile>          SSL key to use, if any, for API requests
     --api-ssl-cert <certfile>        SSL certificate to use, if any, for API requests
     --api-ssl-ca <ca-file>           SSL certificate authority, if any, for API requests
     --api-ssl-request-cert           Request SSL certs to authenticate clients for API requests
     --api-ssl-reject-unauthorized    Reject unauthorized SSL connections (only meaningful if --api-ssl-request-cert is given)
-    
+
     --default-target <host>          Default proxy target (proto://host[:port])
     --error-target <host>            Alternate server for handling proxy errors (proto://host[:port])
     --error-path <path>              Alternate server for handling proxy errors (proto://host[:port])
     --redirect-port <redirect-port>  Redirect HTTP requests on this port to the server on HTTPS
     --pid-file <pid-file>            Write our PID to a file
+
+    --storage-provider <provider>    The storage provider for the route table (defaults to memory)
+    --redis-host <host>              The redis host address (defaults to localhost)
+    --redis-port <port>              The port redis is listening on (defaults to 6379)
+    --redis-db <db>                  The redis db to use (defaults to 0)
+
     --no-x-forward                   Don't add 'X-forward-' headers to proxied requests
     --no-prepend-path                Avoid prepending target paths to proxied requests
     --no-include-prefix              Don't include the routing prefix in proxied requests
+    --auto-rewrite                   Rewrite the Location header host/port in redirect responses
+    --protocol-rewrite <proto>       Rewrite the Location header protocol in redirect responses to the specified protocol
     --insecure                       Disable SSL cert verification
     --host-routing                   Use host routing (host as first level of path)
     --statsd-host <host>             Host to send statsd statistics to
@@ -106,10 +114,9 @@ matching route is found in the proxy table:
     --log-level <loglevel>           Log level (debug, info, warn, error)
 ```
 
-
 ## Using the REST API
 
-The configurable-http-proxy API is documented and available at the 
+The configurable-http-proxy API is documented and available at the
 interactive swagger site, [petstore](http://petstore.swagger.io/?url=https://raw.githubusercontent.com/jupyterhub/configurable-http-proxy/master/doc/rest-api.yml#/default)
 or as a [swagger specification file in this repo](https://github.com/jupyterhub/configurable-http-proxy/blob/master/doc/rest-api.yml).
 
@@ -202,7 +209,7 @@ with their status code:
 - 404: a client has requested a URL for which there is no routing target.
   This can be prevented if a `default target` is specified when starting
   the configurable-http-proxy.
-  
+
 - 503: a route exists, but the upstream server isn't responding.
   This is more common, and can be due to any number of reasons,
   including the target service having died or not finished starting.
@@ -211,10 +218,10 @@ with their status code:
 
 If you specify an error path `--error-path /usr/share/chp-errors` when
 starting the CHP:
- 
+
     configurable-http-proxy --error-path /usr/share/chp-errors
- 
-then when a proxy error occurs, CHP will look in 
+
+then when a proxy error occurs, CHP will look in
 `/usr/share/chp-errors/<CODE>.html` (where CODE is the status code number)
 for an html page to serve, e.g. `404.html` or `503.html`.
 
@@ -225,7 +232,7 @@ If you specify an error path, make sure you also create `error.html`.
 
 When starting the CHP, you can pass a command line option for `--error-target`.
 If you specify `--error-target http://localhost:1234`,
-then when the proxy encounters an error, it will make a GET request to 
+then when the proxy encounters an error, it will make a GET request to
 this server, with URL `/CODE`, and the URL of the failing request
 escaped in a URL parameter, e.g.:
 
@@ -247,3 +254,20 @@ first part of the URL path, e.g.:
   "/otherdomain.biz": "http://10.0.1.4:5555",
 }
 ```
+
+## Using Redis
+
+If you require multiple instances of CHP to be running and kept in sync, you can use [redis] to to store the route table
+rather than the default in memory option.
+
+To do so, you'll need to pass in a couple of options to CHP when launching it.  Specifically, you'll need to set the
+storage provider and any [redis options] you need.
+
+For example, to use [redis] on `localhost:6379` and the default database, you can run the following:
+
+    configurable-http-proxy --storage-provider redis
+
+If you are running [redis] on a different host, or need to customize the port/db, see [redis options].
+
+[redis]: http://redis.io/
+[redis options]: https://github.com/jupyterhub/configurable-http-proxy#command-line-options

bin/configurable-http-proxy

@@ -38,6 +38,11 @@ args
     .option('--error-path <path>', 'Alternate server for handling proxy errors (proto://host[:port])')
     .option('--redirect-port <redirect-port>', 'Redirect HTTP requests on this port to the server on HTTPS')
     .option('--pid-file <pid-file>', 'Write our PID to a file')
+    // storage configuration
+    .option('--storage-provider <provider>', 'The storage provider for the route table (defaults to memory)')
+    .option('--redis-host <host>', 'The redis host address (defaults to localhost)')
+    .option('--redis-port <port>', 'The port redis is listening on (defaults to 6379)')
+    .option('--redis-db <db>', 'The redis db to use (defaults to 0)')
     // passthrough http-proxy options
     .option('--no-x-forward', "Don't add 'X-forward-' headers to proxied requests")
     .option('--no-prepend-path', "Avoid prepending target paths to proxied requests")
@@ -158,6 +163,12 @@ options.host_routing = args.hostRouting;
 options.auth_token = process.env.CONFIGPROXY_AUTH_TOKEN;
 options.redirectPort = args.redirectPort;
 
+// storage options
+options.storageProvider = args.storageProvider;
+options.redisHost       = args.redisHost;
+options.redisPort       = args.redisPort;
+options.redisDb         = args.redisDb;
+
 // statsd options
 if (args.statsdHost) {
     var lynx = require('lynx');

lib/configproxy.js

@@ -97,11 +97,35 @@ function parse_host (req) {
     return host;
 }
 
+function getStorageProvider (options) {
+    var provider = null;
+    var type = options.storageProvider || "memory";
+
+    switch (type.toLowerCase()) {
+        case "memory":
+            provider = store.MemoryStore();
+            break;
+        case "redis":
+            provider = store.RedisStore(
+              options.redisHost,
+              options.redisPort,
+              options.redisDb
+            );
+            break;
+        default:
+            log.warn("Unknown storage provider '%s', falling back to memory", options.storageProvider);
+            provider = store.MemoryStore();
+            break;
+    }
+
+    return provider;
+}
+
 function ConfigurableProxy (options) {
     var that = this;
     this.options = options || {};
 
-    this._routes = store.MemoryStore();
+    this._routes = getStorageProvider(options);
     this.auth_token = this.options.auth_token;
     this.includePrefix = options.includePrefix === undefined ? true : options.includePrefix;
     this.host_routing = this.options.host_routing;

lib/store.js

@@ -1,4 +1,4 @@
-var trie = require("./trie.js");
+var trie = require('./trie.js');
 
 var NotImplemented = function (name) {
   return {
@@ -33,52 +33,11 @@ var BaseStore = Object.create(Object.prototype, {
   }
 });
 
-function MemoryStore () {
-  var routes = {};
-  var urls   = new trie.URLTrie();
 
-  return Object.create(BaseStore, {
-    get: {
-      value: function (path, cb) {
-        this.notify(cb, routes[path]);
-      }
-    },
-    getTarget: {
-      value: function (path, cb) {
-        this.notify(cb, urls.get(path));
-      }
-    },
-    getAll: {
-      value: function (cb) {
-        this.notify(cb, routes);
-      }
-    },
-    add: {
-      value: function (path, data, cb) {
-        routes[path] = data;
-        urls.add(path, data);
-        this.notify(cb);
-      }
-    },
-    update: {
-      value: function (path, data, cb) {
-        Object.assign(routes[path], data);
-        this.notify(cb);
-      }
-    },
-    remove: {
-      value: function (path, cb) {
-        delete routes[path];
-        urls.remove(path);
-        this.notify(cb);
-      }
-    },
-    hasRoute: {
-      value: function (path, cb) {
-        this.notify(cb, routes.hasOwnProperty(path));
-      }
-    }
-  });
-}
+exports.MemoryStore = function () {
+  return require("./store/memory.js").create(BaseStore);
+};
 
-exports.MemoryStore = MemoryStore;
+exports.RedisStore = function (host, port, db) {
+  return require("./store/redis.js").create(BaseStore, host, port, db);
+};

lib/store/memory.js

@@ -0,0 +1,49 @@
+var trie = require("../trie.js");
+
+exports.create = function (BaseStore) {
+  var routes = {};
+  var urls   = new trie.URLTrie();
+
+  return Object.create(BaseStore, {
+    get: {
+      value: function (path, cb) {
+        this.notify(cb, routes[path]);
+      }
+    },
+    getTarget: {
+      value: function (path, cb) {
+        this.notify(cb, urls.get(path));
+      }
+    },
+    getAll: {
+      value: function (cb) {
+        this.notify(cb, routes);
+      }
+    },
+    add: {
+      value: function (path, data, cb) {
+        routes[path] = data;
+        urls.add(path, data);
+        this.notify(cb);
+      }
+    },
+    update: {
+      value: function (path, data, cb) {
+        Object.assign(routes[path], data);
+        this.notify(cb);
+      }
+    },
+    remove: {
+      value: function (path, cb) {
+        delete routes[path];
+        urls.remove(path);
+        this.notify(cb);
+      }
+    },
+    hasRoute: {
+      value: function (path, cb) {
+        this.notify(cb, routes.hasOwnProperty(path));
+      }
+    }
+  });
+};