[WIP] Add the Ability to use Redis for Storage

README.md

@@ -33,7 +33,7 @@ using the npm package manager:
     npm install -g configurable-http-proxy
 
 To install from the source code found in this GitHub repo:
-    
+
     git clone https://github.com/jupyterhub/configurable-http-proxy.git
     cd configurable-http-proxy
     # Use -g for global install
@@ -54,7 +54,7 @@ The configurable proxy runs two HTTP(S) servers:
 
 ### Setting a default target
 
-When you start the proxy from the command line, you can set a 
+When you start the proxy from the command line, you can set a
 default target (`--default-target` option) to be used when no
 matching route is found in the proxy table:
 
@@ -71,7 +71,8 @@ matching route is found in the proxy table:
     -V, --version                    output the version number
     --ip <ip-address>                Public-facing IP of the proxy
     --port <n> (defaults to 8000)    Public-facing port of the proxy
-
+    --storage-command <script>       The optional external executable to use for persistence
+
     --ssl-key <keyfile>              SSL key to use, if any
     --ssl-cert <certfile>            SSL certificate to use, if any
     --ssl-ca <ca-file>               SSL certificate authority, if any
@@ -81,15 +82,15 @@ matching route is found in the proxy table:
     --ssl-ciphers <ciphers>          `:`-separated ssl cipher list. Default excludes RC4
     --ssl-allow-rc4                  Allow RC4 cipher for SSL (disabled by default)
     --ssl-dhparam <dhparam-file>     SSL Diffie-Helman Parameters pem file, if any
-    
+
     --api-ip <ip>                    Inward-facing IP for API requests
     --api-port <n>                   Inward-facing port for API requests (defaults to --port=value+1)
     --api-ssl-key <keyfile>          SSL key to use, if any, for API requests
     --api-ssl-cert <certfile>        SSL certificate to use, if any, for API requests
     --api-ssl-ca <ca-file>           SSL certificate authority, if any, for API requests
     --api-ssl-request-cert           Request SSL certs to authenticate clients for API requests
     --api-ssl-reject-unauthorized    Reject unauthorized SSL connections (only meaningful if --api-ssl-request-cert is given)
-    
+
     --default-target <host>          Default proxy target (proto://host[:port])
     --error-target <host>            Alternate server for handling proxy errors (proto://host[:port])
     --error-path <path>              Alternate server for handling proxy errors (proto://host[:port])
@@ -109,7 +110,7 @@ matching route is found in the proxy table:
 
 ## Using the REST API
 
-The configurable-http-proxy API is documented and available at the 
+The configurable-http-proxy API is documented and available at the
 interactive swagger site, [petstore](http://petstore.swagger.io/?url=https://raw.githubusercontent.com/jupyterhub/configurable-http-proxy/master/doc/rest-api.yml#/default)
 or as a [swagger specification file in this repo](https://github.com/jupyterhub/configurable-http-proxy/blob/master/doc/rest-api.yml).
 
@@ -202,7 +203,7 @@ with their status code:
 - 404: a client has requested a URL for which there is no routing target.
   This can be prevented if a `default target` is specified when starting
   the configurable-http-proxy.
-  
+
 - 503: a route exists, but the upstream server isn't responding.
   This is more common, and can be due to any number of reasons,
   including the target service having died or not finished starting.
@@ -211,10 +212,10 @@ with their status code:
 
 If you specify an error path `--error-path /usr/share/chp-errors` when
 starting the CHP:
- 
+
     configurable-http-proxy --error-path /usr/share/chp-errors
- 
-then when a proxy error occurs, CHP will look in 
+
+then when a proxy error occurs, CHP will look in
 `/usr/share/chp-errors/<CODE>.html` (where CODE is the status code number)
 for an html page to serve, e.g. `404.html` or `503.html`.
 
@@ -225,7 +226,7 @@ If you specify an error path, make sure you also create `error.html`.
 
 When starting the CHP, you can pass a command line option for `--error-target`.
 If you specify `--error-target http://localhost:1234`,
-then when the proxy encounters an error, it will make a GET request to 
+then when the proxy encounters an error, it will make a GET request to
 this server, with URL `/CODE`, and the URL of the failing request
 escaped in a URL parameter, e.g.:
 
@@ -247,3 +248,26 @@ first part of the URL path, e.g.:
   "/otherdomain.biz": "http://10.0.1.4:5555",
 }
 ```
+
+## Using External Storage
+
+By default CHP uses memory in the local process for storing the route table. If you'd prefer to use an external storage
+mechanism, like redis, memcached, mysql, etc. You can specify an executable to be used instead, by passing the path to
+your script in the `--storage-command` option.
+
+To do this, you'll need to ensure that the script is both accessible on the path and executable. The script should
+support the following API, where `<path>` is the incoming request path and `<base64_json>` is a base64 encoded JSON
+object.
+
+```
+$ /path/to/script get <path>
+$ /path/to/script get_all
+$ /path/to/script add <path> <base64_json>
+$ /path/to/script update <path> <base64_json>
+$ /path/to/script remove <path>
+$ /path/to/script exists <path>
+```
+
+For an example, check out [the script] we use for testing this functionality.
+
+[the script]: https://github.com/jupyterhub/configurable-http-proxy/blob/master/test/support/external_store

bin/configurable-http-proxy

@@ -17,6 +17,7 @@ args
     .version(pkg.version)
     .option('--ip <ip-address>', 'Public-facing IP of the proxy')
     .option('--port <n> (defaults to 8000)', 'Public-facing port of the proxy', parseInt)
+    .option('--storage-command <script>', 'An external command to use for storage, if any')
     .option('--ssl-key <keyfile>', 'SSL key to use, if any')
     .option('--ssl-cert <certfile>', 'SSL certificate to use, if any')
     .option('--ssl-ca <ca-file>', 'SSL certificate authority, if any')
@@ -157,6 +158,7 @@ options.error_path = args.errorPath;
 options.host_routing = args.hostRouting;
 options.auth_token = process.env.CONFIGPROXY_AUTH_TOKEN;
 options.redirectPort = args.redirectPort;
+options.storageCommand = args.storageCommand;
 
 // statsd options
 if (args.statsdHost) {

lib/configproxy.js

@@ -101,7 +101,12 @@ function ConfigurableProxy (options) {
     var that = this;
     this.options = options || {};
 
-    this._routes = store.MemoryStore();
+    if (options.storageCommand) {
+      this._routes = store.ExternalStore(options.storageCommand);
+    } else {
+      this._routes = store.MemoryStore();
+    }
+
     this.auth_token = this.options.auth_token;
     this.includePrefix = options.includePrefix === undefined ? true : options.includePrefix;
     this.host_routing = this.options.host_routing;

lib/store.js

@@ -1,4 +1,5 @@
 var trie = require("./trie.js");
+var exec = require('child_process').exec;
 
 var NotImplemented = function (name) {
   return {
@@ -81,4 +82,109 @@ function MemoryStore () {
   });
 }
 
-exports.MemoryStore = MemoryStore;
+function ExternalStore (command) {
+  var encode = function (value) {
+    return new Buffer(JSON.stringify(value)).toString("base64");
+  };
+
+  var decode = function (value) {
+    return JSON.parse(new Buffer(value, "base64").toString("utf8"));
+  };
+
+  var runCommand = function (/*command [,arg1, arg2, ...], callback*/) {
+    var args     = Array.prototype.slice.apply(arguments);
+    var callback = args.pop() || function (err, stdout, stderr) {};
+
+    args.unshift(command);
+    exec(args.join(" "), { timeout: 1000 }, callback);
+  };
+
+  return Object.create(BaseStore, {
+    get: {
+      value: function (path, cb) {
+        var that = this;
+
+        runCommand("get", path, function (error, stdout, stderr) {
+          if (error) {
+            that.notify(cb);
+            return;
+          }
+
+          that.notify(cb, decode(stdout));
+        });
+      }
+    },
+    getTarget: {
+      value: function (path, cb) {
+        this.get(path, function (route) {
+          if (!route) {
+            this.notify(cb);
+            return;
+          }
+
+          this.notify(cb, { prefix: path, data: route });
+        });
+      }
+    },
+    getAll: {
+      value: function (cb) {
+        var that = this;
+
+        runCommand("get_all", function (error, stdout, stderr) {
+          if (error) {
+            that.notify(cb, {});
+            return;
+          }
+
+          var routes  = JSON.parse(stdout);
+          var results = {};
+
+          Object.keys(routes).forEach(function (key) {
+            results[key] = decode(routes[key]);
+          });
+
+          that.notify(cb, results);
+        });
+      }
+    },
+    add: {
+      value: function (path, data, cb) {
+        var that  = this;
+
+        runCommand("add", path, encode(data), function(error, stdout, stderr) {
+          that.notify(cb);
+        });
+      }
+    },
+    update: {
+      value: function (path, data, cb) {
+        var that  = this;
+
+        runCommand("update", path, encode(data), function(error, stdout, stderr) {
+          that.notify(cb);
+        });
+      }
+    },
+    remove: {
+      value: function (path, cb) {
+        var that = this;
+
+        runCommand("remove", path, function(error, stdout, stderr) {
+          that.notify(cb);
+        });
+      }
+    },
+    hasRoute: {
+      value: function (path, cb) {
+        var that = this;
+
+        runCommand("exists", path, function(error, stdout, stderr) {
+          that.notify(cb, !error);
+        });
+      }
+    }
+  });
+}
+
+exports.MemoryStore   = MemoryStore;
+exports.ExternalStore = ExternalStore;