Merge tag '1.38.2' into cdb

Version 1.38.2

2016-10-13

Bug fixes:
 * Batch queries: release redis clients to pool from locker and seeker.

Author: rochoa

.travis.yml

@@ -1,6 +1,6 @@
-before_script:
+before_install:
   - lsb_release -a
-  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
+  - sudo mv /etc/apt/sources.list.d/pgdg.list* /tmp
   - sudo apt-get -qq purge postgis* postgresql*
   - sudo rm -Rf /var/lib/postgresql /etc/postgresql
   - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.5

Makefile

@@ -13,9 +13,11 @@ jshint:
 	@echo "***jshint***"
 	@./node_modules/.bin/jshint app/ batch/ test/ app.js
 
-TEST_SUITE := $(shell find test/{acceptance,unit} -name "*.js")
+TEST_SUITE := $(shell find test/{unit,integration,acceptance} -name "*.js")
 TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")
+TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js")
 TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js")
+TEST_SUITE_BATCH := $(shell find test/*/batch -name "*.js")
 
 test:
 	@echo "***tests***"
@@ -25,10 +27,18 @@ test-unit:
 	@echo "***unit tests***"
 	@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_UNIT)
 
+test-integration:
+	@echo "***integration tests***"
+	@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_INTEGRATION)
+
 test-acceptance:
 	@echo "***acceptance tests***"
 	@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_ACCEPTANCE)
 
+test-batch:
+	@echo "***batch queries tests***"
+	@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_BATCH)
+
 test-all: jshint test
 
 coverage:

NEWS.md

@@ -1,3 +1,86 @@
+1.38.2 - 2016-10-13
+-------------------
+
+Bug fixes:
+ * Batch queries: release redis clients to pool from locker and seeker.
+
+
+1.38.1 - 2016-10-13
+-------------------
+
+Enhancements:
+ * Batch queries: improvements over leader locking.
+
+
+1.38.0 - 2016-10-11
+-------------------
+
+Announcements:
+ * Allow to set statement timeout per query in multi query batch queries.
+ * Batch queries default statement timeout set to 12 hours.
+ * Multiple queries jobs pushed as first job between queries.
+
+
+1.37.1 - 2016-10-05
+-------------------
+
+Bug fixes:
+ * Body parser accepting multipart requests.
+
+
+1.37.0 - 2016-10-04
+-------------------
+
+Enhancements:
+ * Migrate to Express.js 4.x series.
+
+
+1.36.2 - 2016-10-03
+-------------------
+
+Bug fixes:
+ - Batch Queries logs: use path instead of stream to be able to reopen FD.
+
+
+1.36.1 - 2016-09-30
+-------------------
+
+Enhancements:
+ * Tag fallback jobs logs.
+
+
+1.36.0 - 2016-09-30
+-------------------
+
+New features:
+ * Log queries from batch fallback jobs.
+
+Enhancements:
+ * assert.response following callback(err, obj) pattern.
+
+
+1.35.0 - 2016-09-15
+-------------------
+
+New features:
+ * Allow to use `--config /path/to/config.js` to specify configuration file.
+   - Environment will be loaded from config file if `environment` key is present, otherwise it keeps current behaviour.
+
+Bug fixes:
+ * Allow to use absolute paths for log files.
+
+Announcements:
+ * Removes support for optional rollbar logging.
+
+
+1.34.2 - 2016-08-30
+-------------------
+
+Announcements:
+ * Upgrades cartodb-redis to 0.13.1.
+ * Set TTL of finished job to 2h
+
+
 1.34.1 - 2016-07-11
 -------------------
 

app.js

@@ -9,65 +9,67 @@
 * environments: [development, test, production]
 *
 */
-var _ = require('underscore');
 var fs = require('fs');
 var path = require('path');
 
-var ENV = process.env.NODE_ENV || 'development';
-
-if (process.argv[2]) {
-    ENV = process.argv[2];
+var argv = require('yargs')
+    .usage('Usage: $0 <environment> [options]')
+    .help('h')
+    .example(
+        '$0 production -c /etc/sql-api/config.js',
+        'start server in production environment with /etc/sql-api/config.js as config file'
+    )
+    .alias('h', 'help')
+    .alias('c', 'config')
+    .nargs('c', 1)
+    .describe('c', 'Load configuration from path')
+    .argv;
+
+var environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
+var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js');
+if (!fs.existsSync(configurationFile)) {
+    console.error('Configuration file "%s" does not exist', configurationFile);
+    process.exit(1);
 }
 
-process.env.NODE_ENV = ENV;
+global.settings = require(configurationFile);
+var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.settings.environment;
+process.env.NODE_ENV = ENVIRONMENT;
 
 var availableEnvironments = ['development', 'production', 'test', 'staging'];
 
 // sanity check arguments
-if (availableEnvironments.indexOf(ENV) === -1) {
-  console.error("\nnode app.js [environment]");
-  console.error("environments: " + availableEnvironments.join(', '));
+if (availableEnvironments.indexOf(ENVIRONMENT) === -1) {
+  console.error("node app.js [environment]");
+  console.error("Available environments: " + availableEnvironments.join(', '));
   process.exit(1);
 }
 
-// set Node.js app settings and boot
-global.settings  = require(__dirname + '/config/settings');
-var env          = require(__dirname + '/config/environments/' + ENV);
-env.api_hostname = require('os').hostname().split('.')[0];
-_.extend(global.settings, env);
+global.settings.api_hostname = require('os').hostname().split('.')[0];
 
 global.log4js = require('log4js');
-var log4js_config = {
-  appenders: [],
-  replaceConsole:true
+var log4jsConfig = {
+    appenders: [],
+    replaceConsole: true
 };
 
-if ( env.log_filename ) {
-    var logdir = path.dirname(env.log_filename);
-    // See cwd inlog4js.configure call below
-    logdir = path.resolve(__dirname, logdir);
-    if ( ! fs.existsSync(logdir) ) {
-        console.error("Log filename directory does not exist: " + logdir);
+if ( global.settings.log_filename ) {
+    var logFilename = path.resolve(global.settings.log_filename);
+    var logDirectory = path.dirname(logFilename);
+    if (!fs.existsSync(logDirectory)) {
+        console.error("Log filename directory does not exist: " + logDirectory);
         process.exit(1);
     }
-    console.log("Logs will be written to " + env.log_filename);
-    log4js_config.appenders.push(
-        { type: "file", filename: env.log_filename }
+    console.log("Logs will be written to " + logFilename);
+    log4jsConfig.appenders.push(
+        { type: "file", absolute: true, filename: logFilename }
     );
 } else {
-    log4js_config.appenders.push(
+    log4jsConfig.appenders.push(
         { type: "console", layout: { type:'basic' } }
     );
 }
-
-if ( global.settings.rollbar ) {
-  log4js_config.appenders.push({
-    type: __dirname + "/app/models/log4js_rollbar.js",
-    options: global.settings.rollbar
-  });
-}
-
-global.log4js.configure(log4js_config, { cwd: __dirname });
+global.log4js.configure(log4jsConfig);
 global.logger = global.log4js.getLogger();
 
 
@@ -78,12 +80,14 @@ if ( ! global.settings.base_url ) {
 
 var version = require("./package").version;
 
-var app = require(global.settings.app_root + '/app/app')();
-app.listen(global.settings.node_port, global.settings.node_host, function() {
-  console.log(
-      "CartoDB SQL API %s listening on %s:%s with base_url %s (%s)",
-      version, global.settings.node_host, global.settings.node_port, global.settings.base_url, ENV
-  );
+var server = require('./app/server')();
+var listener = server.listen(global.settings.node_port, global.settings.node_host);
+listener.on('listening', function() {
+    console.info('Using configuration file "%s"', configurationFile);
+    console.log(
+        "CartoDB SQL API %s listening on %s:%s PID=%d (%s)",
+        version, global.settings.node_host, global.settings.node_port, process.pid, ENVIRONMENT
+    );
 });
 
 process.on('uncaughtException', function(err) {
@@ -92,15 +96,19 @@ process.on('uncaughtException', function(err) {
 
 process.on('SIGHUP', function() {
     global.log4js.clearAndShutdownAppenders(function() {
-        global.log4js.configure(log4js_config);
+        global.log4js.configure(log4jsConfig);
         global.logger = global.log4js.getLogger();
         console.log('Log files reloaded');
     });
+
+    if (server.batch && server.batch.logger) {
+        server.batch.logger.reopenFileStreams();
+    }
 });
 
 process.on('SIGTERM', function () {
-    app.batch.stop();
-    app.batch.drain(function (err) {
+    server.batch.stop();
+    server.batch.drain(function (err) {
         if (err) {
             console.log('Exit with error');
             return process.exit(1);

app/controllers/health_check_controller.js

@@ -24,11 +24,11 @@ HealthCheckController.prototype.handleHealthCheck = function (req, res) {
             if (err) {
                 response.err = err.message;
             }
-            res.send(response, ok ? 200 : 503);
+            res.status(ok ? 200 : 503).send(response);
 
         });
     } else {
-        res.send({enabled: false, ok: true}, 200);
+        res.status(200).send({enabled: false, ok: true});
     }
 };