From e1f5ad6933f3b7ac40506123c24245c99db5df83 Mon Sep 17 00:00:00 2001 From: Siddhi Pai Date: Mon, 5 Dec 2016 23:20:41 -0800 Subject: [PATCH 01/10] Update paid support URL --- .github/ISSUE_TEMPLATE.md | 36 ++++++++++++++++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 24 +++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..ccc915a7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,36 @@ + + +### Bug or feature request + + + +- [ ] Bug +- [ ] Feature request + +### Description of feature (or steps to reproduce if bug) + + + +### Link to sample repo to reproduce issue (if bug) + + + +### Expected result + + + +### Actual result (if bug) + + + +### Additional information (Node.js version, LoopBack version, etc) + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..d2b240f5 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,24 @@ +### Description + + +#### Related issues + + + +- None + +### Checklist + + + +- [ ] New tests added or existing tests modified to cover all changes +- [ ] Code conforms with the [style + guide](http://loopback.io/doc/en/contrib/style-guide.html) From 187e395e7d08d1e6eb1293491d7fc2e9150b0e10 Mon Sep 17 00:00:00 2001 From: Rand McKinney Date: Wed, 11 Jan 2017 10:26:01 -0800 Subject: [PATCH 02/10] Move info from docs into README (#199) Move info from docs into README --- README.md | 470 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 323 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index ee2a927e..9471d82e 100644 --- a/README.md +++ b/README.md @@ -1,176 +1,352 @@ # loopback-connector-postgresql -The official PostgreSQL connector for the LoopBack framework. +[PostgreSQL](https://www.postgresql.org/), is a popular open-source object-relational database. +The `loopback-connector-postgresql` module is the PostgreSQL connector for the LoopBack framework. -Please see the [official documentation](http://loopback.io/doc/en/lb2/PostgreSQL-connector.html). +
For more information, see the documentation. +

+NOTE: The PostgreSQL connector requires PostgreSQL 8.x or 9.x. +
+## Installation -## Connector settings +In your application root directory, enter this command to install the connector: -The connector can be configured using the following settings from the data source. -* url: The URL to the database, such as 'postgres://test:mypassword@localhost:5432/dev' -* host or hostname (default to 'localhost'): The host name or ip address of the PostgreSQL DB server -* port (default to 5432): The port number of the PostgreSQL DB server -* username or user: The user name to connect to the PostgreSQL DB -* password: The password -* database: The PostgreSQL database name -* debug (default to false) +```shell +$ npm install loopback-connector-postgresql --save +``` -**NOTE**: By default, the 'public' schema is used for all tables. +This installs the module from npm and adds it as a dependency to the application's `package.json` file. -The PostgreSQL connector uses [node-postgres](https://github.com/brianc/node-postgres) as the driver. See more -information about configuration parameters, check [https://github.com/brianc/node-postgres/wiki/Client#constructors](https://github.com/brianc/node-postgres/wiki/Client#constructors). +If you create a PostgreSQL data source using the data source generator as described below, you don't have to do this, since the generator will run `npm install` for you. -## Discovering Models +## Creating a data source -PostgreSQL data sources allow you to discover model definition information from existing postgresql databases. See the following APIs: +Use the [Data source generator](http://loopback.io/doc/en/lb3/Data-source-generator.html) to add a PostgreSQL data source to your application. +The generator will prompt for the database server hostname, port, and other settings +required to connect to a PostgreSQL database. It will also run the `npm install` command above for you. - - [dataSource.discoverModelDefinitions([username], fn)](https://github.com/strongloop/loopback#datasourcediscovermodeldefinitionsusername-fn) - - [dataSource.discoverSchema([owner], name, fn)](https://github.com/strongloop/loopback#datasourcediscoverschemaowner-name-fn) +The entry in the application's `/server/datasources.json` will look like this: +{% include code-caption.html content="/server/datasources.json" %} +```javascript +"mydb": { + "name": "mydb", + "connector": "postgresql" -## Model definition for PostgreSQL + "host": "mydbhost", + "port": 5432, + "url": "postgres://admin:admin@myhost/db", + "database": "db1", + "password": "admin", + "user": "admin" +} +``` -The model definition consists of the following properties: +Edit `datasources.json` to add other properties that enable you to connect the data source to a PostgreSQL database. + +### Properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeDescription
connectorString + Connector name, either "loopback-connector-postgresql" or "postgresql" +
databaseStringDatabase name
debugBooleanIf true, turn on verbose mode to debug database queries and lifecycle.
hostStringDatabase host name
passwordStringPassword to connect to database
portNumberDatabase TCP port
urlStringUse instead of thehost,port,user,password, + anddatabaseproperties. For example:'postgres://test:mypassword@localhost:5432/dev'. +
usernameStringUsername to connect to database
-* name: Name of the model, by default, it's the camel case of the table -* options: Model level operations and mapping to PostgreSQL schema/table -* properties: Property definitions, including mapping to PostgreSQL column +**NOTE**: By default, the 'public' schema is used for all tables. -```json +The PostgreSQL connector uses [node-postgres](https://github.com/brianc/node-postgres) as the driver. For more +information about configuration parameters, see [node-postgres documentation](https://github.com/brianc/node-postgres/wiki/Client#constructors). + +### Connecting to UNIX domain socket + +A common PostgreSQL configuration is to connect to the UNIX domain socket `/var/run/postgresql/.s.PGSQL.5432` instead of using the TCP/IP port. For example: + +```javascript +{ + "postgres": { + "host": "/var/run/postgresql/", + "port": "5432", + "database": "dbname", + "username": "dbuser", + "password": "dbpassword", + "name": "postgres", + "debug": true, + "connector": "postgresql" + } +} +``` - {"name": "Inventory", "options": { - "idInjection": false, +## Defining models + +The model definition consists of the following properties. + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyDefaultDescription
nameCamel-case of the database table nameName of the model.
optionsN/AModel level operations and mapping to PostgreSQL schema/table
propertiesN/AProperty definitions, including mapping to PostgreSQL column
+ +For example: + +{% include code-caption.html content="/common/models/model.json" %} +```javascript +{ + "name": "Inventory", + "options": { + "idInjection": false, + "postgresql": { + "schema": "strongloop", + "table": "inventory" + } + }, + "properties": { + "id": { + "type": "String", + "required": false, + "length": 64, + "precision": null, + "scale": null, + "postgresql": { + "columnName": "id", + "dataType": "character varying", + "dataLength": 64, + "dataPrecision": null, + "dataScale": null, + "nullable": "NO" + } + }, + "productId": { + "type": "String", + "required": false, + "length": 20, + "precision": null, + "scale": null, + "id": 1, "postgresql": { - "schema": "strongloop", - "table": "inventory" + "columnName": "product_id", + "dataType": "character varying", + "dataLength": 20, + "dataPrecision": null, + "dataScale": null, + "nullable": "YES" } - }, "properties": { - "id": { - "type": "String", - "required": false, - "length": 64, - "precision": null, - "scale": null, - "postgresql": { - "columnName": "id", - "dataType": "character varying", - "dataLength": 64, - "dataPrecision": null, - "dataScale": null, - "nullable": "NO" - } - }, - "productId": { - "type": "String", - "required": false, - "length": 20, - "precision": null, - "scale": null, - "id": 1, - "postgresql": { - "columnName": "product_id", - "dataType": "character varying", - "dataLength": 20, - "dataPrecision": null, - "dataScale": null, - "nullable": "YES" - } - }, - "locationId": { - "type": "String", - "required": false, - "length": 20, - "precision": null, - "scale": null, - "id": 1, - "postgresql": { - "columnName": "location_id", - "dataType": "character varying", - "dataLength": 20, - "dataPrecision": null, - "dataScale": null, - "nullable": "YES" - } - }, - "available": { - "type": "Number", - "required": false, - "length": null, - "precision": 32, - "scale": 0, - "postgresql": { - "columnName": "available", - "dataType": "integer", - "dataLength": null, - "dataPrecision": 32, - "dataScale": 0, - "nullable": "YES" - } - }, - "total": { - "type": "Number", - "required": false, - "length": null, - "precision": 32, - "scale": 0, - "postgresql": { - "columnName": "total", - "dataType": "integer", - "dataLength": null, - "dataPrecision": 32, - "dataScale": 0, - "nullable": "YES" - } + }, + "locationId": { + "type": "String", + "required": false, + "length": 20, + "precision": null, + "scale": null, + "id": 1, + "postgresql": { + "columnName": "location_id", + "dataType": "character varying", + "dataLength": 20, + "dataPrecision": null, + "dataScale": null, + "nullable": "YES" } - }} - + }, + "available": { + "type": "Number", + "required": false, + "length": null, + "precision": 32, + "scale": 0, + "postgresql": { + "columnName": "available", + "dataType": "integer", + "dataLength": null, + "dataPrecision": 32, + "dataScale": 0, + "nullable": "YES" + } + }, + "total": { + "type": "Number", + "required": false, + "length": null, + "precision": 32, + "scale": 0, + "postgresql": { + "columnName": "total", + "dataType": "integer", + "dataLength": null, + "dataPrecision": 32, + "dataScale": 0, + "nullable": "YES" + } + } + } +} ``` -## Type Mapping - - - Number - - Boolean - - String - - Object - - Date - - Array - - Buffer - -### JSON to PostgreSQL Types - -* String|JSON|Text|default: VARCHAR, default length is 1024 -* Number: INTEGER -* Date: TIMESTAMP WITH TIME ZONE -* Timestamp: TIMESTAMP WITH TIME ZONE -* Boolean: BOOLEAN - -### PostgreSQL Types to JSON - -* BOOLEAN: Boolean -* VARCHAR|CHARACTER VARYING|CHARACTER|CHAR|TEXT: String -* BYTEA: Binary; -* SMALLINT|INTEGER|BIGINT|DECIMAL|NUMERIC|REAL|DOUBLE|SERIAL|BIGSERIAL: Number -* DATE|TIMESTAMP|TIME: Date -* POINT: GeoPoint - -## Destroying Models - -Destroying models may result in errors due to foreign key integrity. Make sure -to delete any related models first before calling delete on model's with -relationships. - -## Auto Migrate / Auto Update - -After making changes to your model properties you must call `Model.automigrate()` -or `Model.autoupdate()`. Only call `Model.automigrate()` on new models -as it will drop existing tables. +## Type mapping + +See [LoopBack types](http://loopback.io/doc/en/lb3/LoopBack-types.html) for details on LoopBack's data types. + +### LoopBack to PostgreSQL types + + + + + + + + + + + + + + + + + + + + + + + + +
LoopBack TypePostgreSQL Type
String
JSON
Text
Default		 + VARCHAR2
+ Default length is 1024 +
NumberINTEGER
DateTIMESTAMP WITH TIME ZONE
BooleanBOOLEAN
+ +### PostgreSQL types to LoopBack + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PostgreSQL TypeLoopBack Type
BOOLEANBoolean
+ VARCHAR
CHARACTER VARYING
CHARACTER
CHAR
TEXT + 		String
BYTEANode.js Buffer object
SMALLINT
INTEGER
BIGINT
DECIMAL
NUMERIC
REAL
DOUBLE
SERIAL
BIGSERIAL		Number
DATE
TIMESTAMP
TIME		Date
POINTGeoPoint
+ +## Discovery and auto-migration + +### Model discovery + +The PostgreSQL connector supports _model discovery_ that enables you to create LoopBack models +based on an existing database schema using the unified [database discovery API](http://apidocs.strongloop.com/loopback-datasource-juggler/#datasource-prototype-discoverandbuildmodels). For more information on discovery, see [Discovering models from relational databases](https://loopback.io/doc/en/lb3/Discovering-models-from-relational-databases.html). + +### Auto-migratiion + +The PostgreSQL connector also supports _auto-migration_ that enables you to create a database schema +from LoopBack models using the [LoopBack automigrate method](http://apidocs.strongloop.com/loopback-datasource-juggler/#datasource-prototype-automigrate). + +For more information on auto-migration, see [Creating a database schema from models](https://loopback.io/doc/en/lb3/Creating-a-database-schema-from-models.html) for more information. LoopBack PostgreSQL connector creates the following schema objects for a given -model: +model: a table, for example, PRODUCT under the 'public' schema within the database. + +The auto-migrate method: -* A table, for example, PRODUCT under the 'public' schema within the database +* Defines a primary key for the properties whose `id` property is true (or a positive number). +* Creates a column with 'SERIAL' type if the `generated` property of the `id` property is true. +Destroying models may result in errors due to foreign key integrity. First delete any related models by calling delete on models with relationships. ## Running tests From ba457cd9b54b2f19f77b5fe0b571ce14ed8767bc Mon Sep 17 00:00:00 2001 From: Loay Date: Thu, 19 Jan 2017 10:36:51 -0500 Subject: [PATCH 03/10] Use unique param for affectedRows --- lib/postgresql.js | 4 ++-- test/postgresql.test.js | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/postgresql.js b/lib/postgresql.js index 329a4ef6..9b404324 100644 --- a/lib/postgresql.js +++ b/lib/postgresql.js @@ -153,7 +153,7 @@ PostgreSQL.prototype.executeSQL = function(sql, params, options, callback) { switch (data.command) { case 'DELETE': case 'UPDATE': - result = {count: data.rowCount}; + result = {affectedRows: data.rowCount, count: data.rowCount}; break; default: result = data.rows; @@ -520,7 +520,7 @@ PostgreSQL.prototype.getPlaceholderForValue = function(key) { }; PostgreSQL.prototype.getCountForAffectedRows = function(model, info) { - return info && info.count; + return info && info.affectedRows; }; require('./discovery')(PostgreSQL); diff --git a/test/postgresql.test.js b/test/postgresql.test.js index f8893b7b..ecf05800 100644 --- a/test/postgresql.test.js +++ b/test/postgresql.test.js @@ -90,6 +90,21 @@ describe('postgresql connector', function() { }); }); + it('should preserve property `count` after query execution', function(done) { + Post.create( + {title: 'T10', content: 'C10'}, + function(err, p) { + if (err) return done(err); + post = p; + var query = "UPDATE PostWithBoolean SET title ='T20' WHERE id=" + post.id; + db.connector.execute(query, function(err, results) { + results.should.have.property('count', 1); + results.should.have.property('affectedRows', 1); + done(err); + }); + }); + }); + it('should support updating boolean types with false value', function(done) { Post.update({id: post.id}, {approved: false}, function(err) { should.not.exists(err); From e0f2814133117ce6be4c0b946def45244ebb21f6 Mon Sep 17 00:00:00 2001 From: tmclouisluk Date: Thu, 2 Feb 2017 10:27:22 +0800 Subject: [PATCH 04/10] Fix bug when using postgresql 8.x BUG: error: unterminated quoted string at or near \"'\\' ORDER BY \"uuid\"\" when using like filter. eg. { where: {name: {like: "%abc%"}}} --- lib/postgresql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/postgresql.js b/lib/postgresql.js index 9b404324..a9a3a625 100644 --- a/lib/postgresql.js +++ b/lib/postgresql.js @@ -387,10 +387,10 @@ PostgreSQL.prototype.buildExpression = function(columnName, operator, operatorValue, propertyDefinition) { switch (operator) { case 'like': - return new ParameterizedSQL(columnName + " LIKE ? ESCAPE '\\'", + return new ParameterizedSQL(columnName + " LIKE ? ESCAPE '\\\\'", [operatorValue]); case 'nlike': - return new ParameterizedSQL(columnName + " NOT LIKE ? ESCAPE '\\'", + return new ParameterizedSQL(columnName + " NOT LIKE ? ESCAPE '\\\\'", [operatorValue]); case 'regexp': if (operatorValue.global) From 50e9568b4b4838ea203e91f8ba6904ea2c986a6b Mon Sep 17 00:00:00 2001 From: tmclouisluk Date: Thu, 2 Feb 2017 14:42:17 +0800 Subject: [PATCH 05/10] Update postgresql.js --- lib/postgresql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/postgresql.js b/lib/postgresql.js index a9a3a625..13d78f0a 100644 --- a/lib/postgresql.js +++ b/lib/postgresql.js @@ -387,10 +387,10 @@ PostgreSQL.prototype.buildExpression = function(columnName, operator, operatorValue, propertyDefinition) { switch (operator) { case 'like': - return new ParameterizedSQL(columnName + " LIKE ? ESCAPE '\\\\'", + return new ParameterizedSQL(columnName + " LIKE ? ESCAPE E'\\\\'", [operatorValue]); case 'nlike': - return new ParameterizedSQL(columnName + " NOT LIKE ? ESCAPE '\\\\'", + return new ParameterizedSQL(columnName + " NOT LIKE ? ESCAPE E'\\\\'", [operatorValue]); case 'regexp': if (operatorValue.global) From 4304d16ecf21b2a4b23f13a52db537e31285c3e8 Mon Sep 17 00:00:00 2001 From: Diana Lau Date: Fri, 17 Feb 2017 11:33:37 -0500 Subject: [PATCH 06/10] update README for local postgres setup --- README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/README.md b/README.md index 9471d82e..2d5a6c13 100644 --- a/README.md +++ b/README.md @@ -356,3 +356,38 @@ to run them using our preconfigured test server. 1. Ask a core developer for instructions on how to set up test server credentials on your machine 2. `npm test` + +If you wish to run the tests using your own test database instance, + +__Set up the database__ + +1. Go to pgAdmin. +By default, the local database is one of the servers under Server Groups > Servers. +2. Under Login Roles, add a user called ```strongloop```. + +__Change configuration for database connection__ + +In ```test\init.js```, change the value of ```config``` to be pointing to the local database. For example, +``` + var config = { + host: 'localhost', + port: '5432', + database:'strongloop', + username: 'postgres', + password: 'postgres', + }; +``` + +__Troubleshooting__ + +When running npm test, it runs the ```pretest.js``` which eventually runs ```schema.sql``` to set up the database and tables. +If there is problem, you can run the ```schema.sql``` manually. To do this: + +1. Go to SQL Shell (psql) +2. Run: +``` +\i <> + +For example on Windows, +\i c:\somepath\test\schema.sql +``` \ No newline at end of file From cbb0393d5151f7453e6f5eb58160cc7fd3aba88a Mon Sep 17 00:00:00 2001 From: Zak Barbuto Date: Sat, 18 Feb 2017 07:38:24 +1030 Subject: [PATCH 07/10] Add test env information to README --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2d5a6c13..33815beb 100644 --- a/README.md +++ b/README.md @@ -350,8 +350,7 @@ Destroying models may result in errors due to foreign key integrity. First delet ## Running tests -The tests in this repository are mainly integration tests, meaning you will need -to run them using our preconfigured test server. +The tests in this repository are mainly integration tests, meaning you will need to run them using our preconfigured test server. 1. Ask a core developer for instructions on how to set up test server credentials on your machine @@ -378,9 +377,11 @@ In ```test\init.js```, change the value of ```config``` to be pointing to the lo }; ``` +2. (`Linux Only`) `CI=true PGHOST=localhost PGPORT= PGDATABASE= PGUSER= PGPASSWORD= npm test` + __Troubleshooting__ -When running npm test, it runs the ```pretest.js``` which eventually runs ```schema.sql``` to set up the database and tables. +When running `npm test`, it runs the ```pretest.js``` which eventually runs ```schema.sql``` to set up the database and tables. If there is problem, you can run the ```schema.sql``` manually. To do this: 1. Go to SQL Shell (psql) From ae9eccd9ebca8f08bfe3506b03ce8d11de1efea9 Mon Sep 17 00:00:00 2001 From: Zak Barbuto Date: Tue, 10 Jan 2017 16:08:19 +1030 Subject: [PATCH 08/10] Use pool.pool.release over pool.release (#109) --- lib/transaction.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/transaction.js b/lib/transaction.js index 953dee3b..07dbedc1 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -69,9 +69,9 @@ function mixinTransaction(PostgreSQL) { } else { var pool = this.pg; if (err) { - pool.destroy(connection); + pool.pool.destroy(connection); } else { - pool.release(connection); + pool.pool.release(connection); } } }; From c7fdeb9d1cd590765cb5ff9b18bfa7427a232ac3 Mon Sep 17 00:00:00 2001 From: Zak Barbuto Date: Tue, 10 Jan 2017 16:08:45 +1030 Subject: [PATCH 09/10] Use pg callback over connection.release (#109) --- lib/transaction.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/transaction.js b/lib/transaction.js index 07dbedc1..9db790fb 100644 --- a/lib/transaction.js +++ b/lib/transaction.js @@ -21,6 +21,7 @@ function mixinTransaction(PostgreSQL) { debug('Begin a transaction with isolation level: %s', isolationLevel); this.pg.connect(function(err, connection, done) { if (err) return cb(err); + connection.autorelease = done; connection.query('BEGIN TRANSACTION ISOLATION LEVEL ' + isolationLevel, function(err) { if (err) return cb(err); @@ -63,9 +64,9 @@ function mixinTransaction(PostgreSQL) { }; PostgreSQL.prototype.releaseConnection = function(connection, err) { - if (typeof connection.release === 'function') { - connection.release(err); - connection.release = null; + if (typeof connection.autorelease === 'function') { + connection.autorelease(err); + connection.autorelease = null; } else { var pool = this.pg; if (err) { From bbdeb90113eab867ceb0072cbf96285fa91db87d Mon Sep 17 00:00:00 2001 From: Zak Barbuto Date: Fri, 17 Feb 2017 09:08:16 +1030 Subject: [PATCH 10/10] Add test for bulk transactions --- test/postgresql.transaction.test.js | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/postgresql.transaction.test.js b/test/postgresql.transaction.test.js index ce4783aa..a9926498 100644 --- a/test/postgresql.transaction.test.js +++ b/test/postgresql.transaction.test.js @@ -58,6 +58,41 @@ describe('transactions', function() { }; } + describe('bulk', function() { + it('should work with bulk transactions', function(done) { + var completed = 0; + var concurrent = 20; + for (var i = 0; i <= concurrent; i++) { + var post = {title: 'tb' + i, content: 'cb' + i}; + var create = createPostInTx(post); + Transaction.begin(db.connector, Transaction.SERIALIZABLE, + function(err, tx) { + if (err) return done(err); + Post.create(post, {transaction: tx}, + function(err, p) { + if (err) { + done(err); + } else { + tx.commit(function(err) { + if (err) { + done(err); + } + completed++; + checkResults(); + }); + } + }); + }); + } + + function checkResults() { + if (completed === concurrent) { + done(); + } + } + }); + }); + describe('commit', function() { var post = {title: 't1', content: 'c1'}; before(createPostInTx(post));