Use postgres-pool instead of node-postgres (#438)

robknight · ichub · robknight · commit 8dd76f5514b6 · 2023-08-21T09:33:34.000+02:00
This fixes various issues with node-postgres, specifically that the connection pooling implementation does not work well: brianc/node-postgres#1611 brianc/node-postgres#2112 --------- Co-authored-by: Ivan Chub <ichub@users.noreply.github.com>
diff --git a/apps/passport-server/package.json b/apps/passport-server/package.json
@@ -39,6 +39,7 @@
     "p-queue": "6",
     "pg": "^8.8.0",
     "postgres-migrations": "^5.3.0",
+    "postgres-pool": "^7.0.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "request": "^2.88.2",
diff --git a/apps/passport-server/src/apis/devconnect/organizer.ts b/apps/passport-server/src/apis/devconnect/organizer.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { PretixOrganizersConfig } from "../../database/models";
 import { fetchPretixConfiguration } from "../../database/queries/pretix_config/fetchPretixConfiguration";
 import { logger } from "../../util/logger";
@@ -37,7 +37,7 @@ function pretixConfigDBToDevconnectPretixConfig(
             superuserItemIds: eventDB.superuser_item_ids
           })),
           token: organizerDB.token
-        }) satisfies DevconnectPretixOrganizerConfig
+        } satisfies DevconnectPretixOrganizerConfig)
     )
   };
 }
diff --git a/apps/passport-server/src/database/postgresConfiguration.ts b/apps/passport-server/src/database/postgresConfiguration.ts
@@ -1,4 +1,5 @@
-import { ClientConfig, PoolConfig } from "pg";
+import { ClientConfig } from "pg";
+import { PoolOptionsExplicit, SslSettings } from "postgres-pool";
 
 export interface DBConfiguration extends ClientConfig {
   user: string;
@@ -8,7 +9,7 @@ export interface DBConfiguration extends ClientConfig {
   port: number;
 }
 
-export function getDatabaseConfiguration(): PoolConfig {
+export function getDatabaseConfiguration(): PoolOptionsExplicit & SslSettings {
   if (process.env.DATABASE_USERNAME === undefined) {
     throw new Error("Missing environment variable: DATABASE_USERNAME");
   }
@@ -32,11 +33,14 @@ export function getDatabaseConfiguration(): PoolConfig {
     host: process.env.DATABASE_HOST,
     database: process.env.DATABASE_DB_NAME,
     port: 5432,
-    ssl: process.env.DATABASE_SSL === "true",
+    ssl:
+      process.env.DATABASE_SSL === "true"
+        ? { rejectUnauthorized: false }
+        : undefined,
 
     // Pool configuration
-    connectionTimeoutMillis: 1_000,
-    idleTimeoutMillis: 0,
-    max: 8,
+    connectionTimeoutMillis: 16_000,
+    idleTimeoutMillis: 8_000,
+    poolSize: 8
   };
 }
diff --git a/apps/passport-server/src/database/postgresPool.ts b/apps/passport-server/src/database/postgresPool.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { logger } from "../util/logger";
 import { getDatabaseConfiguration } from "./postgresConfiguration";
 import { migrateDatabase } from "./postgresMigrations";
diff --git a/apps/passport-server/src/database/queries/commitments.ts b/apps/passport-server/src/database/queries/commitments.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { CommitmentRow } from "../models";
 import { sqlQuery } from "../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/devconnect_pretix_tickets/fetchDevconnectPretixTicket.ts b/apps/passport-server/src/database/queries/devconnect_pretix_tickets/fetchDevconnectPretixTicket.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import {
   DevconnectPretixTicketDB,
   DevconnectPretixTicketDBWithEmailAndItem,
diff --git a/apps/passport-server/src/database/queries/devconnect_pretix_tickets/insertDevconnectPretixTicket.ts b/apps/passport-server/src/database/queries/devconnect_pretix_tickets/insertDevconnectPretixTicket.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { DevconnectPretixTicket } from "../../models";
 import { sqlQuery } from "../../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/devconnect_pretix_tickets/softDeleteDevconnectPretixTicket.ts b/apps/passport-server/src/database/queries/devconnect_pretix_tickets/softDeleteDevconnectPretixTicket.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { DevconnectPretixTicketDB } from "../../models";
 import { sqlQuery } from "../../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/devconnect_pretix_tickets/updateDevconnectPretixTicket.ts b/apps/passport-server/src/database/queries/devconnect_pretix_tickets/updateDevconnectPretixTicket.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { DevconnectPretixTicket } from "../../models";
 import { sqlQuery } from "../../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/e2ee.ts b/apps/passport-server/src/database/queries/e2ee.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { EncryptedStorageModel } from "../models";
 import { sqlQuery } from "../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/emailToken.ts b/apps/passport-server/src/database/queries/emailToken.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { sqlQuery } from "../sqlQuery";
 
 /**
diff --git a/apps/passport-server/src/database/queries/fetchStatus.ts b/apps/passport-server/src/database/queries/fetchStatus.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { sqlQuery } from "../sqlQuery";
 
 /**
diff --git a/apps/passport-server/src/database/queries/historicSemaphore.ts b/apps/passport-server/src/database/queries/historicSemaphore.ts
@@ -1,4 +1,5 @@
-import { Pool, QueryResultRow } from "pg";
+import { QueryResultRow } from "pg";
+import { Pool } from "postgres-pool";
 import { HistoricSemaphoreGroup } from "../models";
 import { sqlQuery } from "../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/pretixEventInfo.ts b/apps/passport-server/src/database/queries/pretixEventInfo.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { PretixEventInfo, PretixItemInfo } from "../models";
 import { sqlQuery } from "../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/pretixItemInfo.ts b/apps/passport-server/src/database/queries/pretixItemInfo.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { PretixItemInfo } from "../models";
 import { sqlQuery } from "../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/pretix_config/fetchPretixConfiguration.ts b/apps/passport-server/src/database/queries/pretix_config/fetchPretixConfiguration.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { PretixOrganizersConfig } from "../../models";
 import { sqlQuery } from "../../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/pretix_config/insertConfiguration.ts b/apps/passport-server/src/database/queries/pretix_config/insertConfiguration.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { PretixOrganizerRow } from "../../models";
 import { sqlQuery } from "../../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/saveCommitment.ts b/apps/passport-server/src/database/queries/saveCommitment.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { logger } from "../../util/logger";
 import { sqlQuery } from "../sqlQuery";
 
diff --git a/apps/passport-server/src/database/queries/zuzalu_pretix_tickets/deleteZuzaluUser.ts b/apps/passport-server/src/database/queries/zuzalu_pretix_tickets/deleteZuzaluUser.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { sqlQuery } from "../../sqlQuery";
 
 /**
@@ -10,7 +10,7 @@ export async function deleteZuzaluUser(
   email: string
 ): Promise<void> {
   await sqlQuery(client, `delete from zuzalu_pretix_tickets where email = $1`, [
-    email,
+    email
   ]);
 
   await sqlQuery(client, `delete from commitments where email = $1`, [email]);
diff --git a/apps/passport-server/src/database/queries/zuzalu_pretix_tickets/fetchZuzaluUser.ts b/apps/passport-server/src/database/queries/zuzalu_pretix_tickets/fetchZuzaluUser.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { LoggedInZuzaluUser, ZuzaluUser } from "../../models";
 import { sqlQuery } from "../../sqlQuery";
 
@@ -85,7 +85,8 @@ where c.uuid = $1;`,
 export async function fetchAllLoggedInZuzaluUsers(
   client: Pool
 ): Promise<LoggedInZuzaluUser[]> {
-  const result = await client.query(
+  const result = await sqlQuery(
+    client,
     `\
 select 
     c.uuid,
diff --git a/apps/passport-server/src/database/queries/zuzalu_pretix_tickets/insertZuzaluPretixTicket.ts b/apps/passport-server/src/database/queries/zuzalu_pretix_tickets/insertZuzaluPretixTicket.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { ZuzaluPretixTicket } from "../../models";
 import { sqlQuery } from "../../sqlQuery";
 
@@ -23,7 +23,7 @@ on conflict do nothing;`,
       params.order_id,
       params.visitor_date_ranges === undefined
         ? undefined
-        : JSON.stringify(params.visitor_date_ranges),
+        : JSON.stringify(params.visitor_date_ranges)
     ]
   );
   return result.rowCount;
diff --git a/apps/passport-server/src/database/queries/zuzalu_pretix_tickets/updateZuzaluPretixTicket.ts b/apps/passport-server/src/database/queries/zuzalu_pretix_tickets/updateZuzaluPretixTicket.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { ZuzaluPretixTicket } from "../../models";
 import { sqlQuery } from "../../sqlQuery";
 
@@ -21,7 +21,7 @@ where email=$1;`,
       params.visitor_date_ranges === undefined
         ? undefined
         : JSON.stringify(params.visitor_date_ranges),
-      params.name,
+      params.name
     ]
   );
   return result.rowCount;
diff --git a/apps/passport-server/src/database/sqlQuery.ts b/apps/passport-server/src/database/sqlQuery.ts
@@ -1,4 +1,5 @@
-import { Pool, QueryResult } from "pg";
+import { QueryResult } from "pg";
+import { Pool } from "postgres-pool";
 import { traced } from "../services/telemetryService";
 
 /**
diff --git a/apps/passport-server/src/services/devconnectPretixSyncService.ts b/apps/passport-server/src/services/devconnectPretixSyncService.ts
@@ -1,4 +1,4 @@
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import {
   DevconnectPretixEvent,
   DevconnectPretixEventSettings,
diff --git a/apps/passport-server/src/services/pretixSyncService.ts b/apps/passport-server/src/services/pretixSyncService.ts
@@ -1,5 +1,5 @@
 import { DateRange } from "@pcd/passport-interface";
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { IPretixAPI, PretixOrder, PretixSubevent } from "../apis/pretixAPI";
 import { ZuzaluPretixTicket, ZuzaluUserRole } from "../database/models";
 import { deleteZuzaluUser } from "../database/queries/zuzalu_pretix_tickets/deleteZuzaluUser";
@@ -10,7 +10,7 @@ import { ApplicationContext } from "../types";
 import { logger } from "../util/logger";
 import {
   pretixTicketsDifferent,
-  ticketsToMapByEmail,
+  ticketsToMapByEmail
 } from "../util/zuzaluUser";
 import { RollbarService } from "./rollbarService";
 import { SemaphoreService } from "./semaphoreService";
@@ -322,7 +322,7 @@ export class PretixSyncService {
           (subEvent) =>
             ({
               date_from: subEvent?.date_from,
-              date_to: subEvent?.date_to,
+              date_to: subEvent?.date_to
             } satisfies DateRange)
         );
 
@@ -331,7 +331,7 @@ export class PretixSyncService {
           email: (o.email || o.positions[0].attendee_email).toLowerCase(),
           name: o.positions[0].attendee_name,
           order_id: o.code,
-          visitor_date_ranges: visitorDateRanges,
+          visitor_date_ranges: visitorDateRanges
         } satisfies ZuzaluPretixTicket;
       });
 
diff --git a/apps/passport-server/src/services/semaphoreService.ts b/apps/passport-server/src/services/semaphoreService.ts
@@ -1,6 +1,6 @@
 import { serializeSemaphoreGroup } from "@pcd/semaphore-group-pcd";
 import { Group } from "@semaphore-protocol/group";
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import {
   CommitmentRow,
   HistoricSemaphoreGroup,
diff --git a/apps/passport-server/src/types.ts b/apps/passport-server/src/types.ts
@@ -1,7 +1,7 @@
 import { Application } from "express";
 import * as http from "http";
 import Libhoney from "libhoney";
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { IDevconnectPretixAPI } from "./apis/devconnect/devconnectPretixAPI";
 import { IEmailAPI } from "./apis/emailAPI";
 import { IPretixAPI } from "./apis/pretixAPI";
diff --git a/apps/passport-server/test/database.spec.ts b/apps/passport-server/test/database.spec.ts
@@ -2,29 +2,29 @@ import { Identity } from "@semaphore-protocol/identity";
 import { expect } from "chai";
 import "mocha";
 import { step } from "mocha-steps";
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { ZuzaluPretixTicket, ZuzaluUserRole } from "../src/database/models";
 import { getDB } from "../src/database/postgresPool";
 import {
   fetchAllCommitments,
   fetchCommitment,
   fetchCommitmentByPublicCommitment,
-  removeCommitment,
+  removeCommitment
 } from "../src/database/queries/commitments";
 import {
   fetchEncryptedStorage,
-  insertEncryptedStorage,
+  insertEncryptedStorage
 } from "../src/database/queries/e2ee";
 import {
   fetchEmailToken,
-  insertEmailToken,
+  insertEmailToken
 } from "../src/database/queries/emailToken";
 import { insertCommitment } from "../src/database/queries/saveCommitment";
 import { deleteZuzaluUser } from "../src/database/queries/zuzalu_pretix_tickets/deleteZuzaluUser";
 import {
   fetchAllLoggedInZuzaluUsers,
   fetchLoggedInZuzaluUser,
-  fetchZuzaluUser,
+  fetchZuzaluUser
 } from "../src/database/queries/zuzalu_pretix_tickets/fetchZuzaluUser";
 import { insertZuzaluPretixTicket } from "../src/database/queries/zuzalu_pretix_tickets/insertZuzaluPretixTicket";
 import { updateZuzaluPretixTicket } from "../src/database/queries/zuzalu_pretix_tickets/updateZuzaluPretixTicket";
@@ -71,7 +71,7 @@ describe("database reads and writes", function () {
       name: "bob shmob",
       order_id: "ASD12",
       role: ZuzaluUserRole.Organizer,
-      visitor_date_ranges: null,
+      visitor_date_ranges: null
     };
     otherRole = ZuzaluUserRole.Visitor;
 
@@ -101,7 +101,7 @@ describe("database reads and writes", function () {
       const newCommitment = newIdentity.commitment.toString();
       const newUuid = await insertCommitment(db, {
         email: testTicket.email,
-        commitment: newCommitment,
+        commitment: newCommitment
       });
 
       const loggedinUser = await fetchLoggedInZuzaluUser(db, { uuid: newUuid });
@@ -154,8 +154,8 @@ describe("database reads and writes", function () {
       email: testTicket.email,
       role: otherRole,
       visitor_date_ranges: [
-        { date_from: new Date().toString(), date_to: new Date().toString() },
-      ],
+        { date_from: new Date().toString(), date_to: new Date().toString() }
+      ]
     };
 
     await updateZuzaluPretixTicket(db, update);
@@ -220,7 +220,7 @@ describe("database reads and writes", function () {
     const commitment = new Identity().commitment.toString();
     const uuid = await insertCommitment(db, {
       commitment,
-      email,
+      email
     });
     if (!uuid) {
       throw new Error("expected to be able to insert a commitment");
diff --git a/apps/passport-server/test/devconnect.spec.ts b/apps/passport-server/test/devconnect.spec.ts
@@ -16,7 +16,7 @@ import { expect } from "chai";
 import _ from "lodash";
 import "mocha";
 import NodeRSA from "node-rsa";
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import {
   DevconnectPretixConfig,
   getDevconnectPretixConfig
diff --git a/apps/passport-server/test/devconnectdb.spec.ts b/apps/passport-server/test/devconnectdb.spec.ts
@@ -1,7 +1,7 @@
 import { expect } from "chai";
 import "mocha";
 import { step } from "mocha-steps";
-import { Pool } from "pg";
+import { Pool } from "postgres-pool";
 import { v4 as uuid } from "uuid";
 import {
   DevconnectPretixTicket,
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import { Pool } from "pg";`
	`1`	`+import { Pool } from "postgres-pool";`
`2`	`2`	`import { PretixOrganizersConfig } from "../../database/models";`
`3`	`3`	`import { fetchPretixConfiguration } from "../../database/queries/pretix_config/fetchPretixConfiguration";`
`4`	`4`	`import { logger } from "../../util/logger";`
`@@ -37,7 +37,7 @@ function pretixConfigDBToDevconnectPretixConfig(`
`37`	`37`	`superuserItemIds: eventDB.superuser_item_ids`
`38`	`38`	`})),`
`39`	`39`	`token: organizerDB.token`
`40`		`- }) satisfies DevconnectPretixOrganizerConfig`
	`40`	`+ } satisfies DevconnectPretixOrganizerConfig)`
`41`	`41`	`)`
`42`	`42`	`};`
`43`	`43`	`}`