Skip to content

Add MSSQL DB client #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 47 commits into from
Nov 3, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
a2489e5
add mssql db type and client boilerplate
Oct 26, 2022
255e3ee
docker-compose setup for test database
Oct 22, 2022
decfeda
init logic
Oct 22, 2022
a46f38f
check endpoint for mssql + test runner
Oct 26, 2022
bfa60cb
query stream
Oct 24, 2022
32b5a0a
data type schema implementation
Oct 24, 2022
ad73ffc
missing package
Oct 26, 2022
895d108
missing errors message
Oct 24, 2022
2b3157d
import
Oct 24, 2022
800ee28
testing
Oct 26, 2022
05627d3
action for unit-test
Oct 24, 2022
ef7b65e
action
Oct 24, 2022
12588d3
only one version of node
Oct 24, 2022
f8585cd
keep alive
Oct 24, 2022
54038be
tweaked response schema
Oct 24, 2022
cc3be3d
testing with docker containers
Oct 26, 2022
7f6c873
tests for check and simple query
Oct 25, 2022
f62b3d0
cleanup
Oct 25, 2022
3e8f72b
removed type
Oct 25, 2022
1caf1d2
removed gh actions
Oct 25, 2022
907c9aa
simplify mocha config
Oct 25, 2022
ca10aab
test for stream
Oct 25, 2022
6ce331d
added testing commands
Oct 26, 2022
ca7d154
cleanup
Oct 25, 2022
46e5c4a
fix wait-on position
Oct 25, 2022
033681b
export mssql client
Oct 25, 2022
a2c2635
no support for /query endpoint
Oct 26, 2022
dbd1590
post module merge adjustments
Oct 26, 2022
6710555
post module merge 2.0
Oct 26, 2022
03e2f1c
mock at the right place
Oct 26, 2022
339069b
fix UncaughtError
Oct 26, 2022
2c9e0bd
fix tests
Oct 27, 2022
d3782b1
tests for check
Oct 27, 2022
f22d334
support for cell references in SQL queries
Oct 27, 2022
ce49f93
cleanup unused
Oct 27, 2022
178be27
removed SHOW verbs from READ_ONLY
Oct 31, 2022
724e017
renamed query to queryStream
Oct 31, 2022
b2f8754
removed confusing comment
Oct 31, 2022
b71e96b
cleaner error msg on missing creds
Oct 31, 2022
d37510c
throwing not implemented instead of not found
Oct 31, 2022
721fc8b
queryTag to be handled on the client side
Oct 31, 2022
e739301
type adjust
Nov 1, 2022
468995a
comment to justify check function
Nov 2, 2022
827b752
make origin of bak file obvious
Nov 2, 2022
a9f565a
fix data type schema lookup
Nov 2, 2022
5aff01f
tests
Nov 2, 2022
440543f
cleanup TODO
Nov 3, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .babelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"presets": ["@babel/preset-env"]
}
3 changes: 3 additions & 0 deletions .env.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
NODE_ENV=test
MSSQL_CREDENTIALS:Server=mssql,1433;Database=master;User Id=sa;Password=Pass@word;trustServerCertificate=true;
MSSQL_CREDENTIALS_READ_ONLY:Server=mssql,1433;Database=master;User Id=reader;Password=re@derP@ssw0rd;trustServerCertificate=true;
11 changes: 11 additions & 0 deletions .env.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@

export const MSSQL_CREDENTIALS = env("MSSQL_CREDENTIALS");
export const MSSQL_CREDENTIALS_READ_ONLY = env("MSSQL_CREDENTIALS_READ_ONLY");
export const NODE_ENV = env("NODE_ENV");

function env(key, defaultValue) {
const value = process.env[key]; // eslint-disable-line no-process-env
if (value !== undefined) return value;
if (defaultValue !== undefined) return defaultValue;
throw new Error(`Missing environment variable: ${key}`);
}
8 changes: 8 additions & 0 deletions .mocharc.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
module.exports = {
color: true,
extension: ["js"],
global: [],
ignore: [],
require: "@babel/register",
spec: ["test/**/*.test.js"],
};
17 changes: 17 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
FROM node:16.17.0-alpine

RUN mkdir /app
WORKDIR /app

RUN apk --no-cache add bash

COPY package.json yarn.lock /app/
RUN \
yarn --frozen-lockfile && \
yarn cache clean

ENV PATH="/app/node_modules/.bin:${PATH}"

COPY . /app/

CMD yarn test
Binary file added data/AdventureWorks2019.bak
Binary file not shown.
32 changes: 32 additions & 0 deletions data/seed.mssql.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import mssql from "mssql";
import {MSSQL_CREDENTIALS} from "../.env.test.js";

const credentials = MSSQL_CREDENTIALS;

const seed = async () => {
await mssql.connect(credentials);

await mssql.query`RESTORE DATABASE test
FROM DISK = '/var/opt/mssql/backup/test.bak'
WITH REPLACE, RECOVERY,
MOVE 'AdventureWorksLT2012_Data' TO '/var/opt/mssql/data/aw2019.mdf',
MOVE 'AdventureWorksLT2012_Log' TO '/var/opt/mssql/data/aw2019.ldf';`;

await mssql.query`IF NOT EXISTS(SELECT name
FROM sys.syslogins
WHERE name='reader')
BEGIN
CREATE LOGIN reader WITH PASSWORD = 're@derP@ssw0rd'
CREATE USER reader FOR LOGIN reader
END`;
};

seed()
.then(() => {
console.log(`MS_SQL DB seeded.`);
process.exit(0);
})
.catch((err) => {
console.error(err.message, err);
process.exit(1);
});
7 changes: 7 additions & 0 deletions docker-compose.local.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
version: "3.7"

services:
mssql:
image: mcr.microsoft.com/azure-sql-edge
expose:
- "1433"
30 changes: 30 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
version: "3.7"

services:
test:
build: .
depends_on:
- mssql
env_file:
- .env.test
networks:
- db_proxy_test
command: sh -c "set -o pipefail && wait-on -d 10000 -t 30000 tcp:mssql:1433 && node ./data/seed.mssql.js && TZ=UTC NODE_ENV=TEST node_modules/.bin/mocha"

mssql:
image: mcr.microsoft.com/mssql/server:2019-latest
environment:
- MSSQL_SA_PASSWORD=Pass@word
- ACCEPT_EULA=Y
- MSSQL_DATABASE=test
- MSSQL_SLEEP=7
volumes:
- ./data/AdventureWorks2019.bak:/var/opt/mssql/backup/test.bak
ports:
- "1433:1433"
networks:
- db_proxy_test

networks:
db_proxy_test:
name: db_proxy_test
12 changes: 9 additions & 3 deletions lib/errors.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
import {createError} from "micro";
export const unauthorized = error => createError(401, "Unauthorized", error);
export const notFound = error => createError(404, "Not Found", error);
export const exit = message => {
export const unauthorized = (error) => createError(401, "Unauthorized", error);
export const notFound = (error) => createError(404, "Not Found", error);
export const notImplemented = (error) =>
createError(501, "Not Implemented", error);
export const badRequest = (error) =>
createError(400, typeof error === "string" ? error : "Bad request", error);
export const failedCheck = (error) =>
createError(200, typeof error === "string" ? error : "Failed check", error);
export const exit = (message) => {
console.error(message); // eslint-disable-line no-console
process.exit(1);
};
205 changes: 205 additions & 0 deletions lib/mssql.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
import Ajv from "ajv";
import JSONStream from "JSONStream";
import {json} from "micro";
import mssql from "mssql";
import {failedCheck, badRequest, notImplemented} from "./errors.js";

const TYPES = mssql.TYPES;

const pools = new Map();

const READ_ONLY = new Set(["SELECT", "USAGE"]);

const ajv = new Ajv();
const validate = ajv.compile({
type: "object",
additionalProperties: false,
required: ["sql"],
properties: {
sql: {type: "string", minLength: 1},
params: {type: "array"},
},
});

// See: https://tediousjs.github.io/node-mssql/#connection-pools
export const mssqlPool = {
get: (name, config) => {
if (!pools.has(name)) {
if (!config) {
throw new Error("Database configuration required");
}

const pool = new mssql.ConnectionPool(config);
const close = pool.close.bind(pool);
pool.close = (...args) => {
pools.delete(name);
return close(...args);
};

pools.set(name, pool.connect());
}

return pools.get(name);
},

closeAll: () =>
Promise.all(
Array.from(pools.values()).map((connect) => {
return connect.then((pool) => pool.close());
})
),
};

export async function queryStream(req, res, pool) {
const db = await pool;
const body = await json(req);

if (!validate(body)) throw badRequest();

res.setHeader("Content-Type", "text/plain");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't seem like the right content-type? I can't remember if there is a reason for this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ping on this

Copy link
Author

@Sylvestre67 Sylvestre67 Nov 3, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reason for this header lies on how the compression library is filtering the request it needs to compress or not. Compression happens in data connector (here) and is filtered/applied by the compression library following this /^text\/|\+(?:json|text|xml)$/i re-pattern (see here)

const keepAlive = setInterval(() => res.write("\n"), 25e3);

let {sql, params = []} = body;

try {
await new Promise((resolve, reject) => {
const request = new mssql.Request(db);
const stream = request.toReadableStream();

params.forEach((param, idx) => {
request.input(`${idx + 1}`, param);
});

request.query(sql);
request.once("recordset", () => clearInterval(keepAlive));
request.on("recordset", (columns) => {
const schema = {
type: "array",
items: {
type: "object",
properties: Object.entries(columns).reduce(
(schema, [name, props]) => {
return {
...schema,
...{[name]: dataTypeSchema({type: props.type.name})},
};
},
{}
),
},
};

res.write(`${JSON.stringify(schema)}`);
res.write("\n");
});

stream.pipe(JSONStream.stringify("", "\n", "\n")).pipe(res);
stream.on("done", () => {
resolve();
});
stream.on("error", (error) => {
if (!request.canceled) {
request.cancel();
}
reject(error);
});
});
} catch (error) {
if (!error.statusCode) error.statusCode = 400;
throw error;
} finally {
clearInterval(keepAlive);
}

res.end();
}

/*
* This function is checking for the permission of the given credentials. It alerts the user setting
* them up that these may be too permissive.
* */
export async function check(req, res, pool) {
const db = await pool;

// See: https://learn.microsoft.com/en-us/sql/relational-databases/system-functions/sys-fn-my-permissions-transact-sql
const rows = await db.request().query(
`USE ${db.config.database};
SELECT * FROM fn_my_permissions (NULL, 'DATABASE');`
);

const grants = rows.recordset.map((rs) => rs.permission_name);
const permissive = grants.filter((g) => !READ_ONLY.has(g));

if (permissive.length)
throw failedCheck(
`User has too permissive grants: ${permissive.join(", ")}`
);

return {ok: true};
}

export default (credentials) => async (req, res) => {
const pool = mssqlPool.get(JSON.stringify(credentials), credentials);

if (req.method === "POST") {
if (req.url === "/check") {
return check(req, res, pool);
}

if (["/query-stream"].includes(req.url)) {
return queryStream(req, res, pool);
}

throw notImplemented();
}
};

// See https://github.com/tediousjs/node-mssql/blob/66587d97c9ce21bffba8ca360c72a540f2bc47a6/lib/datatypes.js#L6
const boolean = ["null", "boolean"],
integer = ["null", "integer"],
number = ["null", "number"],
object = ["null", "object"],
string = ["null", "string"];
export function dataTypeSchema({type}) {
switch (type) {
case TYPES.Bit.name:
return {type: boolean};
case TYPES.TinyInt.name:
return {type: integer, tiny: true};
case TYPES.SmallInt.name:
return {type: integer, short: true};
case TYPES.BigInt.name:
return {type: integer, long: true};
case TYPES.Int.name:
return {type: integer};
case TYPES.Float.name:
return {type: number, float: true};
case TYPES.Numeric.name:
return {type: number};
case TYPES.Decimal.name:
return {type: number, decimal: true};
case TYPES.Real.name:
return {type: number};
case TYPES.Date.name:
case TYPES.DateTime.name:
case TYPES.DateTime2.name:
case TYPES.DateTimeOffset.name:
case TYPES.SmallDateTime.name:
case TYPES.Time.name:
return {type: string, date: true};
case TYPES.Binary.name:
case TYPES.VarBinary.name:
case TYPES.Image.name:
return {type: object, buffer: true};
case TYPES.SmallMoney.name: // TODO
case TYPES.Money.name: //TODO
case TYPES.Xml.name: //TODO
case TYPES.TVP.name: //TODO
case TYPES.UDT.name: //TODO
case TYPES.Geography.name: //TODO
case TYPES.Geometry.name: //TODO
case TYPES.Variant.name: //TODO
default:
return {type: string};
}
}
7 changes: 5 additions & 2 deletions lib/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {notFound, unauthorized, exit} from "./errors.js";
import mysql from "./mysql.js";
import postgres from "./postgres.js";
import snowflake from "./snowflake.js";
import mssql from "./mssql.js";

export function server(config, argv) {
const development = process.env.NODE_ENV === "development";
Expand All @@ -22,7 +23,7 @@ export function server(config, argv) {
url,
ssl = "disabled",
host = "127.0.0.1",
port = 2899
port = 2899,
} = config;

const handler =
Expand All @@ -32,6 +33,8 @@ export function server(config, argv) {
? postgres(url)
: type === "snowflake"
? snowflake(url)
: type === "mssql"
? mssql(url)
: null;
if (!handler) {
return exit(`Unknown database type: ${type}`);
Expand Down Expand Up @@ -85,7 +88,7 @@ export function server(config, argv) {

const [payload, hmac] = authorization
.split(".")
.map(encoded => Buffer.from(encoded, "base64"));
.map((encoded) => Buffer.from(encoded, "base64"));
const {name} = JSON.parse(payload);
if (config.name !== name) throw notFound();
const {origin, secret} = config;
Expand Down
Loading