From 03f04fd8d5cdbe1ed4c1ae897f50b4ba7d01b728 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Wed, 3 Jul 2019 22:44:34 +1000 Subject: [PATCH] replace express-static-gzip with gunzipping ver. express-static-gzip doesn't gunzip gzipped files for clients that don't have 'Accept-Encoding: gzip', so it has been replaced by a simple clone that does the same thing as the original module and also uses zlib to gunzip the .gz file. This middleware is compatible with files that aren't .gz in the rootFolder, although it will not gzip them on the fly (to my knowledge). --- packages/server/package.json | 5 +- packages/server/src/server.ts | 8 +- packages/server/src/static.ts | 142 ++++++++++++++++++++++++++++++++++ packages/server/yarn.lock | 107 ++++++++++++++++++++++--- 4 files changed, 245 insertions(+), 17 deletions(-) create mode 100644 packages/server/src/static.ts diff --git a/packages/server/package.json b/packages/server/package.json index 88e71d9d021f..eca4991f24d6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -12,18 +12,19 @@ "@coder/nbin": "^1.1.2", "commander": "^2.19.0", "express": "^4.16.4", - "express-static-gzip": "^1.1.3", "httpolyglot": "^0.1.2", - "mime-types": "^2.1.21", + "mime-types": "^2.1.24", "node-netstat": "^1.6.0", "pem": "^1.14.1", "promise.prototype.finally": "^3.1.0", "safe-compare": "^1.1.4", + "serve-static": "^1.14.1", "ws": "^6.1.2", "xhr2": "^0.1.4" }, "devDependencies": { "@types/commander": "^2.12.2", + "@types/etag": "^1.8.0", "@types/express": "^4.16.0", "@types/fs-extra": "^5.0.4", "@types/mime-types": "^2.1.0", diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index b7d9a12cdbee..e55e9961d06b 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -5,7 +5,6 @@ import { TunnelCloseCode } from "@coder/tunnel/src/common"; import { handle as handleTunnel } from "@coder/tunnel/src/server"; import * as express from "express"; //@ts-ignore -import * as expressStaticGzip from "express-static-gzip"; import * as fs from "fs"; import { mkdirp } from "fs-extra"; import * as http from "http"; @@ -22,6 +21,7 @@ import * as url from "url"; import * as ws from "ws"; import { buildDir } from "./constants"; import { createPortScanner } from "./portScanner"; +import { staticGzip } from "./static"; import safeCompare = require("safe-compare"); interface CreateAppOptions { @@ -210,9 +210,6 @@ export const createApp = async (options: CreateAppOptions): Promise<{ return res.redirect(code, newUrlString); }; - const baseDir = buildDir || path.join(__dirname, ".."); - const staticGzip = expressStaticGzip(path.join(baseDir, "build/web")); - app.use((req, res, next) => { logger.trace(`\u001B[1m${req.method} ${res.statusCode} \u001B[0m${req.originalUrl}`, field("host", req.hostname), field("ip", req.ip)); @@ -325,7 +322,8 @@ export const createApp = async (options: CreateAppOptions): Promise<{ }); // Everything else just pulls from the static build directory. - app.use(staticGzip); + const baseDir = buildDir || path.join(__dirname, ".."); + app.use(staticGzip(path.join(baseDir, "build/web"))); return { express: app, diff --git a/packages/server/src/static.ts b/packages/server/src/static.ts new file mode 100644 index 000000000000..e1e744d36e72 --- /dev/null +++ b/packages/server/src/static.ts @@ -0,0 +1,142 @@ +import { field, logger } from "@coder/logger"; +import * as getETag from "etag"; +import * as express from "express"; +import * as fs from "fs"; +import * as mime from "mime-types"; +import * as path from "path"; +import * as serveStatic from "serve-static"; +import * as zlib from "zlib"; + +const defaultContentType = "application/octet-stream"; + +interface FileData { + readonly etag: string; + readonly lastModified: Date; +} + +const listGzFilesRecursive = (rootFolder: string, subFolder: string): { [s: string]: FileData } => { + let files: { [s: string]: FileData } = {}; + + const fullDir = path.join(rootFolder, subFolder); + const contents = fs.readdirSync(fullDir); + for (let file of contents) { + let filePath = path.join(subFolder, file); + const fileFullPath = path.join(fullDir, file); + + const stats = fs.statSync(fileFullPath); + if (stats.isDirectory()) { + files = { ...files, ...listGzFilesRecursive(rootFolder, filePath) }; + } else if (filePath.endsWith(".gz")) { + try { + filePath = filePath.replace(/\.gz$/i, ""); + const etag = getETag(stats); + const mtime = Math.round(stats.mtime/1000)*1000; + const lastModified = new Date(mtime); + files[filePath] = { etag, lastModified }; + } catch (err) { + logger.warn("failed to stat file in listGzFilesRecursive", field("filePath", fileFullPath), field("error", err.message)); + } + } + } + + return files; +}; + +const setPath = (req: express.Request, path: string): void => { + let query = req.url.split("?").splice(1).join("?"); + if (query !== "") { + query = "?" + query; + } + req.url = path + query; +}; + +// staticGzip returns middleware that serves pre-gzipped files from rootFolder. +// If the client doesn't support gzipped responses, the file will be +// gunzipped. After initializing staticGzip, files in rootFolder shouldn't be +// modified (and new files shouldn't be added) as changes won't be reflected in +// responses. +export const staticGzip = (rootFolder: string): express.RequestHandler => { + if (!fs.existsSync(rootFolder)) { + throw new Error("staticGzip: rootFolder does not exist"); + } + + const expressStatic = serveStatic(rootFolder); + const gzipFiles = listGzFilesRecursive(rootFolder, "/"); + + return (req: express.Request, res: express.Response, next: express.NextFunction): void => { + if (req.method !== "GET" && req.method !== "HEAD") { + return next(); + } + + if (req.path.endsWith("/")) { + setPath(req, req.path + "index.html"); + } + + // Check for 404, and let expressStatic handle it if it is. This + // also allows for clients to download raw .gz files or files that aren't gzipped. + if (!gzipFiles.hasOwnProperty(req.path)) { + return expressStatic(req, res, next); + } + + // If we're going to try serving a gzip using serve-static, we + // need to remove the 'Range' header as you can't gunzip partial + // gzips. Unfourtunately, 'Accept-Ranges' is still returned in serve-static responses. + req.headers["range"] = ""; + + let contentType = mime.contentType(path.extname(req.path)); + if (contentType === false) { + contentType = defaultContentType; + } + res.setHeader("Content-Type", contentType); + res.setHeader("Vary", "Accept-Encoding"); + + // Send .gz file directly from disk. + if (req.acceptsEncodings("gzip")) { + setPath(req, req.path + ".gz"); + res.setHeader("Content-Encoding", "gzip"); + + return expressStatic(req, res, next); + } + + const filePath = path.join(rootFolder, req.path + ".gz"); + const fileData = gzipFiles[req.path]; + + // Set 'ETag' and 'Last-Modified' headers. + res.setHeader("ETag", fileData.etag); + res.setHeader("Last-Modified", fileData.lastModified.toUTCString()); + + // Try to send 304 Not Modified response by checking 'If-Match' + // and 'If-Modified-Since' headers. + const ifMatch = req.get("if-match"); + const ifModifiedSince = req.get("if-modified-since"); + if (ifMatch === fileData.etag) { + res.status(304); // Not Modified + + return res.end(); + } + if (ifModifiedSince) { + const ifModifiedSinceDate = new Date(ifModifiedSince); + if (fileData.lastModified.getTime() <= ifModifiedSinceDate.getTime()) { + res.status(304); // Not Modified + + return res.end(); + } + } + + // Gunzip and return stream. We don't know the resulting + // filesize, so we don't set 'Content-Length'. + try { + const file = fs.createReadStream(filePath); + const gunzip = zlib.createGunzip(); + const stream = gunzip.pipe(res); + stream.on("end", () => { + file.close(); + gunzip.close(); + res.end(); + }); + file.pipe(gunzip); + } catch (err) { + next(err); + } + }; +}; diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index b97e1daaed03..e005ef65d5a7 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -40,6 +40,13 @@ dependencies: "@types/node" "*" +"@types/etag@^1.8.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@types/etag/-/etag-1.8.0.tgz#37f0b1f3ea46da7ae319bbedb607e375b4c99f7e" + integrity sha512-EdSN0x+Y0/lBv7YAb8IU4Jgm6DWM+Bqtz7o5qozl96fzaqdqbdfHS5qjdpFeIv7xQ8jSLyjMMNShgYtMajEHyQ== + dependencies: + "@types/node" "*" + "@types/events@*": version "1.2.0" resolved "https://registry.yarnpkg.com/@types/events/-/events-1.2.0.tgz#81a6731ce4df43619e5c8c945383b3e62a89ea86" @@ -426,13 +433,6 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= -express-static-gzip@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/express-static-gzip/-/express-static-gzip-1.1.3.tgz#345ea02637d9d5865777d6fb57ccc0884abcda65" - integrity sha512-k8Q4Dx4PDpzEb8kth4uiPWrBeJWJYSgnWMzNdjQUOsEyXfYKbsyZDkU/uXYKcorRwOie5Vzp4RMEVrJLMfB6rA== - dependencies: - serve-static "^1.12.3" - express@^4.16.4: version "4.16.4" resolved "https://registry.yarnpkg.com/express/-/express-4.16.4.tgz#fddef61926109e24c515ea97fd2f1bdbf62df12e" @@ -562,6 +562,17 @@ http-errors@1.6.3, http-errors@~1.6.2, http-errors@~1.6.3: setprototypeof "1.1.0" statuses ">= 1.4.0 < 2" +http-errors@~1.7.2: + version "1.7.3" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" + integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.1.1" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + httpolyglot@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/httpolyglot/-/httpolyglot-0.1.2.tgz#e4d347fe8984a62f467d4060df527f1851f6997b" @@ -587,6 +598,11 @@ inherits@2, inherits@2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= +inherits@2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + ipaddr.js@1.8.0: version "1.8.0" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.8.0.tgz#eaa33d6ddd7ace8f7f6fe0c9ca0440e706738b1e" @@ -696,12 +712,24 @@ methods@~1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= +mime-db@1.40.0: + version "1.40.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32" + integrity sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA== + mime-db@~1.37.0: version "1.37.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg== -mime-types@^2.1.21, mime-types@~2.1.18: +mime-types@^2.1.24: + version "2.1.24" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.24.tgz#b6f8d0b3e951efb77dedeca194cff6d16f676f81" + integrity sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ== + dependencies: + mime-db "1.40.0" + +mime-types@~2.1.18: version "2.1.21" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg== @@ -713,6 +741,11 @@ mime@1.4.1: resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" integrity sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ== +mime@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" @@ -747,6 +780,11 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= +ms@2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" + integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== + negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" @@ -824,6 +862,11 @@ parseurl@~1.3.2: resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" integrity sha1-/CidTtiZMRlGDBViUyYs3I3mW/M= +parseurl@~1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" + integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== + path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" @@ -871,6 +914,11 @@ range-parser@~1.2.0: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e" integrity sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4= +range-parser@~1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== + raw-body@2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.3.tgz#1b324ece6b5706e153855bc1148c65bb7f6ea0c3" @@ -925,7 +973,26 @@ send@0.16.2: range-parser "~1.2.0" statuses "~1.4.0" -serve-static@1.13.2, serve-static@^1.12.3: +send@0.17.1: + version "0.17.1" + resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" + integrity sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg== + dependencies: + debug "2.6.9" + depd "~1.1.2" + destroy "~1.0.4" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + fresh "0.5.2" + http-errors "~1.7.2" + mime "1.6.0" + ms "2.1.1" + on-finished "~2.3.0" + range-parser "~1.2.1" + statuses "~1.5.0" + +serve-static@1.13.2: version "1.13.2" resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.2.tgz#095e8472fd5b46237db50ce486a43f4b86c6cec1" integrity sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw== @@ -935,11 +1002,26 @@ serve-static@1.13.2, serve-static@^1.12.3: parseurl "~1.3.2" send "0.16.2" +serve-static@^1.14.1: + version "1.14.1" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.1.tgz#666e636dc4f010f7ef29970a88a674320898b2f9" + integrity sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg== + dependencies: + encodeurl "~1.0.2" + escape-html "~1.0.3" + parseurl "~1.3.3" + send "0.17.1" + setprototypeof@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" integrity sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ== +setprototypeof@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" + integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== + signal-exit@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" @@ -965,7 +1047,7 @@ source-map@~0.1.38: dependencies: amdefine ">=0.0.4" -"statuses@>= 1.4.0 < 2": +"statuses@>= 1.4.0 < 2", "statuses@>= 1.5.0 < 2", statuses@~1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= @@ -1013,6 +1095,11 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" +toidentifier@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" + integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== + ts-node@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-7.0.1.tgz#9562dc2d1e6d248d24bc55f773e3f614337d9baf"