diff --git a/packages/server/package.json b/packages/server/package.json index 88e71d9d021f..eca4991f24d6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -12,18 +12,19 @@ "@coder/nbin": "^1.1.2", "commander": "^2.19.0", "express": "^4.16.4", - "express-static-gzip": "^1.1.3", "httpolyglot": "^0.1.2", - "mime-types": "^2.1.21", + "mime-types": "^2.1.24", "node-netstat": "^1.6.0", "pem": "^1.14.1", "promise.prototype.finally": "^3.1.0", "safe-compare": "^1.1.4", + "serve-static": "^1.14.1", "ws": "^6.1.2", "xhr2": "^0.1.4" }, "devDependencies": { "@types/commander": "^2.12.2", + "@types/etag": "^1.8.0", "@types/express": "^4.16.0", "@types/fs-extra": "^5.0.4", "@types/mime-types": "^2.1.0", diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index b7d9a12cdbee..e55e9961d06b 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -5,7 +5,6 @@ import { TunnelCloseCode } from "@coder/tunnel/src/common"; import { handle as handleTunnel } from "@coder/tunnel/src/server"; import * as express from "express"; //@ts-ignore -import * as expressStaticGzip from "express-static-gzip"; import * as fs from "fs"; import { mkdirp } from "fs-extra"; import * as http from "http"; @@ -22,6 +21,7 @@ import * as url from "url"; import * as ws from "ws"; import { buildDir } from "./constants"; import { createPortScanner } from "./portScanner"; +import { staticGzip } from "./static"; import safeCompare = require("safe-compare"); interface CreateAppOptions { @@ -210,9 +210,6 @@ export const createApp = async (options: CreateAppOptions): Promise<{ return res.redirect(code, newUrlString); }; - const baseDir = buildDir || path.join(__dirname, ".."); - const staticGzip = expressStaticGzip(path.join(baseDir, "build/web")); - app.use((req, res, next) => { logger.trace(`\u001B[1m${req.method} ${res.statusCode} \u001B[0m${req.originalUrl}`, field("host", req.hostname), field("ip", req.ip)); @@ -325,7 +322,8 @@ export const createApp = async (options: CreateAppOptions): Promise<{ }); // Everything else just pulls from the static build directory. - app.use(staticGzip); + const baseDir = buildDir || path.join(__dirname, ".."); + app.use(staticGzip(path.join(baseDir, "build/web"))); return { express: app, diff --git a/packages/server/src/static.ts b/packages/server/src/static.ts new file mode 100644 index 000000000000..e1e744d36e72 --- /dev/null +++ b/packages/server/src/static.ts @@ -0,0 +1,142 @@ +import { field, logger } from "@coder/logger"; +import * as getETag from "etag"; +import * as express from "express"; +import * as fs from "fs"; +import * as mime from "mime-types"; +import * as path from "path"; +import * as serveStatic from "serve-static"; +import * as zlib from "zlib"; + +const defaultContentType = "application/octet-stream"; + +interface FileData { + readonly etag: string; + readonly lastModified: Date; +} + +const listGzFilesRecursive = (rootFolder: string, subFolder: string): { [s: string]: FileData } => { + let files: { [s: string]: FileData } = {}; + + const fullDir = path.join(rootFolder, subFolder); + const contents = fs.readdirSync(fullDir); + for (let file of contents) { + let filePath = path.join(subFolder, file); + const fileFullPath = path.join(fullDir, file); + + const stats = fs.statSync(fileFullPath); + if (stats.isDirectory()) { + files = { ...files, ...listGzFilesRecursive(rootFolder, filePath) }; + } else if (filePath.endsWith(".gz")) { + try { + filePath = filePath.replace(/\.gz$/i, ""); + const etag = getETag(stats); + const mtime = Math.round(stats.mtime/1000)*1000; + const lastModified = new Date(mtime); + files[filePath] = { etag, lastModified }; + } catch (err) { + logger.warn("failed to stat file in listGzFilesRecursive", field("filePath", fileFullPath), field("error", err.message)); + } + } + } + + return files; +}; + +const setPath = (req: express.Request, path: string): void => { + let query = req.url.split("?").splice(1).join("?"); + if (query !== "") { + query = "?" + query; + } + req.url = path + query; +}; + +// staticGzip returns middleware that serves pre-gzipped files from rootFolder. +// If the client doesn't support gzipped responses, the file will be +// gunzipped. After initializing staticGzip, files in rootFolder shouldn't be +// modified (and new files shouldn't be added) as changes won't be reflected in +// responses. +export const staticGzip = (rootFolder: string): express.RequestHandler => { + if (!fs.existsSync(rootFolder)) { + throw new Error("staticGzip: rootFolder does not exist"); + } + + const expressStatic = serveStatic(rootFolder); + const gzipFiles = listGzFilesRecursive(rootFolder, "/"); + + return (req: express.Request, res: express.Response, next: express.NextFunction): void => { + if (req.method !== "GET" && req.method !== "HEAD") { + return next(); + } + + if (req.path.endsWith("/")) { + setPath(req, req.path + "index.html"); + } + + // Check for 404, and let expressStatic handle it if it is. This + // also allows for clients to download raw .gz files or files that aren't gzipped. + if (!gzipFiles.hasOwnProperty(req.path)) { + return expressStatic(req, res, next); + } + + // If we're going to try serving a gzip using serve-static, we + // need to remove the 'Range' header as you can't gunzip partial + // gzips. Unfourtunately, 'Accept-Ranges' is still returned in serve-static responses. + req.headers["range"] = ""; + + let contentType = mime.contentType(path.extname(req.path)); + if (contentType === false) { + contentType = defaultContentType; + } + res.setHeader("Content-Type", contentType); + res.setHeader("Vary", "Accept-Encoding"); + + // Send .gz file directly from disk. + if (req.acceptsEncodings("gzip")) { + setPath(req, req.path + ".gz"); + res.setHeader("Content-Encoding", "gzip"); + + return expressStatic(req, res, next); + } + + const filePath = path.join(rootFolder, req.path + ".gz"); + const fileData = gzipFiles[req.path]; + + // Set 'ETag' and 'Last-Modified' headers. + res.setHeader("ETag", fileData.etag); + res.setHeader("Last-Modified", fileData.lastModified.toUTCString()); + + // Try to send 304 Not Modified response by checking 'If-Match' + // and 'If-Modified-Since' headers. + const ifMatch = req.get("if-match"); + const ifModifiedSince = req.get("if-modified-since"); + if (ifMatch === fileData.etag) { + res.status(304); // Not Modified + + return res.end(); + } + if (ifModifiedSince) { + const ifModifiedSinceDate = new Date(ifModifiedSince); + if (fileData.lastModified.getTime() <= ifModifiedSinceDate.getTime()) { + res.status(304); // Not Modified + + return res.end(); + } + } + + // Gunzip and return stream. We don't know the resulting + // filesize, so we don't set 'Content-Length'. + try { + const file = fs.createReadStream(filePath); + const gunzip = zlib.createGunzip(); + const stream = gunzip.pipe(res); + stream.on("end", () => { + file.close(); + gunzip.close(); + res.end(); + }); + file.pipe(gunzip); + } catch (err) { + next(err); + } + }; +}; diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index b97e1daaed03..e005ef65d5a7 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -40,6 +40,13 @@ dependencies: "@types/node" "*" +"@types/etag@^1.8.0": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@types/etag/-/etag-1.8.0.tgz#37f0b1f3ea46da7ae319bbedb607e375b4c99f7e" + integrity sha512-EdSN0x+Y0/lBv7YAb8IU4Jgm6DWM+Bqtz7o5qozl96fzaqdqbdfHS5qjdpFeIv7xQ8jSLyjMMNShgYtMajEHyQ== + dependencies: + "@types/node" "*" + "@types/events@*": version "1.2.0" resolved "https://registry.yarnpkg.com/@types/events/-/events-1.2.0.tgz#81a6731ce4df43619e5c8c945383b3e62a89ea86" @@ -426,13 +433,6 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= -express-static-gzip@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/express-static-gzip/-/express-static-gzip-1.1.3.tgz#345ea02637d9d5865777d6fb57ccc0884abcda65" - integrity sha512-k8Q4Dx4PDpzEb8kth4uiPWrBeJWJYSgnWMzNdjQUOsEyXfYKbsyZDkU/uXYKcorRwOie5Vzp4RMEVrJLMfB6rA== - dependencies: - serve-static "^1.12.3" - express@^4.16.4: version "4.16.4" resolved "https://registry.yarnpkg.com/express/-/express-4.16.4.tgz#fddef61926109e24c515ea97fd2f1bdbf62df12e" @@ -562,6 +562,17 @@ http-errors@1.6.3, http-errors@~1.6.2, http-errors@~1.6.3: setprototypeof "1.1.0" statuses ">= 1.4.0 < 2" +http-errors@~1.7.2: + version "1.7.3" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" + integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.1.1" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + httpolyglot@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/httpolyglot/-/httpolyglot-0.1.2.tgz#e4d347fe8984a62f467d4060df527f1851f6997b" @@ -587,6 +598,11 @@ inherits@2, inherits@2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= +inherits@2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + ipaddr.js@1.8.0: version "1.8.0" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.8.0.tgz#eaa33d6ddd7ace8f7f6fe0c9ca0440e706738b1e" @@ -696,12 +712,24 @@ methods@~1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= +mime-db@1.40.0: + version "1.40.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32" + integrity sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA== + mime-db@~1.37.0: version "1.37.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg== -mime-types@^2.1.21, mime-types@~2.1.18: +mime-types@^2.1.24: + version "2.1.24" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.24.tgz#b6f8d0b3e951efb77dedeca194cff6d16f676f81" + integrity sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ== + dependencies: + mime-db "1.40.0" + +mime-types@~2.1.18: version "2.1.21" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg== @@ -713,6 +741,11 @@ mime@1.4.1: resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" integrity sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ== +mime@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" @@ -747,6 +780,11 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= +ms@2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" + integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== + negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" @@ -824,6 +862,11 @@ parseurl@~1.3.2: resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" integrity sha1-/CidTtiZMRlGDBViUyYs3I3mW/M= +parseurl@~1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" + integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== + path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" @@ -871,6 +914,11 @@ range-parser@~1.2.0: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e" integrity sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4= +range-parser@~1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== + raw-body@2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.3.tgz#1b324ece6b5706e153855bc1148c65bb7f6ea0c3" @@ -925,7 +973,26 @@ send@0.16.2: range-parser "~1.2.0" statuses "~1.4.0" -serve-static@1.13.2, serve-static@^1.12.3: +send@0.17.1: + version "0.17.1" + resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" + integrity sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg== + dependencies: + debug "2.6.9" + depd "~1.1.2" + destroy "~1.0.4" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + fresh "0.5.2" + http-errors "~1.7.2" + mime "1.6.0" + ms "2.1.1" + on-finished "~2.3.0" + range-parser "~1.2.1" + statuses "~1.5.0" + +serve-static@1.13.2: version "1.13.2" resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.2.tgz#095e8472fd5b46237db50ce486a43f4b86c6cec1" integrity sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw== @@ -935,11 +1002,26 @@ serve-static@1.13.2, serve-static@^1.12.3: parseurl "~1.3.2" send "0.16.2" +serve-static@^1.14.1: + version "1.14.1" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.1.tgz#666e636dc4f010f7ef29970a88a674320898b2f9" + integrity sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg== + dependencies: + encodeurl "~1.0.2" + escape-html "~1.0.3" + parseurl "~1.3.3" + send "0.17.1" + setprototypeof@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" integrity sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ== +setprototypeof@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" + integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== + signal-exit@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" @@ -965,7 +1047,7 @@ source-map@~0.1.38: dependencies: amdefine ">=0.0.4" -"statuses@>= 1.4.0 < 2": +"statuses@>= 1.4.0 < 2", "statuses@>= 1.5.0 < 2", statuses@~1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= @@ -1013,6 +1095,11 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" +toidentifier@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" + integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== + ts-node@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-7.0.1.tgz#9562dc2d1e6d248d24bc55f773e3f614337d9baf"