package updates / npm vulnerability clean up (microsoft#10927)

* pass 1

* axios, request

* gulp, cssnano, mocha

* bump dompurify

* gulp5 breaking change, need to explicitly output after merge, as streams close immediately

* sanitize html

* pin terser ver to clear uglifyify vuln

* clear smoothie vuln

* remove gulp-header; lodash vuln tagged in way that makes it unpinnable, and implementation is small

* sanitize-html seems useless when we have dompurify and use it elsewhere for same thing

* update marked (pxtlib/webapp portions mostly from codex, review)

* fix error on pxt serve

* attempt to fix gulp travis?

* cull remaining uglify, fix test failure

* Clean up multiple tags as well

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* remove legacy az media support

* fix spacing typo

* update smoothie.js

* add back in sanitize-html

* fix code blocks

* restore summary parsing

* docs fixed but need to clean up

* marked -> 17

* partially back to clean, next to figure out docsrender w/o restoreCodeLanguageClassesFromTokens

* fix cli server issue without mangling webapp logic

* clean up other attempts at fixes

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

Author: jwunderl

README.md

@@ -135,12 +135,13 @@ Currently, the following languages are included:
 * JavaScript
 * HTML,XML
 * Markdown
+* JSON
 
 If you need to add other languages or update existing ones,
 you can find the distribution at [https://highlightjs.org/download/](https://highlightjs.org/download/);
 select all the languages you want to include (including the ones above!),
 download and unzip,
-and finally copy over `highlight.pack.js` into `webapp/public/highlight.js/`.
+and finally copy over `highlight.min.js` as `webapp/public/highlight.js/highlight.pack.js`.
 
 ## Tests
 

cli/cli.ts

@@ -34,7 +34,30 @@ import { SUB_WEBAPPS } from './subwebapp';
 
 const rimraf: (f: string, opts: any, cb: (err: Error, res: any) => void) => void = require('rimraf');
 
-pxt.docs.requireDOMSanitizer = () => require("sanitize-html");
+// dompurify requires a DOM implementation; sanitize-html does not.
+// sanitize-html is a bit more aggressive and culls class names by default from code snippets
+// Need to wrap to prevent that. Possibly worth swapping to jsdom + dompurify later for consistency.
+pxt.docs.requireDOMSanitizer = () => {
+    const sanitizeHtml = require("sanitize-html");
+    const defaults = sanitizeHtml.defaults || {};
+    const baseAllowedAttrs = defaults.allowedAttributes || {};
+
+    const mergeClassAttribute = (tag: string) => {
+        const existing: string[] = baseAllowedAttrs[tag] || [];
+        return Array.from(new Set<string>([...existing, "class"]));
+    };
+
+    const options = {
+        ...defaults,
+        allowedAttributes: {
+            ...baseAllowedAttrs,
+            code: mergeClassAttribute("code"),
+            pre: mergeClassAttribute("pre"),
+        },
+    };
+
+    return (html: string) => sanitizeHtml(html, options);
+};
 
 let forceCloudBuild = process.env["KS_FORCE_CLOUD"] !== "no";
 let forceLocalBuild = !!process.env["PXT_FORCE_LOCAL"];

cli/nodeutil.ts

@@ -57,7 +57,7 @@ export function readResAsync(g: events.EventEmitter) {
 
         g.on("error", (err: any) => reject(err))
 
-        g.on('end', () => resolve(Buffer.concat(bufs)))
+        g.on('end', () => resolve(Buffer.concat(bufs as unknown as Uint8Array[])))
     })
 }
 
@@ -92,7 +92,7 @@ export function spawnWithPipeAsync(opts: SpawnOptions) {
         ch.on('close', (code: number) => {
             if (code != 0 && !opts.allowNonZeroExit)
                 reject(new Error("Exit code: " + code + " from " + info))
-            resolve(Buffer.concat(bufs))
+            resolve(Buffer.concat(bufs as unknown as Uint8Array[]))
         });
         if (opts.input != null)
             ch.stdin.end(opts.input, "utf8")

cli/server.ts

@@ -433,7 +433,8 @@ export function expandHtml(html: string, params?: pxt.Map<string>, appTheme?: px
         // so we don't do them here either.
     }
     pxt.docs.prepTemplate(d)
-    return d.finish().replace(/@-(\w+)-@/g, (f, w) => "@" + w + "@")
+    const expanded = d.finish().replace(/@-(\w+)-@/g, (f, w) => "@" + w + "@")
+    return expanded
 }
 
 export function expandDocTemplateCore(template: string) {

gulpfile.js

@@ -1,12 +1,13 @@
 const fs = require("fs");
 const os = require("os");
 const path = require("path");
+const { Transform, PassThrough } = require("stream");
+const { minify } = require("terser");
 
 const gulp = require("gulp");
 const ts = require("gulp-typescript");
 const merge = require("merge-stream");
 const concat = require("gulp-concat");
-const header = require("gulp-header");
 const replace = require("gulp-replace");
 const ju = require("./jakeutil");
 const yargs = require('yargs');
@@ -21,6 +22,63 @@ const clean = () => rimraf("built").then(() => rimraf("temp"));
 const update = () => exec("git pull", true).then(() => exec("npm install", true))
 const noop = () => Promise.resolve();
 
+function prependHeader(headerText) {
+    const headerBuffer = Buffer.from(headerText);
+
+    return new Transform({
+        objectMode: true,
+        transform(file, _enc, callback) {
+            if (file.isNull()) {
+                return callback(null, file);
+            }
+
+            if (file.isBuffer && file.isBuffer()) {
+                file.contents = Buffer.concat([headerBuffer, file.contents]);
+                return callback(null, file);
+            }
+
+            if (file.isStream && file.isStream()) {
+                const combined = new PassThrough();
+                combined.write(headerBuffer);
+                file.contents.on('error', err => combined.emit('error', err));
+                file.contents.pipe(combined, { end: true });
+                file.contents = combined;
+                return callback(null, file);
+            }
+
+            return callback(null, file);
+        }
+    });
+}
+
+function terserMinifyTransform(options = {}) {
+    return new Transform({
+        objectMode: true,
+        transform(file, _enc, callback) {
+            if (file.isNull()) {
+                return callback(null, file);
+            }
+
+            if (file.isStream && file.isStream()) {
+                return callback(new Error("Streaming not supported for terserMinifyTransform"));
+            }
+
+            const source = file.contents.toString("utf8");
+
+            minify(source, options)
+                .then(result => {
+                    if (!result || typeof result.code !== "string") {
+                        throw new Error("Terser produced no output");
+                    }
+
+                    file.contents = Buffer.from(result.code, "utf8");
+                    callback(null, file);
+                })
+                .catch(err => callback(err));
+        }
+    });
+}
+
 const SUB_WEBAPPS = require("./cli/webapps-config.json").webapps;
 
 /** onlineline */
@@ -71,7 +129,7 @@ const pxtworker = () => gulp.src([
     "built/pxtpy.js"
 ])
     .pipe(concat("pxtworker.js"))
-    .pipe(header(`"use strict";\n`))
+    .pipe(prependHeader(`"use strict";\n`))
     .pipe(gulp.dest("built/web"));
 
 const pxtembed = () => gulp.src([
@@ -96,7 +154,7 @@ const buildpxtjs = () => gulp.src([
     "built/cli.js"
 ])
     .pipe(concat("pxt.js"))
-    .pipe(header(`
+    .pipe(prependHeader(`
         "use strict";
         // make sure TypeScript doesn't overwrite our module.exports
         global.savedModuleExports = module.exports;
@@ -137,7 +195,7 @@ function initWatch() {
         webapp,
         browserifyWebapp,
         browserifyAssetEditor,
-        gulp.parallel(semanticjs, copyJquery, copyWebapp, copySemanticFonts, copyMonaco),
+        gulp.parallel(semanticjs, copyJquery, copyMarked, copySmoothie, copyWebapp, copySemanticFonts, copyMonaco),
         notifyBuildComplete
     ];
 
@@ -203,10 +261,23 @@ function compileTsProject(dirname, destination, useOutdir, filename) {
     let tsResult = tsProject.src()
         .pipe(tsProject());
 
-    return merge(
-        tsResult.js.pipe(gulp.dest(destination)),
-        tsResult.dts.pipe(gulp.dest(destination))
-    );
+    const outputStreams = [];
+
+    if (tsResult.js) {
+        outputStreams.push(tsResult.js);
+    }
+
+    if (tsResult.dts) {
+        outputStreams.push(tsResult.dts);
+    }
+
+    const mergedOutput = outputStreams.length > 1 ? merge(...outputStreams) : outputStreams[0];
+
+    if (!mergedOutput) {
+        return Promise.resolve();
+    }
+
+    return mergedOutput.pipe(gulp.dest(destination));
 }
 
 // TODO: Copied from Jakefile; should be async
@@ -381,6 +452,18 @@ const copyJquery = () => gulp.src("node_modules/jquery/dist/jquery.min.js")
     .pipe(concat("jquery.js"))
     .pipe(gulp.dest("built/web"));
 
+const copyMarked = () =>
+    gulp.src("node_modules/marked/lib/marked.umd.js")
+        .pipe(concat("marked.min.js"))
+        .pipe(gulp.dest("built/web/marked"))
+        .pipe(gulp.dest("webapp/public/marked"));
+
+const copySmoothie = () =>
+    gulp.src("node_modules/smoothie/smoothie.js")
+        .pipe(terserMinifyTransform())
+        .pipe(concat("smoothie_compressed.js"))
+        .pipe(gulp.dest("webapp/public/smoothie"));
+
 const copyWebapp = () =>
     gulp.src([
         "node_modules/@microsoft/applicationinsights-web/browser/ai.2.min.js",
@@ -399,8 +482,10 @@ const copyWebapp = () =>
 const copySemanticFonts = () => gulp.src("node_modules/semantic-ui-less/themes/default/assets/fonts/*")
     .pipe(gulp.dest("built/web/fonts"))
 
+// NOTE: Uglifyify no longer runs here because it crashes on modern syntax emitted in
+// production builds; minification is handled later in runUglify via Terser.
 const execBrowserify = (entryPoint, outfile) => process.env.PXT_ENV == 'production' ?
-    exec(`node node_modules/browserify/bin/cmd ${entryPoint} -g [ envify --NODE_ENV production ] -g [ uglifyify --ignore '**/node_modules/@blockly/**' ] -o ${outfile}`) :
+    exec(`node node_modules/browserify/bin/cmd ${entryPoint} -g [ envify --NODE_ENV production ] -o ${outfile}`) :
     exec(`node node_modules/browserify/bin/cmd ${entryPoint} -o ${outfile} --debug`);
 
 const browserifyWebapp = () => execBrowserify("./built/webapp/src/app.js", "./built/web/main.js");
@@ -719,11 +804,11 @@ function testTask(testFolder, testFile, additionalFiles) {
 
     const buildTestRunner = () => gulp.src(src)
         .pipe(concat("runner.js"))
-        .pipe(header(`
+        .pipe(prependHeader(`
             "use strict";
             // make sure TypeScript doesn't overwrite our module.exports
             global.savedModuleExports = module.exports;
-            module.exports = null;
+            module.exports = {};
         `))
         .pipe(gulp.dest('built/tests/' + testFolder));
 
@@ -766,7 +851,7 @@ const buildAll = gulp.series(
     webapp,
     browserifyWebapp,
     browserifyAssetEditor,
-    gulp.parallel(semanticjs, copyJquery, copyWebapp, copySemanticFonts, copyMonaco),
+    gulp.parallel(semanticjs, copyJquery, copyMarked, copySmoothie, copyWebapp, copySemanticFonts, copyMonaco),
     buildBlocksTestRunner,
     gulp.parallel(browserifyBlocksTestRunner, browserifyBlocksPrep),
     runUglify

package.json

@@ -49,15 +49,14 @@
     "tests/blocks-test/karma.conf.js",
     "built/tests/blocksrunner.js",
     "built/tests/blockssetup.js",
-    "scripts/patchUglifyify.js",
     "docfiles",
     "theme",
     "common-docs",
     "react-common"
   ],
   "main": "built/pxt.js",
   "engines": {
-    "node": ">=16.0.0",
+    "node": ">=18.0.0",
     "npm": ">=8.0.0"
   },
   "dependencies": {
@@ -71,34 +70,33 @@
     "@types/diff-match-patch": "^1.0.32",
     "@zip.js/zip.js": "2.4.20",
     "adm-zip": "^0.5.12",
-    "axios": "^1.6.8",
+    "axios": "^1.12.2",
     "blockly": "12.2.0",
     "browserify": "17.0.0",
     "chai": "^3.5.0",
     "chalk": "^4.1.2",
-    "cssnano": "4.1.10",
+    "cssnano": "7.1.1",
     "dashjs": "^4.4.0",
     "diff-match-patch": "^1.0.5",
-    "dompurify": "2.0.17",
+    "dompurify": "3.3.0",
     "faye-websocket": "0.11.1",
     "karma": "6.4.4",
     "karma-chai": "0.1.0",
     "karma-chrome-launcher": "3.2.0",
     "karma-mocha": "2.0.1",
     "less": "3.13.1",
     "lzma": "2.3.2",
-    "marked": "0.3.19",
-    "mocha": "8.4.0",
+    "marked": "17.0.0",
+    "mocha": "10.8.2",
     "pngjs": "3.4.0",
-    "postcss": "6.0.21",
+    "postcss": "8.5.6",
     "promise.prototype.finally": "^3.1.2",
     "puppeteer": "23.11.1",
-    "request": "2.88.0",
     "rimraf": "2.5.4",
-    "rtlcss": "2.2.1",
-    "sanitize-html": "2.3.2",
+    "rtlcss": "4.3.0",
+    "sanitize-html": "2.17.0",
     "semantic-ui-less": "2.4.1",
-    "terser": "5.6.0"
+    "terser": "5.44.0"
   },
   "lazyDependencies": {
     "node-hid": "^0.7.2",
@@ -111,15 +109,13 @@
     "@types/fuse": "2.5.2",
     "@types/highlight.js": "9.12.2",
     "@types/jquery": "3.3.29",
-    "@types/marked": "0.3.0",
-    "@types/mocha": "^2.2.44",
+    "@types/mocha": "^10.0.10",
     "@types/node": "10.14.2",
     "@types/react": "17.0.50",
     "@types/react-dom": "17.0.17",
     "@types/react-modal": "3.13.1",
     "@types/react-redux": "7.1.24",
     "@types/react-tooltip": "3.3.6",
-    "@types/request": "2.48.5",
     "@types/resize-observer-browser": "^0.1.5",
     "@types/web-bluetooth": "0.0.4",
     "@typescript-eslint/eslint-plugin": "^4.20.0",
@@ -132,32 +128,37 @@
     "eslint-plugin-jsx-a11y": "^6.4.1",
     "eslint-plugin-react": "^7.23.1",
     "fuse.js": "3.2.0",
-    "gulp": "4.0.0",
+    "gulp": "5.0.1",
+    "gulp-cli": "3.1.0",
     "gulp-concat": "2.6.1",
-    "gulp-header": "2.0.9",
-    "gulp-replace": "1.0.0",
+    "gulp-replace": "1.1.4",
     "gulp-typescript": "5.0.1",
-    "highlight.js": "9.12.0",
+    "highlight.js": "11.11.1",
     "jquery": "3.5.0",
     "merge-stream": "2.0.0",
     "monaco-editor": "0.24.0",
-    "pouchdb": "7.2.1",
-    "pouchdb-adapter-memory": "7.2.1",
+    "pouchdb": "7.3.1",
+    "pouchdb-adapter-memory": "7.3.1",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-modal": "3.15.1",
     "react-redux": "7.2.9",
     "react-tooltip": "3.9.0",
     "redux": "4.2.0",
-    "smoothie": "1.35.0",
+    "smoothie": "1.36.1",
     "typescript": "4.8.3",
-    "uglifyify": "5.0.2",
     "yargs": "^17.7.2"
   },
   "overrides": {
     "combine-source-map": {
       "source-map": "0.4.4"
-    }
+    },
+    "sha.js": "2.4.12",
+    "cipher-base": "1.0.7",
+    "pbkdf2": "3.1.5",
+    "elliptic": "6.6.1",
+    "tar-fs": "3.1.1",
+    "terser": "5.44.0"
   },
   "scripts": {
     "build": "gulp",
@@ -175,7 +176,6 @@
     "test:tutorials": "gulp testtutorials",
     "update": "gulp update",
     "watch-streamer": "cd docs/static/streamer && tsc -t es6 --watch",
-    "prepare": "node ./scripts/npm-prepare.js",
-    "postinstall": "node ./scripts/patchUglifyify.js"
+    "prepare": "node ./scripts/npm-prepare.js"
   }
 }