[pull] master from microsoft:master

gulpfile.js

@@ -738,6 +738,7 @@ const testlang = testTask("compile-test", "compilerunner.js");
 const testhelpers = testTask("helpers-test", "helperrunner.js");
 const testerr = testTask("errors-test", "errorrunner.js");
 const testfmt = testTask("format-test", "formatrunner.js");
+const testembed = testTask("embed-test", "embedrunner.js");
 const testpydecomp = testTask("pydecompile-test", "pydecompilerunner.js");
 const testpycomp = testTask("pyconverter-test", "pyconvertrunner.js");
 const testpytraces = testTask("runtime-trace-tests", "tracerunner.js");
@@ -775,6 +776,7 @@ const testAll = gulp.series(
     testhelpers,
     testerr,
     testfmt,
+    testembed,
     testpydecomp,
     testpycomp,
     testpytraces,
@@ -886,6 +888,7 @@ exports.testdecompiler = testdecompiler;
 exports.testlang = testlang;
 exports.testerr = testerr;
 exports.testfmt = testfmt;
+exports.testembed = testembed;
 exports.testpydecomp = testpydecomp;
 exports.testpycomp = testpycomp;
 exports.testpytraces = testpytraces;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pxt-core",
-  "version": "13.1.7",
+  "version": "13.1.8",
   "description": "Microsoft MakeCode provides Blocks / JavaScript / Python tools and editors",
   "keywords": [
     "TypeScript",

pxtcompiler/emitter/hexfile.ts

@@ -647,7 +647,7 @@ namespace ts.pxtc {
         }
     }
 
-    function saveSourceToUF2(uf2: UF2.BlockFile, bin: Binary) {
+    export function saveSourceToUF2(uf2: UF2.BlockFile, bin: Binary) {
         if (!bin.packedSource)
             return
         let addr = (uf2.currPtr + 0x1000) & ~0xff
@@ -1094,16 +1094,19 @@ _stored_program: .hex ${res}
 `
     }
 
-    function packSource(meta: string, binstring: string) {
+    export function packSource(meta: string, binstring: string) {
         let metablob = Util.toUTF8(meta)
         let totallen = metablob.length + binstring.length
 
         let res = "\x41\x14\x0E\x2F\xB8\x2F\xA2\xBB"
 
+        // Binary header as specified in docs/source-embedding.md: 2-byte meta
+        // length, 4-byte text length, 2 reserved bytes, all little endian.
         res += U.uint8ArrayToString([
             metablob.length & 0xff, metablob.length >> 8,
-            binstring.length & 0xff, binstring.length >> 8,
-            0, 0, 0, 0
+            binstring.length & 0xff, (binstring.length >> 8) & 0xff,
+            (binstring.length >> 16) & 0xff, (binstring.length >> 24) & 0xff,
+            0, 0
         ])
 
         res += metablob

pxtlib/cpp.ts

@@ -1222,7 +1222,7 @@ int main() {
             let m = /^:10....0[0E]41140E2FB82FA2BB(....)(....)(....)(....)(..)/.exec(ln)
             if (m) {
                 metaLen = parseInt(swapBytes(m[1]), 16)
-                textLen = parseInt(swapBytes(m[2]), 16)
+                textLen = parseInt(swapBytes(m[2]), 16) + parseInt(swapBytes(m[3]), 16) * 0x10000
                 toGo = metaLen + textLen
                 buf = <any>new Uint8Array(toGo)
             } else if (toGo > 0) {

pxtsim/simdriver.ts

@@ -422,10 +422,16 @@ namespace pxsim {
                             // not found, spin a new one
                             if (!messageFrame) {
                                 const url = new URL(simulatorExtension.url);
-                                if (this.options.parentOrigin)
-                                    url.searchParams.set("parentOrigin", encodeURIComponent(this.options.parentOrigin));
+                                // The extension's parent window is this editor, so the origin it
+                                // should trust is the editor's own. options.parentOrigin is the
+                                // editor's *embedder* origin (controller mode) - the extension's
+                                // grandparent - so an extension that validates messages against
+                                // it rejects everything the editor sends when the editor is
+                                // embedded cross-origin.
+                                // searchParams.set() encodes, so don't encode the values here.
+                                url.searchParams.set("parentOrigin", window.location.origin);
                                 if (this.options.userLanguage)
-                                    url.searchParams.set("language", encodeURIComponent(this.options.userLanguage));
+                                    url.searchParams.set("language", this.options.userLanguage);
                                 startSimulatorExtension(url.toString(), simulatorExtension.permanent, simulatorExtension.aspectRatio);
                             }
                             // not running the current run, restart

tests/embed-test/embedrunner.ts

@@ -0,0 +1,117 @@
+/// <reference path="../../built/pxtcompiler.d.ts"/>
+
+import * as chai from "chai";
+import "mocha";
+
+function initGlobals() {
+  let g = global as any
+  g.pxt = pxt;
+  g.ts = ts;
+  g.pxtc = pxtc;
+  g.btoa = (str: string) => Buffer.from(str, "binary").toString("base64");
+  g.atob = (str: string) => Buffer.from(str, "base64").toString("binary");
+}
+
+initGlobals();
+
+const MAGIC = [0x41, 0x14, 0x0E, 0x2F, 0xB8, 0x2F, 0xA2, 0xBB];
+
+// No compression field on purpose so we can control the length.
+const testMeta = JSON.stringify({ editor: "tsprj", name: "embed-test" });
+
+// Deterministic non-repeating text so that a misread length or offset cannot
+// accidentally round-trip.
+function makeText(len: number): string {
+  const chars: string[] = new Array(len);
+  for (let i = 0; i < len; ++i)
+    chars[i] = String.fromCharCode(0x20 + ((i * 31 + (i >> 8)) % 95));
+  return chars.join("");
+}
+
+// Mirrors the appended-source record emission in patchHex
+// (pxtcompiler/emitter/hexfile.ts).
+function toHexRecords(packed: string): string {
+  const lines: string[] = [];
+  let addr = 0;
+  for (let i = 0; i < packed.length; i += 16) {
+    const bytes = [0x10, (addr >> 8) & 0xff, addr & 0xff, 0x0E];
+    for (let j = 0; j < 16; ++j)
+      bytes.push((packed.charCodeAt(i + j) || 0) & 0xff);
+    lines.push(pxtc.hexfile.hexBytes(bytes));
+    addr += 16;
+  }
+  lines.push(":00000001FF");
+  return lines.join("\n");
+}
+
+function headerOf(packed: string): number[] {
+  const r: number[] = [];
+  for (let i = 0; i < 16; ++i)
+    r.push(packed.charCodeAt(i));
+  return r;
+}
+
+function assertRoundTrip(res: pxt.cpp.HexFile, text: string) {
+  chai.assert(
+    res.source === text,
+    `embedded text did not round-trip: wrote ${text.length} chars, read back ${res.source ? res.source.length : 0}`
+  );
+}
+
+async function hexRoundTrip(textLen: number) {
+  const text = makeText(textLen);
+  const packed = pxtc.packSource(testMeta, text);
+  const hex = toHexRecords(packed);
+  const res = await pxt.cpp.unpackSourceFromHexAsync(new Uint8Array(Buffer.from(hex, "utf8")));
+  assertRoundTrip(res, text);
+}
+
+async function uf2RoundTrip(textLen: number) {
+  const text = makeText(textLen);
+  const packed = pxtc.packSource(testMeta, text);
+  const uf2 = pxtc.UF2.newBlockFile();
+  // Stand-in for the program, which patchHex writes before the source.
+  pxtc.UF2.writeBytes(uf2, 0, new Array(256).fill(0));
+  pxtc.saveSourceToUF2(uf2, { packedSource: packed } as any);
+  const dat = Uint8Array.from(pxtc.UF2.serializeFile(uf2), c => c.charCodeAt(0) & 0xff);
+  const res = await pxt.cpp.unpackSourceFromHexAsync(dat);
+  assertRoundTrip(res, text);
+}
+
+describe("source embedding binary header (docs/source-embedding.md)", () => {
+  it("is unchanged for projects under 64 KiB", () => {
+    const packed = pxtc.packSource(testMeta, makeText(1000));
+    chai.expect(headerOf(packed)).to.deep.equal([
+      ...MAGIC,
+      testMeta.length, 0, // offsets 8-9: meta length
+      0xe8, 0x03,         // offsets 10-11: text length 1000, low 16 bits
+      0, 0,               // offsets 12-13: text length, high 16 bits
+      0, 0,               // offsets 14-15: reserved
+    ]);
+  });
+
+  it("stores bits 16-31 of the text length at offsets 12-13", () => {
+    const packed = pxtc.packSource(testMeta, makeText(74688)); // 0x123C0
+    chai.expect(headerOf(packed)).to.deep.equal([
+      ...MAGIC,
+      testMeta.length, 0,
+      0xc0, 0x23,
+      0x01, 0x00,
+      0, 0,
+    ]);
+  });
+});
+
+describe("source embedding round-trip through a hex file", () => {
+  it("works under the 2-byte length limit", () => hexRoundTrip(1000));
+  it("works at the 2-byte length limit (65,535 bytes)", () => hexRoundTrip(0xffff));
+  it("works at 64 KiB (65,536 bytes)", () => hexRoundTrip(0x10000));
+  it("works above 64 KiB (74,688 bytes)", () => hexRoundTrip(74688));
+});
+
+describe("source embedding round-trip through a UF2 file", () => {
+  it("works under the 2-byte length limit", () => uf2RoundTrip(1000));
+  it("works at the 2-byte length limit (65,535 bytes)", () => uf2RoundTrip(0xffff));
+  it("works at 64 KiB (65,536 bytes)", () => uf2RoundTrip(0x10000));
+  it("works above 64 KiB (74,688 bytes)", () => uf2RoundTrip(74688));
+});

tests/embed-test/tsconfig.json

@@ -0,0 +1,22 @@
+{
+    "compilerOptions": {
+        "target": "es2017",
+        "noImplicitAny": true,
+        "noImplicitReturns": true,
+        "declaration": true,
+        "outDir": "../../built/tests",
+        "rootDir": "..",
+        "newLine": "LF",
+        "module": "commonjs",
+        "types": [
+            "chai",
+            "mocha",
+            "node"
+        ],
+        "sourceMap": false,
+        "skipLibCheck": true
+    },
+    "files": [
+        "embedrunner.ts"
+    ]
+}