Skip to content

Commit 36e2082

Browse files
committed
add a cached compile mode for faster MSL loading times..
Signed-off-by: yeus <thomas.meschede@web.de>
1 parent eb9864d commit 36e2082

3 files changed

Lines changed: 351 additions & 4 deletions

File tree

‎crates/rumoca-bind-wasm/src/lib.rs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -414,10 +414,8 @@ pub(crate) fn compile_requested_model(
414414
session: &mut Session,
415415
model_name: &str,
416416
) -> Result<CompilationResult, JsValue> {
417-
let mut report = session.compile_model_with_mode(
418-
model_name,
419-
CompilationMode::StrictReachableUncachedWithRecovery,
420-
);
417+
let mut report =
418+
session.compile_model_with_mode(model_name, CompilationMode::StrictReachableWithRecovery);
421419
if !report.failures.is_empty() {
422420
return Err(JsValue::from_str(&format!(
423421
"Compilation error: {}",

‎crates/rumoca-compile/src/source_root_cache.rs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,51 @@ mod tests {
524524
assert!(second.cache_file.is_none());
525525
}
526526

527+
#[test]
528+
fn warm_source_root_cache_hit_skips_reparse_timing() {
529+
let temp = tempfile::tempdir().expect("tempdir");
530+
let lib_dir = temp.path().join("lib");
531+
let cache_dir = temp.path().join("cache");
532+
std::fs::create_dir_all(&lib_dir).expect("mkdir");
533+
std::fs::write(lib_dir.join("package.mo"), "package Lib\nend Lib;\n")
534+
.expect("write package");
535+
for index in 0..24 {
536+
std::fs::write(
537+
lib_dir.join(format!("M{index}.mo")),
538+
format!(
539+
"within Lib;\nmodel M{index}\n Real x;\nequation\n der(x) = {index}.0;\nend M{index};\n"
540+
),
541+
)
542+
.expect("write model");
543+
}
544+
545+
let first =
546+
parse_source_root_with_cache_in(&lib_dir, Some(&cache_dir)).expect("first parse");
547+
assert_eq!(first.cache_status, SourceRootCacheStatus::Miss);
548+
assert_eq!(first.file_count, 25);
549+
550+
let second =
551+
parse_source_root_with_cache_in(&lib_dir, Some(&cache_dir)).expect("second parse");
552+
assert_eq!(second.cache_status, SourceRootCacheStatus::Hit);
553+
assert_eq!(second.file_count, first.file_count);
554+
assert_eq!(
555+
second.timing.parse_files_ms, 0,
556+
"warm source-root cache hit must not fall back to parsing files"
557+
);
558+
assert_eq!(
559+
second.timing.cache_write_ms, 0,
560+
"warm source-root cache hit should not rewrite parsed artifacts"
561+
);
562+
assert!(
563+
second.timing.total_ms
564+
>= second.timing.collect_files_ms
565+
+ second.timing.hash_inputs_ms
566+
+ second.timing.cache_deserialize_ms
567+
+ second.timing.validate_layout_ms,
568+
"reported warm timing should account for cache hydration work"
569+
);
570+
}
571+
527572
#[test]
528573
fn resolve_source_root_cache_dir_is_absolute_and_stable() {
529574
let path = resolve_source_root_cache_dir().expect("cache dir should resolve by default");
Lines changed: 304 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
import { inflateRawSync } from "node:zlib";
2+
import { readdir, readFile, stat } from "node:fs/promises";
3+
import path from "node:path";
4+
import { performance } from "node:perf_hooks";
5+
import { ensureNodeSelfForWasmBindgenRayon } from "./node_rayon_shim.mjs";
6+
7+
const END_OF_CENTRAL_DIRECTORY_SIGNATURE = 0x06054b50;
8+
const CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE = 0x02014b50;
9+
const LOCAL_FILE_HEADER_SIGNATURE = 0x04034b50;
10+
11+
function argValue(name, fallback) {
12+
const index = process.argv.indexOf(name);
13+
return index >= 0 ? process.argv[index + 1] : fallback;
14+
}
15+
16+
function argValues(name) {
17+
const values = [];
18+
for (let index = 0; index < process.argv.length; index++) {
19+
if (process.argv[index] === name && process.argv[index + 1]) {
20+
values.push(process.argv[index + 1]);
21+
}
22+
}
23+
return values;
24+
}
25+
26+
function assert(condition, message) {
27+
if (!condition) {
28+
throw new Error(message);
29+
}
30+
}
31+
32+
async function collectMoFiles(root, prefix = "") {
33+
const entries = await readdir(root, { withFileTypes: true });
34+
entries.sort((lhs, rhs) => lhs.name.localeCompare(rhs.name));
35+
const files = [];
36+
for (const entry of entries) {
37+
const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
38+
if (relative.includes("Test") || relative.includes("Obsolete")) {
39+
continue;
40+
}
41+
const absolute = path.join(root, entry.name);
42+
if (entry.isDirectory()) {
43+
files.push(...await collectMoFiles(absolute, relative));
44+
} else if (entry.isFile() && entry.name.endsWith(".mo")) {
45+
files.push([absolute, relative]);
46+
}
47+
}
48+
return files;
49+
}
50+
51+
async function readMslSources(mslArchiveRoot) {
52+
const roots = [
53+
["Modelica", path.join(mslArchiveRoot, "Modelica 4.1.0")],
54+
["ModelicaServices", path.join(mslArchiveRoot, "ModelicaServices 4.1.0")],
55+
];
56+
const sources = {};
57+
for (const [logicalRoot, root] of roots) {
58+
const rootStat = await stat(root);
59+
assert(rootStat.isDirectory(), `missing MSL source-root directory: ${root}`);
60+
for (const [absolute, relative] of await collectMoFiles(root)) {
61+
sources[`${logicalRoot}/${relative}`] = await readFile(absolute, "utf8");
62+
}
63+
}
64+
65+
const complexPath = path.join(mslArchiveRoot, "Complex.mo");
66+
sources["Complex.mo"] = await readFile(complexPath, "utf8");
67+
return sources;
68+
}
69+
70+
function findEndOfCentralDirectory(zip) {
71+
const minOffset = Math.max(0, zip.length - 0xffff - 22);
72+
for (let offset = zip.length - 22; offset >= minOffset; offset--) {
73+
if (zip.readUInt32LE(offset) === END_OF_CENTRAL_DIRECTORY_SIGNATURE) {
74+
return offset;
75+
}
76+
}
77+
throw new Error("ZIP end-of-central-directory record not found");
78+
}
79+
80+
function zipEntries(zip) {
81+
const eocdOffset = findEndOfCentralDirectory(zip);
82+
const entryCount = zip.readUInt16LE(eocdOffset + 10);
83+
let offset = zip.readUInt32LE(eocdOffset + 16);
84+
const entries = [];
85+
86+
for (let index = 0; index < entryCount; index++) {
87+
assert(
88+
zip.readUInt32LE(offset) === CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE,
89+
`invalid ZIP central directory header at ${offset}`,
90+
);
91+
const compressionMethod = zip.readUInt16LE(offset + 10);
92+
const compressedSize = zip.readUInt32LE(offset + 20);
93+
const uncompressedSize = zip.readUInt32LE(offset + 24);
94+
const fileNameLength = zip.readUInt16LE(offset + 28);
95+
const extraLength = zip.readUInt16LE(offset + 30);
96+
const commentLength = zip.readUInt16LE(offset + 32);
97+
const localHeaderOffset = zip.readUInt32LE(offset + 42);
98+
const nameStart = offset + 46;
99+
const name = zip.toString("utf8", nameStart, nameStart + fileNameLength);
100+
entries.push({
101+
name,
102+
compressionMethod,
103+
compressedSize,
104+
uncompressedSize,
105+
localHeaderOffset,
106+
});
107+
offset = nameStart + fileNameLength + extraLength + commentLength;
108+
}
109+
110+
return entries;
111+
}
112+
113+
function normalizeArchivePath(relativePath) {
114+
const parts = relativePath.split("/");
115+
if (parts.length > 1 && /(?:Standard)?Library|^MSL/i.test(parts[0])) {
116+
return parts.slice(1).join("/");
117+
}
118+
if (parts.length > 0) {
119+
parts[0] = parts[0].replace(/[\s-][\d.]+$/, "");
120+
return parts.join("/");
121+
}
122+
return relativePath;
123+
}
124+
125+
function inflateZipEntryToString(zip, entry) {
126+
assert(
127+
zip.readUInt32LE(entry.localHeaderOffset) === LOCAL_FILE_HEADER_SIGNATURE,
128+
`invalid ZIP local file header at ${entry.localHeaderOffset}`,
129+
);
130+
const fileNameLength = zip.readUInt16LE(entry.localHeaderOffset + 26);
131+
const extraLength = zip.readUInt16LE(entry.localHeaderOffset + 28);
132+
const dataStart = entry.localHeaderOffset + 30 + fileNameLength + extraLength;
133+
const compressed = zip.subarray(dataStart, dataStart + entry.compressedSize);
134+
if (entry.compressionMethod === 0) {
135+
return compressed.toString("utf8");
136+
}
137+
if (entry.compressionMethod === 8) {
138+
const inflated = inflateRawSync(compressed);
139+
assert(
140+
inflated.length === entry.uncompressedSize,
141+
`inflated size mismatch for ${entry.name}`,
142+
);
143+
return inflated.toString("utf8");
144+
}
145+
throw new Error(
146+
`unsupported ZIP compression method ${entry.compressionMethod} for ${entry.name}`,
147+
);
148+
}
149+
150+
async function readMslZipSources(mslZipPath) {
151+
const readStarted = performance.now();
152+
const zip = await readFile(mslZipPath);
153+
const zipReadMs = ms(readStarted);
154+
155+
const parseStarted = performance.now();
156+
const moEntries = zipEntries(zip).filter((entry) => {
157+
return (
158+
entry.name.endsWith(".mo") &&
159+
!entry.name.includes("Test") &&
160+
!entry.name.includes("Obsolete")
161+
);
162+
});
163+
const zipParseMs = ms(parseStarted);
164+
165+
const extractStarted = performance.now();
166+
const sources = {};
167+
for (const entry of moEntries) {
168+
sources[normalizeArchivePath(entry.name)] = inflateZipEntryToString(zip, entry);
169+
}
170+
171+
return {
172+
sources,
173+
zipReadMs,
174+
zipParseMs,
175+
zipExtractMs: ms(extractStarted),
176+
};
177+
}
178+
179+
function ms(msStarted) {
180+
return Math.round((performance.now() - msStarted) * 10) / 10;
181+
}
182+
183+
function compileTiming(raw) {
184+
const parsed = JSON.parse(raw);
185+
return {
186+
jsonBytes: Buffer.byteLength(raw),
187+
balanced: parsed.balance?.is_balanced === true,
188+
equations: parsed.balance?.num_equations ?? null,
189+
unknowns: parsed.balance?.num_unknowns ?? null,
190+
phaseTiming: parsed.__compile_phase_timing ?? null,
191+
};
192+
}
193+
194+
function compileModels(wasmModule, models) {
195+
const source = "model WasmBenchmarkInput\nend WasmBenchmarkInput;\n";
196+
return models.map((model) => {
197+
const started = performance.now();
198+
try {
199+
const raw = wasmModule.compile(source, model);
200+
return {
201+
model,
202+
ok: true,
203+
compileMs: ms(started),
204+
...compileTiming(raw),
205+
};
206+
} catch (error) {
207+
return {
208+
model,
209+
ok: false,
210+
compileMs: ms(started),
211+
error: error?.message || String(error),
212+
};
213+
}
214+
});
215+
}
216+
217+
async function main() {
218+
ensureNodeSelfForWasmBindgenRayon();
219+
const pkgSubdir = argValue("--pkg-subdir", "release-full-web");
220+
const mslRoot = argValue(
221+
"--msl-root",
222+
"target/msl/ModelicaStandardLibrary-4.1.0",
223+
);
224+
const mslZip = argValue("--msl-zip", "");
225+
const models = argValues("--compile-model");
226+
if (models.length === 0) {
227+
models.push(
228+
"Modelica.Blocks.Examples.BooleanNetwork1",
229+
"Modelica.Electrical.Analog.Examples.Resistor",
230+
"Modelica.Blocks.Examples.PID_Controller",
231+
);
232+
}
233+
const wasmModule = await import(`../../../pkg/${pkgSubdir}/rumoca_bind_wasm.js`);
234+
const wasmBytes = await readFile(
235+
new URL(`../../../pkg/${pkgSubdir}/rumoca_bind_wasm_bg.wasm`, import.meta.url),
236+
);
237+
await wasmModule.default({ module_or_path: wasmBytes });
238+
239+
wasmModule.clear_source_root_cache();
240+
241+
let started = performance.now();
242+
const zipTimings = mslZip
243+
? await readMslZipSources(path.resolve(mslZip))
244+
: null;
245+
const sources = zipTimings
246+
? zipTimings.sources
247+
: await readMslSources(path.resolve(mslRoot));
248+
const readMs = zipTimings ? undefined : ms(started);
249+
const uris = Object.keys(sources).sort((lhs, rhs) => lhs.localeCompare(rhs));
250+
251+
started = performance.now();
252+
const sourceRootsJson = JSON.stringify(sources);
253+
const stringifyMs = ms(started);
254+
255+
started = performance.now();
256+
const loadResult = JSON.parse(wasmModule.load_source_roots(sourceRootsJson));
257+
const loadMs = ms(started);
258+
const documentCountAfterLoad = wasmModule.get_source_root_document_count();
259+
260+
const compileAfterTextLoad = compileModels(wasmModule, models);
261+
262+
started = performance.now();
263+
const binaryCache = wasmModule.export_parsed_source_roots_binary(JSON.stringify(uris));
264+
const exportBinaryMs = ms(started);
265+
266+
wasmModule.clear_source_root_cache();
267+
started = performance.now();
268+
const restoredCount = wasmModule.merge_parsed_source_roots_binary(binaryCache);
269+
const restoreBinaryMs = ms(started);
270+
const documentCountAfterRestore = wasmModule.get_source_root_document_count();
271+
272+
const compileAfterBinaryRestore = compileModels(wasmModule, models);
273+
274+
const report = {
275+
pkgSubdir,
276+
mslRoot: path.resolve(mslRoot),
277+
fileCount: uris.length,
278+
sourceJsonBytes: Buffer.byteLength(sourceRootsJson),
279+
binaryCacheBytes: binaryCache.length,
280+
readMs,
281+
zipReadMs: zipTimings?.zipReadMs,
282+
zipParseMs: zipTimings?.zipParseMs,
283+
zipExtractMs: zipTimings?.zipExtractMs,
284+
stringifyMs,
285+
wasmLoadSourceRootsMs: loadMs,
286+
exportParsedBinaryMs: exportBinaryMs,
287+
restoreParsedBinaryMs: restoreBinaryMs,
288+
parsedCount: loadResult.parsed_count,
289+
errorCount: loadResult.error_count,
290+
documentCountAfterLoad,
291+
compileAfterTextLoad,
292+
restoredCount,
293+
documentCountAfterRestore,
294+
compileAfterBinaryRestore,
295+
};
296+
297+
console.log(JSON.stringify(report, null, 2));
298+
}
299+
300+
main().catch((error) => {
301+
console.error("[wasm-benchmark] failed:");
302+
console.error(error);
303+
process.exit(1);
304+
});

0 commit comments

Comments
 (0)