WIP: perf: add stream download/upload mem benchmark

.gitignore

@@ -123,3 +123,6 @@ run
 !test/ctx_register.js
 
 .egg/
+
+# Benchmark test files
+benchmark/stream_download/nginx/50mb_ones.txt

benchmark/stream_download/.gitignore

@@ -0,0 +1 @@
+tmp

benchmark/stream_download/Dockerfile

@@ -0,0 +1,43 @@
+FROM node:24.12.0
+
+# 安装 nginx 和其他必要工具
+RUN apt-get update && apt-get install -y \
+    nginx \
+    curl \
+    vim \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# 创建 nginx 配置目录
+RUN mkdir -p /etc/nginx/conf.d
+
+# 复制 nginx 配置文件
+COPY nginx.conf /etc/nginx/sites-available/default
+
+# 创建 nginx 工作目录
+RUN mkdir -p /var/www/html
+
+# 创建启动脚本
+COPY start-nginx.sh /usr/local/bin/start-nginx.sh
+RUN chmod +x /usr/local/bin/start-nginx.sh
+
+# 暴露端口
+EXPOSE 80 9229
+
+# 设置工作目录
+WORKDIR /var/www/html
+
+# 健康检查
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost/health || exit 1
+
+RUN mkdir -p /root/workspace
+
+COPY gc.js /root/workspace/gc.js
+COPY benchmark.js /root/workspace/benchmark.js
+COPY benchmark_undici.js /root/workspace/benchmark_undici.js
+
+RUN cd /root/workspace && npm i urllib --registry https://registry.npmmirror.com
+
+# 启动命令
+CMD ["/usr/local/bin/start-nginx.sh"]

benchmark/stream_download/README.md

@@ -0,0 +1,51 @@
+# Nginx 下载/上传测试服务器
+
+## 快速开始
+
+> **注意**: 请先切换到 `benchmark/stream_download` 目录下执行以下命令
+
+### 构建镜像
+
+```bash
+docker build --platform linux/amd64 -t nginx-node-benchmark .
+```
+
+### 运行容器
+
+```bash
+docker run --rm -d --platform linux/amd64 \
+  --name nginx-node-benchmark \
+  -p 8080:80 \
+  -v $(pwd)/nginx:/var/www/html \
+  nginx-node-benchmark
+```
+
+### 测试
+
+```bash
+# 下载测试
+curl -v http://localhost:8080/download/test-file.txt
+
+# 上传测试
+curl -v -X POST -d "test" http://localhost:8080/upload/
+```
+
+### 停止
+
+```bash
+docker stop nginx-node-benchmark && docker rm nginx-node-benchmark
+```
+
+### 运行生成大文件
+
+```bash
+sh generate_50mb_file.sh
+```
+
+### 运行 node 测试
+
+```bash
+docker exec -ti nginx-node-benchmark bash
+cd /root/workspace
+node benchmark.js
+```

benchmark/stream_download/benchmark.js

@@ -0,0 +1,60 @@
+const urllib = require('urllib');
+const fs = require('fs');
+const crypto = require('crypto');
+const path = require('path');
+const { setTimeout } = require('timers/promises');
+
+require('./gc.js');
+
+const tmp = path.join(__dirname, 'tmp');
+fs.mkdirSync(tmp, { recursive: true });
+
+const BASE_URL = process.argv[2] || 'http://127.0.0.1';
+let count = 0;
+async function downloadAndUpload() {
+  const tmpFilePath = path.join(tmp, `${crypto.randomUUID()}.txt`);
+  const downloadResponse = await urllib.request(`${BASE_URL}/download/50mb_ones.txt`, {
+    writeStream: fs.createWriteStream(tmpFilePath),
+  });
+  const uploadResponse = await urllib.request(`${BASE_URL}/upload/`, {
+    method: 'POST',
+    stream: fs.createReadStream(tmpFilePath),
+  });
+  await fs.promises.rm(tmpFilePath);
+  count++;
+  if (count % 100 === 0) {
+    console.log(
+      `Downloaded and uploaded ${count} times, downloadResponse: ${downloadResponse.status}, uploadResponse: ${uploadResponse.status}`,
+    );
+  }
+}
+
+let downloading = true;
+(async () => {
+  while (true) {
+    if (downloading) {
+      await Promise.all([downloadAndUpload(), downloadAndUpload(), downloadAndUpload()]);
+    } else {
+      await setTimeout(100);
+      if (globalThis.gc) {
+        globalThis.gc();
+      }
+    }
+  }
+})();
+(async () => {
+  while (true) {
+    if (downloading) {
+      await Promise.all([downloadAndUpload(), downloadAndUpload(), downloadAndUpload()]);
+    } else {
+      await setTimeout(100);
+      if (globalThis.gc) {
+        globalThis.gc();
+      }
+    }
+  }
+})();
+
+process.on('SIGUSR2', () => {
+  downloading = !downloading;
+});

benchmark/stream_download/benchmark_undici.js

@@ -0,0 +1,39 @@
+const { request } = require('undici');
+const fs = require('fs');
+const crypto = require('crypto');
+const path = require('path');
+const { pipeline } = require('stream/promises');
+
+async function downloadAndUpload() {
+  const tmpFilePath = path.join(__dirname, `${crypto.randomUUID()}.txt`);
+
+  // Download file
+  const downloadResponse = await request('http://127.0.0.1/download/50mb_ones.txt');
+  await pipeline(downloadResponse.body, fs.createWriteStream(tmpFilePath));
+
+  // Upload file
+  await request('http://127.0.0.1/upload/', {
+    method: 'POST',
+    body: fs.createReadStream(tmpFilePath),
+  });
+
+  await fs.promises.rm(tmpFilePath);
+}
+
+let downloading = true;
+(async () => {
+  while (true) {
+    if (downloading) {
+      await downloadAndUpload();
+    } else {
+      await setTimeout(100);
+      if (globalThis.gc) {
+        globalThis.gc();
+      }
+    }
+  }
+})();
+
+process.on('SIGUSR2', () => {
+  downloading = !downloading;
+});

benchmark/stream_download/curl-format.txt

@@ -0,0 +1,16 @@
+time_namelookup:  %{time_namelookup}\n
+       time_connect:  %{time_connect}\n
+    time_appconnect:  %{time_appconnect}\n
+   time_pretransfer:  %{time_pretransfer}\n
+      time_redirect:  %{time_redirect}\n
+ time_starttransfer:  %{time_starttransfer}\n
+                    ----------\n
+         time_total:  %{time_total}\n
+                    ----------\n
+       size_download:  %{size_download}\n
+      speed_download:  %{speed_download}\n
+       size_request:   %{size_request}\n
+      speed_upload:     %{speed_upload}\n
+     content_type:     %{content_type}\n
+    num_connects:     %{num_connects}\n
+   num_redirects:      %{num_redirects}

benchmark/stream_download/docker-compose.nginx.yml

@@ -0,0 +1,23 @@
+version: '3.8'
+
+services:
+  nginx-benchmark:
+    image: nginx:alpine
+    container_name: nginx-benchmark-server
+    ports:
+      - '8080:80'
+    volumes:
+      - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
+      - ./nginx:/usr/share/nginx/html:ro
+    restart: unless-stopped
+
+  # 可选：使用 openresty 支持 Lua 模块
+  # openresty-benchmark:
+  #   image: openresty/openresty:alpine
+  #   container_name: openresty-benchmark-server
+  #   ports:
+  #     - "8080:80"
+  #   volumes:
+  #     - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
+  #     - ./nginx:/usr/share/nginx/html:ro
+  #   restart: unless-stopped

benchmark/stream_download/gc.js

@@ -0,0 +1,79 @@
+const { PerformanceObserver, constants } = require('node:perf_hooks');
+
+const gcStats = {
+  totalGCDuration: 0, // ms
+  count: 0,
+  byKind: {
+    scavenge: 0, // minor GC
+    markSweepCompact: 0, // major GC
+    incremental: 0,
+    weakc: 0,
+    unknown: 0,
+  },
+};
+
+// kind meaning: https://nodejs.org/api/perf_hooks.html#performancegc_kind
+// 1: scavenge
+// 2: mark-sweep-compact
+// 4: incremental
+// 8: weak callbacks
+function kindToString(kind) {
+  switch (kind) {
+    case constants.NODE_PERFORMANCE_GC_MAJOR:
+      return 'markSweepCompact';
+    case constants.NODE_PERFORMANCE_GC_MINOR:
+      return 'scavenge';
+    case constants.NODE_PERFORMANCE_GC_INCREMENTAL:
+      return 'incremental';
+    case constants.NODE_PERFORMANCE_GC_WEAKCB:
+      return 'weakc';
+    default:
+      return 'unknown';
+  }
+}
+
+const obs = new PerformanceObserver((list) => {
+  const entries = list.getEntries();
+  for (const entry of entries) {
+    gcStats.totalGCDuration += entry.duration;
+    gcStats.count += 1;
+
+    const kindCode = entry.detail?.kind;
+    const kind = kindToString(kindCode);
+    if (!gcStats.byKind[kind]) gcStats.byKind[kind] = 0;
+    gcStats.byKind[kind] += entry.duration;
+  }
+});
+
+obs.observe({ entryTypes: ['gc'] });
+
+// for other modules to use
+function getGCStats() {
+  return {
+    totalGCDuration: gcStats.totalGCDuration,
+    count: gcStats.count,
+    avgDuration: gcStats.count ? gcStats.totalGCDuration / gcStats.count : 0,
+    byKind: { ...gcStats.byKind },
+  };
+}
+
+// only print GC stats if the GC environment variable is set
+if (process.env.GC || true) {
+  setInterval(() => {
+    const stats = getGCStats();
+    console.log('');
+    console.log(
+      '[GC]',
+      'total(ms)=',
+      stats.totalGCDuration.toFixed(2),
+      'count=',
+      stats.count,
+      'avg(ms)=',
+      stats.avgDuration.toFixed(2),
+      'byKind=',
+      stats.byKind,
+    );
+    // process memory usage
+    console.log('process memory usage=', process.memoryUsage());
+  }, 2000);
+}

benchmark/stream_download/generate_50mb_file.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# 生成一个50MB的txt文件，内容都是1
+# 文件名为: 50mb_ones.txt
+
+OUTPUT_FILE="nginx/50mb_ones.txt"
+TARGET_SIZE_MB=50
+TARGET_SIZE_BYTES=$((TARGET_SIZE_MB * 1024 * 1024))
+
+# 检查文件是否已存在
+if [ -f "$OUTPUT_FILE" ]; then
+    echo "文件 $OUTPUT_FILE 已存在，正在删除..."
+    rm -f "$OUTPUT_FILE"
+fi
+
+echo "正在生成 $TARGET_SIZE_MB MB 的文件，内容都是1..."
+
+# 使用dd命令生成文件，每块1KB，共50*1024块
+dd if=/dev/zero bs=1024 count=$((TARGET_SIZE_MB * 1024)) | tr '\0' '1' > "$OUTPUT_FILE"
+
+# 验证文件大小
+ACTUAL_SIZE=$(stat -f%z "$OUTPUT_FILE" 2>/dev/null || stat -c%s "$OUTPUT_FILE" 2>/dev/null)
+if [ "$ACTUAL_SIZE" -eq "$TARGET_SIZE_BYTES" ]; then
+    echo "成功生成文件: $OUTPUT_FILE"
+    echo "文件大小: $(ls -lh "$OUTPUT_FILE" | awk '{print $5}')"
+else
+    echo "警告: 文件大小不匹配，期望: $TARGET_SIZE_BYTES 字节，实际: $ACTUAL_SIZE 字节"
+fi
+
+echo "文件路径: $(pwd)/$OUTPUT_FILE"