Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,4 @@
version: 2
registries:
public-openai:
type: npm-registry
url: https://registry.npmjs.org
# Hosted updaters cannot use the fixtures' local Verdaccio registry.
scope: "@openai"
updates:
# The root pnpm workspace includes packages/*, examples/*, and docs.
- package-ecosystem: "npm"
Expand All @@ -28,7 +22,6 @@ updates:
- "patch"
# These fixtures install independently from Verdaccio, outside the pnpm workspace.
- package-ecosystem: "npm"
registries: [public-openai]
directories:
- "/integration-tests/bun"
- "/integration-tests/cloudflare-workers/worker"
Expand Down Expand Up @@ -61,7 +54,6 @@ updates:
update-types: ["minor", "patch"]
- package-ecosystem: "npm"
directory: "/integration-tests/node-openai-client-types"
registries: [public-openai]
schedule:
interval: "monthly"
time: "09:00"
Expand Down
2 changes: 2 additions & 0 deletions integration-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ This project hosts packages to test the different environments that the Agents S

It is intentionally not part of the `pnpm` workspace and instead installs the packages from a local package registry using verdaccio.

Committed fixture `.npmrc` files use public npm so Dependabot can resolve dependencies without registry credentials. The integration-test global setup temporarily routes `@openai` packages to the local Verdaccio registry and restores each original `.npmrc` during teardown, including after a partial setup failure. Run fixtures through the integration-test commands below to use locally published packages.

## How to run integration tests

1. **Requirements:**
Expand Down
139 changes: 139 additions & 0 deletions integration-tests/_helpers/setup.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
import { execFileSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { writeFile } from 'node:fs/promises';
import { setup } from './setup';

vi.mock('node:fs/promises', async (importOriginal) => {
const actual = await importOriginal<typeof import('node:fs/promises')>();
return { ...actual, writeFile: vi.fn(actual.writeFile) };
});

const fs =
await vi.importActual<typeof import('node:fs/promises')>('node:fs/promises');
const repositoryRoot = process.cwd();
const dependabotConfig = await fs.readFile(
path.join(repositoryRoot, '.github/dependabot.yml'),
'utf8',
);
const fixtureDirectories = [
...dependabotConfig.matchAll(/"\/integration-tests\/([^"\n]+)"/g),
].map((match) => match[1]);

let temporaryRoot: string;
let originals: Map<string, Buffer>;

beforeEach(async () => {
temporaryRoot = await fs.mkdtemp(path.join(tmpdir(), 'agents-registry-'));
originals = new Map();
for (const directory of fixtureDirectories) {
const file = path.join(
temporaryRoot,
'integration-tests',
directory,
'.npmrc',
);
const content = Buffer.from(
'# Custom fixture configuration\n@openai:registry=https://registry.npmjs.org\npackage-lock=false',
);
await fs.mkdir(path.dirname(file), { recursive: true });
await fs.writeFile(file, content);
originals.set(file, content);
}
vi.spyOn(process, 'cwd').mockReturnValue(temporaryRoot);
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ status: 200 }));
vi.mocked(writeFile).mockClear();
});

afterEach(async () => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
vi.mocked(writeFile).mockImplementation(fs.writeFile);
await fs.rm(temporaryRoot, { recursive: true, force: true });
});

async function expectOriginals() {
for (const [file, content] of originals) {
expect(await fs.readFile(file)).toEqual(content);
}
}

describe('integration registry setup', () => {
it('keeps every hosted updater on public npm without a custom registry', async () => {
expect(fixtureDirectories).toHaveLength(13);
expect(dependabotConfig).not.toMatch(/^registries:/m);
expect(dependabotConfig).not.toContain('registries: [');
for (const directory of fixtureDirectories) {
const config = await fs.readFile(
path.join(repositoryRoot, 'integration-tests', directory, '.npmrc'),
'utf8',
);
expect(config).toContain('@openai:registry=https://registry.npmjs.org');
expect(config).not.toContain('localhost');
}
});

it('routes all fixtures locally for a run, then restores exact original bytes', async () => {
const teardown = await setup();
for (const file of originals.keys()) {
expect(await fs.readFile(file, 'utf8')).toContain(
'\n@openai:registry=http://localhost:4873\n',
);
}
// Verify npm's effective configuration, including duplicate-key precedence.
const fixture = path.join(temporaryRoot, 'integration-tests/node');
expect(
execFileSync('npm', ['config', 'get', '@openai:registry'], {
cwd: fixture,
encoding: 'utf8',
stdio: 'pipe',
}).trim(),
).toBe('http://localhost:4873');
await teardown();
await expectOriginals();
});

it('does not edit fixtures when the registry is unavailable', async () => {
vi.mocked(fetch).mockResolvedValue({ status: 503 } as Response);
await expect(setup()).rejects.toThrow('Local npm registry not running');
expect(writeFile).not.toHaveBeenCalled();
await expectOriginals();
});

it('reads all originals before performing any writes', async () => {
await fs.rm([...originals.keys()].at(-1)!);
await expect(setup()).rejects.toThrow();
expect(writeFile).not.toHaveBeenCalled();
});

it('restores partial writes when setup fails', async () => {
const failure = new Error('Fixture write failed');
vi.mocked(writeFile)
.mockImplementationOnce(fs.writeFile)
.mockImplementationOnce(async (file) => {
await fs.writeFile(file, 'partial');
throw failure;
});
await expect(setup()).rejects.toBe(failure);
await expectOriginals();
});

it('attempts every restoration and preserves setup and cleanup errors', async () => {
const setupFailure = new Error('Fixture write failed');
const cleanupFailure = new Error('Fixture restoration failed');
vi.mocked(writeFile)
.mockImplementationOnce(fs.writeFile)
.mockRejectedValueOnce(setupFailure)
.mockRejectedValueOnce(cleanupFailure);
const error = await setup().catch((error: unknown) => error);
expect(error).toBeInstanceOf(AggregateError);
expect((error as AggregateError).cause).toBe(setupFailure);
const errors = (error as AggregateError).errors;
expect(errors[0]).toBe(setupFailure);
expect((errors[1] as AggregateError).errors).toContain(cleanupFailure);
const secondFile = [...originals.keys()][1];
expect(await fs.readFile(secondFile)).toEqual(originals.get(secondFile));
expect(writeFile).toHaveBeenCalledTimes(4);
});
});
76 changes: 66 additions & 10 deletions integration-tests/_helpers/setup.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,72 @@
import { readFile, writeFile } from 'node:fs/promises';
import path from 'node:path';

const fixtureDirectories = [
'bun',
'cloudflare-workers/worker',
'deno',
'node',
'node-ai-sdk-v2-ts',
'node-ai-sdk-v3-ts',
'node-ai-sdk-v4-ts',
'node-openai-client-types',
'node-sdk-behavior',
'node-zod3',
'node-zod4',
'node-zod4-ts',
'vite-react',
];

export async function setup() {
try {
const response = await fetch('http://localhost:4873/');
if (response.status === 200) {
console.log('Local npm registry already running');
return () => {};
}
} catch (err) {
const response = await fetch('http://localhost:4873/');
if (response.status !== 200) {
throw new Error('Local npm registry not running');
}

return () => {
// console.log('Shutting down local npm registry');
// verdaccioServer.kill();
// Read every original before changing any fixture, including custom settings.
const originals = await Promise.all(
fixtureDirectories.map(async (directory) => {
const file = path.resolve('integration-tests', directory, '.npmrc');
return { file, content: await readFile(file) };
}),
);
const changed: typeof originals = [];
const restore = async () => {
const results = await Promise.allSettled(
changed.map(({ file, content }) => writeFile(file, content)),
);
const errors = results.flatMap((result) =>
result.status === 'rejected' ? [result.reason] : [],
);
if (errors.length > 0) {
throw new AggregateError(
errors,
'Could not restore integration npm configs',
);
}
};

try {
for (const original of originals) {
// Register before writing so a partial write is also restored.
changed.push(original);
await writeFile(
original.file,
`${original.content.toString()}\n@openai:registry=http://localhost:4873\n`,
);
}
} catch (error) {
try {
await restore();
} catch (restoreError) {
throw new AggregateError(
[error, restoreError],
'Integration npm config setup and restoration failed',
{ cause: error },
);
}
throw error;
}

return restore;
}
2 changes: 1 addition & 1 deletion integration-tests/bun/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/cloudflare-workers/worker/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/deno/.npmrc
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
registry=https://registry.npmjs.org/
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node-ai-sdk-v2-ts/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node-ai-sdk-v3-ts/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node-ai-sdk-v4-ts/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node-openai-client-types/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node-sdk-behavior/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node-zod3/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node-zod4-ts/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node-zod4/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/node/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
2 changes: 1 addition & 1 deletion integration-tests/vite-react/.npmrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
@openai:registry=http://localhost:4873
@openai:registry=https://registry.npmjs.org
package-lock=false
1 change: 1 addition & 0 deletions vitest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ function createProjects(reviewTestProfile: boolean) {
maxConcurrency: 4,
include: [
'helpers/tests/consoleGuard.test.ts',
'integration-tests/_helpers/setup.test.ts',
'integration-tests/cloudflare-workers/worker.test.ts',
'helpers/vitest/codeChangeVerificationPolicy.test.ts',
'helpers/vitest/reviewTestProfile.test.ts',
Expand Down
Loading