Skip to content
Open
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 53 additions & 37 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
],
"main": "lib/src/extension.js",
"dependencies": {
"@salesforce/agents": "^1.2.0",
"@salesforce/agents": "^1.6.4",
"@salesforce/core": "^8.28.3",
"@salesforce/kit": "^3.2.6",
"@salesforce/types": "^1.7.1",
Expand Down
18 changes: 18 additions & 0 deletions src/types/MetadataDefinitions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,21 @@ export type AgentTestCase = {
{ name: 'expectedOutcome'; expectedValue: string }
];
};

// until @salesforce/types has AiTestingDefinition
export type AgentforceStudioTestCaseMetadata = {
number: string;
inputs: { utterance: string };
};

// until @salesforce/types has AiTestingDefinition
export type AiTestingDefinition = {
AiTestingDefinition: {
description: string;
name: string;
subjectType: 'AGENT';
subjectName: string;
subjectVersion: string;
testCase: AgentforceStudioTestCaseMetadata | AgentforceStudioTestCaseMetadata[];
};
};
7 changes: 6 additions & 1 deletion src/types/TestNodes.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import * as vscode from 'vscode';
import { TestStatus } from '@salesforce/agents';
import { TestStatus, type TestRunnerType } from '@salesforce/agents';
import { getTestOutlineProvider } from '../views/testOutlineProvider';

/**
Expand Down Expand Up @@ -66,8 +66,13 @@ export abstract class TestNode extends vscode.TreeItem {
* has children AgentTestNode for individual test cases
*/
export class AgentTestGroupNode extends TestNode {
/** The actual metadata API name, which may differ from the display label when there's a naming conflict. */
public testDefinitionName: string;
public runnerType: TestRunnerType = 'testing-center';

constructor(label: string, location?: vscode.Location) {
super(label, vscode.TreeItemCollapsibleState.Expanded, location ?? null);
this.testDefinitionName = label;
}

public contextValue = 'agentTestGroup';
Expand Down
121 changes: 89 additions & 32 deletions src/views/testOutlineProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,50 +18,107 @@ import { basename } from 'path';
import * as vscode from 'vscode';
import * as xml from 'fast-xml-parser';
import { Commands } from '../enums/commands';
import { AgentTestGroupNode, AgentTestNode, AiEvaluationDefinition, TestNode } from '../types';
import { AgentTestGroupNode, AgentTestNode, AiEvaluationDefinition, AiTestingDefinition, TestNode } from '../types';
import type { TestRunnerType } from '@salesforce/agents';

const NO_TESTS_MESSAGE = 'no tests found';
const NO_TESTS_DESCRIPTION = 'no test description';
const AGENT_TESTS = 'AgentTests';

const buildTestGroupNode = (
definitionUri: vscode.Uri,
definitionApiName: string,
label: string,
runnerType: TestRunnerType,
testCases: Array<{ number: string; inputs: { utterance: string } }>,
fileContent: string
): AgentTestGroupNode => {
const testDefinitionNode = new AgentTestGroupNode(
label,
new vscode.Location(definitionUri, new vscode.Position(0, 0))
);
testDefinitionNode.testDefinitionName = definitionApiName;
testDefinitionNode.runnerType = runnerType;
const splitContent = fileContent.split(EOL);
testCases.forEach(test => {
const line = splitContent.findIndex(l => l.includes(`<number>${test.number}</number`));
Comment thread
andresrivas-sf marked this conversation as resolved.
Outdated
Comment thread
andresrivas-sf marked this conversation as resolved.
Outdated
const testcaseNode = new AgentTestNode(
`#${test.number}`,
new vscode.Location(definitionUri, new vscode.Position(line < 0 ? 0 : line, 8))
);
testcaseNode.parentName = label;
testcaseNode.description = test.inputs.utterance;
testDefinitionNode.children.push(testcaseNode);
});
testDefinitionNode.children.sort((a, b) => a.name.localeCompare(b.name));
return testDefinitionNode;
};

export const parseAgentTestsFromProject = async (): Promise<Map<string, AgentTestGroupNode>> => {
const aiTestDefs = await vscode.workspace.findFiles('**/*.aiEvaluationDefinition-meta.xml');
//from the aiTestDef files, parse the xml using fast-xml-parser, find the testSetName that it points to
const aggregator = new Map<string, AgentTestGroupNode>();
const [aiEvalDefs, aiTestingDefs] = await Promise.all([
vscode.workspace.findFiles('**/*.aiEvaluationDefinition-meta.xml'),
vscode.workspace.findFiles('**/*.aiTestingDefinition-meta.xml')
]);
const parser = new xml.XMLParser();
await Promise.all(
aiTestDefs.map(async definition => {

// Parse both sets independently, keyed by API name
const evalNodes = new Map<string, AgentTestGroupNode>();
const agentforceStudioNodes = new Map<string, AgentTestGroupNode>();

await Promise.all([
...aiEvalDefs.map(async definition => {
const fileContent = (await vscode.workspace.fs.readFile(definition)).toString();
const testDefinition = parser.parse(fileContent) as AiEvaluationDefinition;
const definitionApiName = basename(definition.fsPath, '.aiEvaluationDefinition-meta.xml');

const testDefinitionNode = new AgentTestGroupNode(
const rawTestCases = testDefinition.AiEvaluationDefinition?.testCase;
if (!rawTestCases) return;
const testCases = Array.isArray(rawTestCases) ? rawTestCases : [rawTestCases];
evalNodes.set(
definitionApiName,
new vscode.Location(definition, new vscode.Position(0, 0))
buildTestGroupNode(definition, definitionApiName, definitionApiName, 'testing-center', testCases, fileContent)
);
}),
...aiTestingDefs.map(async definition => {
const fileContent = (await vscode.workspace.fs.readFile(definition)).toString();
const testDefinition = parser.parse(fileContent) as AiTestingDefinition;
const definitionApiName = basename(definition.fsPath, '.aiTestingDefinition-meta.xml');
const rawTestCases = testDefinition.AiTestingDefinition?.testCase;
if (!rawTestCases) return;
const testCases = Array.isArray(rawTestCases) ? rawTestCases : [rawTestCases];
agentforceStudioNodes.set(
definitionApiName,
buildTestGroupNode(
definition,
definitionApiName,
definitionApiName,
'agentforce-studio',
testCases,
fileContent
)
);

const splitContent = fileContent.split(EOL);

(Array.isArray(testDefinition.AiEvaluationDefinition.testCase)
? // xml parser will not parse single node into array
testDefinition.AiEvaluationDefinition.testCase
: [testDefinition.AiEvaluationDefinition.testCase]
).map(test => {
const line = splitContent.findIndex(l => l.includes(`<number>${test.number}</number`));
const testcaseNode = new AgentTestNode(
`#${test.number}`,
new vscode.Location(definition, new vscode.Position(line, 8))
);
testcaseNode.parentName = definitionApiName;
testcaseNode.description = test.inputs.utterance;
testDefinitionNode.children.push(testcaseNode);
});

// Sort test cases alphabetically by name
testDefinitionNode.children.sort((a, b) => a.name.localeCompare(b.name));

aggregator.set(testDefinitionNode.name, testDefinitionNode);
})
);
]);

// Apply disambiguation labels for names that appear in both sets
const conflicts = new Set([...evalNodes.keys()].filter(k => agentforceStudioNodes.has(k)));
for (const name of conflicts) {
const evalNode = evalNodes.get(name)!;
const afsNode = agentforceStudioNodes.get(name)!;
const evalLabel = `${name} (testing-center)`;
const afsLabel = `${name} (agentforce-studio)`;
evalNode.label = evalLabel;
evalNode.name = evalLabel;
evalNode.children.forEach(c => (c.parentName = evalLabel));
afsNode.label = afsLabel;
afsNode.name = afsLabel;
afsNode.children.forEach(c => (c.parentName = afsLabel));
}

// Merge into a single map keyed by the (possibly suffixed) label
const aggregator = new Map<string, AgentTestGroupNode>();
for (const node of [...evalNodes.values(), ...agentforceStudioNodes.values()]) {
aggregator.set(node.name, node);
}

return aggregator;
};
Expand Down
Loading
Loading