Skip to content

Commit c7ba1af

Browse files
SergeyMenshykhCopilot
SergeyMenshykh
and
Copilot
authored
.Net: Temporarily skip direct OpenAI integration tests (#14507)
### Motivation and Context The .NET integration job in https://github.com/microsoft/semantic-kernel/actions/runs/36585640900/job/109465158271 failed on direct OpenAI API calls because the configured key was rejected (401) and the account had exhausted its credits (429). Temporarily pause those cases while credentials and billing are restored; Azure OpenAI failures are separate and are not addressed here. ### Description - Skip direct OpenAI connector tests in code with an explicit temporary reason. - Skip only the OpenAI-backed data rows of response-agent tests; retain Azure-backed rows. - Temporarily disable the OpenAI response-agent conformance classes that require the direct API. Keep their imports and declarations available for re-enabling. - Leave the integration-test workflow unchanged. Focused validation: the integration-test project compiled, and the selected OpenAI test cases reported 12 skipped, 0 failed. Azure OpenAI service errors from the original run can still cause the job to fail. ### Contribution Checklist - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent c5868e5 commit c7ba1af

19 files changed

Lines changed: 251 additions & 109 deletions

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AgentThreadConformance/OpenAIResponseAgentThreadTests.cs‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,43 @@ namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.Agen
77

88
public class OpenAIResponseAgentThreadTests() : AgentThreadTests(() => new OpenAIResponseAgentFixture())
99
{
10-
[Fact]
10+
private const string TempOAIKeySkipReason = "Temporarily disabled until the OpenAI API key and credits are restored.";
11+
12+
[Fact(Skip = TempOAIKeySkipReason)]
1113
public override Task OnNewMessageWithServiceFailureThrowsAgentOperationExceptionAsync()
1214
{
1315
// Test not applicable since we cannot add a message to the thread we can only respond to a message.
1416
return Task.CompletedTask;
1517
}
1618

17-
[Fact]
19+
[Fact(Skip = TempOAIKeySkipReason)]
1820
public override Task UsingThreadBeforeCreateCreatesAsync()
1921
{
2022
// Test not applicable since we cannot create a thread we can only respond to a message.
2123
return Task.CompletedTask;
2224
}
25+
26+
[Fact(Skip = TempOAIKeySkipReason)]
27+
public override Task DeletingThreadTwiceDoesNotThrowAsync()
28+
{
29+
return base.DeletingThreadTwiceDoesNotThrowAsync();
30+
}
31+
32+
[Fact(Skip = TempOAIKeySkipReason)]
33+
public override Task UsingThreadAfterDeleteThrowsAsync()
34+
{
35+
return base.UsingThreadAfterDeleteThrowsAsync();
36+
}
37+
38+
[Fact(Skip = TempOAIKeySkipReason)]
39+
public override Task DeleteThreadBeforeCreateThrowsAsync()
40+
{
41+
return base.DeleteThreadBeforeCreateThrowsAsync();
42+
}
43+
44+
[Fact(Skip = TempOAIKeySkipReason)]
45+
public override Task DeleteThreadWithServiceFailureThrowsAgentOperationExceptionAsync()
46+
{
47+
return base.DeleteThreadWithServiceFailureThrowsAgentOperationExceptionAsync();
48+
}
2349
}

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/AgentWithAIContextProviderConformance/OpenAIResponseAgentWithAIContextProviderTests.cs‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,35 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using System.Threading.Tasks;
4+
using Xunit;
5+
36
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.AgentWithStatePartConformance;
47

5-
public class OpenAIResponseAgentWithAIContextProviderTests() : AgentWithAIContextProviderTests<OpenAIResponseAgentFixture>(() => new OpenAIResponseAgentFixture());
8+
public class OpenAIResponseAgentWithAIContextProviderTests() : AgentWithAIContextProviderTests<OpenAIResponseAgentFixture>(() => new OpenAIResponseAgentFixture())
9+
{
10+
private const string TempOAIKeySkipReason = "Temporarily disabled until the OpenAI API key and credits are restored.";
11+
12+
[Fact(Skip = TempOAIKeySkipReason)]
13+
public override Task StatePartReceivesMessagesFromAgentAsync()
14+
{
15+
return base.StatePartReceivesMessagesFromAgentAsync();
16+
}
17+
18+
[Fact(Skip = TempOAIKeySkipReason)]
19+
public override Task StatePartReceivesMessagesFromAgentWhenStreamingAsync()
20+
{
21+
return base.StatePartReceivesMessagesFromAgentWhenStreamingAsync();
22+
}
23+
24+
[Fact(Skip = TempOAIKeySkipReason)]
25+
public override Task StatePartPreInvokeStateIsUsedByAgentAsync()
26+
{
27+
return base.StatePartPreInvokeStateIsUsedByAgentAsync();
28+
}
29+
30+
[Fact(Skip = TempOAIKeySkipReason)]
31+
public override Task StatePartPreInvokeStateIsUsedByAgentWhenStreamingAsync()
32+
{
33+
return base.StatePartPreInvokeStateIsUsedByAgentWhenStreamingAsync();
34+
}
35+
}

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeConformance/OpenAIResponseAgentInvokeTests.cs‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,33 @@
33
using System.ClientModel;
44
using System.Threading.Tasks;
55
using Microsoft.SemanticKernel.Agents.OpenAI;
6+
using xRetry;
67
using Xunit;
78

89
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeConformance;
910

1011
public class OpenAIResponseAgentInvokeTests() : InvokeTests(() => new OpenAIResponseAgentFixture())
1112
{
13+
private const string TempOAIKeySkipReason = "Temporarily disabled until the OpenAI API key and credits are restored.";
14+
15+
[RetryFact(3, 5000, Skip = TempOAIKeySkipReason)]
16+
public override Task InvokeReturnsResultAsync()
17+
{
18+
return base.InvokeReturnsResultAsync();
19+
}
20+
21+
[RetryFact(3, 5000, Skip = TempOAIKeySkipReason)]
22+
public override Task InvokeWithoutThreadCreatesThreadAsync()
23+
{
24+
return base.InvokeWithoutThreadCreatesThreadAsync();
25+
}
26+
27+
[RetryFact(3, 5000, Skip = TempOAIKeySkipReason)]
28+
public override Task MultiStepInvokeWithPluginAndArgOverridesAsync()
29+
{
30+
return base.MultiStepInvokeWithPluginAndArgOverridesAsync();
31+
}
32+
1233
[Fact(Skip = $"{nameof(OpenAIResponseAgent)} excludes the final response from the remote history.")]
1334
public override Task ConversationMaintainsHistoryAsync()
1435
{
@@ -18,7 +39,7 @@ public override Task ConversationMaintainsHistoryAsync()
1839
/// <summary>
1940
/// <see cref="OpenAIResponseAgent"/> must be invoked with a message.
2041
/// </summary>
21-
[Fact]
42+
[Fact(Skip = TempOAIKeySkipReason)]
2243
public override Task InvokeWithoutMessageCreatesThreadAsync()
2344
{
2445
return Assert.ThrowsAsync<ClientResultException>(() => base.InvokeWithoutMessageCreatesThreadAsync());

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/InvokeStreamingConformance/OpenAIResponseAgentInvokeStreamingTests.cs‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,40 @@
33
using System.ClientModel;
44
using System.Threading.Tasks;
55
using Microsoft.SemanticKernel.Agents.OpenAI;
6+
using xRetry;
67
using Xunit;
78

89
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.InvokeStreamingConformance;
910

1011
[Collection("Sequential")]
1112
public class OpenAIResponseAgentInvokeStreamingTests() : InvokeStreamingTests(() => new OpenAIResponseAgentFixture())
1213
{
14+
private const string TempOAIKeySkipReason = "Temporarily disabled until the OpenAI API key and credits are restored.";
15+
16+
[RetryFact(3, 10_000, Skip = TempOAIKeySkipReason)]
17+
public override Task InvokeStreamingAsyncReturnsResultAsync()
18+
{
19+
return base.InvokeStreamingAsyncReturnsResultAsync();
20+
}
21+
22+
[RetryFact(3, 10_000, Skip = TempOAIKeySkipReason)]
23+
public override Task InvokeStreamingAsyncWithoutThreadCreatesThreadAsync()
24+
{
25+
return base.InvokeStreamingAsyncWithoutThreadCreatesThreadAsync();
26+
}
27+
28+
[RetryFact(3, 10_000, Skip = TempOAIKeySkipReason)]
29+
public override Task MultiStepInvokeStreamingAsyncWithPluginAndArgOverridesAsync()
30+
{
31+
return base.MultiStepInvokeStreamingAsyncWithPluginAndArgOverridesAsync();
32+
}
33+
34+
[RetryFact(3, 10_000, Skip = TempOAIKeySkipReason)]
35+
public override Task InvokeStreamingWithPluginNotifiesForAllMessagesAsync()
36+
{
37+
return base.InvokeStreamingWithPluginNotifiesForAllMessagesAsync();
38+
}
39+
1340
[Fact(Skip = $"{nameof(OpenAIResponseAgent)} excludes the final response from the remote history.")]
1441
public override Task ConversationMaintainsHistoryAsync()
1542
{
@@ -19,7 +46,7 @@ public override Task ConversationMaintainsHistoryAsync()
1946
/// <summary>
2047
/// <see cref="OpenAIResponseAgent"/> must be invoked with a message.
2148
/// </summary>
22-
[Fact]
49+
[Fact(Skip = TempOAIKeySkipReason)]
2350
public override Task InvokeStreamingAsyncWithoutMessageCreatesThreadAsync()
2451
{
2552
return Assert.ThrowsAsync<ClientResultException>(() => base.InvokeStreamingAsyncWithoutMessageCreatesThreadAsync());

‎dotnet/src/IntegrationTests/Agents/CommonInterfaceConformance/SemanticKernelAIAgentConformance/OpenAIResponseAgentAdapterTests.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,17 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using System.Threading.Tasks;
4+
using Xunit;
5+
36
namespace SemanticKernel.IntegrationTests.Agents.CommonInterfaceConformance.SemanticKernelAIAgentConformance;
47

58
public class OpenAIResponseAgentAdapterTests() : SemanticKernelAIAgentTests(() => new OpenAIResponseAgentFixture())
69
{
10+
private const string TempOAIKeySkipReason = "Temporarily disabled until the OpenAI API key and credits are restored.";
11+
12+
[Fact(Skip = TempOAIKeySkipReason)]
13+
public override Task ConvertAndRunAgentAsync()
14+
{
15+
return base.ConvertAndRunAgentAsync();
16+
}
717
}

‎dotnet/src/IntegrationTests/Agents/OpenAIResponseAgentTests.cs‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ namespace SemanticKernel.IntegrationTests.Agents;
2626

2727
public sealed class OpenAIResponseAgentTests(ITestOutputHelper output)
2828
{
29+
private const string TempOAIKeySkipReason = "Temporarily disabled until the OpenAI API key and credits are restored.";
30+
2931
private readonly IConfigurationRoot _configuration = new ConfigurationBuilder()
3032
.AddJsonFile(path: "testsettings.json", optional: true, reloadOnChange: true)
3133
.AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true)
@@ -37,8 +39,8 @@ public sealed class OpenAIResponseAgentTests(ITestOutputHelper output)
3739
/// Integration test for <see cref="OpenAIResponseAgent"/>.
3840
/// </summary>
3941
[RetryTheory(typeof(HttpOperationException))]
40-
[InlineData("What is the capital of France?", "Paris", true, true)]
41-
[InlineData("What is the capital of France?", "Paris", true, false)]
42+
[InlineData("What is the capital of France?", "Paris", true, true, Skip = TempOAIKeySkipReason)]
43+
[InlineData("What is the capital of France?", "Paris", true, false, Skip = TempOAIKeySkipReason)]
4244
[InlineData("What is the capital of France?", "Paris", false, true)]
4345
[InlineData("What is the capital of France?", "Paris", false, false)]
4446
public async Task OpenAIResponseAgentInvokeAsync(string input, string expectedAnswerContains, bool isOpenAI, bool storeEnabled)
@@ -57,8 +59,8 @@ await this.ExecuteAgentAsync(
5759
/// Integration test for <see cref="OpenAIResponseAgent"/> using a thread.
5860
/// </summary>
5961
[RetryTheory(typeof(HttpOperationException))]
60-
[InlineData("What is the capital of France?", "Paris", true, true)]
61-
[InlineData("What is the capital of France?", "Paris", true, false)]
62+
[InlineData("What is the capital of France?", "Paris", true, true, Skip = TempOAIKeySkipReason)]
63+
[InlineData("What is the capital of France?", "Paris", true, false, Skip = TempOAIKeySkipReason)]
6264
[InlineData("What is the capital of France?", "Paris", false, true)]
6365
[InlineData("What is the capital of France?", "Paris", false, false)]
6466
public async Task OpenAIResponseAgentInvokeWithThreadAsync(string input, string expectedAnswerContains, bool isOpenAI, bool storeEnabled)
@@ -106,8 +108,8 @@ public async Task OpenAIResponseAgentInvokeWithThreadAsync(string input, string
106108
/// Integration test for <see cref="OpenAIResponseAgent"/> using a function calling.
107109
/// </summary>
108110
[RetryTheory(typeof(HttpOperationException))]
109-
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", true, true)]
110-
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", true, false)]
111+
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", true, true, Skip = TempOAIKeySkipReason)]
112+
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", true, false, Skip = TempOAIKeySkipReason)]
111113
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", false, true)]
112114
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", false, false)]
113115
public async Task OpenAIResponseAgentInvokeWithFunctionCallingAsync(string input, string expectedAnswerContains, bool isOpenAI, bool storeEnabled)
@@ -156,8 +158,8 @@ public async Task OpenAIResponseAgentInvokeWithFunctionCallingAsync(string input
156158
/// Integration test for <see cref="OpenAIResponseAgent"/> using streaming.
157159
/// </summary>
158160
[RetryTheory(typeof(HttpOperationException))]
159-
[InlineData("What is the capital of France?", "Paris", true, true)]
160-
[InlineData("What is the capital of France?", "Paris", true, false)]
161+
[InlineData("What is the capital of France?", "Paris", true, true, Skip = TempOAIKeySkipReason)]
162+
[InlineData("What is the capital of France?", "Paris", true, false, Skip = TempOAIKeySkipReason)]
161163
[InlineData("What is the capital of France?", "Paris", false, true)]
162164
[InlineData("What is the capital of France?", "Paris", false, false)]
163165
public async Task OpenAIResponseAgentInvokeStreamingAsync(string input, string expectedAnswerContains, bool isOpenAI, bool storeEnabled)
@@ -176,8 +178,8 @@ await this.ExecuteStreamingAgentAsync(
176178
/// Integration test for <see cref="OpenAIResponseAgent"/> adding override instructions to a thread on invocation via custom options.
177179
/// </summary>
178180
[RetryTheory(typeof(HttpOperationException))]
179-
[InlineData(true, false)]
180-
[InlineData(true, false)]
181+
[InlineData(true, false, Skip = TempOAIKeySkipReason)]
182+
[InlineData(true, false, Skip = TempOAIKeySkipReason)]
181183
[InlineData(false, true)]
182184
[InlineData(false, false)]
183185
public async Task OpenAIResponseAgentInvokeStreamingWithThreadAsync(bool isOpenAI, bool storeEnabled)
@@ -224,8 +226,8 @@ public async Task OpenAIResponseAgentInvokeStreamingWithThreadAsync(bool isOpenA
224226
/// Integration test for <see cref="OpenAIResponseAgent"/> adding override instructions to a thread on invocation via custom options.
225227
/// </summary>
226228
[RetryTheory(typeof(HttpOperationException))]
227-
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", true, true)]
228-
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", true, false)]
229+
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", true, true, Skip = TempOAIKeySkipReason)]
230+
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", true, false, Skip = TempOAIKeySkipReason)]
229231
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", false, true)]
230232
[InlineData("What is the special soup and how much does it cost?", "Clam Chowder", false, false)]
231233
public async Task OpenAIResponseAgentInvokeStreamingWithFunctionCallingAsync(string input, string expectedAnswerContains, bool isOpenAI, bool storeEnabled)

‎dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIAudioToTextTests.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,16 @@ namespace SemanticKernel.IntegrationTests.Connectors.OpenAI;
1515

1616
public sealed class OpenAIAudioToTextTests()
1717
{
18+
private const string TempOAIKeySkipReason = "Temporarily disabled until the OpenAI API key and credits are restored.";
19+
1820
private readonly IConfigurationRoot _configuration = new ConfigurationBuilder()
1921
.AddJsonFile(path: "testsettings.json", optional: true, reloadOnChange: true)
2022
.AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true)
2123
.AddEnvironmentVariables()
2224
.AddUserSecrets<OpenAIAudioToTextTests>()
2325
.Build();
2426

25-
[RetryFact] //(Skip = "OpenAI will often throttle requests. This test is for manual verification.")]
27+
[RetryFact(Skip = TempOAIKeySkipReason)]
2628
public async Task OpenAIAudioToTextTestAsync()
2729
{
2830
// Arrange

‎dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIChatClientTests.cs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ namespace SemanticKernel.IntegrationTests.Connectors.OpenAI;
2121

2222
public sealed class OpenAIChatClientTests : BaseIntegrationTest
2323
{
24-
[Fact]
24+
private const string TempOAIKeySkipReason = "Temporarily disabled until the OpenAI API key and credits are restored.";
25+
26+
[Fact(Skip = TempOAIKeySkipReason)]
2527
public async Task ItCanUseOpenAiChatForTextGenerationAsync()
2628
{
2729
// Arrange
@@ -40,7 +42,7 @@ public async Task ItCanUseOpenAiChatForTextGenerationAsync()
4042
Assert.Contains("Uranus", result.GetValue<string>(), StringComparison.InvariantCultureIgnoreCase);
4143
}
4244

43-
[Fact]
45+
[Fact(Skip = TempOAIKeySkipReason)]
4446
public async Task OpenAIStreamingTestAsync()
4547
{
4648
// Arrange
@@ -106,7 +108,7 @@ public async Task OpenAIHttpRetryPolicyTestAsync()
106108
Assert.Equal(HttpStatusCode.Unauthorized, exception.StatusCode);
107109
}
108110

109-
[Fact]
111+
[Fact(Skip = TempOAIKeySkipReason)]
110112
public async Task OpenAIShouldReturnUsageAsync()
111113
{
112114
// Arrange
@@ -148,7 +150,7 @@ public async Task CompletionWithDifferentLineEndingsAsync(string lineEnding)
148150
Assert.Contains("<result>John</result>", actual.GetValue<string>(), StringComparison.OrdinalIgnoreCase);
149151
}
150152

151-
[Fact]
153+
[Fact(Skip = TempOAIKeySkipReason)]
152154
public async Task ChatSystemPromptIsNotIgnoredAsync()
153155
{
154156
// Arrange
@@ -163,7 +165,7 @@ public async Task ChatSystemPromptIsNotIgnoredAsync()
163165
Assert.Contains("I don't know", result.ToString(), StringComparison.OrdinalIgnoreCase);
164166
}
165167

166-
[Fact]
168+
[Fact(Skip = TempOAIKeySkipReason)]
167169
public async Task SemanticKernelVersionHeaderIsSentAsync()
168170
{
169171
// Arrange

0 commit comments

Comments
 (0)