Skip to content

Commit dde4ea8

Browse files
authored
Integration tests stabilization (#559)
1 parent e07a70a commit dde4ea8

File tree

2 files changed

+29
-28
lines changed

2 files changed

+29
-28
lines changed

integration-tests/src/jvmTest/kotlin/ai/koog/integration/tests/AIAgentIntegrationTest.kt

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -981,14 +981,13 @@ class AIAgentIntegrationTest {
981981
@MethodSource("openAIModels", "anthropicModels", "googleModels")
982982
fun integration_AgentWithToolsWithoutParamsTest(model: LLModel) = runTest(timeout = 120.seconds) {
983983
assumeTrue(model.capabilities.contains(LLMCapability.Tools), "Model $model does not support tools")
984-
assumeTrue(
985-
model.id != GoogleModels.Gemini2_0Flash.id,
986-
"gemini-2.0-flash returns flaky results and fails to call tools on a permanent basis"
987-
)
988-
assumeTrue(
989-
model.id != GoogleModels.Gemini2_0Flash001.id,
990-
"gemini-2.0-flash-001 returns flaky results and fails to call tools on a permanent basis"
984+
val flakyModels = listOf(
985+
GoogleModels.Gemini2_0Flash.id,
986+
GoogleModels.Gemini2_0Flash001.id,
987+
GoogleModels.Gemini2_0FlashLite.id,
988+
GoogleModels.Gemini2_0FlashLite001.id
991989
)
990+
assumeTrue(!flakyModels.contains(model.id), "Model $model is flaky and fails to call tools")
992991

993992
val registry = ToolRegistry {
994993
tool(CalculatorToolNoArgs)

integration-tests/src/jvmTest/kotlin/ai/koog/integration/tests/SingleLLMPromptExecutorIntegrationTest.kt

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -716,32 +716,34 @@ class SingleLLMPromptExecutorIntegrationTest {
716716
}
717717
}
718718

719-
try {
720-
val response = executor.execute(prompt, model)
721-
when (scenario) {
722-
MarkdownTestScenario.MALFORMED_SYNTAX,
723-
MarkdownTestScenario.MATH_NOTATION,
724-
MarkdownTestScenario.BROKEN_LINKS,
725-
MarkdownTestScenario.IRREGULAR_TABLES -> {
726-
checkResponseBasic(response)
727-
}
719+
withRetry {
720+
try {
721+
val response = executor.execute(prompt, model)
722+
when (scenario) {
723+
MarkdownTestScenario.MALFORMED_SYNTAX,
724+
MarkdownTestScenario.MATH_NOTATION,
725+
MarkdownTestScenario.BROKEN_LINKS,
726+
MarkdownTestScenario.IRREGULAR_TABLES -> {
727+
checkResponseBasic(response)
728+
}
728729

729-
else -> {
730-
checkExecutorMediaResponse(response)
730+
else -> {
731+
checkExecutorMediaResponse(response)
732+
}
731733
}
732-
}
733-
} catch (e: Exception) {
734-
when (scenario) {
735-
MarkdownTestScenario.EMPTY_MARKDOWN -> {
736-
when (model.provider) {
737-
LLMProvider.Google -> {
738-
println("Expected exception for ${scenario.name.lowercase()} image: ${e.message}")
734+
} catch (e: Exception) {
735+
when (scenario) {
736+
MarkdownTestScenario.EMPTY_MARKDOWN -> {
737+
when (model.provider) {
738+
LLMProvider.Google -> {
739+
println("Expected exception for ${scenario.name.lowercase()} image: ${e.message}")
740+
}
739741
}
740742
}
741-
}
742743

743-
else -> {
744-
throw e
744+
else -> {
745+
throw e
746+
}
745747
}
746748
}
747749
}

0 commit comments

Comments
 (0)