From d81410d9a0d2fcd465e292711a2867c6bc69bb99 Mon Sep 17 00:00:00 2001 From: Jansoon Date: Wed, 1 Jul 2026 02:54:48 +0900 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20@Async=20Executor=EC=97=90=20?= =?UTF-8?q?=EA=B1=B0=EB=B6=80=20=EC=A0=95=EC=B1=85=20=EB=B0=8F=20graceful?= =?UTF-8?q?=20shutdown=20=EC=B6=94=EA=B0=80=20(#256)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 부하 시 자원 사용을 제어하기 위해 거부 정책(CallerRunsPolicy)과 graceful shutdown을 적용. 풀 사이즈는 보수적으로 설정(core 5, max 20, queue 100)하고 메트릭 모니터링을 통해 추후 튜닝 예정. --- .../sofa/linkiving/global/config/AsyncConfig.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/sofa/linkiving/global/config/AsyncConfig.java b/src/main/java/com/sofa/linkiving/global/config/AsyncConfig.java index 5ec86e63..5192db05 100644 --- a/src/main/java/com/sofa/linkiving/global/config/AsyncConfig.java +++ b/src/main/java/com/sofa/linkiving/global/config/AsyncConfig.java @@ -1,6 +1,7 @@ package com.sofa.linkiving.global.config; import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -24,11 +25,19 @@ public AsyncConfig(TaskDecorator taskDecorator) { @Bean public ThreadPoolTaskExecutor applicationTaskExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(10); - executor.setMaxPoolSize(50); - executor.setQueueCapacity(200); + + executor.setCorePoolSize(5); + executor.setMaxPoolSize(20); + executor.setQueueCapacity(100); executor.setThreadNamePrefix("async-"); + + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(30); + executor.setTaskDecorator(taskDecorator); + executor.initialize(); return executor; } From 0e711341e4e36f74481a52aa8062f0fe2bfb56f9 Mon Sep 17 00:00:00 2001 From: Jansoon Date: Wed, 1 Jul 2026 02:55:01 +0900 Subject: [PATCH 2/3] =?UTF-8?q?refactor:=20AI=20=EC=9D=91=EB=8B=B5=20?= =?UTF-8?q?=EC=8B=A4=ED=8C=A8=EB=A5=BC=20null=20=EB=B0=98=ED=99=98?= =?UTF-8?q?=EC=97=90=EC=84=9C=20typed=20exception=EC=9C=BC=EB=A1=9C=20=20(?= =?UTF-8?q?#256)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AI 연동 실패를 명시적 예외로 전환하여 실패 흐름을 명확화. - 통신 오류는 EXTERNAL_API_COMMUNICATION_ERROR로 전파 - 빈 응답은 EmptyAiResponseException으로 구분, 재시도에서 제외 - 변경된 실패 처리에 맞게 단위 테스트 수정 --- .../domain/chat/ai/RagAnswerClient.java | 34 ++++++---- .../domain/link/ai/RagSummaryClient.java | 65 +++++++++++-------- .../domain/link/worker/SummaryWorker.java | 15 ++--- .../global/fillter/RequestLoggingFilter.java | 3 +- .../infra/feign/EmptyAiResponseException.java | 9 +++ .../domain/chat/ai/RagAnswerClientTest.java | 29 +++++---- .../domain/link/ai/RagSummaryClientTest.java | 56 ++++++++++------ .../domain/link/worker/SummaryWorkerTest.java | 9 +-- 8 files changed, 128 insertions(+), 92 deletions(-) create mode 100644 src/main/java/com/sofa/linkiving/infra/feign/EmptyAiResponseException.java diff --git a/src/main/java/com/sofa/linkiving/domain/chat/ai/RagAnswerClient.java b/src/main/java/com/sofa/linkiving/domain/chat/ai/RagAnswerClient.java index 6fe928c8..a9f80cfa 100644 --- a/src/main/java/com/sofa/linkiving/domain/chat/ai/RagAnswerClient.java +++ b/src/main/java/com/sofa/linkiving/domain/chat/ai/RagAnswerClient.java @@ -7,6 +7,9 @@ import com.sofa.linkiving.domain.chat.dto.request.RagAnswerReq; import com.sofa.linkiving.domain.chat.dto.response.RagAnswerRes; +import com.sofa.linkiving.global.error.exception.BusinessException; +import com.sofa.linkiving.infra.feign.EmptyAiResponseException; +import com.sofa.linkiving.infra.feign.ExternalApiErrorCode; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; @@ -43,23 +46,26 @@ private Counter buildCounter(String result) { @Override public RagAnswerRes generateAnswer(RagAnswerReq request) { + List ragAnswerRes; try { - List ragAnswerRes = ragAnswerFeign.generateAnswer(request); - - if (ragAnswerRes == null || ragAnswerRes.isEmpty()) { - log.warn("RagAnswerClient generateAnswer empty response"); - emptyCounter.increment(); - return null; - } - - log.info("RagAnswerClient generateAnswer ragAnswerRes={}", ragAnswerRes); - successCounter.increment(); - return ragAnswerRes.get(0); - + ragAnswerRes = ragAnswerFeign.generateAnswer(request); + } catch (BusinessException e) { + failureCounter.increment(); + log.warn("[AI Server] generateAnswer failed - code={}", e.getErrorCode().getCode()); + throw e; } catch (Exception e) { - log.error("RagAnswerClient generateAnswer error", e); failureCounter.increment(); - return null; + log.warn("[AI Server] generateAnswer failed - reason={}", e.getMessage()); + throw new BusinessException(ExternalApiErrorCode.EXTERNAL_API_COMMUNICATION_ERROR); } + + if (ragAnswerRes == null || ragAnswerRes.isEmpty()) { + emptyCounter.increment(); + log.warn("[AI Server] generateAnswer empty response"); + throw new EmptyAiResponseException(); + } + + successCounter.increment(); + return ragAnswerRes.get(0); } } diff --git a/src/main/java/com/sofa/linkiving/domain/link/ai/RagSummaryClient.java b/src/main/java/com/sofa/linkiving/domain/link/ai/RagSummaryClient.java index 752dc4fa..1e30beb8 100644 --- a/src/main/java/com/sofa/linkiving/domain/link/ai/RagSummaryClient.java +++ b/src/main/java/com/sofa/linkiving/domain/link/ai/RagSummaryClient.java @@ -9,6 +9,9 @@ import com.sofa.linkiving.domain.link.dto.request.RagRegenerateSummaryReq; import com.sofa.linkiving.domain.link.dto.response.RagInitialSummaryRes; import com.sofa.linkiving.domain.link.dto.response.RagRegenerateSummaryRes; +import com.sofa.linkiving.global.error.exception.BusinessException; +import com.sofa.linkiving.infra.feign.EmptyAiResponseException; +import com.sofa.linkiving.infra.feign.ExternalApiErrorCode; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; @@ -51,47 +54,53 @@ private Counter buildCounter(String operation, String result) { @Override public RagInitialSummaryRes initialSummary(Long linkId, Long userId, String title, String url, String memo) { + List response; try { RagInitialSummaryReq req = new RagInitialSummaryReq(linkId, userId, title, url, memo); - List response = ragSummaryFeign.requestInitialSummary(req); - - if (response != null && !response.isEmpty()) { - log.info("[AI Server] Initial Summary Requested Success. LinkId: {}", linkId); - initialSuccess.increment(); - return response.get(0); - } - - initialEmpty.increment(); - return null; - + response = ragSummaryFeign.requestInitialSummary(req); + } catch (BusinessException e) { + initialFailure.increment(); + log.warn("[AI Server] Initial summary failed - linkId={}, code={}", linkId, e.getErrorCode().getCode()); + throw e; } catch (Exception e) { - log.error("[AI Server Error] Failed to request initial summary for LinkId: {}. Error: {}", linkId, - e.getMessage()); initialFailure.increment(); - return null; + log.warn("[AI Server] Initial summary failed - linkId={}, reason={}", linkId, e.getMessage()); + throw new BusinessException(ExternalApiErrorCode.EXTERNAL_API_COMMUNICATION_ERROR); } + + if (response == null || response.isEmpty()) { + initialEmpty.increment(); + log.warn("[AI Server] Initial summary empty response - linkId={}", linkId); + throw new EmptyAiResponseException(); + } + + initialSuccess.increment(); + return response.get(0); } @Override public RagRegenerateSummaryRes regenerateSummary(Long linkId, Long userId, String url, String existingSummary) { + List response; try { RagRegenerateSummaryReq req = new RagRegenerateSummaryReq(linkId, userId, url, existingSummary); - List response = ragSummaryFeign.requestRegenerateSummary(req); - - if (response != null && !response.isEmpty()) { - log.info("[AI Server] Regenerate Summary Success. LinkId: {}", linkId); - regenerateSuccess.increment(); - return response.get(0); - } - - regenerateEmpty.increment(); - return null; - + response = ragSummaryFeign.requestRegenerateSummary(req); + } catch (BusinessException e) { + regenerateFailure.increment(); + log.warn("[AI Server] Regenerate summary failed - linkId={}, code={}", linkId, e.getErrorCode().getCode()); + throw e; } catch (Exception e) { - log.error("[AI Server Error] Failed to regenerate summary for LinkId: {}. Error: {}", linkId, - e.getMessage()); regenerateFailure.increment(); - return null; + log.warn("[AI Server] Regenerate summary failed - linkId={}, reason={}", linkId, e.getMessage()); + throw new BusinessException(ExternalApiErrorCode.EXTERNAL_API_COMMUNICATION_ERROR); } + + if (response == null || response.isEmpty()) { + regenerateEmpty.increment(); + log.warn("[AI Server] Regenerate summary empty response - linkId={}", linkId); + throw new EmptyAiResponseException(); + } + + regenerateSuccess.increment(); + return response.get(0); } } diff --git a/src/main/java/com/sofa/linkiving/domain/link/worker/SummaryWorker.java b/src/main/java/com/sofa/linkiving/domain/link/worker/SummaryWorker.java index f6963117..5dd61f61 100644 --- a/src/main/java/com/sofa/linkiving/domain/link/worker/SummaryWorker.java +++ b/src/main/java/com/sofa/linkiving/domain/link/worker/SummaryWorker.java @@ -20,6 +20,7 @@ import com.sofa.linkiving.domain.link.event.SummaryStatusEvent; import com.sofa.linkiving.domain.link.facade.SummaryWorkerFacade; import com.sofa.linkiving.global.logging.LogContext; +import com.sofa.linkiving.infra.feign.EmptyAiResponseException; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; @@ -95,8 +96,6 @@ private void processQueue() throws InterruptedException { String userEmail = null; log.info("Processing link for summary - linkId: {}", linkId); - generateFailureCounter.increment(); - try { Link link = summaryWorkerFacade.getLinkWithMember(linkId); userEmail = link.getMember().getEmail(); @@ -123,6 +122,7 @@ private void processQueue() throws InterruptedException { )); } } catch (Exception e) { + generateFailureCounter.increment(); log.error("Failed to generate summary for linkId: {}", linkId, e); try { @@ -146,25 +146,20 @@ private void processQueue() throws InterruptedException { } @Retryable( - value = {Exception.class}, + retryFor = {Exception.class}, + noRetryFor = {EmptyAiResponseException.class}, maxAttempts = 3, backoff = @Backoff(delay = 2000) ) public RagInitialSummaryRes callAiServerWithRetry(Link link) { log.info("Attempting summary request to AI server - linkId: {}", link.getId()); - RagInitialSummaryRes res = summaryClient.initialSummary( + return summaryClient.initialSummary( link.getId(), link.getMember().getId(), link.getTitle(), link.getUrl(), link.getMemo() ); - - if (res == null) { - throw new RuntimeException("Received a null response from the AI server."); - } - - return res; } } diff --git a/src/main/java/com/sofa/linkiving/global/fillter/RequestLoggingFilter.java b/src/main/java/com/sofa/linkiving/global/fillter/RequestLoggingFilter.java index d176165e..034b75cf 100644 --- a/src/main/java/com/sofa/linkiving/global/fillter/RequestLoggingFilter.java +++ b/src/main/java/com/sofa/linkiving/global/fillter/RequestLoggingFilter.java @@ -58,7 +58,8 @@ public class RequestLoggingFilter extends OncePerRequestFilter { "/favicon.ico", "/swagger", "/v3/api-docs", - "/health-check" + "/health-check", + "/h2-console" }; private static final int MAX_BODY_LENGTH = 2000; diff --git a/src/main/java/com/sofa/linkiving/infra/feign/EmptyAiResponseException.java b/src/main/java/com/sofa/linkiving/infra/feign/EmptyAiResponseException.java new file mode 100644 index 00000000..4541aed8 --- /dev/null +++ b/src/main/java/com/sofa/linkiving/infra/feign/EmptyAiResponseException.java @@ -0,0 +1,9 @@ +package com.sofa.linkiving.infra.feign; + +import com.sofa.linkiving.global.error.exception.BusinessException; + +public class EmptyAiResponseException extends BusinessException { + public EmptyAiResponseException() { + super(ExternalApiErrorCode.EXTERNAL_API_INVALID_RESPONSE); + } +} diff --git a/src/test/java/com/sofa/linkiving/domain/chat/ai/RagAnswerClientTest.java b/src/test/java/com/sofa/linkiving/domain/chat/ai/RagAnswerClientTest.java index 24413f65..9caae54d 100644 --- a/src/test/java/com/sofa/linkiving/domain/chat/ai/RagAnswerClientTest.java +++ b/src/test/java/com/sofa/linkiving/domain/chat/ai/RagAnswerClientTest.java @@ -17,6 +17,9 @@ import com.sofa.linkiving.domain.chat.dto.request.RagAnswerReq; import com.sofa.linkiving.domain.chat.dto.response.RagAnswerRes; +import com.sofa.linkiving.global.error.exception.BusinessException; +import com.sofa.linkiving.infra.feign.EmptyAiResponseException; +import com.sofa.linkiving.infra.feign.ExternalApiErrorCode; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; @@ -60,34 +63,32 @@ void shouldReturnFirstElement_WhenGenerateAnswerSuccess() { } @Test - @DisplayName("Feign 요청 중 예외가 발생하면 예외를 잡고 null을 반환한다") - void shouldCatchExceptionAndReturnNull_WhenGenerateAnswerThrowsException() { + @DisplayName("Feign 요청 중 예외가 발생하면 통신 오류 예외로 전환하고 failure 로 집계한다") + void shouldThrowCommunicationError_WhenGenerateAnswerThrowsException() { // given RagAnswerReq req = mock(RagAnswerReq.class); given(ragAnswerFeign.generateAnswer(any(RagAnswerReq.class))) .willThrow(new RuntimeException("AI Server Error")); - // when - RagAnswerRes actualRes = ragAnswerClient.generateAnswer(req); - - // then - assertThat(actualRes).isNull(); + // when & then + assertThatThrownBy(() -> ragAnswerClient.generateAnswer(req)) + .isInstanceOf(BusinessException.class) + .extracting("errorCode") + .isEqualTo(ExternalApiErrorCode.EXTERNAL_API_COMMUNICATION_ERROR); assertThat(counterCount("failure")).isEqualTo(1.0); } @Test - @DisplayName("Feign 응답이 비어있으면 null 을 반환하고 empty 로 집계한다") - void shouldReturnNullAndCountEmpty_WhenResponseIsEmpty() { + @DisplayName("Feign 응답이 비어있으면 EmptyAiResponseException 을 던지고 empty 로 집계한다") + void shouldThrowEmptyResponse_WhenResponseIsEmpty() { // given RagAnswerReq req = mock(RagAnswerReq.class); given(ragAnswerFeign.generateAnswer(any(RagAnswerReq.class))) .willReturn(Collections.emptyList()); - // when - RagAnswerRes actualRes = ragAnswerClient.generateAnswer(req); - - // then - assertThat(actualRes).isNull(); + // when & then + assertThatThrownBy(() -> ragAnswerClient.generateAnswer(req)) + .isInstanceOf(EmptyAiResponseException.class); assertThat(counterCount("empty")).isEqualTo(1.0); } } diff --git a/src/test/java/com/sofa/linkiving/domain/link/ai/RagSummaryClientTest.java b/src/test/java/com/sofa/linkiving/domain/link/ai/RagSummaryClientTest.java index 124ac3c4..b4b0f79c 100644 --- a/src/test/java/com/sofa/linkiving/domain/link/ai/RagSummaryClientTest.java +++ b/src/test/java/com/sofa/linkiving/domain/link/ai/RagSummaryClientTest.java @@ -19,6 +19,9 @@ import com.sofa.linkiving.domain.link.dto.request.RagRegenerateSummaryReq; import com.sofa.linkiving.domain.link.dto.response.RagInitialSummaryRes; import com.sofa.linkiving.domain.link.dto.response.RagRegenerateSummaryRes; +import com.sofa.linkiving.global.error.exception.BusinessException; +import com.sofa.linkiving.infra.feign.EmptyAiResponseException; +import com.sofa.linkiving.infra.feign.ExternalApiErrorCode; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; @@ -74,32 +77,30 @@ void shouldReturnInitialSummaryResWhenSuccess() { } @Test - @DisplayName("최초 요약 요청 시 응답이 비어있으면 null을 반환한다") - void shouldReturnNullWhenInitialSummaryResponseIsEmpty() { + @DisplayName("최초 요약 요청 시 응답이 비어있으면 EmptyAiResponseException 을 던지고 empty 로 집계한다") + void shouldThrowEmptyResponse_WhenInitialSummaryResponseIsEmpty() { // given given(ragSummaryFeign.requestInitialSummary(any(RagInitialSummaryReq.class))) .willReturn(Collections.emptyList()); - // when - RagInitialSummaryRes result = ragSummaryClient.initialSummary(1L, 100L, "Title", "URL", "Memo"); - - // then - assertThat(result).isNull(); + // when & then + assertThatThrownBy(() -> ragSummaryClient.initialSummary(1L, 100L, "Title", "URL", "Memo")) + .isInstanceOf(EmptyAiResponseException.class); assertThat(counterCount("initial", "empty")).isEqualTo(1.0); } @Test - @DisplayName("최초 요약 요청 중 예외 발생 시 로그를 남기고 null을 반환한다") - void shouldReturnNullWhenInitialSummaryThrowsException() { + @DisplayName("최초 요약 요청 중 예외 발생 시 통신 오류 예외로 전환하고 failure 로 집계한다") + void shouldThrowCommunicationError_WhenInitialSummaryThrowsException() { // given given(ragSummaryFeign.requestInitialSummary(any(RagInitialSummaryReq.class))) .willThrow(new RuntimeException("AI Server Error")); - // when - RagInitialSummaryRes result = ragSummaryClient.initialSummary(1L, 100L, "Title", "URL", "Memo"); - - // then - assertThat(result).isNull(); + // when & then + assertThatThrownBy(() -> ragSummaryClient.initialSummary(1L, 100L, "Title", "URL", "Memo")) + .isInstanceOf(BusinessException.class) + .extracting("errorCode") + .isEqualTo(ExternalApiErrorCode.EXTERNAL_API_COMMUNICATION_ERROR); assertThat(counterCount("initial", "failure")).isEqualTo(1.0); } @@ -130,17 +131,30 @@ void shouldReturnRegenerateSummaryResWhenSuccess() { } @Test - @DisplayName("요약 재생성 요청 중 예외 발생 시 null을 반환한다") - void shouldReturnNullWhenRegenerateSummaryThrowsException() { + @DisplayName("요약 재생성 요청 중 예외 발생 시 통신 오류 예외로 전환하고 failure 로 집계한다") + void shouldThrowCommunicationError_WhenRegenerateSummaryThrowsException() { // given given(ragSummaryFeign.requestRegenerateSummary(any(RagRegenerateSummaryReq.class))) .willThrow(new RuntimeException("Connection Timeout")); - // when - RagRegenerateSummaryRes result = ragSummaryClient.regenerateSummary(1L, 100L, "URL", "Old"); - - // then - assertThat(result).isNull(); + // when & then + assertThatThrownBy(() -> ragSummaryClient.regenerateSummary(1L, 100L, "URL", "Old")) + .isInstanceOf(BusinessException.class) + .extracting("errorCode") + .isEqualTo(ExternalApiErrorCode.EXTERNAL_API_COMMUNICATION_ERROR); assertThat(counterCount("regenerate", "failure")).isEqualTo(1.0); } + + @Test + @DisplayName("요약 재생성 요청 시 응답이 비어있으면 EmptyAiResponseException 을 던지고 empty 로 집계한다") + void shouldThrowEmptyResponse_WhenRegenerateSummaryResponseIsEmpty() { + // given + given(ragSummaryFeign.requestRegenerateSummary(any(RagRegenerateSummaryReq.class))) + .willReturn(Collections.emptyList()); + + // when & then + assertThatThrownBy(() -> ragSummaryClient.regenerateSummary(1L, 100L, "URL", "Old")) + .isInstanceOf(EmptyAiResponseException.class); + assertThat(counterCount("regenerate", "empty")).isEqualTo(1.0); + } } diff --git a/src/test/java/com/sofa/linkiving/domain/link/worker/SummaryWorkerTest.java b/src/test/java/com/sofa/linkiving/domain/link/worker/SummaryWorkerTest.java index 8d79c3ac..99888815 100644 --- a/src/test/java/com/sofa/linkiving/domain/link/worker/SummaryWorkerTest.java +++ b/src/test/java/com/sofa/linkiving/domain/link/worker/SummaryWorkerTest.java @@ -29,6 +29,7 @@ import com.sofa.linkiving.domain.link.event.SummaryStatusEvent; import com.sofa.linkiving.domain.link.facade.SummaryWorkerFacade; import com.sofa.linkiving.domain.member.entity.Member; +import com.sofa.linkiving.infra.feign.EmptyAiResponseException; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; @@ -276,8 +277,8 @@ void shouldPublishProcessingAndCompletedEvents_WhenSuccess() { } @Test - @DisplayName("AI 응답이 null일 경우 예외가 발생하여 FAILED 이벤트를 발행함") - void shouldPublishFailedEvent_WhenAiResponseIsNull() { + @DisplayName("AI 응답이 비어있어 EmptyAiResponseException 이 발생하면 FAILED 이벤트를 발행함") + void shouldPublishFailedEvent_WhenAiResponseIsEmpty() { // given given(summaryQueue.pollTaskFromQueue()) .willReturn(Optional.of(task(1L))) @@ -285,8 +286,8 @@ void shouldPublishFailedEvent_WhenAiResponseIsNull() { given(summaryWorkerFacade.getLinkWithMember(1L)).willReturn(mockLink); - // AI 응답이 null이면 callAiServerWithRetry 내에서 RuntimeException 발생 - given(summaryClient.initialSummary(anyLong(), anyLong(), any(), any(), any())).willReturn(null); + given(summaryClient.initialSummary(anyLong(), anyLong(), any(), any(), any())) + .willThrow(new EmptyAiResponseException()); // when summaryWorker.startWorker(); From 2e175e5d7c4d86838d9204cea0ef6ab25301a5f5 Mon Sep 17 00:00:00 2001 From: Jansoon Date: Sat, 11 Jul 2026 02:28:41 +0900 Subject: [PATCH 3/3] =?UTF-8?q?chore:=20async=20executor=20=ED=8F=AC?= =?UTF-8?q?=ED=99=94=20=EA=B4=80=EC=B8=A1=20=EC=B6=94=EA=B0=80=20(?= =?UTF-8?q?=EB=8C=80=EC=8B=9C=EB=B3=B4=EB=93=9C=20=ED=8C=A8=EB=84=90=20+?= =?UTF-8?q?=20=EC=95=8C=EB=A6=BC=20=EB=A3=B0)=20(#255)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 전역 async executor(applicationTaskExecutor) 포화를 관측 가능하도록 Grafana 패널과 Prometheus 알림 룰 추가. CallerRunsPolicy 발동(포화) 상황을 그래프·알림으로 감지 가능하게 함. - Grafana: 큐 대기(capacity 100)/스레드 active·pool·max/큐 사용률 게이지 패널 추가 - Prometheus: 큐 포화 임박(queued>=80, 2m), 풀 max 도달(2m) 경고 룰 추가 - 자동계측된 executor_* 지표(name=applicationTaskExecutor) 사용, 코드 변경 없음 --- docker/alerting/alert.rules.yml | 25 ++ .../linkiving-ai-async-overview.json | 231 +++++++++++++++++- 2 files changed, 255 insertions(+), 1 deletion(-) diff --git a/docker/alerting/alert.rules.yml b/docker/alerting/alert.rules.yml index 314505f4..e63477bd 100644 --- a/docker/alerting/alert.rules.yml +++ b/docker/alerting/alert.rules.yml @@ -66,3 +66,28 @@ groups: annotations: summary: "비동기 작업 최종 실패: {{ $labels.task }}" description: "task '{{ $labels.task }}'{{ if $labels.action }} (action {{ $labels.action }}){{ end }} 에서 재시도 소진 후 최종 실패가 발생했습니다. 수동 복구가 필요할 수 있습니다." + + # ── 비동기 executor 큐 포화 임박 ────────────────────── + # applicationTaskExecutor 큐(capacity 100)의 80% 이상(=80건)이 2분 지속되면 경고. + # 큐가 가득 차고 풀도 max면 CallerRunsPolicy가 발동해 호출/이벤트 스레드가 작업을 직접 실행하게 됨. + - alert: AsyncExecutorQueueSaturation + expr: executor_queued_tasks{name="applicationTaskExecutor"} >= 80 + for: 2m + labels: + severity: warning + annotations: + summary: "async executor 큐 포화 임박" + description: "applicationTaskExecutor 대기 작업이 큐 용량(100)의 80%를 2분 이상 초과했습니다 (현재 대기 {{ $value }}건). 큐가 가득 차면 CallerRunsPolicy로 호출 스레드가 작업을 직접 실행하여 응답 지연이 발생할 수 있습니다." + + # ── 비동기 executor 스레드 풀 max 도달 ──────────────── + # 현재 풀 스레드 수가 max(20)에 도달한 상태가 2분 지속되면 경고 (큐까지 차면 포화). + - alert: AsyncExecutorPoolExhausted + expr: | + executor_pool_size_threads{name="applicationTaskExecutor"} + >= executor_pool_max_threads{name="applicationTaskExecutor"} + for: 2m + labels: + severity: warning + annotations: + summary: "async executor 스레드 풀 max 도달" + description: "applicationTaskExecutor 풀이 max({{ $value }})에 2분 이상 도달했습니다. 큐까지 가득 차면 CallerRunsPolicy로 호출/이벤트 스레드가 작업을 직접 실행하게 됩니다." diff --git a/docker/grafana/dashboards/linkiving-ai-async-overview.json b/docker/grafana/dashboards/linkiving-ai-async-overview.json index 6cd96d94..3f670ec6 100644 --- a/docker/grafana/dashboards/linkiving-ai-async-overview.json +++ b/docker/grafana/dashboards/linkiving-ai-async-overview.json @@ -38,7 +38,7 @@ "timezone": "", "title": "Linkiving AI / Async Overview", "uid": "linkiving-ai-async-overview", - "version": 1, + "version": 2, "panels": [ { "id": 1, @@ -344,6 +344,235 @@ "refId": "A" } ] + }, + { + "id": 6, + "title": "Async Executor 큐 대기 (capacity 100)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "lineWidth": 2, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "executor_queued_tasks{name=\"applicationTaskExecutor\"}", + "legendFormat": "queued", + "refId": "A" + } + ] + }, + { + "id": 7, + "title": "Async Executor 스레드 (active / pool / max)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "executor_active_threads{name=\"applicationTaskExecutor\"}", + "legendFormat": "active", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "executor_pool_size_threads{name=\"applicationTaskExecutor\"}", + "legendFormat": "pool", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "executor_pool_max_threads{name=\"applicationTaskExecutor\"}", + "legendFormat": "max", + "refId": "C" + } + ] + }, + { + "id": 8, + "title": "큐 사용률 (현재)", + "type": "gauge", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 30 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "executor_queued_tasks{name=\"applicationTaskExecutor\"}", + "legendFormat": "queued", + "refId": "A" + } + ] } ] }