From 5c81d401b02e98e54fd7f00e7975f2bc5b1b2c74 Mon Sep 17 00:00:00 2001 From: LHkeeper666 Date: Sat, 30 May 2026 10:45:34 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat(retrieve):=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E6=B7=B7=E5=90=88=E6=A3=80=E7=B4=A2=E9=80=9A=E9=81=93=EF=BC=8C?= =?UTF-8?q?=E6=94=AF=E6=8C=81PG=E5=85=A8=E6=96=87=E6=A3=80=E7=B4=A2?= =?UTF-8?q?=E4=B8=8ERRF=E8=9E=8D=E5=90=88=E6=8E=92=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 引入 KeywordSearchChannel 作为第三路检索通道，利用 PostgreSQL tsvector/tsquery 实现关键词精确匹配，弥补向量检索对专有名词、型号等 精确查询的召回不足。新增 HybridFusionPostProcessor，通过 RRF 或加权 求和将向量与关键词两路结果融合，位于去重之后、Rerank 之前。 Co-Authored-By: Claude Opus 4.7 --- .../rag/config/SearchChannelProperties.java | 53 +++++ .../rag/controller/RAGSettingsController.java | 33 +++ .../rag/controller/vo/SystemSettingsVO.java | 69 ++++++- .../channel/KeywordSearchChannel.java | 193 ++++++++++++++++++ .../channel/fusion/FusionStrategy.java | 39 ++++ .../channel/fusion/RRFFusionStrategy.java | 75 +++++++ .../fusion/WeightedSumFusionStrategy.java | 79 +++++++ .../HybridFusionPostProcessor.java | 135 ++++++++++++ .../admin/settings/SystemSettingsPage.tsx | 46 +++++ frontend/src/services/settingsService.ts | 31 +++ resources/database/schema_pg.sql | 16 +- resources/database/upgrade_v1.2_to_v1.3.sql | 23 +++ 12 files changed, 789 insertions(+), 3 deletions(-) create mode 100644 bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java create mode 100644 bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/FusionStrategy.java create mode 100644 bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/RRFFusionStrategy.java create mode 100644 bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/WeightedSumFusionStrategy.java create mode 100644 bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/postprocessor/HybridFusionPostProcessor.java create mode 100644 resources/database/upgrade_v1.2_to_v1.3.sql diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/config/SearchChannelProperties.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/config/SearchChannelProperties.java index 877ae2f4c..2f724ecfa 100644 --- a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/config/SearchChannelProperties.java +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/config/SearchChannelProperties.java @@ -51,6 +51,16 @@ public static class Channels { * 意图定向检索配置 */ private IntentDirected intentDirected = new IntentDirected(); + + /** + * 关键词检索配置 + */ + private Keyword keyword = new Keyword(); + + /** + * 混合检索融合配置 + */ + private Hybrid hybrid = new Hybrid(); } @Data @@ -99,4 +109,47 @@ public static class IntentDirected { */ private int topKMultiplier = 2; } + + @Data + public static class Keyword { + + /** + * 是否启用关键词检索通道 + */ + private boolean enabled = true; + + /** + * TopK 倍数，关键词检索时召回更多候选 + */ + private int topKMultiplier = 3; + + /** + * 融合时的关键词通道权重(仅 WEIGHTED_SUM 模式) + */ + private float boost = 1.0f; + } + + @Data + public static class Hybrid { + + /** + * 是否启用混合融合 + */ + private boolean enabled = true; + + /** + * 融合策略：RRF / WEIGHTED_SUM + */ + private FusionMode fusion = FusionMode.RRF; + + /** + * 向量权重(仅 WEIGHTED_SUM 模式生效) + */ + private float vectorWeight = 0.7f; + } + + public enum FusionMode { + RRF, + WEIGHTED_SUM + } } diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/controller/RAGSettingsController.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/controller/RAGSettingsController.java index 97efb5d6e..062dd888a 100644 --- a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/controller/RAGSettingsController.java +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/controller/RAGSettingsController.java @@ -24,9 +24,13 @@ import com.nageoffer.ai.ragent.rag.config.RAGConfigProperties; import com.nageoffer.ai.ragent.rag.config.RAGDefaultProperties; import com.nageoffer.ai.ragent.rag.config.RAGRateLimitProperties; +import com.nageoffer.ai.ragent.rag.config.SearchChannelProperties; import com.nageoffer.ai.ragent.rag.controller.vo.SystemSettingsVO; import com.nageoffer.ai.ragent.rag.controller.vo.SystemSettingsVO.AISettings; +import com.nageoffer.ai.ragent.rag.controller.vo.SystemSettingsVO.ChannelConfig; +import com.nageoffer.ai.ragent.rag.controller.vo.SystemSettingsVO.ChannelSettings; import com.nageoffer.ai.ragent.rag.controller.vo.SystemSettingsVO.DefaultSettings; +import com.nageoffer.ai.ragent.rag.controller.vo.SystemSettingsVO.HybridChannelConfig; import com.nageoffer.ai.ragent.rag.controller.vo.SystemSettingsVO.MemorySettings; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; @@ -51,6 +55,7 @@ public class RAGSettingsController { private final RAGRateLimitProperties ragRateLimitProperties; private final MemoryProperties memoryProperties; private final AIModelProperties aiModelProperties; + private final SearchChannelProperties searchChannelProperties; @Value("${spring.servlet.multipart.max-file-size:50MB}") private DataSize maxFileSize; @@ -83,6 +88,7 @@ public Result settings() { .build()) .build()) .memory(toMemorySettings(memoryProperties)) + .channels(toChannelSettings(searchChannelProperties)) .build()) .ai(toAISettings(aiModelProperties)) .build(); @@ -160,6 +166,33 @@ private AISettings.ModelGroup toModelGroup(AIModelProperties.ModelGroup group) { .build(); } + private ChannelSettings toChannelSettings(SearchChannelProperties props) { + SearchChannelProperties.Channels channels = props.getChannels(); + return ChannelSettings.builder() + .vectorGlobal(ChannelConfig.builder() + .enabled(channels.getVectorGlobal().isEnabled()) + .confidenceThreshold(channels.getVectorGlobal().getConfidenceThreshold()) + .singleIntentSupplementThreshold(channels.getVectorGlobal().getSingleIntentSupplementThreshold()) + .topKMultiplier(channels.getVectorGlobal().getTopKMultiplier()) + .build()) + .intentDirected(ChannelConfig.builder() + .enabled(channels.getIntentDirected().isEnabled()) + .minIntentScore(channels.getIntentDirected().getMinIntentScore()) + .topKMultiplier(channels.getIntentDirected().getTopKMultiplier()) + .build()) + .keyword(ChannelConfig.builder() + .enabled(channels.getKeyword().isEnabled()) + .topKMultiplier(channels.getKeyword().getTopKMultiplier()) + .boost(channels.getKeyword().getBoost()) + .build()) + .hybrid(HybridChannelConfig.builder() + .enabled(channels.getHybrid().isEnabled()) + .fusion(channels.getHybrid().getFusion().name()) + .vectorWeight(channels.getHybrid().getVectorWeight()) + .build()) + .build(); + } + private String maskApiKey(String apiKey) { if (!StringUtils.hasText(apiKey)) { return null; diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/controller/vo/SystemSettingsVO.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/controller/vo/SystemSettingsVO.java index 109b9db8c..af0eb6c2c 100644 --- a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/controller/vo/SystemSettingsVO.java +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/controller/vo/SystemSettingsVO.java @@ -159,13 +159,16 @@ public static class RagSettings { private QueryRewriteSettings queryRewrite; private RateLimitSettings rateLimit; private MemorySettings memory; + private ChannelSettings channels; public RagSettings(DefaultSettings defaultConfig, QueryRewriteSettings queryRewrite, - RateLimitSettings rateLimit, MemorySettings memory) { + RateLimitSettings rateLimit, MemorySettings memory, + ChannelSettings channels) { this.defaultConfig = defaultConfig; this.queryRewrite = queryRewrite; this.rateLimit = rateLimit; this.memory = memory; + this.channels = channels; } public static RagSettingsBuilder builder() { @@ -177,6 +180,7 @@ public static class RagSettingsBuilder { private QueryRewriteSettings queryRewrite; private RateLimitSettings rateLimit; private MemorySettings memory; + private ChannelSettings channels; public RagSettingsBuilder defaultConfig(DefaultSettings defaultConfig) { this.defaultConfig = defaultConfig; @@ -198,12 +202,73 @@ public RagSettingsBuilder memory(MemorySettings memory) { return this; } + public RagSettingsBuilder channels(ChannelSettings channels) { + this.channels = channels; + return this; + } + public RagSettings build() { - return new RagSettings(defaultConfig, queryRewrite, rateLimit, memory); + return new RagSettings(defaultConfig, queryRewrite, rateLimit, memory, channels); } } } + @Setter + @Getter + public static class ChannelSettings { + private ChannelConfig vectorGlobal; + private ChannelConfig intentDirected; + private ChannelConfig keyword; + private HybridChannelConfig hybrid; + + public ChannelSettings(ChannelConfig vectorGlobal, ChannelConfig intentDirected, + ChannelConfig keyword, HybridChannelConfig hybrid) { + this.vectorGlobal = vectorGlobal; + this.intentDirected = intentDirected; + this.keyword = keyword; + this.hybrid = hybrid; + } + + public static ChannelSettingsBuilder builder() { + return new ChannelSettingsBuilder(); + } + + public static class ChannelSettingsBuilder { + private ChannelConfig vectorGlobal; + private ChannelConfig intentDirected; + private ChannelConfig keyword; + private HybridChannelConfig hybrid; + + public ChannelSettingsBuilder vectorGlobal(ChannelConfig c) { this.vectorGlobal = c; return this; } + public ChannelSettingsBuilder intentDirected(ChannelConfig c) { this.intentDirected = c; return this; } + public ChannelSettingsBuilder keyword(ChannelConfig c) { this.keyword = c; return this; } + public ChannelSettingsBuilder hybrid(HybridChannelConfig c) { this.hybrid = c; return this; } + + public ChannelSettings build() { + return new ChannelSettings(vectorGlobal, intentDirected, keyword, hybrid); + } + } + } + + @Data + @Builder + public static class ChannelConfig { + private Boolean enabled; + private Double confidenceThreshold; + private Double minIntentScore; + private Double singleIntentSupplementThreshold; + private Integer topKMultiplier; + private Float boost; + } + + @Data + @Builder + public static class HybridChannelConfig { + private Boolean enabled; + private String fusion; + private Float vectorWeight; + } + @Setter @Getter public static class QueryRewriteSettings { diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java new file mode 100644 index 000000000..1f6530d88 --- /dev/null +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nageoffer.ai.ragent.rag.core.retrieve.channel; + +import cn.hutool.core.collection.CollUtil; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.nageoffer.ai.ragent.framework.convention.RetrievedChunk; +import com.nageoffer.ai.ragent.knowledge.dao.entity.KnowledgeBaseDO; +import com.nageoffer.ai.ragent.knowledge.dao.mapper.KnowledgeBaseMapper; +import com.nageoffer.ai.ragent.rag.config.SearchChannelProperties; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; + +/** + * 关键词全文检索通道 + * 使用 PostgreSQL tsvector/tsquery 进行关键词精确匹配检索， + * 弥补向量检索对专有名词、产品型号等精确关键词的召回不足。 + */ +@Slf4j +@Component +@ConditionalOnProperty(name = "rag.vector.type", havingValue = "pg") +public class KeywordSearchChannel implements SearchChannel { + + private final SearchChannelProperties properties; + private final KnowledgeBaseMapper knowledgeBaseMapper; + private final JdbcTemplate jdbcTemplate; + private final Executor innerRetrievalExecutor; + + public KeywordSearchChannel(SearchChannelProperties properties, + KnowledgeBaseMapper knowledgeBaseMapper, + JdbcTemplate jdbcTemplate, + Executor innerRetrievalExecutor) { + this.properties = properties; + this.knowledgeBaseMapper = knowledgeBaseMapper; + this.jdbcTemplate = jdbcTemplate; + this.innerRetrievalExecutor = innerRetrievalExecutor; + } + + @Override + public String getName() { + return "KeywordSearch"; + } + + @Override + public int getPriority() { + return 20; + } + + @Override + public boolean isEnabled(SearchContext context) { + return properties.getChannels().getKeyword().isEnabled() + && context.getMainQuestion() != null + && !context.getMainQuestion().isBlank(); + } + + @Override + public SearchChannelResult search(SearchContext context) { + long startTime = System.currentTimeMillis(); + + try { + String query = context.getMainQuestion(); + log.info("执行关键词检索，问题：{}", query); + + List collections = getAllKBCollections(); + if (collections.isEmpty()) { + log.warn("未找到任何 KB collection，跳过关键词检索"); + return emptyResult(startTime); + } + + int topK = context.getTopK() * properties.getChannels().getKeyword().getTopKMultiplier(); + List allChunks = retrieveFromAllCollections(query, collections, topK); + + long latency = System.currentTimeMillis() - startTime; + log.info("关键词检索完成，检索到 {} 个 Chunk，耗时 {}ms", allChunks.size(), latency); + + return SearchChannelResult.builder() + .channelType(SearchChannelType.KEYWORD_ES) + .channelName(getName()) + .chunks(allChunks) + .latencyMs(latency) + .build(); + + } catch (Exception e) { + log.error("关键词检索失败", e); + return emptyResult(startTime); + } + } + + @Override + public SearchChannelType getType() { + return SearchChannelType.KEYWORD_ES; + } + + private List getAllKBCollections() { + Set collections = new HashSet<>(); + List kbList = knowledgeBaseMapper.selectList( + Wrappers.lambdaQuery(KnowledgeBaseDO.class) + .select(KnowledgeBaseDO::getCollectionName) + .eq(KnowledgeBaseDO::getDeleted, 0) + ); + for (KnowledgeBaseDO kb : kbList) { + String name = kb.getCollectionName(); + if (name != null && !name.isBlank()) { + collections.add(name); + } + } + return new ArrayList<>(collections); + } + + private List retrieveFromAllCollections(String query, + List collections, + int topK) { + List>> futures = collections.stream() + .map(collection -> CompletableFuture.supplyAsync( + () -> searchInCollection(query, collection, topK), + innerRetrievalExecutor + )) + .toList(); + + List allChunks = new ArrayList<>(); + int success = 0; + int failure = 0; + for (int i = 0; i < futures.size(); i++) { + try { + List chunks = futures.get(i).join(); + allChunks.addAll(chunks); + success++; + } catch (Exception e) { + failure++; + log.error("关键词检索失败 - Collection: {}", collections.get(i), e); + } + } + log.info("关键词检索统计 - 总 collection 数: {}, 成功: {}, 失败: {}, Chunk 总数: {}", + collections.size(), success, failure, allChunks.size()); + return allChunks; + } + + private List searchInCollection(String query, String collectionName, int topK) { + // 预处理查询：移除特殊字符避免 tsquery 解析错误 + String sanitized = query.replaceAll("[^\\w\\u4e00-\\u9fff\\s]", " "); + if (sanitized.isBlank()) { + return List.of(); + } + // plainto_tsquery 将输入切分为词后用 & 连接，sanitized 中的空格天然实现 AND 语义 + // 使用带索引的 tsv 列而非实时计算 to_tsvector + // noinspection SqlDialectInspection,SqlNoDataSourceInspection + return jdbcTemplate.query( + "SELECT id, content, ts_rank(tsv, plainto_tsquery('simple', ?)) AS score " + + "FROM t_knowledge_vector " + + "WHERE metadata->>'collection_name' = ? AND tsv @@ plainto_tsquery('simple', ?) " + + "ORDER BY score DESC LIMIT ?", + (rs, rowNum) -> RetrievedChunk.builder() + .id(rs.getString("id")) + .text(rs.getString("content")) + .score(rs.getFloat("score")) + .build(), + sanitized, collectionName, sanitized, topK + ); + } + + private SearchChannelResult emptyResult(long startTime) { + return SearchChannelResult.builder() + .channelType(SearchChannelType.KEYWORD_ES) + .channelName(getName()) + .chunks(List.of()) + .latencyMs(System.currentTimeMillis() - startTime) + .build(); + } +} diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/FusionStrategy.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/FusionStrategy.java new file mode 100644 index 000000000..00d5927bc --- /dev/null +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/FusionStrategy.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nageoffer.ai.ragent.rag.core.retrieve.channel.fusion; + +import com.nageoffer.ai.ragent.framework.convention.RetrievedChunk; + +import java.util.List; +import java.util.Map; + +/** + * 多通道检索结果融合策略接口 + */ +public interface FusionStrategy { + + /** + * 融合多个通道的检索结果 + * + * @param rankedChunks 各通道的排序结果列表，key 为通道标识 + * @param weights 各通道的权重(可选，用于加权融合) + * @return 融合后的 Chunk 列表，按融合分数降序排列 + */ + List fuse(Map> rankedChunks, + Map weights); +} diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/RRFFusionStrategy.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/RRFFusionStrategy.java new file mode 100644 index 000000000..64647c7b8 --- /dev/null +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/RRFFusionStrategy.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nageoffer.ai.ragent.rag.core.retrieve.channel.fusion; + +import com.nageoffer.ai.ragent.framework.convention.RetrievedChunk; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * RRF (Reciprocal Rank Fusion) 融合策略 + * RRF_score(d) = Σ 1/(k + rank_i(d)) + * 不需要分数归一化，仅依赖各通道内的排名。 + */ +public class RRFFusionStrategy implements FusionStrategy { + + static final int K = 60; + + @Override + public List fuse(Map> rankedChunks, + Map weights) { + // chunkKey → RRF score + Map rrfScores = new LinkedHashMap<>(); + // chunkKey → chunk + Map chunkMap = new HashMap<>(); + + for (Map.Entry> entry : rankedChunks.entrySet()) { + List chunks = entry.getValue(); + for (int rank = 0; rank < chunks.size(); rank++) { + RetrievedChunk chunk = chunks.get(rank); + String key = chunk.getId() != null ? chunk.getId() : String.valueOf(chunk.getText().hashCode()); + + double score = 1.0 / (K + rank + 1); + rrfScores.merge(key, score, Double::sum); + + chunkMap.putIfAbsent(key, chunk); + } + } + + List fused = new ArrayList<>(); + for (Map.Entry entry : rrfScores.entrySet()) { + RetrievedChunk chunk = chunkMap.get(entry.getKey()); + if (chunk != null) { + RetrievedChunk fusedChunk = RetrievedChunk.builder() + .id(chunk.getId()) + .text(chunk.getText()) + .score(entry.getValue().floatValue()) + .build(); + fused.add(fusedChunk); + } + } + + fused.sort(Comparator.comparingDouble(RetrievedChunk::getScore).reversed()); + return fused; + } +} diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/WeightedSumFusionStrategy.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/WeightedSumFusionStrategy.java new file mode 100644 index 000000000..892d21b44 --- /dev/null +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/fusion/WeightedSumFusionStrategy.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nageoffer.ai.ragent.rag.core.retrieve.channel.fusion; + +import com.nageoffer.ai.ragent.framework.convention.RetrievedChunk; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 加权求和融合策略 + * weighted_score(d) = w_i * norm(score_i(d)) + * 对各通道的分数做 min-max 归一化后再加权求和。 + */ +public class WeightedSumFusionStrategy implements FusionStrategy { + + @Override + public List fuse(Map> rankedChunks, + Map weights) { + // chunkKey → weighted score + Map weightedScores = new HashMap<>(); + // chunkKey → chunk + Map chunkMap = new HashMap<>(); + + for (Map.Entry> entry : rankedChunks.entrySet()) { + String channel = entry.getKey(); + List chunks = entry.getValue(); + if (chunks.isEmpty()) continue; + + float weight = weights != null && weights.containsKey(channel) ? weights.get(channel) : 1.0f; + + // min-max normalize scores within this channel + double maxScore = chunks.stream().mapToDouble(c -> c.getScore() != null ? c.getScore() : 0).max().orElse(1.0); + double minScore = chunks.stream().mapToDouble(c -> c.getScore() != null ? c.getScore() : 0).min().orElse(0.0); + double range = maxScore - minScore; + if (range == 0) range = 1.0; + + for (RetrievedChunk chunk : chunks) { + String key = chunk.getId() != null ? chunk.getId() : String.valueOf(chunk.getText().hashCode()); + double normScore = ((chunk.getScore() != null ? chunk.getScore() : 0) - minScore) / range; + weightedScores.merge(key, weight * normScore, Double::sum); + chunkMap.putIfAbsent(key, chunk); + } + } + + List fused = new ArrayList<>(); + for (Map.Entry entry : weightedScores.entrySet()) { + RetrievedChunk chunk = chunkMap.get(entry.getKey()); + if (chunk != null) { + fused.add(RetrievedChunk.builder() + .id(chunk.getId()) + .text(chunk.getText()) + .score(entry.getValue().floatValue()) + .build()); + } + } + + fused.sort(Comparator.comparingDouble(RetrievedChunk::getScore).reversed()); + return fused; + } +} diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/postprocessor/HybridFusionPostProcessor.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/postprocessor/HybridFusionPostProcessor.java new file mode 100644 index 000000000..5fdd72ac4 --- /dev/null +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/postprocessor/HybridFusionPostProcessor.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nageoffer.ai.ragent.rag.core.retrieve.postprocessor; + +import com.nageoffer.ai.ragent.framework.convention.RetrievedChunk; +import com.nageoffer.ai.ragent.rag.config.SearchChannelProperties; +import com.nageoffer.ai.ragent.rag.config.SearchChannelProperties.FusionMode; +import com.nageoffer.ai.ragent.rag.core.retrieve.channel.SearchChannelResult; +import com.nageoffer.ai.ragent.rag.core.retrieve.channel.SearchChannelType; +import com.nageoffer.ai.ragent.rag.core.retrieve.channel.SearchContext; +import com.nageoffer.ai.ragent.rag.core.retrieve.channel.fusion.FusionStrategy; +import com.nageoffer.ai.ragent.rag.core.retrieve.channel.fusion.RRFFusionStrategy; +import com.nageoffer.ai.ragent.rag.core.retrieve.channel.fusion.WeightedSumFusionStrategy; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 混合检索融合后置处理器 + *

+ * 将向量检索通道(VECTOR_GLOBAL / INTENT_DIRECTED)和关键词检索通道(KEYWORD_ES)的结果 + * 按照配置的融合策略(RRF 或加权求和)合并重排。 + *

+ * 执行顺序位于去重之后、Rerank 之前，确保融合后的结果能进一步由 Rerank 精排。 + */ +@Slf4j +@Component +public class HybridFusionPostProcessor implements SearchResultPostProcessor { + + private final SearchChannelProperties properties; + + public HybridFusionPostProcessor(SearchChannelProperties properties) { + this.properties = properties; + } + + @Override + public String getName() { + return "HybridFusion"; + } + + @Override + public int getOrder() { + return 5; + } + + @Override + public boolean isEnabled(SearchContext context) { + // 只在混合融合启用 且 存在关键词检索结果时才执行 + if (!properties.getChannels().getHybrid().isEnabled()) { + return false; + } + // 检查 results 中是否同时包含向量和关键词两类结果，在 process() 中判断 + return true; + } + + @Override + public List process(List chunks, + List results, + SearchContext context) { + // 按通道类型分组 + Map> vectorGroups = new LinkedHashMap<>(); + SearchChannelResult keywordResult = null; + + for (SearchChannelResult result : results) { + if (result.getChunks().isEmpty()) continue; + if (result.getChannelType() == SearchChannelType.KEYWORD_ES) { + keywordResult = result; + } else if (isVectorChannel(result.getChannelType())) { + vectorGroups.put(result.getChannelName(), result.getChunks()); + } + } + + // 没有关键词结果或没有向量结果时不需要融合，直接返回原始 chunks + if (keywordResult == null || vectorGroups.isEmpty()) { + log.debug("混合融合跳过：向量通道={}，关键词通道={}", + !vectorGroups.isEmpty(), + keywordResult != null); + return chunks; + } + + // 合并所有向量通道的结果为一路 + Map> inputs = new LinkedHashMap<>(); + inputs.put("vector", chunks.stream() + .filter(c -> vectorGroups.values().stream().anyMatch(v -> v.contains(c))) + .toList()); + inputs.put("keyword", keywordResult.getChunks()); + + // 选择融合策略 + FusionStrategy strategy = createStrategy(); + + // 权重配置 + Map weights = Map.of( + "vector", properties.getChannels().getHybrid().getVectorWeight(), + "keyword", 1.0f - properties.getChannels().getHybrid().getVectorWeight() + ); + + List fused = strategy.fuse(inputs, weights); + log.info("混合融合完成：向量 {} 个 + 关键词 {} 个 → 融合后 {} 个", + inputs.get("vector").size(), + inputs.get("keyword").size(), + fused.size()); + + return fused; + } + + private boolean isVectorChannel(SearchChannelType type) { + return type == SearchChannelType.VECTOR_GLOBAL || type == SearchChannelType.INTENT_DIRECTED; + } + + private FusionStrategy createStrategy() { + FusionMode mode = properties.getChannels().getHybrid().getFusion(); + if (mode == FusionMode.WEIGHTED_SUM) { + return new WeightedSumFusionStrategy(); + } + return new RRFFusionStrategy(); + } +} diff --git a/frontend/src/pages/admin/settings/SystemSettingsPage.tsx b/frontend/src/pages/admin/settings/SystemSettingsPage.tsx index 9db4a9dba..0112ea3f9 100644 --- a/frontend/src/pages/admin/settings/SystemSettingsPage.tsx +++ b/frontend/src/pages/admin/settings/SystemSettingsPage.tsx @@ -258,6 +258,52 @@ export function SystemSettingsPage() { + + + 检索通道配置 + 多路检索通道的启用状态与参数 + + + {rag.channels && ( +

+
+

向量全局检索

+
+ } /> + + + +
+
+
+

意图定向检索

+
+ } /> + + +
+
+
+

关键词检索

+
+ } /> + + +
+
+
+

混合融合

+
+ } /> + + +
+
+
+ )} + + + Rerank 模型配置 diff --git a/frontend/src/services/settingsService.ts b/frontend/src/services/settingsService.ts index 917001244..95a5ccfbf 100644 --- a/frontend/src/services/settingsService.ts +++ b/frontend/src/services/settingsService.ts @@ -30,6 +30,37 @@ export interface SystemSettings { summaryMaxChars: number; titleMaxLength: number; }; + channels: { + vectorGlobal: { + enabled: boolean; + confidenceThreshold: number; + singleIntentSupplementThreshold: number; + topKMultiplier: number; + minIntentScore?: number; + boost?: number; + }; + intentDirected: { + enabled: boolean; + minIntentScore: number; + topKMultiplier: number; + confidenceThreshold?: number; + singleIntentSupplementThreshold?: number; + boost?: number; + }; + keyword: { + enabled: boolean; + topKMultiplier: number; + boost: number; + confidenceThreshold?: number; + minIntentScore?: number; + singleIntentSupplementThreshold?: number; + }; + hybrid: { + enabled: boolean; + fusion: string; + vectorWeight: number; + }; + }; }; ai: { providers: Record< diff --git a/resources/database/schema_pg.sql b/resources/database/schema_pg.sql index 23a4416e9..352f787a9 100644 --- a/resources/database/schema_pg.sql +++ b/resources/database/schema_pg.sql @@ -423,16 +423,30 @@ CREATE TABLE t_knowledge_vector ( id VARCHAR(20) PRIMARY KEY, content TEXT, metadata JSONB, - embedding vector(1536) + embedding vector(1536), + tsv tsvector ); CREATE INDEX idx_kv_metadata ON t_knowledge_vector USING gin(metadata); CREATE INDEX idx_kv_embedding ON t_knowledge_vector USING hnsw (embedding vector_cosine_ops); +CREATE INDEX idx_kv_tsv ON t_knowledge_vector USING GIN(tsv); COMMENT ON TABLE t_knowledge_vector IS '知识库向量存储表'; COMMENT ON COLUMN t_knowledge_vector.id IS '分块ID'; COMMENT ON COLUMN t_knowledge_vector.content IS '分块文本内容'; COMMENT ON COLUMN t_knowledge_vector.metadata IS '元数据'; COMMENT ON COLUMN t_knowledge_vector.embedding IS '向量'; +COMMENT ON COLUMN t_knowledge_vector.tsv IS 'tsvector 全文检索列'; + +-- 触发器：自动维护 tsv +CREATE OR REPLACE FUNCTION kv_tsv_trigger() RETURNS trigger AS $$ +BEGIN + NEW.tsv := to_tsvector('simple', COALESCE(NEW.content, '')); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_kv_tsv BEFORE INSERT OR UPDATE OF content ON t_knowledge_vector + FOR EACH ROW EXECUTE FUNCTION kv_tsv_trigger(); -- ============================================ -- Column Comments diff --git a/resources/database/upgrade_v1.2_to_v1.3.sql b/resources/database/upgrade_v1.2_to_v1.3.sql new file mode 100644 index 000000000..04b8f79fe --- /dev/null +++ b/resources/database/upgrade_v1.2_to_v1.3.sql @@ -0,0 +1,23 @@ +-- ragent v1.2 -> v1.3 升级脚本 +-- t_knowledge_vector 表：新增 tsvector 列及全文检索索引，支持混合检索通道 + +-- 1. 新增 tsvector 列 +ALTER TABLE t_knowledge_vector ADD COLUMN IF NOT EXISTS tsv tsvector; + +-- 2. GIN 索引加速全文检索 +CREATE INDEX IF NOT EXISTS idx_kv_tsv ON t_knowledge_vector USING GIN(tsv); + +-- 3. 触发器：自动维护 tsv 列 +CREATE OR REPLACE FUNCTION kv_tsv_trigger() RETURNS trigger AS $$ +BEGIN + NEW.tsv := to_tsvector('simple', COALESCE(NEW.content, '')); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_kv_tsv ON t_knowledge_vector; +CREATE TRIGGER trg_kv_tsv BEFORE INSERT OR UPDATE OF content ON t_knowledge_vector + FOR EACH ROW EXECUTE FUNCTION kv_tsv_trigger(); + +-- 4. 回填已有数据 +UPDATE t_knowledge_vector SET tsv = to_tsvector('simple', COALESCE(content, '')); From cf10558b7fcbd2a0f2b7397087947822563689f2 Mon Sep 17 00:00:00 2001 From: LHkeeper666 Date: Sun, 31 May 2026 09:04:36 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat(retrieve):=20=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E8=AF=8D=E6=A3=80=E7=B4=A2=E9=80=9A=E9=81=93=E5=88=87=E6=8D=A2?= =?UTF-8?q?=E4=B8=BAzhparser=E4=B8=AD=E6=96=87=E5=88=86=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将全文检索配置从 simple（按字切分）迁移到 zhparser（中文词语切分）， 提升中文关键词检索的召回精度。升级脚本增加 zhparser 扩展安装及文本 检索配置，触发器和存量数据回填同步切换分词引擎。 Co-Authored-By: Claude Opus 4.7 --- .../retrieve/channel/KeywordSearchChannel.java | 4 ++-- resources/database/schema_pg.sql | 11 ++++++++++- resources/database/upgrade_v1.2_to_v1.3.sql | 17 +++++++++++++---- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java index 1f6530d88..57d25ac0a 100644 --- a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java @@ -169,9 +169,9 @@ private List searchInCollection(String query, String collectionN // 使用带索引的 tsv 列而非实时计算 to_tsvector // noinspection SqlDialectInspection,SqlNoDataSourceInspection return jdbcTemplate.query( - "SELECT id, content, ts_rank(tsv, plainto_tsquery('simple', ?)) AS score " + + "SELECT id, content, ts_rank(tsv, plainto_tsquery('zhparser', ?)) AS score " + "FROM t_knowledge_vector " + - "WHERE metadata->>'collection_name' = ? AND tsv @@ plainto_tsquery('simple', ?) " + + "WHERE metadata->>'collection_name' = ? AND tsv @@ plainto_tsquery('zhparser', ?) " + "ORDER BY score DESC LIMIT ?", (rs, rowNum) -> RetrievedChunk.builder() .id(rs.getString("id")) diff --git a/resources/database/schema_pg.sql b/resources/database/schema_pg.sql index 352f787a9..b7f7229f7 100644 --- a/resources/database/schema_pg.sql +++ b/resources/database/schema_pg.sql @@ -4,6 +4,15 @@ -- Enable pgvector extension CREATE EXTENSION IF NOT EXISTS vector; +-- Enable zhparser extension (requires zhparser compiled in PG) +CREATE EXTENSION IF NOT EXISTS zhparser; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_ts_config WHERE cfgname = 'zhparser') THEN + CREATE TEXT SEARCH CONFIGURATION zhparser (PARSER = zhparser); + END IF; +END $$; + -- ============================================ -- User & Conversation Tables -- ============================================ @@ -440,7 +449,7 @@ COMMENT ON COLUMN t_knowledge_vector.tsv IS 'tsvector 全文检索列'; -- 触发器：自动维护 tsv CREATE OR REPLACE FUNCTION kv_tsv_trigger() RETURNS trigger AS $$ BEGIN - NEW.tsv := to_tsvector('simple', COALESCE(NEW.content, '')); + NEW.tsv := to_tsvector('zhparser', COALESCE(NEW.content, '')); RETURN NEW; END; $$ LANGUAGE plpgsql; diff --git a/resources/database/upgrade_v1.2_to_v1.3.sql b/resources/database/upgrade_v1.2_to_v1.3.sql index 04b8f79fe..e4844258d 100644 --- a/resources/database/upgrade_v1.2_to_v1.3.sql +++ b/resources/database/upgrade_v1.2_to_v1.3.sql @@ -7,10 +7,19 @@ ALTER TABLE t_knowledge_vector ADD COLUMN IF NOT EXISTS tsv tsvector; -- 2. GIN 索引加速全文检索 CREATE INDEX IF NOT EXISTS idx_kv_tsv ON t_knowledge_vector USING GIN(tsv); --- 3. 触发器：自动维护 tsv 列 +-- 3. 启用 zhparser 中文分词扩展 +CREATE EXTENSION IF NOT EXISTS zhparser; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_ts_config WHERE cfgname = 'zhparser') THEN + CREATE TEXT SEARCH CONFIGURATION zhparser (PARSER = zhparser); + END IF; +END $$; + +-- 4. 触发器：自动维护 tsv 列（zhparser 分词） CREATE OR REPLACE FUNCTION kv_tsv_trigger() RETURNS trigger AS $$ BEGIN - NEW.tsv := to_tsvector('simple', COALESCE(NEW.content, '')); + NEW.tsv := to_tsvector('zhparser', COALESCE(NEW.content, '')); RETURN NEW; END; $$ LANGUAGE plpgsql; @@ -19,5 +28,5 @@ DROP TRIGGER IF EXISTS trg_kv_tsv ON t_knowledge_vector; CREATE TRIGGER trg_kv_tsv BEFORE INSERT OR UPDATE OF content ON t_knowledge_vector FOR EACH ROW EXECUTE FUNCTION kv_tsv_trigger(); --- 4. 回填已有数据 -UPDATE t_knowledge_vector SET tsv = to_tsvector('simple', COALESCE(content, '')); +-- 5. 回填已有数据（zhparser 分词） +UPDATE t_knowledge_vector SET tsv = to_tsvector('zhparser', COALESCE(content, '')); From 0fc87b8b748c2685e87d2a1713adf0982941be07 Mon Sep 17 00:00:00 2001 From: LHkeeper666 Date: Sun, 31 May 2026 09:32:02 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix(retrieve):=20=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E8=AF=8D=E6=A3=80=E7=B4=A2=E4=BB=8EAND=E8=AF=AD=E4=B9=89?= =?UTF-8?q?=E6=94=B9=E4=B8=BAOR=E8=AF=AD=E4=B9=89=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D0=E5=8F=AC=E5=9B=9E=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit plainto_tsquery 将切词结果用 & 连接，要求所有 token 同时命中， 对中文短句过于严格。改为 to_tsquery + | 运算符实现 OR 语义， 匹配任一词即可返回结果。 Co-Authored-By: Claude Opus 4.7 --- .../core/retrieve/channel/KeywordSearchChannel.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java index 57d25ac0a..35964d9fd 100644 --- a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java @@ -165,20 +165,20 @@ private List searchInCollection(String query, String collectionN if (sanitized.isBlank()) { return List.of(); } - // plainto_tsquery 将输入切分为词后用 & 连接，sanitized 中的空格天然实现 AND 语义 - // 使用带索引的 tsv 列而非实时计算 to_tsvector + // 将空格替换为 | 实现 OR 语义：匹配任一词即可命中 + String tsQuery = sanitized.trim().replaceAll("\\s+", " | "); // noinspection SqlDialectInspection,SqlNoDataSourceInspection return jdbcTemplate.query( - "SELECT id, content, ts_rank(tsv, plainto_tsquery('zhparser', ?)) AS score " + + "SELECT id, content, ts_rank(tsv, to_tsquery('zhparser', ?)) AS score " + "FROM t_knowledge_vector " + - "WHERE metadata->>'collection_name' = ? AND tsv @@ plainto_tsquery('zhparser', ?) " + + "WHERE metadata->>'collection_name' = ? AND tsv @@ to_tsquery('zhparser', ?) " + "ORDER BY score DESC LIMIT ?", (rs, rowNum) -> RetrievedChunk.builder() .id(rs.getString("id")) .text(rs.getString("content")) .score(rs.getFloat("score")) .build(), - sanitized, collectionName, sanitized, topK + tsQuery, collectionName, tsQuery, topK ); } From 9381c4d94fccd3b101620afec9bb15d4a6299ee5 Mon Sep 17 00:00:00 2001 From: LHkeeper666 Date: Sun, 31 May 2026 10:16:15 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix(retrieve):=20=E4=BF=AE=E5=A4=8Dzhparser?= =?UTF-8?q?=E5=88=86=E8=AF=8D=E5=A4=B1=E8=B4=A5=E4=B8=8E=E9=9B=B6=E5=8F=AC?= =?UTF-8?q?=E5=9B=9E=E9=97=AE=E9=A2=98=EF=BC=8C=E6=B8=85=E7=90=86=E8=B0=83?= =?UTF-8?q?=E8=AF=95=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 升级脚本增加 ADD MAPPING FOR n,v,a,i,e,l WITH simple， 修复 zhparser token 类型未映射导致 to_tsvector 返回空的问题 2. 在中英文/数字交界处插入空格，避免混合文本被当成单个 token 导致 to_tsquery 返回空 3. 删除启动诊断和每次请求的验证查询，保留正常的 info 统计日志 Co-Authored-By: Claude Opus 4.7 --- .../rag/core/retrieve/channel/KeywordSearchChannel.java | 8 +++++--- resources/database/schema_pg.sql | 9 +++++++++ resources/database/upgrade_v1.2_to_v1.3.sql | 9 +++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java index 35964d9fd..fe69c1764 100644 --- a/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java +++ b/bootstrap/src/main/java/com/nageoffer/ai/ragent/rag/core/retrieve/channel/KeywordSearchChannel.java @@ -17,7 +17,6 @@ package com.nageoffer.ai.ragent.rag.core.retrieve.channel; -import cn.hutool.core.collection.CollUtil; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.nageoffer.ai.ragent.framework.convention.RetrievedChunk; import com.nageoffer.ai.ragent.knowledge.dao.entity.KnowledgeBaseDO; @@ -37,7 +36,7 @@ /** * 关键词全文检索通道 - * 使用 PostgreSQL tsvector/tsquery 进行关键词精确匹配检索， + * 使用 PostgreSQL tsvector/tsquery + zhparser 中文分词进行关键词匹配， * 弥补向量检索对专有名词、产品型号等精确关键词的召回不足。 */ @Slf4j @@ -160,13 +159,16 @@ private List retrieveFromAllCollections(String query, } private List searchInCollection(String query, String collectionName, int topK) { - // 预处理查询：移除特殊字符避免 tsquery 解析错误 String sanitized = query.replaceAll("[^\\w\\u4e00-\\u9fff\\s]", " "); + // 在中英文/数字交界处插入空格，避免 "Redis介绍" 被当成一个无法解析的 token + sanitized = sanitized.replaceAll( + "(?<=[\\u4e00-\\u9fff])(?=[a-zA-Z0-9])|(?<=[a-zA-Z0-9])(?=[\\u4e00-\\u9fff])", " "); if (sanitized.isBlank()) { return List.of(); } // 将空格替换为 | 实现 OR 语义：匹配任一词即可命中 String tsQuery = sanitized.trim().replaceAll("\\s+", " | "); + // noinspection SqlDialectInspection,SqlNoDataSourceInspection return jdbcTemplate.query( "SELECT id, content, ts_rank(tsv, to_tsquery('zhparser', ?)) AS score " + diff --git a/resources/database/schema_pg.sql b/resources/database/schema_pg.sql index b7f7229f7..6965bd523 100644 --- a/resources/database/schema_pg.sql +++ b/resources/database/schema_pg.sql @@ -13,6 +13,15 @@ BEGIN END IF; END $$; +-- zhparser token 类型映射（n/v/a/i/e/l → simple），否则 token 被丢弃 +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR n; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR v; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR a; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR i; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR e; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR l; +ALTER TEXT SEARCH CONFIGURATION zhparser ADD MAPPING FOR n,v,a,i,e,l WITH simple; + -- ============================================ -- User & Conversation Tables -- ============================================ diff --git a/resources/database/upgrade_v1.2_to_v1.3.sql b/resources/database/upgrade_v1.2_to_v1.3.sql index e4844258d..05a041c18 100644 --- a/resources/database/upgrade_v1.2_to_v1.3.sql +++ b/resources/database/upgrade_v1.2_to_v1.3.sql @@ -16,6 +16,15 @@ BEGIN END IF; END $$; +-- 3a. token 类型映射：zhparser 产出的 n/v/a/i/e/l 必须映射到 simple 词典，否则 PG 丢弃这些 token +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR n; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR v; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR a; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR i; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR e; +ALTER TEXT SEARCH CONFIGURATION zhparser DROP MAPPING IF EXISTS FOR l; +ALTER TEXT SEARCH CONFIGURATION zhparser ADD MAPPING FOR n,v,a,i,e,l WITH simple; + -- 4. 触发器：自动维护 tsv 列（zhparser 分词） CREATE OR REPLACE FUNCTION kv_tsv_trigger() RETURNS trigger AS $$ BEGIN