(improvement)(chat) Clean up unused code and optimize some code and make the tests pass (#595)

chat/server/pom.xml

@@ -51,6 +51,12 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-inline</artifactId>
+            <version>${mockito-inline.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java

@@ -4,21 +4,19 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO;
 import com.tencent.supersonic.chat.server.persistence.repository.StatisticsRepository;
 import com.tencent.supersonic.chat.server.service.StatisticsService;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
 
-@Service("statisticsService")
+@Service
 @Slf4j
 public class StatisticsServiceImpl implements StatisticsService {
 
+    @Autowired
     private StatisticsRepository statisticsRepository;
 
-    public StatisticsServiceImpl(StatisticsRepository statisticsRepository) {
-        this.statisticsRepository = statisticsRepository;
-    }
-
     @Async
     @Override
     public void batchSaveStatistics(List<StatisticsDO> list) {

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java

@@ -3,20 +3,14 @@ package com.tencent.supersonic.chat.server.util;
 import com.tencent.supersonic.chat.core.corrector.SemanticCorrector;
 import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.mapper.SchemaMapper;
-import com.tencent.supersonic.chat.core.parser.JavaLLMProxy;
-import com.tencent.supersonic.chat.core.parser.LLMProxy;
 import com.tencent.supersonic.chat.core.parser.SemanticParser;
-import com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver;
 import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor;
 import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor;
-import com.tencent.supersonic.common.util.ContextUtils;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.core.io.support.SpringFactoriesLoader;
 
 @Slf4j
@@ -26,11 +20,8 @@ public class ComponentFactory {
     private static List<SemanticParser> semanticParsers = new ArrayList<>();
     private static List<SemanticCorrector> semanticCorrectors = new ArrayList<>();
     private static SemanticInterpreter semanticInterpreter;
-
-    private static LLMProxy llmProxy;
     private static List<ParseResultProcessor> parseProcessors = new ArrayList<>();
     private static List<ExecuteResultProcessor> executeProcessors = new ArrayList<>();
-    private static ModelResolver modelResolver;
 
     public static List<SchemaMapper> getSchemaMappers() {
         return CollectionUtils.isEmpty(schemaMappers) ? init(SchemaMapper.class, schemaMappers) : schemaMappers;
@@ -62,32 +53,6 @@ public class ComponentFactory {
         return semanticInterpreter;
     }
 
-    public static LLMProxy getLLMProxy() {
-        //1.Preferentially retrieve from environment variables
-        String llmProxyEnv = System.getenv("llmProxy");
-        if (StringUtils.isNotBlank(llmProxyEnv)) {
-            Map<String, LLMProxy> implementations = ContextUtils.getBeansOfType(LLMProxy.class);
-            llmProxy = implementations.entrySet().stream()
-                    .filter(entry -> entry.getKey().equalsIgnoreCase(llmProxyEnv))
-                    .map(Map.Entry::getValue)
-                    .findFirst()
-                    .orElse(null);
-        }
-        //2.default JavaLLMProxy
-        if (Objects.isNull(llmProxy)) {
-            llmProxy = ContextUtils.getBean(JavaLLMProxy.class);
-        }
-        log.info("llmProxy:{}", llmProxy);
-        return llmProxy;
-    }
-
-    public static ModelResolver getModelResolver() {
-        if (Objects.isNull(modelResolver)) {
-            modelResolver = init(ModelResolver.class);
-        }
-        return modelResolver;
-    }
-
     private static <T> List<T> init(Class<T> factoryType, List list) {
         list.addAll(SpringFactoriesLoader.loadFactories(factoryType,
                 Thread.currentThread().getContextClassLoader()));

chat/server/src/test/java/com/tencent/supersonic/chat/server/application/parser/TimeRangeParserTest.java

@@ -1,24 +0,0 @@
-package com.tencent.supersonic.chat.server.application.parser;
-
-import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
-import com.tencent.supersonic.chat.core.parser.sql.rule.TimeRangeParser;
-import com.tencent.supersonic.chat.core.pojo.ChatContext;
-import com.tencent.supersonic.chat.core.pojo.QueryContext;
-import org.junit.jupiter.api.Test;
-
-
-class TimeRangeParserTest {
-
-    @Test
-    void parse() {
-        TimeRangeParser timeRangeParser = new TimeRangeParser();
-
-        QueryReq queryRequest = new QueryReq();
-        ChatContext chatCtx = new ChatContext();
-
-        queryRequest.setQueryText("supersonic最近30天访问次数");
-
-        timeRangeParser.parse(QueryContext.builder().request(queryRequest).semanticSchema(null).build(), chatCtx);
-
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/application/parser/aggregate/AggregateTypeParserTest.java

@@ -1,38 +0,0 @@
-package com.tencent.supersonic.chat.server.application.parser.aggregate;
-
-
-import com.tencent.supersonic.chat.core.parser.sql.rule.AggregateTypeParser;
-import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
-import org.junit.Assert;
-import org.junit.jupiter.api.Test;
-
-class AggregateTypeParserTest {
-
-    @Test
-    void getAggregateParser() {
-        AggregateTypeParser aggregateParser = new AggregateTypeParser();
-        AggregateTypeEnum aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数最大值");
-        Assert.assertEquals(aggregateType, AggregateTypeEnum.MAX);
-
-        aggregateType = aggregateParser.resolveAggregateType("supsersonic产品pv");
-        Assert.assertEquals(aggregateType, AggregateTypeEnum.COUNT);
-
-        aggregateType = aggregateParser.resolveAggregateType("supsersonic产品uv");
-        Assert.assertEquals(aggregateType, AggregateTypeEnum.DISTINCT);
-
-        aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数最大值");
-        Assert.assertEquals(aggregateType, AggregateTypeEnum.MAX);
-
-        aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数最小值");
-        Assert.assertEquals(aggregateType, AggregateTypeEnum.MIN);
-
-        aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数平均值");
-        Assert.assertEquals(aggregateType, AggregateTypeEnum.AVG);
-
-        aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数topN");
-        Assert.assertEquals(aggregateType, AggregateTypeEnum.TOPN);
-
-        aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数汇总");
-        Assert.assertEquals(aggregateType, AggregateTypeEnum.SUM);
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/application/search/SearchServiceImplTest.java

@@ -1,15 +0,0 @@
-package com.tencent.supersonic.chat.server.application.search;
-
-import org.junit.jupiter.api.Test;
-
-
-class SearchServiceImplTest {
-
-    @Test
-    void search() {
-    }
-
-    @Test
-    void filerMetricsByModel() {
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/mapper/LoadRemoveServiceTest.java

@@ -1,15 +0,0 @@
-package com.tencent.supersonic.chat.server.mapper;
-
-
-import com.hankcs.hanlp.algorithm.EditDistance;
-import org.junit.Assert;
-import org.junit.jupiter.api.Test;
-
-class LoadRemoveServiceTest {
-
-    @Test
-    void edit() {
-        int compute = EditDistance.compute("在", "在你的身边");
-        Assert.assertEquals(compute, 4);
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/mapper/match/HanlpDictMatchStrategyTest.java

@@ -1,18 +0,0 @@
-package com.tencent.supersonic.chat.server.mapper.match;
-
-import com.tencent.supersonic.chat.server.test.context.ContextTest;
-import org.junit.jupiter.api.Test;
-
-/**
- * MatchStrategyImplTest
- */
-class HanlpDictMatchStrategyTest extends ContextTest {
-
-    @Test
-    void match() {
-    }
-
-    @Test
-    void testMatch() {
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/parser/llm/s2sql/LLMResponseServiceTest.java

@@ -1,55 +0,0 @@
-package com.tencent.supersonic.chat.server.parser.llm.s2sql;
-
-import com.tencent.supersonic.chat.core.parser.sql.llm.LLMResponseService;
-import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp;
-import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp;
-import java.util.HashMap;
-import java.util.Map;
-import org.junit.Assert;
-import org.junit.jupiter.api.Test;
-
-class LLMResponseServiceTest {
-
-    @Test
-    void deduplicationSqlWeight() {
-        String sql1 = "SELECT a,b,c,d FROM table1 WHERE column1 = 1 AND column2 = 2 order by a";
-        String sql2 = "SELECT d,c,b,a FROM table1 WHERE column2 = 2 AND column1 = 1 order by a";
-
-        LLMResp llmResp = new LLMResp();
-        Map<String, LLMSqlResp> sqlWeight = new HashMap<>();
-        sqlWeight.put(sql1, LLMSqlResp.builder().sqlWeight(0.20).build());
-        sqlWeight.put(sql2, LLMSqlResp.builder().sqlWeight(0.80).build());
-
-        llmResp.setSqlRespMap(sqlWeight);
-        LLMResponseService llmResponseService = new LLMResponseService();
-        Map<String, LLMSqlResp> deduplicationSqlResp = llmResponseService.getDeduplicationSqlResp(llmResp);
-
-        Assert.assertEquals(deduplicationSqlResp.size(), 1);
-
-        sql1 = "SELECT a,b,c,d FROM table1 WHERE column1 = 1 AND column2 = 2 order by a";
-        sql2 = "SELECT d,c,b,a FROM table1 WHERE column2 = 2 AND column1 = 1 order by a";
-
-        LLMResp llmResp2 = new LLMResp();
-        Map<String, LLMSqlResp> sqlWeight2 = new HashMap<>();
-        sqlWeight2.put(sql1, LLMSqlResp.builder().sqlWeight(0.20).build());
-        sqlWeight2.put(sql2, LLMSqlResp.builder().sqlWeight(0.80).build());
-
-        llmResp2.setSqlRespMap(sqlWeight2);
-        deduplicationSqlResp = llmResponseService.getDeduplicationSqlResp(llmResp2);
-
-        Assert.assertEquals(deduplicationSqlResp.size(), 1);
-
-        sql1 = "SELECT a,b,c,d,e FROM table1 WHERE column1 = 1 AND column2 = 2 order by a";
-        sql2 = "SELECT d,c,b,a FROM table1 WHERE column2 = 2 AND column1 = 1 order by a";
-
-        LLMResp llmResp3 = new LLMResp();
-        Map<String, LLMSqlResp> sqlWeight3 = new HashMap<>();
-        sqlWeight3.put(sql1, LLMSqlResp.builder().sqlWeight(0.20).build());
-        sqlWeight3.put(sql2, LLMSqlResp.builder().sqlWeight(0.80).build());
-        llmResp3.setSqlRespMap(sqlWeight3);
-        deduplicationSqlResp = llmResponseService.getDeduplicationSqlResp(llmResp3);
-
-        Assert.assertEquals(deduplicationSqlResp.size(), 2);
-
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/parser/llm/s2sql/LLMS2SQLParserTest.java

@@ -1,64 +0,0 @@
-package com.tencent.supersonic.chat.server.parser.llm.s2sql;
-
-import static org.mockito.Mockito.when;
-
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
-import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
-import com.tencent.supersonic.chat.server.service.impl.SchemaService;
-import com.tencent.supersonic.common.util.ContextUtils;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import org.junit.jupiter.api.Test;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
-
-class LLMS2SQLParserTest {
-
-    @Test
-    void setFilter() {
-        MockedStatic<ContextUtils> mockContextUtils = Mockito.mockStatic(ContextUtils.class);
-
-        SchemaService mockSchemaService = Mockito.mock(SchemaService.class);
-        SemanticSchema mockSemanticSchema = Mockito.mock(SemanticSchema.class);
-
-        List<SchemaElement> dimensions = new ArrayList<>();
-        List<SchemaValueMap> schemaValueMaps = new ArrayList<>();
-        SchemaValueMap value1 = new SchemaValueMap();
-        value1.setBizName("杰伦");
-        value1.setTechName("周杰伦");
-        value1.setAlias(Arrays.asList("周杰倫", "Jay Chou", "周董", "周先生"));
-        schemaValueMaps.add(value1);
-
-        SchemaElement schemaElement = SchemaElement.builder()
-                .bizName("singer_name")
-                .name("歌手名")
-                .model(2L)
-                .schemaValueMaps(schemaValueMaps)
-                .build();
-        dimensions.add(schemaElement);
-
-        SchemaElement schemaElement2 = SchemaElement.builder()
-                .bizName("publish_time")
-                .name("发布时间")
-                .model(2L)
-                .build();
-        dimensions.add(schemaElement2);
-
-        when(mockSemanticSchema.getDimensions()).thenReturn(dimensions);
-
-        List<SchemaElement> metrics = new ArrayList<>();
-        SchemaElement metric = SchemaElement.builder()
-                .bizName("play_count")
-                .name("播放量")
-                .model(2L)
-                .build();
-        metrics.add(metric);
-
-        when(mockSemanticSchema.getMetrics()).thenReturn(metrics);
-
-        when(mockSchemaService.getSemanticSchema()).thenReturn(mockSemanticSchema);
-        mockContextUtils.when(() -> ContextUtils.getBean(SchemaService.class)).thenReturn(mockSchemaService);
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java

@@ -106,7 +106,7 @@ class MetricCheckProcessorTest {
         SemanticParseInfo parseInfo = mockParseInfo(correctSql);
         String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo,
                 mockModelSchemaNoDimensionSetting());
-        String expectedProcessedSql = "select 部门, count(*) from 超音数 group by 部门";
+        String expectedProcessedSql = "SELECT count(*) FROM 超音数";
         Assertions.assertEquals(expectedProcessedSql, actualProcessedSql);
     }
 

chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/ContextTest.java

@@ -1,35 +0,0 @@
-
-package com.tencent.supersonic.chat.server.test.context;
-
-import com.tencent.supersonic.chat.core.knowledge.semantic.RemoteSemanticInterpreter;
-import com.tencent.supersonic.chat.core.utils.ComponentFactory;
-import com.tencent.supersonic.chat.server.persistence.mapper.ChatContextMapper;
-import com.tencent.supersonic.chat.server.persistence.repository.impl.ChatContextRepositoryImpl;
-import com.tencent.supersonic.chat.server.service.QueryService;
-import com.tencent.supersonic.chat.server.test.ChatBizLauncher;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.MetricService;
-import com.tencent.supersonic.headless.server.service.ModelService;
-import org.junit.runner.RunWith;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.test.context.junit4.SpringRunner;
-import org.springframework.web.client.RestTemplate;
-
-@MockBean(ChatContextRepositoryImpl.class)
-@MockBean(QueryService.class)
-@MockBean(DimensionService.class)
-@MockBean(MetricService.class)
-@MockBean(ModelService.class)
-@MockBean(ChatContextMapper.class)
-@MockBean(RestTemplate.class)
-@MockBean(RemoteSemanticInterpreter.class)
-@MockBean(ComponentFactory.class)
-@RunWith(SpringRunner.class)
-@SpringBootTest(classes = ChatBizLauncher.class)
-public class ContextTest {
-
-    protected final Logger logger = LoggerFactory.getLogger(ContextTest.class);
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java

@@ -53,8 +53,9 @@ class QueryReqBuilderTest {
 
         QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq);
         Assert.assertEquals(
-                "SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
-                        + "GROUP BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
+                "SELECT department, SUM(pv) AS pv FROM 内容库 "
+                        + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP "
+                        + "BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
 
         queryStructReq.setQueryType(QueryType.TAG);
         queryS2SQLReq = queryStructReq.convert(queryStructReq);