(improvement)(chat) Clean up unused code and optimize some code and make the tests pass (#595)

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java

@@ -1,15 +1,15 @@
 package com.tencent.supersonic.chat.api.pojo;
 
-import org.springframework.util.CollectionUtils;
-
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import org.springframework.util.CollectionUtils;
 
 public class SemanticSchema implements Serializable {
 
@@ -160,7 +160,8 @@ public class SemanticSchema implements Serializable {
     private Optional<SchemaElement> getElementsByNameOrAlias(String name, List<SchemaElement> elements) {
         return elements.stream()
                 .filter(schemaElement ->
-                    name.equals(schemaElement.getName()) || schemaElement.getAlias().contains(name)
+                        name.equals(schemaElement.getName()) || (Objects.nonNull(schemaElement.getAlias())
+                                && schemaElement.getAlias().contains(name))
                 ).findFirst();
     }
 

chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/SearchService.java

@@ -20,10 +20,8 @@ import java.util.TreeSet;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
-@Service
 @Slf4j
 public class SearchService {
 

chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java

@@ -5,7 +5,6 @@ import com.tencent.supersonic.chat.core.parser.JavaLLMProxy;
 import com.tencent.supersonic.chat.core.parser.LLMProxy;
 import com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver;
 import com.tencent.supersonic.common.util.ContextUtils;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import lombok.extern.slf4j.Slf4j;
@@ -52,12 +51,6 @@ public class ComponentFactory {
         return modelResolver;
     }
 
-    private static <T> List<T> init(Class<T> factoryType, List list) {
-        list.addAll(SpringFactoriesLoader.loadFactories(factoryType,
-                Thread.currentThread().getContextClassLoader()));
-        return list;
-    }
-
     private static <T> T init(Class<T> factoryType) {
         return SpringFactoriesLoader.loadFactories(factoryType,
                 Thread.currentThread().getContextClassLoader()).get(0);

chat/core/src/test/java/com/tencent/supersonic/chat/core/mapper/LoadRemoveServiceTest.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.server.mapper;
+package com.tencent.supersonic.chat.core.mapper;
 
 
 import com.hankcs.hanlp.algorithm.EditDistance;

chat/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/AggregateTypeParserTest.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.server.application.parser.aggregate;
+package com.tencent.supersonic.chat.core.parser.aggregate;
 
 
 import com.tencent.supersonic.chat.core.parser.sql.rule.AggregateTypeParser;

chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMResponseServiceTest.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.server.parser.llm.s2sql;
+package com.tencent.supersonic.chat.core.s2sql;
 
 import com.tencent.supersonic.chat.core.parser.sql.llm.LLMResponseService;
 import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp;

chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java

@@ -1,26 +1,20 @@
-package com.tencent.supersonic.chat.server.parser.llm.s2sql;
+package com.tencent.supersonic.chat.core.s2sql;
 
 import static org.mockito.Mockito.when;
 
 import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
-import com.tencent.supersonic.chat.server.service.impl.SchemaService;
-import com.tencent.supersonic.common.util.ContextUtils;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import org.junit.jupiter.api.Test;
-import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 
-class LLMS2SQLParserTest {
+class LLMSqlParserTest {
 
     @Test
     void setFilter() {
-        MockedStatic<ContextUtils> mockContextUtils = Mockito.mockStatic(ContextUtils.class);
-
-        SchemaService mockSchemaService = Mockito.mock(SchemaService.class);
         SemanticSchema mockSemanticSchema = Mockito.mock(SemanticSchema.class);
 
         List<SchemaElement> dimensions = new ArrayList<>();
@@ -57,8 +51,5 @@ class LLMS2SQLParserTest {
         metrics.add(metric);
 
         when(mockSemanticSchema.getMetrics()).thenReturn(metrics);
-
-        when(mockSchemaService.getSemanticSchema()).thenReturn(mockSemanticSchema);
-        mockContextUtils.when(() -> ContextUtils.getBean(SchemaService.class)).thenReturn(mockSchemaService);
     }
 }

chat/server/pom.xml

@@ -51,6 +51,12 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-inline</artifactId>
+            <version>${mockito-inline.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/StatisticsServiceImpl.java

@@ -4,21 +4,19 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.StatisticsDO;
 import com.tencent.supersonic.chat.server.persistence.repository.StatisticsRepository;
 import com.tencent.supersonic.chat.server.service.StatisticsService;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
 
-@Service("statisticsService")
+@Service
 @Slf4j
 public class StatisticsServiceImpl implements StatisticsService {
 
+    @Autowired
     private StatisticsRepository statisticsRepository;
 
-    public StatisticsServiceImpl(StatisticsRepository statisticsRepository) {
-        this.statisticsRepository = statisticsRepository;
-    }
-
     @Async
     @Override
     public void batchSaveStatistics(List<StatisticsDO> list) {

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java

@@ -3,20 +3,14 @@ package com.tencent.supersonic.chat.server.util;
 import com.tencent.supersonic.chat.core.corrector.SemanticCorrector;
 import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.mapper.SchemaMapper;
-import com.tencent.supersonic.chat.core.parser.JavaLLMProxy;
-import com.tencent.supersonic.chat.core.parser.LLMProxy;
 import com.tencent.supersonic.chat.core.parser.SemanticParser;
-import com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver;
 import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor;
 import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor;
-import com.tencent.supersonic.common.util.ContextUtils;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.core.io.support.SpringFactoriesLoader;
 
 @Slf4j
@@ -26,11 +20,8 @@ public class ComponentFactory {
     private static List<SemanticParser> semanticParsers = new ArrayList<>();
     private static List<SemanticCorrector> semanticCorrectors = new ArrayList<>();
     private static SemanticInterpreter semanticInterpreter;
-
-    private static LLMProxy llmProxy;
     private static List<ParseResultProcessor> parseProcessors = new ArrayList<>();
     private static List<ExecuteResultProcessor> executeProcessors = new ArrayList<>();
-    private static ModelResolver modelResolver;
 
     public static List<SchemaMapper> getSchemaMappers() {
         return CollectionUtils.isEmpty(schemaMappers) ? init(SchemaMapper.class, schemaMappers) : schemaMappers;
@@ -62,32 +53,6 @@ public class ComponentFactory {
         return semanticInterpreter;
     }
 
-    public static LLMProxy getLLMProxy() {
-        //1.Preferentially retrieve from environment variables
-        String llmProxyEnv = System.getenv("llmProxy");
-        if (StringUtils.isNotBlank(llmProxyEnv)) {
-            Map<String, LLMProxy> implementations = ContextUtils.getBeansOfType(LLMProxy.class);
-            llmProxy = implementations.entrySet().stream()
-                    .filter(entry -> entry.getKey().equalsIgnoreCase(llmProxyEnv))
-                    .map(Map.Entry::getValue)
-                    .findFirst()
-                    .orElse(null);
-        }
-        //2.default JavaLLMProxy
-        if (Objects.isNull(llmProxy)) {
-            llmProxy = ContextUtils.getBean(JavaLLMProxy.class);
-        }
-        log.info("llmProxy:{}", llmProxy);
-        return llmProxy;
-    }
-
-    public static ModelResolver getModelResolver() {
-        if (Objects.isNull(modelResolver)) {
-            modelResolver = init(ModelResolver.class);
-        }
-        return modelResolver;
-    }
-
     private static <T> List<T> init(Class<T> factoryType, List list) {
         list.addAll(SpringFactoriesLoader.loadFactories(factoryType,
                 Thread.currentThread().getContextClassLoader()));

chat/server/src/test/java/com/tencent/supersonic/chat/server/application/parser/TimeRangeParserTest.java

@@ -1,24 +0,0 @@
-package com.tencent.supersonic.chat.server.application.parser;
-
-import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
-import com.tencent.supersonic.chat.core.parser.sql.rule.TimeRangeParser;
-import com.tencent.supersonic.chat.core.pojo.ChatContext;
-import com.tencent.supersonic.chat.core.pojo.QueryContext;
-import org.junit.jupiter.api.Test;
-
-
-class TimeRangeParserTest {
-
-    @Test
-    void parse() {
-        TimeRangeParser timeRangeParser = new TimeRangeParser();
-
-        QueryReq queryRequest = new QueryReq();
-        ChatContext chatCtx = new ChatContext();
-
-        queryRequest.setQueryText("supersonic最近30天访问次数");
-
-        timeRangeParser.parse(QueryContext.builder().request(queryRequest).semanticSchema(null).build(), chatCtx);
-
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/application/search/SearchServiceImplTest.java

@@ -1,15 +0,0 @@
-package com.tencent.supersonic.chat.server.application.search;
-
-import org.junit.jupiter.api.Test;
-
-
-class SearchServiceImplTest {
-
-    @Test
-    void search() {
-    }
-
-    @Test
-    void filerMetricsByModel() {
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/mapper/match/HanlpDictMatchStrategyTest.java

@@ -1,18 +0,0 @@
-package com.tencent.supersonic.chat.server.mapper.match;
-
-import com.tencent.supersonic.chat.server.test.context.ContextTest;
-import org.junit.jupiter.api.Test;
-
-/**
- * MatchStrategyImplTest
- */
-class HanlpDictMatchStrategyTest extends ContextTest {
-
-    @Test
-    void match() {
-    }
-
-    @Test
-    void testMatch() {
-    }
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/processor/MetricCheckProcessorTest.java

@@ -106,7 +106,7 @@ class MetricCheckProcessorTest {
         SemanticParseInfo parseInfo = mockParseInfo(correctSql);
         String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo,
                 mockModelSchemaNoDimensionSetting());
-        String expectedProcessedSql = "select 部门, count(*) from 超音数 group by 部门";
+        String expectedProcessedSql = "SELECT count(*) FROM 超音数";
         Assertions.assertEquals(expectedProcessedSql, actualProcessedSql);
     }
 

chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/ContextTest.java

@@ -1,35 +0,0 @@
-
-package com.tencent.supersonic.chat.server.test.context;
-
-import com.tencent.supersonic.chat.core.knowledge.semantic.RemoteSemanticInterpreter;
-import com.tencent.supersonic.chat.core.utils.ComponentFactory;
-import com.tencent.supersonic.chat.server.persistence.mapper.ChatContextMapper;
-import com.tencent.supersonic.chat.server.persistence.repository.impl.ChatContextRepositoryImpl;
-import com.tencent.supersonic.chat.server.service.QueryService;
-import com.tencent.supersonic.chat.server.test.ChatBizLauncher;
-import com.tencent.supersonic.headless.server.service.DimensionService;
-import com.tencent.supersonic.headless.server.service.MetricService;
-import com.tencent.supersonic.headless.server.service.ModelService;
-import org.junit.runner.RunWith;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.test.context.junit4.SpringRunner;
-import org.springframework.web.client.RestTemplate;
-
-@MockBean(ChatContextRepositoryImpl.class)
-@MockBean(QueryService.class)
-@MockBean(DimensionService.class)
-@MockBean(MetricService.class)
-@MockBean(ModelService.class)
-@MockBean(ChatContextMapper.class)
-@MockBean(RestTemplate.class)
-@MockBean(RemoteSemanticInterpreter.class)
-@MockBean(ComponentFactory.class)
-@RunWith(SpringRunner.class)
-@SpringBootTest(classes = ChatBizLauncher.class)
-public class ContextTest {
-
-    protected final Logger logger = LoggerFactory.getLogger(ContextTest.class);
-}

chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java

@@ -53,8 +53,9 @@ class QueryReqBuilderTest {
 
         QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq);
         Assert.assertEquals(
-                "SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+                "SELECT department, SUM(pv) AS pv FROM 内容库 "
-                        + "GROUP BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
+                        + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP "
+                        + "BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
 
         queryStructReq.setQueryType(QueryType.TAG);
         queryS2SQLReq = queryStructReq.convert(queryStructReq);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java

@@ -37,28 +37,21 @@ import com.tencent.supersonic.headless.api.response.MetricResp;
 import com.tencent.supersonic.headless.api.response.ModelResp;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
 import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
-import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
+import com.tencent.supersonic.headless.core.executor.QueryExecutor;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
-import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
-import com.tencent.supersonic.headless.server.service.QueryService;
-import com.tencent.supersonic.headless.server.service.SchemaService;
-import com.tencent.supersonic.headless.server.utils.StatUtils;
 import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck;
 import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission;
 import com.tencent.supersonic.headless.server.annotation.StructDataPermission;
 import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
-import com.tencent.supersonic.headless.core.executor.QueryExecutor;
 import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
 import com.tencent.supersonic.headless.server.service.AppService;
 import com.tencent.supersonic.headless.server.service.Catalog;
+import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
+import com.tencent.supersonic.headless.server.service.QueryService;
+import com.tencent.supersonic.headless.server.service.SchemaService;
+import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
 import com.tencent.supersonic.headless.server.utils.QueryUtils;
-import lombok.SneakyThrows;
+import com.tencent.supersonic.headless.server.utils.StatUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections.CollectionUtils;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import javax.servlet.http.HttpServletRequest;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -66,6 +59,13 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import javax.servlet.http.HttpServletRequest;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Service;
 
 
 @Service
@@ -92,7 +92,7 @@ public class QueryServiceImpl implements QueryService {
             CacheUtils cacheUtils,
             QueryUtils queryUtils,
             QueryReqConverter queryReqConverter,
-            HeadlessQueryEngine headlessQueryEngine,
+            @Lazy HeadlessQueryEngine headlessQueryEngine,
             Catalog catalog,
             AppService appService) {
         this.statUtils = statUtils;
@@ -283,7 +283,7 @@ public class QueryServiceImpl implements QueryService {
     @Override
     @ApiHeaderCheck
     public ItemQueryResultResp metricDataQueryById(QueryItemReq queryItemReq,
-                                                   HttpServletRequest request) throws Exception {
+            HttpServletRequest request) throws Exception {
         AppDetailResp appDetailResp = getAppDetailResp(request);
         authCheck(appDetailResp, queryItemReq.getIds(), ApiItemType.METRIC);
         List<SingleItemQueryResult> results = Lists.newArrayList();
@@ -323,7 +323,7 @@ public class QueryServiceImpl implements QueryService {
     }
 
     private QueryStructReq buildQueryStructReq(List<DimensionResp> dimensionResps,
-                                               MetricResp metricResp, DateConf dateConf, Long limit) {
+            MetricResp metricResp, DateConf dateConf, Long limit) {
         Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
         modelIds.add(metricResp.getModelId());
         QueryStructReq queryStructReq = new QueryStructReq();