[release][project] supersonic 0.7.3 version backend update (#40)

* [improvement] add some features

* [improvement] revise CHANGELOG

---------

Co-authored-by: zuopengge <hwzuopengge@tencent.com>

chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeRangeParserTest.java

@@ -2,7 +2,6 @@ package com.tencent.supersonic.chat.application.parser;
 
 import com.tencent.supersonic.chat.api.pojo.ChatContext;
 import com.tencent.supersonic.chat.api.pojo.QueryContext;
-import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.parser.rule.TimeRangeParser;
 import org.junit.jupiter.api.Test;
@@ -10,35 +9,15 @@ import org.junit.jupiter.api.Test;
 
 class TimeRangeParserTest {
 
-//    private HeuristicQuerySelector voteStrategy = new HeuristicQuerySelector() {
-//        @Override
-//        public void init(List<SemanticParser> semanticParsers) {
-//            List<String> queryMode = new ArrayList<>(Arrays.asList(EntityDetailQuery.QUERY_MODE));
-//            for(SemanticParser semanticParser : semanticParsers) {
-//                if(semanticParser.getName().equals(TimeRangeParser.PARSER_MODE)) {
-//                    semanticParser.getQueryModes().clear();
-//                    semanticParser.getQueryModes().addAll(queryMode);
-//                }
-//            }
-//        }
-//    };
-
     @Test
     void parse() {
         TimeRangeParser timeRangeParser = new TimeRangeParser();
 
         QueryReq queryRequest = new QueryReq();
         ChatContext chatCtx = new ChatContext();
-        SchemaMapInfo schemaMap = new SchemaMapInfo();
 
         queryRequest.setQueryText("supersonic最近30天访问次数");
-        //voteStrategy.init(new ArrayList<>(Arrays.asList(timeRangeParser)));
         timeRangeParser.parse(new QueryContext(queryRequest), chatCtx);
 
-        //DateConf dateInfo = queryContext.getParseInfo(timeRangeParser.getQueryModes().get(0))
-        //       .getDateInfo();
-
-        //System.out.println(dateInfo);
-
     }
-}
+}

chat/core/src/test/java/com/tencent/supersonic/chat/mapper/LoadRemoveServiceTest.java

@@ -5,7 +5,7 @@ import com.hankcs.hanlp.algorithm.EditDistance;
 import org.junit.Assert;
 import org.junit.jupiter.api.Test;
 
-class MapperHelperTest {
+class LoadRemoveServiceTest {
 
 
     @Test
@@ -13,4 +13,4 @@ class MapperHelperTest {
         int compute = EditDistance.compute("在", "在你的身边");
         Assert.assertEquals(compute, 4);
     }
-}
+}

chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java

@@ -0,0 +1,12 @@
+package com.tencent.supersonic.chat.parser.llm.dsl;
+
+import org.junit.jupiter.api.Test;
+
+class LLMDslParserTest {
+
+
+    @Test
+    void getDimensionFilter() {
+    }
+
+}

chat/core/src/test/java/com/tencent/supersonic/chat/query/llm/dsl/corrector/DateFieldCorrectorTest.java

@@ -1,8 +1,9 @@
-package com.tencent.supersonic.chat.query.dsl.optimizer;
+package com.tencent.supersonic.chat.query.llm.dsl.corrector;
 
 import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
 import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.chat.corrector.DateFieldCorrector;
 import org.junit.Assert;
 import org.junit.jupiter.api.Test;
 
@@ -20,18 +21,18 @@ class DateFieldCorrectorTest {
                 .parseInfo(parseInfo)
                 .build();
 
-        CorrectionInfo rewriter = dateFieldCorrector.rewriter(correctionInfo);
+        CorrectionInfo rewriter = dateFieldCorrector.corrector(correctionInfo);
 
         Assert.assertEquals("SELECT count(歌曲名) FROM 歌曲库 WHERE 数据日期 = '2023-08-14'", rewriter.getSql());
 
-         correctionInfo = CorrectionInfo.builder()
+        correctionInfo = CorrectionInfo.builder()
                 .sql("select count(歌曲名) from 歌曲库 where 数据日期 = '2023-08-14'")
                 .parseInfo(parseInfo)
                 .build();
 
-        rewriter = dateFieldCorrector.rewriter(correctionInfo);
+        rewriter = dateFieldCorrector.corrector(correctionInfo);
 
         Assert.assertEquals("select count(歌曲名) from 歌曲库 where 数据日期 = '2023-08-14'", rewriter.getSql());
 
     }
-}
+}

chat/core/src/test/java/com/tencent/supersonic/chat/query/llm/dsl/corrector/FieldValueCorrectorTest.java

@@ -0,0 +1,66 @@
+package com.tencent.supersonic.chat.query.llm.dsl.corrector;
+
+import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
+import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.chat.corrector.FieldValueCorrector;
+import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
+import com.tencent.supersonic.chat.query.llm.dsl.LLMReq;
+import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue;
+import com.tencent.supersonic.common.pojo.Constants;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+class FieldValueCorrectorTest {
+
+    @Test
+    void rewriter() {
+
+        FieldValueCorrector corrector = new FieldValueCorrector();
+        CorrectionInfo correctionInfo = CorrectionInfo.builder()
+                .sql("select 歌曲名 from 歌曲库 where 专辑照片 = '七里香' and 专辑名 = '流行' and 数据日期 = '2023-08-19'")
+                .build();
+
+        SemanticParseInfo parseInfo = new SemanticParseInfo();
+
+        DSLParseResult dslParseResult = new DSLParseResult();
+        LLMReq llmReq = new LLMReq();
+        List<ElementValue> linking = new ArrayList<>();
+        ElementValue elementValue = new ElementValue();
+        elementValue.setFieldValue("流行");
+        elementValue.setFieldName("歌曲风格");
+        linking.add(elementValue);
+
+        ElementValue elementValue2 = new ElementValue();
+        elementValue2.setFieldValue("七里香");
+        elementValue2.setFieldName("歌曲名");
+        linking.add(elementValue2);
+
+        ElementValue elementValue3 = new ElementValue();
+        elementValue3.setFieldValue("周杰伦");
+        elementValue3.setFieldName("歌手名");
+        linking.add(elementValue3);
+
+        ElementValue elementValue4 = new ElementValue();
+        elementValue4.setFieldValue("流行");
+        elementValue4.setFieldName("歌曲流派");
+        linking.add(elementValue4);
+
+        llmReq.setLinking(linking);
+        dslParseResult.setLlmReq(llmReq);
+
+        Map<String, Object> properties = new HashMap<>();
+        properties.put(Constants.CONTEXT, dslParseResult);
+
+        parseInfo.setProperties(properties);
+        correctionInfo.setParseInfo(parseInfo);
+
+        CorrectionInfo rewriter = corrector.corrector(correctionInfo);
+
+        Assert.assertEquals("SELECT 歌曲名 FROM 歌曲库 WHERE 歌曲名 = '七里香' AND 歌曲流派 = '流行' AND 数据日期 = '2023-08-19'",
+                rewriter.getSql());
+    }
+}

chat/core/src/test/java/com/tencent/supersonic/chat/query/llm/dsl/corrector/SelectFieldAppendCorrectorTest.java

@@ -1,8 +1,7 @@
-package com.tencent.supersonic.chat.query.dsl.optimizer;
-
-import static org.junit.jupiter.api.Assertions.*;
+package com.tencent.supersonic.chat.query.llm.dsl.corrector;
 
 import com.tencent.supersonic.chat.api.pojo.CorrectionInfo;
+import com.tencent.supersonic.chat.corrector.SelectFieldAppendCorrector;
 import org.junit.Assert;
 import org.junit.jupiter.api.Test;
 
@@ -12,14 +11,16 @@ class SelectFieldAppendCorrectorTest {
     void rewriter() {
         SelectFieldAppendCorrector corrector = new SelectFieldAppendCorrector();
         CorrectionInfo correctionInfo = CorrectionInfo.builder()
-                .sql("select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' and sys_imp_date = '2023-08-09' and 歌曲发布时 = '2023-08-01' order by 播放量 desc limit 11")
+                .sql("select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' "
+                        + "and sys_imp_date = '2023-08-09' and 歌曲发布时 = '2023-08-01' order by 播放量 desc limit 11")
                 .build();
 
-        CorrectionInfo rewriter = corrector.rewriter(correctionInfo);
+        CorrectionInfo rewriter = corrector.corrector(correctionInfo);
 
         Assert.assertEquals(
-                "SELECT 歌曲名, 歌手名, 歌曲发布时, 发布日期 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 AND 歌手名 = '邓紫棋' AND sys_imp_date = '2023-08-09' AND 歌曲发布时 = '2023-08-01' ORDER BY 播放量 DESC LIMIT 11",
-                rewriter.getSql());
+                "SELECT 歌曲名, 歌手名, 歌曲发布时, 发布日期 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 "
+                        + "AND 歌手名 = '邓紫棋' AND sys_imp_date = '2023-08-09' "
+                        + "AND 歌曲发布时 = '2023-08-01' ORDER BY 播放量 DESC LIMIT 11", rewriter.getSql());
 
     }
-}
+}

chat/core/src/test/java/com/tencent/supersonic/chat/test/ChatBizLauncher.java

@@ -6,9 +6,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.annotation.ComponentScan;
 
 
-@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic.chat"}
-        //  , exclude = {DataSourceAutoConfiguration.class}
-)
+@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic.chat"})
 @ComponentScan("com.tencent.supersonic.chat")
 @MapperScan("com.tencent.supersonic.chat")
 public class ChatBizLauncher {

chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java

@@ -4,28 +4,26 @@ import static org.mockito.ArgumentMatchers.anyList;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.when;
 
-import com.tencent.supersonic.chat.api.component.SemanticLayer;
 import com.tencent.supersonic.chat.api.pojo.ChatContext;
+import com.tencent.supersonic.chat.api.component.SemanticLayer;
 import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
-import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
-import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp;
 import com.tencent.supersonic.chat.config.DefaultMetric;
 import com.tencent.supersonic.chat.config.DefaultMetricInfo;
 import com.tencent.supersonic.chat.config.EntityInternalDetail;
-import com.tencent.supersonic.chat.persistence.mapper.ChatContextMapper;
 import com.tencent.supersonic.chat.persistence.repository.impl.ChatContextRepositoryImpl;
-import com.tencent.supersonic.chat.service.ChatService;
 import com.tencent.supersonic.chat.service.QueryService;
-import com.tencent.supersonic.chat.service.impl.ConfigServiceImpl;
-import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
+import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
-import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
+import com.tencent.supersonic.chat.service.impl.ConfigServiceImpl;
+import com.tencent.supersonic.chat.service.ChatService;
+import com.tencent.supersonic.chat.persistence.mapper.ChatContextMapper;
+import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.semantic.model.domain.DimensionService;
-import com.tencent.supersonic.semantic.model.domain.MetricService;
 import com.tencent.supersonic.semantic.model.domain.ModelService;
+import com.tencent.supersonic.semantic.model.domain.MetricService;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -45,17 +43,13 @@ public class MockBeansConfiguration {
 
     public static void buildHttpSemanticServiceImpl(SemanticLayer httpSemanticLayer, List<DimSchemaResp> dimensionDescs,
             List<MetricSchemaResp> metricDescs) {
-        ChatConfigRichResp chaConfigRichDesc = new ChatConfigRichResp();
         DefaultMetric defaultMetricDesc = new DefaultMetric();
         defaultMetricDesc.setUnit(3);
         defaultMetricDesc.setPeriod(Constants.DAY);
-//        chaConfigRichDesc.setDefaultMetrics(new ArrayList<>(Arrays.asList(defaultMetricDesc)));
-        EntityRichInfoResp entityDesc = new EntityRichInfoResp();
         List<DimSchemaResp> dimensionDescs1 = new ArrayList<>();
         DimSchemaResp dimensionDesc = new DimSchemaResp();
         dimensionDesc.setId(162L);
         dimensionDescs1.add(dimensionDesc);
-//        entityDesc.setEntityIds(dimensionDescs1);
 
         DimSchemaResp dimensionDesc2 = new DimSchemaResp();
         dimensionDesc2.setId(163L);
@@ -69,14 +63,11 @@ public class MockBeansConfiguration {
         metricDesc.setBizName("js_play_cnt");
         metricDesc.setName("结算播放量");
         entityInternalDetailDesc.setMetricList(new ArrayList<>(Arrays.asList(metricDesc)));
-//        entityDesc.setEntityInternalDetailDesc(entityInternalDetailDesc);
 
-//        chaConfigRichDesc.setEntity(entityDesc);
-//        when(httpSemanticLayer.getChatConfigRichInfo(anyLong())).thenReturn(chaConfigRichDesc);
         ModelSchemaResp modelSchemaDesc = new ModelSchemaResp();
         modelSchemaDesc.setDimensions(dimensionDescs);
         modelSchemaDesc.setMetrics(metricDescs);
-//        when(httpSemanticLayer.getModelSchemaInfo(anyLong())).thenReturn(modelSchemaDesc);
+
     }
 
     public static void getModelExtendMock(ConfigServiceImpl configService) {
@@ -87,7 +78,6 @@ public class MockBeansConfiguration {
         defaultMetricInfos.add(defaultMetricInfo);
 
         ChatConfigResp chaConfigDesc = new ChatConfigResp();
-//        chaConfigDesc.setDefaultMetrics(defaultMetricInfos);
         when(configService.fetchConfigByModelId(anyLong())).thenReturn(chaConfigDesc);
     }