[improvement][project] global refactor , code format , support llm , support fuzzy detect ,support query filter and so on.

This commit is contained in:
lexluo
2023-07-08 15:00:03 +08:00
parent 5ffd617431
commit 404163f391
329 changed files with 21050 additions and 5036 deletions

View File

@@ -3,12 +3,24 @@ package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.common.pojo.DateConf;
import org.junit.jupiter.api.Test;
class TimeSemanticParserTest {
// private HeuristicQuerySelector voteStrategy = new HeuristicQuerySelector() {
// @Override
// public void init(List<SemanticParser> semanticParsers) {
// List<String> queryMode = new ArrayList<>(Arrays.asList(EntityDetail.QUERY_MODE));
// for(SemanticParser semanticParser : semanticParsers) {
// if(semanticParser.getName().equals(TimeSemanticParser.PARSER_MODE)) {
// semanticParser.getQueryModes().clear();
// semanticParser.getQueryModes().addAll(queryMode);
// }
// }
// }
// };
@Test
void parse() {
TimeSemanticParser timeSemanticParser = new TimeSemanticParser();
@@ -16,12 +28,15 @@ class TimeSemanticParserTest {
QueryContextReq queryContext = new QueryContextReq();
ChatContext chatCtx = new ChatContext();
SchemaMapInfo schemaMap = new SchemaMapInfo();
queryContext.setQueryText("supersonic最近30天访问次数");
//voteStrategy.init(new ArrayList<>(Arrays.asList(timeSemanticParser)));
timeSemanticParser.parse(queryContext, chatCtx);
boolean parse = timeSemanticParser.parse(queryContext, chatCtx);
DateConf dateInfo = queryContext.getParseInfo().getDateInfo();
//DateConf dateInfo = queryContext.getParseInfo(timeSemanticParser.getQueryModes().get(0))
// .getDateInfo();
//System.out.println(dateInfo);
}
}

View File

@@ -0,0 +1,38 @@
package com.tencent.supersonic.chat.application.parser.aggregate;
import static org.junit.Assert.assertEquals;
import com.tencent.supersonic.chat.application.parser.AggregateSemanticParser;
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
import org.junit.jupiter.api.Test;
class AggregateSemanticParserTest {
@Test
void getAggregateParser() {
AggregateSemanticParser aggregateParser = new AggregateSemanticParser();
AggregateTypeEnum aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数最大值");
assertEquals(aggregateType, AggregateTypeEnum.MAX);
aggregateType = aggregateParser.resolveAggregateType("supsersonic产品pv");
assertEquals(aggregateType, AggregateTypeEnum.COUNT);
aggregateType = aggregateParser.resolveAggregateType("supsersonic产品uv");
assertEquals(aggregateType, AggregateTypeEnum.DISTINCT);
aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数最大值");
assertEquals(aggregateType, AggregateTypeEnum.MAX);
aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数最小值");
assertEquals(aggregateType, AggregateTypeEnum.MIN);
aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数平均值");
assertEquals(aggregateType, AggregateTypeEnum.AVG);
aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数topN");
assertEquals(aggregateType, AggregateTypeEnum.TOPN);
aggregateType = aggregateParser.resolveAggregateType("supsersonic产品访问次数汇总");
assertEquals(aggregateType, AggregateTypeEnum.SUM);
}
}

View File

@@ -1,38 +0,0 @@
package com.tencent.supersonic.chat.application.parser.aggregate;
import static org.junit.Assert.assertEquals;
import com.tencent.supersonic.chat.application.parser.resolver.RegexAggregateTypeResolver;
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
import org.junit.jupiter.api.Test;
class RegexAggregateTypeEnumResolverTest {
@Test
void getAggregateParser() {
RegexAggregateTypeResolver regexAggregateParser = new RegexAggregateTypeResolver();
AggregateTypeEnum aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数最大值");
assertEquals(aggregateType, AggregateTypeEnum.MAX);
aggregateType = regexAggregateParser.resolve("supsersonic产品pv");
assertEquals(aggregateType, AggregateTypeEnum.COUNT);
aggregateType = regexAggregateParser.resolve("supsersonic产品uv");
assertEquals(aggregateType, AggregateTypeEnum.DISTINCT);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数最大值");
assertEquals(aggregateType, AggregateTypeEnum.MAX);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数最小值");
assertEquals(aggregateType, AggregateTypeEnum.MIN);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数平均值");
assertEquals(aggregateType, AggregateTypeEnum.AVG);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数topN");
assertEquals(aggregateType, AggregateTypeEnum.TOPN);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数汇总");
assertEquals(aggregateType, AggregateTypeEnum.SUM);
}
}

View File

@@ -0,0 +1,28 @@
package com.tencent.supersonic.chat.domain.utils;
import com.tencent.supersonic.semantic.api.core.enums.TimeDimensionEnum;
import java.text.MessageFormat;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
/**
* @author lex luo
* @date 2023/6/29 16:05
*/
class DslToSemanticInfoTest {
@Test
void search() {
Integer domainId = 1;
String dayField = TimeDimensionEnum.DAY.getName();
String startDate = "2023-04-01";
String endDate = "2023-06-01";
String format = MessageFormat.format(DslToSemanticInfo.SUB_TABLE, domainId, dayField, startDate, endDate);
Assert.assertEquals(format,
" ( select * from t_1 where sys_imp_date >= '2023-04-01' and sys_imp_date <= '2023-06-01' ) as t_sub_1");
}
}

View File

@@ -6,11 +6,11 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic"}
@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic.chat"}
// , exclude = {DataSourceAutoConfiguration.class}
)
@ComponentScan("com.tencent.supersonic")
@MapperScan("com.tencent.supersonic")
@ComponentScan("com.tencent.supersonic.chat")
@MapperScan("com.tencent.supersonic.chat")
public class ChatBizLauncher {
public static void main(String[] args) {

View File

@@ -1,14 +1,16 @@
package com.tencent.supersonic.chat.test.context;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.chat.infrastructure.mapper.ChatContextMapper;
import com.tencent.supersonic.chat.infrastructure.repository.ChatContextRepositoryImpl;
import com.tencent.supersonic.chat.infrastructure.semantic.DefaultSemanticLayerImpl;
import com.tencent.supersonic.chat.infrastructure.semantic.RemoteSemanticLayerImpl;
import com.tencent.supersonic.chat.test.ChatBizLauncher;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.semantic.query.domain.QueryService;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.test.context.SpringBootTest;
@@ -23,7 +25,8 @@ import org.springframework.web.client.RestTemplate;
@MockBean(DomainService.class)
@MockBean(ChatContextMapper.class)
@MockBean(RestTemplate.class)
@MockBean(DefaultSemanticLayerImpl.class)
@MockBean(RemoteSemanticLayerImpl.class)
@MockBean(ComponentFactory.class)
//@MybatisTest
//@AutoConfigureMybatis
//@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)

View File

@@ -5,7 +5,8 @@ import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.when;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.service.SemanticLayer;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.domain.service.QueryService;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
@@ -20,7 +21,6 @@ import com.tencent.supersonic.chat.domain.pojo.config.EntityInternalDetail;
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
import com.tencent.supersonic.chat.domain.service.ChatService;
import com.tencent.supersonic.chat.domain.service.QueryService;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.chat.infrastructure.mapper.ChatContextMapper;
import com.tencent.supersonic.chat.infrastructure.repository.ChatContextRepositoryImpl;
@@ -39,51 +39,6 @@ import org.springframework.web.client.RestTemplate;
@Configuration
public class MockBeansConfiguration {
@Bean
public ChatContextRepositoryImpl getChatContextRepository() {
return Mockito.mock(ChatContextRepositoryImpl.class);
}
@Bean
public QueryService getQueryService() {
return Mockito.mock(QueryService.class);
}
@Bean
public DimensionService getDimensionService() {
return Mockito.mock(DimensionService.class);
}
@Bean
public MetricService getMetricService() {
return Mockito.mock(MetricService.class);
}
@Bean
public DomainService getDomainService() {
return Mockito.mock(DomainService.class);
}
@Bean
public ChatContextMapper getChatContextMapper() {
return Mockito.mock(ChatContextMapper.class);
}
@Bean
public ConfigServiceImpl getDomainExtendService() {
return Mockito.mock(ConfigServiceImpl.class);
}
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
// @Bean
// public SemanticLayer getSemanticService() {
// return Mockito.mock(HttpSemanticServiceImpl.class);
// }
public static void getOrCreateContextMock(ChatService chatService) {
ChatContext context = new ChatContext();
context.setChatId(1);
@@ -142,8 +97,6 @@ public class MockBeansConfiguration {
when(configService.fetchConfigByDomainId(anyLong())).thenReturn(chaConfigDesc);
}
//queryDimensionDescs
public static void dimensionDescBuild(DimensionService dimensionService, List<DimensionResp> dimensionDescs) {
when(dimensionService.getDimensions(anyList())).thenReturn(dimensionDescs);
}
@@ -167,4 +120,50 @@ public class MockBeansConfiguration {
dimensionDesc.setBizName(bizName);
return dimensionDesc;
}
@Bean
public ChatContextRepositoryImpl getChatContextRepository() {
return Mockito.mock(ChatContextRepositoryImpl.class);
}
// @Bean
// public SemanticLayer getSemanticService() {
// return Mockito.mock(HttpSemanticServiceImpl.class);
// }
@Bean
public QueryService getQueryService() {
return Mockito.mock(QueryService.class);
}
@Bean
public DimensionService getDimensionService() {
return Mockito.mock(DimensionService.class);
}
@Bean
public MetricService getMetricService() {
return Mockito.mock(MetricService.class);
}
//queryDimensionDescs
@Bean
public DomainService getDomainService() {
return Mockito.mock(DomainService.class);
}
@Bean
public ChatContextMapper getChatContextMapper() {
return Mockito.mock(ChatContextMapper.class);
}
@Bean
public ConfigServiceImpl getDomainExtendService() {
return Mockito.mock(ConfigServiceImpl.class);
}
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
}

View File

@@ -0,0 +1,177 @@
package com.tencent.supersonic.chat.test.context;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.application.mapper.HanlpSchemaMapper;
import com.tencent.supersonic.chat.application.mapper.QueryMatchStrategy;
import com.tencent.supersonic.chat.application.parser.DomainSemanticParser;
import com.tencent.supersonic.chat.domain.service.ChatService;
import com.tencent.supersonic.chat.test.ChatBizLauncher;
import com.tencent.supersonic.knowledge.infrastructure.nlp.Suggester;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Bean;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.List;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ChatBizLauncher.class)
public class QueryServiceImplTest {
private static final Logger LOGGER = LoggerFactory.getLogger(QueryServiceImplTest.class);
// @MockBean
// private QueryService queryService;
//private SemanticLayer semanticLayer = mock(SemanticLayer.class);
// @MockBean
// private DefaultSemanticLayerImpl semanticLayer;
//SemanticLayer
@Autowired
public QueryMatchStrategy queryMatchStrategy;
@Autowired(required = false)
public Suggester suggester;
@MockBean
private SemanticLayer semanticLayer;//= ComponentFactory.getSemanticLayer();
// @Before
// public void setUp() {
// //List<DomainSchemaResp> getDomainSchemaInfo(List<Long> ids)
// List<DomainSchemaResp> domainSchemaRespList=MockUtils.getChatContext();
// Mockito.when(semanticLayer.getDomainSchemaInfo(Mockito.anyList())).thenReturn(domainSchemaRespList);
// }
@MockBean
private DomainSemanticParser domainSemanticParser;
@MockBean
private HanlpSchemaMapper hanlpSchemaMapper;
@MockBean
private ChatService chatService;
//// @Autowired(required = false)
//// private ChatService chatService;
//private SemanticLayer semanticLayer ;
@Test
public void test() throws Exception {
QueryContextReq queryContextReq = getQueryContextReq("超音数访问次数");
//hanlpSchemaMapper.map(queryContextReq);
// List<Term> terms = HanlpHelper.getSegment().seg(queryContextReq.getQueryText().toLowerCase()).stream()
// .collect(Collectors.toList());
// LOGGER.info("terms::::{}",terms);
// MockUtils.putSuggester();
// List<MapResult> matches = queryMatchStrategy.match(queryContextReq.getQueryText(), terms, queryContextReq.getDomainId());
// HanlpHelper.transLetterOriginal(matches);
// HanlpSchemaMapperHelper.convertTermsToSchemaMapInfo(matches, queryContextReq.getMapInfo());
HanlpSchemaMapper hanlpSchemaMapper = new HanlpSchemaMapper();
hanlpSchemaMapper.map(queryContextReq);
//QueryContextReq queryContextReq=MockUtils.getQueryContextReq("METRIC_FILTER");
LOGGER.info("QueryContextReq::::{}", queryContextReq.getMapInfo().getMatchedDomains());
LOGGER.info("QueryContextReq::::{}", queryContextReq.getMapInfo().getDomainElementMatches());
LOGGER.info("QueryContextReq::::{}", queryContextReq);
// //chatService=new ChatServiceImpl();
// ChatContext chatCtx = chatService.getOrCreateContext(queryContextReq.getChatId());
// if (chatCtx == null) {
// chatCtx=new ChatContext();
// chatCtx.setChatId(queryContextReq.getChatId());
// }
// LOGGER.info("chatService::::{}",chatService);
// LOGGER.info("ChatContext::::{}",chatCtx);
ChatContext chatCtx = new ChatContext();//MockUtils.getChatContext1();
//semanticLayer = ComponentFactory.getSemanticLayer();
// DomainSemanticParser domainSemanticParser=new DomainSemanticParser();
// domainSemanticParser.parse(queryContextReq,chatCtx);
//DomainSemanticParser
//domainSemanticParser=new DomainSemanticParser();
LOGGER.info("domainSemanticParser::::{}", domainSemanticParser);
//SemanticLayer semanticLayer= mock(SemanticLayer.class);
//List<DomainSchemaResp> domainSchemaRespList=MockUtils.getChatContext();
// //SemanticLayer semanticLayer = mock(SemanticLayer.class);
// when(semanticLayer.getDomainSchemaInfo(Mockito.anyList())).thenReturn(domainSchemaRespList);
// domainSemanticParser.parse(queryContextReq,chatCtx);
// LOGGER.info("QueryContextReq::::{}",queryContextReq);
// TimeSemanticParser timeSemanticParser=new TimeSemanticParser();
// timeSemanticParser.parse(queryContextReq,chatCtx);
// AggregateSemanticParser aggregateSemanticParser=new AggregateSemanticParser();
// aggregateSemanticParser.parse(queryContextReq,chatCtx);
// //PickStrategy pickStrategy = ComponentFactory.getPickStrategy();
// LOGGER.info("pickStrategy::::{}",pickStrategy);
// pickStrategy=new ScoreBasedPickStrategy();
// SemanticParseInfo semanticParse = pickStrategy.pick(queryContextReq, chatCtx);
// LOGGER.info("semanticParse::::{}",semanticParse);
//SemanticQueryExecutorHelper semanticQueryExecutorHelper=new SemanticQueryExecutorHelper();
//semanticQueryExecutorHelper.execute(queryContextReq.getParseInfo(),queryContextReq.getUser());
// LOGGER.info("queryContextReq::::{}",queryContextReq.getMapInfo().getMatchedDomains());
// LOGGER.info("queryContextReq::::{}",queryContextReq.getMapInfo().getDomainElementMatches());
// LOGGER.info("queryContextReq::::{}",queryContextReq.getCandidateParseInfos());
// LOGGER.info("queryContextReq::::{}",queryContextReq.getDomainId());
// List<QueryExecutor> queryExecutors = ComponentFactory.getQueryExecutors();
// QueryResultResp queryResponse=new QueryResultResp();
// for (QueryExecutor executor : queryExecutors) {
// queryResponse = executor.execute(semanticParse, queryContextReq.getUser());
// if (queryResponse != null) {
// // update chat context after a successful semantic query
// if (queryContextReq.isSaveAnswer() && queryResponse.getQueryState() == QueryState.NORMAL.getState()) {
// chatService.updateContext(chatCtx, queryContextReq, semanticParse);
// }
// queryResponse.setChatContext(chatCtx.getParseInfo());
// chatService.addQuery(queryResponse, queryContextReq, chatCtx);
// break;
// }
// }
//assertThat(found.getName()).isEqualTo(name);
}
public QueryContextReq getQueryContextReq(String query) {
QueryContextReq queryContextReq = new QueryContextReq();
queryContextReq.setQueryText(query);//"alice的访问次数"
queryContextReq.setChatId(1);
queryContextReq.setUser(new User(1L, "admin", "admin", "admin@email"));
return queryContextReq;
}
@TestConfiguration
static class EmployeeServiceImplTestContextConfiguration {
@Bean
public QueryMatchStrategy queryMatchStrategyService() {
return new QueryMatchStrategy();
}
// @Bean
// public SemanticLayer querySemanticLayer() {
// return new DefaultSemanticLayerImpl();
// }
//@Bean
//public DomainSemanticParser queryDomainSemanticParser() {
// return new DomainSemanticParser();
//}
// @Bean
// public RestTemplate getRestTemplate(){
// return new RestTemplate();
// }
//
// @Bean
// public DefaultSemanticInternalUtils getUtils(){
// return new DefaultSemanticInternalUtils();
// }
}
}

View File

@@ -1 +1 @@
CustomDictionaryPath=data/dictionary/custom/23_285.txt
CustomDictionaryPath=data/dictionary/custom/DimValue_1_2.txt