first commit

This commit is contained in:
jerryjzhang
2023-06-12 18:44:01 +08:00
commit dc4fc69b57
879 changed files with 573090 additions and 0 deletions

View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.chat.application.mapper;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.test.context.ContextTest;
import org.junit.jupiter.api.Test;
/**
* HanlpSchemaMapperTest
*/
class HanlpSchemaMapperTest extends ContextTest {
@Test
void map() {
QueryContextReq searchCtx = new QueryContextReq();
searchCtx.setChatId(1);
searchCtx.setDomainId(2);
searchCtx.setQueryText("supersonic按部门访问次数");
HanlpSchemaMapper hanlpSchemaMapper = new HanlpSchemaMapper();
hanlpSchemaMapper.map(searchCtx);
}
}

View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.chat.application.mapper.match;
import com.tencent.supersonic.chat.test.context.ContextTest;
import org.junit.jupiter.api.Test;
/**
* MatchStrategyImplTest
*/
class QueryMatchStrategyTest extends ContextTest {
@Test
void match() {
}
@Test
void testMatch() {
}
}

View File

@@ -0,0 +1,27 @@
package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.common.pojo.DateConf;
import org.junit.jupiter.api.Test;
class TimeSemanticParserTest {
@Test
void parse() {
TimeSemanticParser timeSemanticParser = new TimeSemanticParser();
QueryContextReq searchCtx = new QueryContextReq();
ChatContext chatCtx = new ChatContext();
SchemaMapInfo schemaMap = new SchemaMapInfo();
searchCtx.setQueryText("supersonic最近30天访问次数");
boolean parse = timeSemanticParser.parse(searchCtx, chatCtx);
DateConf dateInfo = searchCtx.getParseInfo().getDateInfo();
}
}

View File

@@ -0,0 +1,38 @@
package com.tencent.supersonic.chat.application.parser.aggregate;
import static org.junit.Assert.assertEquals;
import com.tencent.supersonic.chat.application.parser.resolver.RegexAggregateTypeResolver;
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
import org.junit.jupiter.api.Test;
class RegexAggregateTypeEnumResolverTest {
@Test
void getAggregateParser() {
RegexAggregateTypeResolver regexAggregateParser = new RegexAggregateTypeResolver();
AggregateTypeEnum aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数最大值");
assertEquals(aggregateType, AggregateTypeEnum.MAX);
aggregateType = regexAggregateParser.resolve("supsersonic产品pv");
assertEquals(aggregateType, AggregateTypeEnum.COUNT);
aggregateType = regexAggregateParser.resolve("supsersonic产品uv");
assertEquals(aggregateType, AggregateTypeEnum.DISTINCT);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数最大值");
assertEquals(aggregateType, AggregateTypeEnum.MAX);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数最小值");
assertEquals(aggregateType, AggregateTypeEnum.MIN);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数平均值");
assertEquals(aggregateType, AggregateTypeEnum.AVG);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数topN");
assertEquals(aggregateType, AggregateTypeEnum.TOPN);
aggregateType = regexAggregateParser.resolve("supsersonic产品访问次数汇总");
assertEquals(aggregateType, AggregateTypeEnum.SUM);
}
}

View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.chat.application.search;
import org.junit.jupiter.api.Test;
class SearchServiceImplTest {
@Test
void search() {
}
@Test
void filerMetricsByDomain() {
}
}

View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.chat.test;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
@SpringBootApplication(scanBasePackages = {"com.tencent.supersonic"}
// , exclude = {DataSourceAutoConfiguration.class}
)
@ComponentScan("com.tencent.supersonic")
@MapperScan("com.tencent.supersonic")
public class ChatBizLauncher {
public static void main(String[] args) {
SpringApplication.run(ChatBizLauncher.class, args);
}
}

View File

@@ -0,0 +1,35 @@
package com.tencent.supersonic.chat.test.context;
import com.tencent.supersonic.chat.infrastructure.mapper.ChatContextMapper;
import com.tencent.supersonic.chat.infrastructure.repository.ChatContextRepositoryImpl;
import com.tencent.supersonic.chat.infrastructure.semantic.DefaultSemanticLayerImpl;
import com.tencent.supersonic.chat.test.ChatBizLauncher;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.semantic.query.domain.QueryService;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.web.client.RestTemplate;
@MockBean(ChatContextRepositoryImpl.class)
@MockBean(QueryService.class)
@MockBean(DimensionService.class)
@MockBean(MetricService.class)
@MockBean(DomainService.class)
@MockBean(ChatContextMapper.class)
@MockBean(RestTemplate.class)
@MockBean(DefaultSemanticLayerImpl.class)
//@MybatisTest
//@AutoConfigureMybatis
//@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ChatBizLauncher.class)
public class ContextTest {
protected final Logger logger = LoggerFactory.getLogger(ContextTest.class);
}

View File

@@ -0,0 +1,170 @@
package com.tencent.supersonic.chat.test.context;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.when;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.service.SemanticLayer;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
import com.tencent.supersonic.chat.application.ConfigServiceImpl;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigInfo;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
import com.tencent.supersonic.chat.domain.pojo.config.DefaultMetric;
import com.tencent.supersonic.chat.domain.pojo.config.DefaultMetricInfo;
import com.tencent.supersonic.chat.domain.pojo.config.EntityInternalDetail;
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
import com.tencent.supersonic.chat.domain.pojo.semantic.DomainInfos;
import com.tencent.supersonic.chat.domain.service.ChatService;
import com.tencent.supersonic.chat.domain.service.QueryService;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.chat.infrastructure.mapper.ChatContextMapper;
import com.tencent.supersonic.chat.infrastructure.repository.ChatContextRepositoryImpl;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.mockito.Mockito;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.client.RestTemplate;
@Configuration
public class MockBeansConfiguration {
@Bean
public ChatContextRepositoryImpl getChatContextRepository() {
return Mockito.mock(ChatContextRepositoryImpl.class);
}
@Bean
public QueryService getQueryService() {
return Mockito.mock(QueryService.class);
}
@Bean
public DimensionService getDimensionService() {
return Mockito.mock(DimensionService.class);
}
@Bean
public MetricService getMetricService() {
return Mockito.mock(MetricService.class);
}
@Bean
public DomainService getDomainService() {
return Mockito.mock(DomainService.class);
}
@Bean
public ChatContextMapper getChatContextMapper() {
return Mockito.mock(ChatContextMapper.class);
}
@Bean
public ConfigServiceImpl getDomainExtendService() {
return Mockito.mock(ConfigServiceImpl.class);
}
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
// @Bean
// public SemanticLayer getSemanticService() {
// return Mockito.mock(HttpSemanticServiceImpl.class);
// }
public static void getOrCreateContextMock(ChatService chatService) {
ChatContext context = new ChatContext();
context.setChatId(1);
when(chatService.getOrCreateContext(1)).thenReturn(context);
}
public static void buildHttpSemanticServiceImpl(SemanticLayer httpSemanticLayer, List<DimSchemaResp> dimensionDescs,
List<MetricSchemaResp> metricDescs) {
ChatConfigRichInfo chaConfigRichDesc = new ChatConfigRichInfo();
DefaultMetric defaultMetricDesc = new DefaultMetric();
defaultMetricDesc.setUnit(3);
defaultMetricDesc.setPeriod(Constants.DAY);
chaConfigRichDesc.setDefaultMetrics(new ArrayList<>(Arrays.asList(defaultMetricDesc)));
EntityRichInfo entityDesc = new EntityRichInfo();
List<DimSchemaResp> dimensionDescs1 = new ArrayList<>();
DimSchemaResp dimensionDesc = new DimSchemaResp();
dimensionDesc.setId(162L);
dimensionDescs1.add(dimensionDesc);
entityDesc.setEntityIds(dimensionDescs1);
DimSchemaResp dimensionDesc2 = new DimSchemaResp();
dimensionDesc2.setId(163L);
dimensionDesc2.setBizName("song_name");
dimensionDesc2.setName("歌曲名");
EntityInternalDetail entityInternalDetailDesc = new EntityInternalDetail();
entityInternalDetailDesc.setDimensionList(new ArrayList<>(Arrays.asList(dimensionDesc2)));
MetricSchemaResp metricDesc = new MetricSchemaResp();
metricDesc.setId(877L);
metricDesc.setBizName("js_play_cnt");
metricDesc.setName("结算播放量");
entityInternalDetailDesc.setMetricList(new ArrayList<>(Arrays.asList(metricDesc)));
entityDesc.setEntityInternalDetailDesc(entityInternalDetailDesc);
chaConfigRichDesc.setEntity(entityDesc);
// when(httpSemanticLayer.getChatConfigRichInfo(anyLong())).thenReturn(chaConfigRichDesc);
DomainSchemaResp domainSchemaDesc = new DomainSchemaResp();
domainSchemaDesc.setDimensions(dimensionDescs);
domainSchemaDesc.setMetrics(metricDescs);
when(httpSemanticLayer.getDomainSchemaInfo(anyLong())).thenReturn(domainSchemaDesc);
DomainInfos domainInfos = new DomainInfos();
when(SchemaInfoConverter.convert(httpSemanticLayer.getDomainSchemaInfo(anyList()))).thenReturn(domainInfos);
}
public static void getDomainExtendMock(ConfigServiceImpl configService) {
DefaultMetricInfo defaultMetricInfo = new DefaultMetricInfo();
defaultMetricInfo.setUnit(3);
defaultMetricInfo.setPeriod(Constants.DAY);
List<DefaultMetricInfo> defaultMetricInfos = new ArrayList<>();
defaultMetricInfos.add(defaultMetricInfo);
ChatConfigInfo chaConfigDesc = new ChatConfigInfo();
chaConfigDesc.setDefaultMetrics(defaultMetricInfos);
when(configService.fetchConfigByDomainId(anyLong())).thenReturn(chaConfigDesc);
}
//queryDimensionDescs
public static void dimensionDescBuild(DimensionService dimensionService, List<DimensionResp> dimensionDescs) {
when(dimensionService.getDimensions(anyList())).thenReturn(dimensionDescs);
}
public static void metricDescBuild(MetricService dimensionService, List<MetricResp> metricDescs) {
when(dimensionService.getMetrics(anyList())).thenReturn(metricDescs);
}
public static DimSchemaResp getDimensionDesc(Long id, String bizName, String name) {
DimSchemaResp dimensionDesc = new DimSchemaResp();
dimensionDesc.setId(id);
dimensionDesc.setName(name);
dimensionDesc.setBizName(bizName);
return dimensionDesc;
}
public static MetricSchemaResp getMetricDesc(Long id, String bizName, String name) {
MetricSchemaResp dimensionDesc = new MetricSchemaResp();
dimensionDesc.setId(id);
dimensionDesc.setName(name);
dimensionDesc.setBizName(bizName);
return dimensionDesc;
}
}

View File

@@ -0,0 +1,119 @@
package com.tencent.supersonic.chat.test.context;
import com.google.gson.Gson;
import com.tencent.supersonic.chat.api.pojo.Filter;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
public class SemanticParseObjectHelper {
public static SemanticParseInfo copy(SemanticParseInfo semanticParseInfo) {
Gson g = new Gson();
return g.fromJson(g.toJson(semanticParseInfo), SemanticParseInfo.class);
}
public static SemanticParseInfo getSemanticParseInfo(String json) {
Gson gson = new Gson();
SemanticParseJson semanticParseJson = gson.fromJson(json, SemanticParseJson.class);
if (semanticParseJson != null) {
return getSemanticParseInfo(semanticParseJson);
}
return null;
}
private static SemanticParseInfo getSemanticParseInfo(SemanticParseJson semanticParseJson) {
Long domain = semanticParseJson.getDomain();
List<SchemaItem> dimensionList = new ArrayList<>();
List<SchemaItem> metricList = new ArrayList<>();
List<Filter> chatFilters = new ArrayList<>();
if (semanticParseJson.getFilter() != null && semanticParseJson.getFilter().size() > 0) {
for (List<String> filter : semanticParseJson.getFilter()) {
chatFilters.add(getChatFilter(filter));
}
}
for (String dim : semanticParseJson.getDimensions()) {
dimensionList.add(getDimension(dim, domain));
}
for (String metric : semanticParseJson.getMetrics()) {
metricList.add(getMetric(metric, domain));
}
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setDimensionFilters(chatFilters);
semanticParseInfo.setAggType(semanticParseJson.getAggregateType());
semanticParseInfo.setDomainId(domain);
semanticParseInfo.setQueryMode(semanticParseJson.getQueryMode());
semanticParseInfo.setMetrics(metricList);
semanticParseInfo.setDimensions(dimensionList);
DateConf dateInfo = getDateInfoAgo(semanticParseJson.getDay());
semanticParseInfo.setDateInfo(dateInfo);
return semanticParseInfo;
}
private static DateConf getDateInfoAgo(int dayAgo) {
if (dayAgo > 0) {
DateConf dateInfo = new DateConf();
dateInfo.setUnit(dayAgo);
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
return dateInfo;
}
return null;
}
private static Filter getChatFilter(List<String> filters) {
if (filters.size() > 1) {
Filter chatFilter = new Filter();
chatFilter.setBizName(filters.get(1));
chatFilter.setOperator(FilterOperatorEnum.getSqlOperator(filters.get(2)));
if (filters.size() > 4) {
List<String> valuse = new ArrayList<>();
valuse.addAll(filters.subList(3, filters.size()));
chatFilter.setValue(valuse);
} else {
chatFilter.setValue(filters.get(3));
}
return chatFilter;
}
return null;
}
private static SchemaItem getMetric(String bizName, Long domainId) {
SchemaItem metric = new SchemaItem();
metric.setBizName(bizName);
//metric.set(domainId);
return metric;
}
private static SchemaItem getDimension(String bizName, Long domainId) {
SchemaItem dimension = new SchemaItem();
dimension.setBizName(bizName);
//dimension.setDomainId(domainId);
return dimension;
}
@Data
public static class SemanticParseJson {
private Long domain;
private String queryMode;
private AggregateTypeEnum aggregateType;
private Integer day;
private List<String> dimensions;
private List<String> metrics;
private List<List<String>> filter;
}
}

View File

@@ -0,0 +1,17 @@
mybatis:
mapper-locations: classpath:mapper/*.xml
spring:
h2:
console:
path: /h2-console/semantic
# enabled web
enabled: true
datasource:
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false
username: root
password: semantic
schema: classpath:db/chat-schema-h2.sql
data: classpath:db/chat-data-h2.sql

View File

@@ -0,0 +1,2 @@
[InternetShortcut]
URL=https://github.com/hankcs/HanLP/

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,6 @@
tom _1_2 5
alice _1_2 6
lucy _1_2 4
dean _1_2 2
john _1_2 8
jack _1_2 8

View File

@@ -0,0 +1 @@
1.7.5

View File

@@ -0,0 +1 @@
insert into chat_context (chat_id, modified_at , `user`, `query_text`, `semantic_parse` ,ext_data) VALUES(1, '2023-05-24 00:00:00', 'admin', '超音数访问次数', '', 'admin');

View File

@@ -0,0 +1,59 @@
CREATE TABLE `chat_context`
(
`chat_id` BIGINT NOT NULL , -- context chat id
`modified_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP , -- row modify time
`user` varchar(64) DEFAULT NULL , -- row modify user
`query_text` LONGVARCHAR DEFAULT NULL , -- query text
`semantic_parse` LONGVARCHAR DEFAULT NULL , -- parse data
`ext_data` LONGVARCHAR DEFAULT NULL , -- extend data
PRIMARY KEY (`chat_id`)
);
CREATE TABLE `chat`
(
`chat_id` BIGINT NOT NULL ,-- AUTO_INCREMENT,
`chat_name` varchar(100) DEFAULT NULL,
`create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
`last_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
`creator` varchar(30) DEFAULT NULL,
`last_question` varchar(200) DEFAULT NULL,
`is_delete` INT DEFAULT '0' COMMENT 'is deleted',
`is_top` INT DEFAULT '0' COMMENT 'is top',
PRIMARY KEY (`chat_id`)
) ;
CREATE TABLE `chat_query`
(
`id` BIGINT NOT NULL ,--AUTO_INCREMENT,
`question_id` BIGINT DEFAULT NULL,
`create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`user_name` varchar(150) DEFAULT NULL COMMENT '',
`question` varchar(300) DEFAULT NULL COMMENT '',
`query_result` LONGVARCHAR,
`time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
`state` int(1) DEFAULT NULL,
`data_content` varchar(30) DEFAULT NULL,
`name` varchar(100) DEFAULT NULL,
`scene_type` int(2) DEFAULT NULL,
`query_type` int(2) DEFAULT NULL,
`is_deleted` int(1) DEFAULT NULL,
`module` varchar(30) DEFAULT NULL,
`entity` LONGVARCHAR COMMENT '',
`chat_id` BIGINT DEFAULT NULL COMMENT 'chat id',
`recommend` text,
`aggregator` varchar(20) DEFAULT 'trend',
`top_num` int DEFAULT NULL,
`start_time` varchar(30) DEFAULT NULL,
`end_time` varchar(30) DEFAULT NULL,
`compare_recommend` LONGVARCHAR,
`compare_entity` LONGVARCHAR,
`query_sql` LONGVARCHAR,
`columns` varchar(2000) DEFAULT NULL,
`result_list` LONGVARCHAR,
`main_entity` varchar(5000) DEFAULT NULL,
`semantic_text` varchar(5000) DEFAULT NULL,
`score` int DEFAULT '0',
`feedback` varchar(1024) DEFAULT '',
PRIMARY KEY (`id`)
) ;

View File

@@ -0,0 +1 @@
CustomDictionaryPath=data/dictionary/custom/23_285.txt