diff --git a/common/pom.xml b/common/pom.xml index c3cfd33c0..5345f0409 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -111,20 +111,6 @@ ${transmittable.version} compile - - - - com.github.plexpt - chatgpt - 4.1.2 - - - com.knuddels - jtokkit - - - - com.github.pagehelper pagehelper-spring-boot-starter diff --git a/common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java deleted file mode 100644 index baddb87e2..000000000 --- a/common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java +++ /dev/null @@ -1,156 +0,0 @@ -package com.tencent.supersonic.common.util; - - -import cn.hutool.core.util.StrUtil; -import com.plexpt.chatgpt.ChatGPT; -import com.plexpt.chatgpt.entity.chat.ChatCompletion; -import com.plexpt.chatgpt.entity.chat.ChatCompletionResponse; -import com.plexpt.chatgpt.entity.chat.Message; -import com.plexpt.chatgpt.util.Proxys; -import java.net.Proxy; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - - -@Component -@Slf4j -public class ChatGptHelper { - - @Value("${llm.chatgpt.apikey:}") - private String apiKey; - - @Value("${llm.chatgpt.apiHost:}") - private String apiHost; - - @Value("${llm.chatgpt.proxyIp:default}") - private String proxyIp; - - @Value("${llm.chatgpt.proxyPort:}") - private Integer proxyPort; - - public ChatGPT getChatGPT() { - Proxy proxy = null; - if (!"default".equals(proxyIp)) { - proxy = Proxys.http(proxyIp, proxyPort); - } - return ChatGPT.builder() - .apiKey(apiKey) - .proxy(proxy) - .timeout(900) - .apiHost(apiHost) //反向代理地址 - .build() - .init(); - } - - public Message getChatCompletion(Message system, Message message) { - List messages; - if (StrUtil.isBlank(system.getContent())) { - messages = Arrays.asList(message); - } else { - messages = Arrays.asList(system, message); - } - ChatCompletion chatCompletion = ChatCompletion.builder() - .model(ChatCompletion.Model.GPT_3_5_TURBO_16K.getName()) - .messages(messages) - .maxTokens(10000) - .temperature(0.9) - .build(); - ChatCompletionResponse response = getChatGPT().chatCompletion(chatCompletion); - return response.getChoices().get(0).getMessage(); - } - - public String inferredTime(String queryText) { - long nowTime = System.currentTimeMillis(); - Date date = new Date(nowTime); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - String formattedDate = sdf.format(date); - Message system = Message.ofSystem("现在时间 " + formattedDate + "，你是一个专业的数据分析师，你的任务是基于数据，专业的解答用户的问题。" - + "你需要遵守以下规则：\n" - + "1.返回规范的数据格式，json，如： 输入：近 10 天的日活跃数，输出：{\"start\":\"2023-07-21\",\"end\":\"2023-07-31\"}" - + "2.你对时间数据要求规范，能从近 10 天，国庆节，端午节，获取到相应的时间，填写到 json 中。\n" - + "3.你的数据时间，只有当前及之前时间即可,超过则回复去年\n" - + "4.只需要解析出时间，时间可以是时间月和年或日、日历采用公历\n" - + "5.时间给出要是绝对正确，不能瞎编\n" - ); - Message message = Message.of("输入：" + queryText + "，输出："); - Message res = getChatCompletion(system, message); - return res.getContent(); - } - - public String mockAlias(String mockType, - String name, - String bizName, - String table, - String desc, - Boolean isPercentage) { - String msg = "Assuming you are a professional data analyst specializing in metrics and dimensions, " - + "you have a vast amount of data analysis metrics content. You are familiar with the basic" - + " format of the content,Now, Construct your answer Based on the following json-schema.\n" - + "{\n" - + "\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n" - + "\"type\": \"array\",\n" - + "\"minItems\": 2,\n" - + "\"maxItems\": 4,\n" - + "\"items\": {\n" - + "\"type\": \"string\",\n" - + "\"description\": \"Assuming you are a data analyst and give a defined " - + mockType - + " name: " - + name + "," - + "this " - + mockType - + " is from database and table: " - + table + ",This " - + mockType - + " calculates the field source: " - + bizName - + ", The description of this metrics is: " - + desc - + ", provide some aliases for this, please take chinese or english," - + "You must adhere to the following rules:\n" - + "1. Please do not generate aliases like xxx1, xxx2, xxx3.\n" - + "2. Please do not generate aliases that are the same as the original names of metrics/dimensions.\n" - + "3. Please pay attention to the quality of the generated aliases and " - + " avoid creating aliases that look like test data.\n" - + "4. Please generate more Chinese aliases." - + "},\n" - + "\"additionalProperties\":false}\n" - + "Please double-check whether the answer conforms to the format described in the JSON-schema.\n" - + "ANSWER JSON:"; - log.info("msg:{}", msg); - Message system = Message.ofSystem(""); - Message message = Message.of(msg); - Message res = getChatCompletion(system, message); - return res.getContent(); - } - - public String mockDimensionValueAlias(String json) { - String msg = "Assuming you are a professional data analyst specializing in indicators,for you a json list，" - + "the required content to follow is as follows: " - + "1. The format of JSON," - + "2. Only return in JSON format," - + "3. the array item > 1 and < 5,more alias," - + "for example：input:[\"qq_music\",\"kugou_music\"]," - + "out:{\"tran\":[\"qq音乐\",\"酷狗音乐\"],\"alias\":{\"qq_music\":[\"q音\",\"qq音乐\"]," - + "\"kugou_music\":[\"kugou\",\"酷狗\"]}}," - + "now input: " - + json + "," - + "answer json:"; - log.info("msg:{}", msg); - Message system = Message.ofSystem(""); - Message message = Message.of(msg); - Message res = getChatCompletion(system, message); - return res.getContent(); - } - - public static void main(String[] args) { - ChatGptHelper chatGptHelper = new ChatGptHelper(); - System.out.println(chatGptHelper.mockAlias("", "", "", "", "", false)); - } - -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 940422281..29c366dc4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -15,7 +15,7 @@ import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; -import com.tencent.supersonic.common.util.ChatGptHelper; +import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; @@ -69,7 +69,7 @@ public class DimensionServiceImpl implements DimensionService { private ModelService modelService; - private ChatGptHelper chatGptHelper; + private AliasGenerateHelper chatGptHelper; private DatabaseService databaseService; @@ -85,7 +85,7 @@ public class DimensionServiceImpl implements DimensionService { public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService, - ChatGptHelper chatGptHelper, + AliasGenerateHelper chatGptHelper, DatabaseService databaseService, ModelRelaService modelRelaService, DataSetService dataSetService, @@ -341,7 +341,7 @@ public class DimensionServiceImpl implements DimensionService { @Override public List mockAlias(DimensionReq dimensionReq, String mockType, User user) { - String mockAlias = chatGptHelper.mockAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(), + String mockAlias = chatGptHelper.generateAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(), "", dimensionReq.getDescription(), false); return JSONObject.parseObject(mockAlias, new TypeReference>() { }); @@ -363,7 +363,7 @@ public class DimensionServiceImpl implements DimensionService { String value = (String) stringObjectMap.get(dimensionReq.getBizName()); valueList.add(value); } - String json = chatGptHelper.mockDimensionValueAlias(JSON.toJSONString(valueList)); + String json = chatGptHelper.generateDimensionValueAlias(JSON.toJSONString(valueList)); log.info("return llm res is :{}", json); JSONObject jsonObject = JSON.parseObject(json); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index 4f917c5a5..e1b8ca132 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -17,7 +17,7 @@ import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; -import com.tencent.supersonic.common.util.ChatGptHelper; +import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.Measure; @@ -97,7 +97,7 @@ public class MetricServiceImpl implements MetricService { private DimensionService dimensionService; - private ChatGptHelper chatGptHelper; + private AliasGenerateHelper chatGptHelper; private CollectService collectService; @@ -111,7 +111,7 @@ public class MetricServiceImpl implements MetricService { public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, - ChatGptHelper chatGptHelper, + AliasGenerateHelper chatGptHelper, CollectService collectService, DataSetService dataSetService, ApplicationEventPublisher eventPublisher, @@ -535,7 +535,7 @@ public class MetricServiceImpl implements MetricService { @Override public List mockAlias(MetricBaseReq metricReq, String mockType, User user) { - String mockAlias = chatGptHelper.mockAlias(mockType, metricReq.getName(), metricReq.getBizName(), "", + String mockAlias = chatGptHelper.generateAlias(mockType, metricReq.getName(), metricReq.getBizName(), "", metricReq.getDescription(), !"".equals(metricReq.getDataFormatType())); return JSONObject.parseObject(mockAlias, new TypeReference>() { }); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java new file mode 100644 index 000000000..61f2daa09 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java @@ -0,0 +1,86 @@ +package com.tencent.supersonic.headless.server.utils; + + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.output.Response; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + + +@Component +@Slf4j +public class AliasGenerateHelper { + + @Autowired + private ChatLanguageModel chatLanguageModel; + + public String getChatCompletion(String message) { + SystemMessage from = SystemMessage.from(message); + Response response = chatLanguageModel.generate(from); + log.info("message:{}\n response:{}", message, response); + return response.content().text(); + } + + public String generateAlias(String mockType, + String name, + String bizName, + String table, + String desc, + Boolean isPercentage) { + String msg = "Assuming you are a professional data analyst specializing in metrics and dimensions, " + + "you have a vast amount of data analysis metrics content. You are familiar with the basic" + + " format of the content,Now, Construct your answer Based on the following json-schema.\n" + + "{\n" + + "\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n" + + "\"type\": \"array\",\n" + + "\"minItems\": 2,\n" + + "\"maxItems\": 4,\n" + + "\"items\": {\n" + + "\"type\": \"string\",\n" + + "\"description\": \"Assuming you are a data analyst and give a defined " + + mockType + + " name: " + + name + "," + + "this " + + mockType + + " is from database and table: " + + table + ",This " + + mockType + + " calculates the field source: " + + bizName + + ", The description of this metrics is: " + + desc + + ", provide some aliases for this, please take chinese or english," + + "You must adhere to the following rules:\n" + + "1. Please do not generate aliases like xxx1, xxx2, xxx3.\n" + + "2. Please do not generate aliases that are the same as the original names of metrics/dimensions.\n" + + "3. Please pay attention to the quality of the generated aliases and " + + " avoid creating aliases that look like test data.\n" + + "4. Please generate more Chinese aliases." + + "},\n" + + "\"additionalProperties\":false}\n" + + "Please double-check whether the answer conforms to the format described in the JSON-schema.\n" + + "ANSWER JSON:"; + log.info("msg:{}", msg); + return getChatCompletion(msg); + } + + public String generateDimensionValueAlias(String json) { + String msg = "Assuming you are a professional data analyst specializing in indicators,for you a json list，" + + "the required content to follow is as follows: " + + "1. The format of JSON," + + "2. Only return in JSON format," + + "3. the array item > 1 and < 5,more alias," + + "for example：input:[\"qq_music\",\"kugou_music\"]," + + "out:{\"tran\":[\"qq音乐\",\"酷狗音乐\"],\"alias\":{\"qq_music\":[\"q音\",\"qq音乐\"]," + + "\"kugou_music\":[\"kugou\",\"酷狗\"]}}," + + "now input: " + + json + "," + + "answer json:"; + log.info("msg:{}", msg); + return getChatCompletion(msg); + } +} diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java index cc3cd7b67..e13e1aef4 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; -import com.tencent.supersonic.common.util.ChatGptHelper; +import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.MeasureParam; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; @@ -64,7 +64,7 @@ public class MetricServiceImplTest { private MetricService mockMetricService(MetricRepository metricRepository, ModelService modelService) { - ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class); + AliasGenerateHelper chatGptHelper = Mockito.mock(AliasGenerateHelper.class); CollectService collectService = Mockito.mock(CollectService.class); ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class); DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);