(improvement)(launchers) Demo data initialization optimization (#1015)

This commit is contained in:
LXW
2024-05-20 18:26:26 +08:00
committed by GitHub
parent eaec7b4663
commit d513b6d2cc
17 changed files with 274 additions and 604 deletions

View File

@@ -6,7 +6,7 @@ import java.util.List;
public interface ChatRepository {
boolean createChat(ChatDO chatDO);
Long createChat(ChatDO chatDO);
List<ChatDO> getAll(String creator, Integer agentId);

View File

@@ -1,14 +1,15 @@
package com.tencent.supersonic.chat.server.persistence.repository.impl;
import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO;
import com.tencent.supersonic.chat.server.persistence.mapper.ChatMapper;
import com.tencent.supersonic.chat.server.persistence.repository.ChatRepository;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
@Primary
@Slf4j
@@ -21,8 +22,9 @@ public class ChatRepositoryImpl implements ChatRepository {
}
@Override
public boolean createChat(ChatDO chatDO) {
return chatMapper.createChat(chatDO);
public Long createChat(ChatDO chatDO) {
chatMapper.createChat(chatDO);
return chatDO.getChatId();
}
@Override

View File

@@ -32,7 +32,8 @@ public class ChatController {
public Boolean save(@RequestParam(value = "chatName") String chatName,
@RequestParam(value = "agentId", required = false) Integer agentId,
HttpServletRequest request, HttpServletResponse response) {
return chatService.addChat(UserHolder.findUser(request, response), chatName, agentId);
chatService.addChat(UserHolder.findUser(request, response), chatName, agentId);
return true;
}
@GetMapping("/getAll")

View File

@@ -8,7 +8,7 @@ public interface AgentService {
List<Agent> getAgents();
void createAgent(Agent agent, User user);
Integer createAgent(Agent agent, User user);
void updateAgent(Agent agent, User user);

View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
import java.util.List;
public interface ChatManageService {
Boolean addChat(User user, String chatName, Integer agentId);
Long addChat(User user, String chatName, Integer agentId);
List<ChatDO> getAll(String userName, Integer agentId);

View File

@@ -26,9 +26,11 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO>
}
@Override
public void createAgent(Agent agent, User user) {
public Integer createAgent(Agent agent, User user) {
agent.createdBy(user.getName());
save(convert(agent));
AgentDO agentDO = convert(agent);
save(agentDO);
return agentDO.getId();
}
@Override

View File

@@ -43,7 +43,7 @@ public class ChatManageServiceImpl implements ChatManageService {
private ChatQueryRepository chatQueryRepository;
@Override
public Boolean addChat(User user, String chatName, Integer agentId) {
public Long addChat(User user, String chatName, Integer agentId) {
ChatDO chatDO = new ChatDO();
chatDO.setChatName(chatName);
chatDO.setCreator(user.getName());

View File

@@ -49,7 +49,7 @@
where chat_id = #{chatId}
</update>
<insert id="createChat" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO">
<insert id="createChat" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO" useGeneratedKeys="true" keyProperty="chatId">
insert into s2_chat
(agent_id, chat_name, create_time, last_time, creator, last_question, is_delete, is_top)
values (#{agentId}, #{chatName}, #{createTime}, #{lastTime}, #{creator}, #{lastQuestion}, #{isDelete}, #{isTop})

View File

@@ -1,11 +1,19 @@
package com.tencent.supersonic.headless.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_database")
public class DatabaseDO {
/**
*
*/
@TableId(type = IdType.AUTO)
private Long id;
/**

View File

@@ -1,71 +1,11 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDOExample;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface DatabaseDOMapper {
/**
*
* @mbg.generated
*/
long countByExample(DatabaseDOExample example);
public interface DatabaseDOMapper extends BaseMapper<DatabaseDO> {
/**
*
* @mbg.generated
*/
int deleteByPrimaryKey(Long id);
/**
*
* @mbg.generated
*/
int insert(DatabaseDO record);
/**
*
* @mbg.generated
*/
int insertSelective(DatabaseDO record);
/**
*
* @mbg.generated
*/
List<DatabaseDO> selectByExampleWithBLOBs(DatabaseDOExample example);
/**
*
* @mbg.generated
*/
List<DatabaseDO> selectByExample(DatabaseDOExample example);
/**
*
* @mbg.generated
*/
DatabaseDO selectByPrimaryKey(Long id);
/**
*
* @mbg.generated
*/
int updateByPrimaryKeySelective(DatabaseDO record);
/**
*
* @mbg.generated
*/
int updateByPrimaryKeyWithBLOBs(DatabaseDO record);
/**
*
* @mbg.generated
*/
int updateByPrimaryKey(DatabaseDO record);
}

View File

@@ -1,18 +0,0 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import java.util.List;
public interface DatabaseRepository {
void createDatabase(DatabaseDO databaseDO);
void updateDatabase(DatabaseDO databaseDO);
DatabaseDO getDatabase(Long id);
List<DatabaseDO> getDatabaseList();
void deleteDatabase(Long id);
}

View File

@@ -1,48 +0,0 @@
package com.tencent.supersonic.headless.server.persistence.repository.impl;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDOExample;
import com.tencent.supersonic.headless.server.persistence.mapper.DatabaseDOMapper;
import com.tencent.supersonic.headless.server.persistence.repository.DatabaseRepository;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
public class DatabaseRepositoryImpl implements DatabaseRepository {
private DatabaseDOMapper databaseDOMapper;
public DatabaseRepositoryImpl(DatabaseDOMapper databaseDOMapper) {
this.databaseDOMapper = databaseDOMapper;
}
@Override
public void createDatabase(DatabaseDO databaseDO) {
databaseDOMapper.insert(databaseDO);
}
@Override
public void updateDatabase(DatabaseDO databaseDO) {
databaseDOMapper.updateByPrimaryKeyWithBLOBs(databaseDO);
}
@Override
public DatabaseDO getDatabase(Long id) {
return databaseDOMapper.selectByPrimaryKey(id);
}
@Override
public List<DatabaseDO> getDatabaseList() {
return databaseDOMapper.selectByExampleWithBLOBs(new DatabaseDOExample());
}
@Override
public void deleteDatabase(Long id) {
databaseDOMapper.deleteByPrimaryKey(id);
}
}

View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
@@ -14,7 +15,7 @@ import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
import com.tencent.supersonic.headless.core.utils.SqlUtils;
import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import com.tencent.supersonic.headless.server.persistence.repository.DatabaseRepository;
import com.tencent.supersonic.headless.server.persistence.mapper.DatabaseDOMapper;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import com.tencent.supersonic.headless.server.pojo.DbParameterFactory;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
@@ -22,6 +23,7 @@ import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -34,20 +36,16 @@ import java.util.stream.Collectors;
@Slf4j
@Service
public class DatabaseServiceImpl implements DatabaseService {
public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseDO>
implements DatabaseService {
private final SqlUtils sqlUtils;
private DatabaseRepository databaseRepository;
@Autowired
private SqlUtils sqlUtils;
@Lazy
@Autowired
private ModelService datasourceService;
public DatabaseServiceImpl(DatabaseRepository databaseRepository,
SqlUtils sqlUtils,
@Lazy ModelService datasourceService) {
this.databaseRepository = databaseRepository;
this.sqlUtils = sqlUtils;
this.datasourceService = datasourceService;
}
@Override
public boolean testConnect(DatabaseReq databaseReq, User user) {
Database database = DatabaseConverter.convert(databaseReq);
@@ -61,21 +59,19 @@ public class DatabaseServiceImpl implements DatabaseService {
if (databaseDO != null) {
database.updatedBy(user.getName());
DatabaseConverter.convert(database, databaseDO);
databaseRepository.updateDatabase(databaseDO);
updateById(databaseDO);
return DatabaseConverter.convertWithPassword(databaseDO);
}
database.createdBy(user.getName());
databaseDO = DatabaseConverter.convert(database);
databaseRepository.createDatabase(databaseDO);
save(databaseDO);
return DatabaseConverter.convertWithPassword(databaseDO);
}
@Override
public List<DatabaseResp> getDatabaseList(User user) {
List<DatabaseResp> databaseResps =
databaseRepository.getDatabaseList()
.stream().map(DatabaseConverter::convert)
.collect(Collectors.toList());
List<DatabaseResp> databaseResps = list().stream().map(DatabaseConverter::convert)
.collect(Collectors.toList());
fillPermission(databaseResps, user);
return databaseResps;
}
@@ -107,12 +103,12 @@ public class DatabaseServiceImpl implements DatabaseService {
String message = String.format("该数据库被模型%s使用无法删除", datasourceNames);
throw new RuntimeException(message);
}
databaseRepository.deleteDatabase(databaseId);
removeById(databaseId);
}
@Override
public DatabaseResp getDatabase(Long id) {
DatabaseDO databaseDO = databaseRepository.getDatabase(id);
DatabaseDO databaseDO = getById(id);
return DatabaseConverter.convertWithPassword(databaseDO);
}
@@ -156,7 +152,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
private DatabaseDO getDatabaseDO(Long id) {
return databaseRepository.getDatabase(id);
return getById(id);
}
@Override

View File

@@ -1,266 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.tencent.supersonic.headless.server.persistence.mapper.DatabaseDOMapper">
<resultMap id="BaseResultMap" type="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO">
<id column="id" jdbcType="BIGINT" property="id" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="description" jdbcType="VARCHAR" property="description" />
<result column="version" jdbcType="VARCHAR" property="version" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt" />
<result column="created_by" jdbcType="VARCHAR" property="createdBy" />
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
<result column="admin" jdbcType="VARCHAR" property="admin" />
<result column="viewer" jdbcType="VARCHAR" property="viewer" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO">
<result column="config" jdbcType="LONGVARCHAR" property="config" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, name, description, version, type, created_at, created_by, updated_at, updated_by,
admin, viewer
</sql>
<sql id="Blob_Column_List">
config
</sql>
<select id="selectByExampleWithBLOBs" parameterType="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDOExample" resultMap="ResultMapWithBLOBs">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from s2_database
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByExample" parameterType="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDOExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
from s2_database
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
<if test="limitStart != null and limitStart>=0">
limit #{limitStart} , #{limitEnd}
</if>
</select>
<select id="selectByPrimaryKey" parameterType="java.lang.Long" resultMap="ResultMapWithBLOBs">
select
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from s2_database
where id = #{id,jdbcType=BIGINT}
</select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
delete from s2_database
where id = #{id,jdbcType=BIGINT}
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO">
insert into s2_database (id, name, description,
version, type, created_at,
created_by, updated_at, updated_by,
admin, viewer, config
)
values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{description,jdbcType=VARCHAR},
#{version,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{createdAt,jdbcType=TIMESTAMP},
#{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP}, #{updatedBy,jdbcType=VARCHAR},
#{admin,jdbcType=VARCHAR}, #{viewer,jdbcType=VARCHAR}, #{config,jdbcType=LONGVARCHAR}
)
</insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO">
insert into s2_database
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="name != null">
name,
</if>
<if test="description != null">
description,
</if>
<if test="version != null">
version,
</if>
<if test="type != null">
type,
</if>
<if test="createdAt != null">
created_at,
</if>
<if test="createdBy != null">
created_by,
</if>
<if test="updatedAt != null">
updated_at,
</if>
<if test="updatedBy != null">
updated_by,
</if>
<if test="admin != null">
admin,
</if>
<if test="viewer != null">
viewer,
</if>
<if test="config != null">
config,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=BIGINT},
</if>
<if test="name != null">
#{name,jdbcType=VARCHAR},
</if>
<if test="description != null">
#{description,jdbcType=VARCHAR},
</if>
<if test="version != null">
#{version,jdbcType=VARCHAR},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
#{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
#{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
#{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
#{updatedBy,jdbcType=VARCHAR},
</if>
<if test="admin != null">
#{admin,jdbcType=VARCHAR},
</if>
<if test="viewer != null">
#{viewer,jdbcType=VARCHAR},
</if>
<if test="config != null">
#{config,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDOExample" resultType="java.lang.Long">
select count(*) from s2_database
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</select>
<update id="updateByPrimaryKeySelective" parameterType="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO">
update s2_database
<set>
<if test="name != null">
name = #{name,jdbcType=VARCHAR},
</if>
<if test="description != null">
description = #{description,jdbcType=VARCHAR},
</if>
<if test="version != null">
version = #{version,jdbcType=VARCHAR},
</if>
<if test="type != null">
type = #{type,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
created_at = #{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
created_by = #{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
updated_by = #{updatedBy,jdbcType=VARCHAR},
</if>
<if test="admin != null">
admin = #{admin,jdbcType=VARCHAR},
</if>
<if test="viewer != null">
viewer = #{viewer,jdbcType=VARCHAR},
</if>
<if test="config != null">
config = #{config,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKeyWithBLOBs" parameterType="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO">
update s2_database
set name = #{name,jdbcType=VARCHAR},
description = #{description,jdbcType=VARCHAR},
version = #{version,jdbcType=VARCHAR},
type = #{type,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
admin = #{admin,jdbcType=VARCHAR},
viewer = #{viewer,jdbcType=VARCHAR},
config = #{config,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO">
update s2_database
set name = #{name,jdbcType=VARCHAR},
description = #{description,jdbcType=VARCHAR},
version = #{version,jdbcType=VARCHAR},
type = #{type,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
admin = #{admin,jdbcType=VARCHAR},
viewer = #{viewer,jdbcType=VARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
</mapper>

View File

@@ -11,17 +11,14 @@ import com.tencent.supersonic.chat.server.agent.AgentToolType;
import com.tencent.supersonic.chat.server.agent.LLMParserTool;
import com.tencent.supersonic.chat.server.agent.MultiTurnConfig;
import com.tencent.supersonic.chat.server.agent.RuleParserTool;
import com.tencent.supersonic.chat.server.plugin.Plugin;
import com.tencent.supersonic.chat.server.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.server.plugin.build.ParamOption;
import com.tencent.supersonic.chat.server.plugin.build.WebBase;
import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.chat.server.service.ChatManageService;
import com.tencent.supersonic.chat.server.service.ChatService;
import com.tencent.supersonic.chat.server.service.PluginService;
import com.tencent.supersonic.common.pojo.SysParameter;
import com.tencent.supersonic.common.pojo.enums.S2ModelProvider;
import com.tencent.supersonic.common.service.SysParameterService;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.LLMConfig;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -31,9 +28,6 @@ import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.List;
@Component
@Slf4j
@Order(3)
@@ -45,8 +39,6 @@ public class ChatDemoLoader implements CommandLineRunner {
@Autowired
private ChatManageService chatManageService;
@Autowired
private PluginService pluginService;
@Autowired
private AgentService agentService;
@Autowired
private SysParameterService sysParameterService;
@@ -69,13 +61,12 @@ public class ChatDemoLoader implements CommandLineRunner {
public void doRun() {
try {
addSysParameter();
addPlugin_1();
addAgent1();
Integer agentId = addAgent1();
addAgent2();
addAgent3();
//addAgent4();
addSampleChats();
addSampleChats2();
addSampleChats(agentId);
addSampleChats2(agentId);
updateQueryScore(1);
updateQueryScore(4);
} catch (Exception e) {
@@ -83,11 +74,11 @@ public class ChatDemoLoader implements CommandLineRunner {
}
}
private void parseAndExecute(int chatId, String queryText) throws Exception {
private void parseAndExecute(int chatId, int agentId, String queryText) throws Exception {
ChatParseReq chatParseReq = new ChatParseReq();
chatParseReq.setQueryText(queryText);
chatParseReq.setChatId(chatId);
chatParseReq.setAgentId(1);
chatParseReq.setAgentId(agentId);
chatParseReq.setUser(User.getFakeUser());
ParseResp parseResp = chatService.performParsing(chatParseReq);
if (CollectionUtils.isEmpty(parseResp.getSelectedParses())) {
@@ -104,20 +95,20 @@ public class ChatDemoLoader implements CommandLineRunner {
chatService.performExecution(executeReq);
}
public void addSampleChats() throws Exception {
chatManageService.addChat(user, "样例对话1", 1);
public void addSampleChats(Integer agentId) throws Exception {
Long chatId = chatManageService.addChat(user, "样例对话1", agentId);
parseAndExecute(1, "超音数 访问次数");
parseAndExecute(1, "按部门统计");
parseAndExecute(1, "查询近30天");
parseAndExecute(chatId.intValue(), agentId, "超音数 访问次数");
parseAndExecute(chatId.intValue(), agentId, "按部门统计");
parseAndExecute(chatId.intValue(), agentId, "查询近30天");
}
public void addSampleChats2() throws Exception {
chatManageService.addChat(user, "样例对话2", 1);
public void addSampleChats2(Integer agentId) throws Exception {
Long chatId = chatManageService.addChat(user, "样例对话2", agentId);
parseAndExecute(2, "alice 停留时长");
parseAndExecute(2, "对比alice和lucy的访问次数");
parseAndExecute(2, "访问次数最高的部门");
parseAndExecute(chatId.intValue(), agentId,"alice 停留时长");
parseAndExecute(chatId.intValue(), agentId,"对比alice和lucy的访问次数");
parseAndExecute(chatId.intValue(), agentId,"访问次数最高的部门");
}
public void addSysParameter() {
@@ -127,32 +118,7 @@ public class ChatDemoLoader implements CommandLineRunner {
sysParameterService.save(sysParameter);
}
private void addPlugin_1() {
Plugin plugin1 = new Plugin();
plugin1.setType("WEB_PAGE");
plugin1.setDataSetList(Arrays.asList(1L));
plugin1.setPattern("用于分析超音数的流量概况包含UV、PV等核心指标的追踪。P.S. 仅作为示例展示,无实际看板");
plugin1.setName("超音数流量分析看板");
PluginParseConfig pluginParseConfig = new PluginParseConfig();
pluginParseConfig.setDescription(plugin1.getPattern());
pluginParseConfig.setName(plugin1.getName());
pluginParseConfig.setExamples(Lists.newArrayList("tom最近访问超音数情况怎么样"));
plugin1.setParseModeConfig(JSONObject.toJSONString(pluginParseConfig));
WebBase webBase = new WebBase();
webBase.setUrl("www.yourbi.com");
ParamOption paramOption = new ParamOption();
paramOption.setKey("name");
paramOption.setParamType(ParamOption.ParamType.SEMANTIC);
paramOption.setElementId(2L);
paramOption.setModelId(1L);
List<ParamOption> paramOptions = Arrays.asList(paramOption);
webBase.setParamOptions(paramOptions);
plugin1.setConfig(JsonUtil.toString(webBase));
pluginService.createPlugin(plugin1, user);
}
private void addAgent1() {
private Integer addAgent1() {
Agent agent = new Agent();
agent.setId(1);
agent.setName("算指标");
@@ -165,7 +131,7 @@ public class ChatDemoLoader implements CommandLineRunner {
RuleParserTool ruleQueryTool = new RuleParserTool();
ruleQueryTool.setType(AgentToolType.NL2SQL_RULE);
ruleQueryTool.setId("0");
ruleQueryTool.setDataSetIds(Lists.newArrayList(1L));
ruleQueryTool.setDataSetIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(ruleQueryTool);
if (demoEnabledNl2SqlLlm) {
LLMParserTool llmParserTool = new LLMParserTool();
@@ -178,6 +144,7 @@ public class ChatDemoLoader implements CommandLineRunner {
MultiTurnConfig multiTurnConfig = new MultiTurnConfig(false);
agent.setMultiTurnConfig(multiTurnConfig);
agentService.createAgent(agent, User.getFakeUser());
return agent.getId();
}
private void addAgent2() {
@@ -192,7 +159,7 @@ public class ChatDemoLoader implements CommandLineRunner {
RuleParserTool ruleQueryTool = new RuleParserTool();
ruleQueryTool.setId("0");
ruleQueryTool.setType(AgentToolType.NL2SQL_RULE);
ruleQueryTool.setDataSetIds(Lists.newArrayList(2L));
ruleQueryTool.setDataSetIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(ruleQueryTool);
if (demoEnabledNl2SqlLlm) {

View File

@@ -1,10 +1,16 @@
package com.tencent.supersonic;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.chat.server.plugin.Plugin;
import com.tencent.supersonic.chat.server.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.server.plugin.build.ParamOption;
import com.tencent.supersonic.chat.server.plugin.build.WebBase;
import com.tencent.supersonic.chat.server.service.PluginService;
import com.tencent.supersonic.common.pojo.JoinCondition;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
@@ -14,6 +20,7 @@ import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig;
import com.tencent.supersonic.headless.api.pojo.DefaultDisplayInfo;
@@ -50,6 +57,14 @@ import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
import com.tencent.supersonic.headless.api.pojo.request.TagObjectReq;
import com.tencent.supersonic.headless.api.pojo.request.TagReq;
import com.tencent.supersonic.headless.api.pojo.request.TermReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.TagObjectResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -70,6 +85,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@Component
@Slf4j
@@ -100,57 +116,64 @@ public class ModelDemoDataLoader {
private TagMetaService tagMetaService;
@Autowired
private TermService termService;
@Autowired
private PluginService pluginService;
public void doRun() {
try {
addDatabase();
addDomain();
addTagObjectUser();
addTagObjectSinger();
addModel_1();
addModel_2();
addMetric_uv();
addMetric_pv_avg();
addModel_3();
addModelRela_1();
addModelRela_2();
addDomain_2();
addModel_4();
updateDimension();
updateMetric();
addTags();
updateMetric_pv();
addDataSet_1();
addDataSet_2();
addAuthGroup_1();
addAuthGroup_2();
addTerm();
addTerm_1();
DatabaseResp databaseResp = addDatabase();
DomainResp s2Domain = addDomain();
TagObjectResp s2TagObject = addTagObjectUser(s2Domain);
ModelResp userModel = addModel_1(s2Domain, databaseResp, s2TagObject);
ModelResp pvUvModel = addModel_2(s2Domain, databaseResp);
DimensionResp userDimension = getDimension("user_name", userModel);
DimensionResp departmentDimension = getDimension("department", userModel);
MetricResp metricUv = addMetric_uv(userModel, departmentDimension);
MetricResp metricPv = getMetric("pv", pvUvModel);
addMetric_pv_avg(metricPv, metricUv, departmentDimension, pvUvModel);
ModelResp stayTimeModel = addModel_3(s2Domain, databaseResp);
addModelRela_1(s2Domain, userModel, pvUvModel);
addModelRela_2(s2Domain, userModel, stayTimeModel);
DomainResp singerDomain = addDomain_2();
TagObjectResp singerTagObject = addTagObjectSinger(singerDomain);
ModelResp singerModel = addModel_4(singerDomain, databaseResp, singerTagObject);
DimensionResp pageDimension = getDimension("page", stayTimeModel);
updateDimension(stayTimeModel, pageDimension);
updateMetric(stayTimeModel, departmentDimension, userDimension);
addTags(singerModel);
updateMetric_pv(pvUvModel, departmentDimension, userDimension, metricPv);
DataSetResp s2DataSet = addDataSet_1(s2Domain);
addDataSet_2(singerDomain, singerModel);
addAuthGroup_1(stayTimeModel);
addAuthGroup_2(stayTimeModel);
addTerm(s2Domain);
addTerm_1(s2Domain);
addPlugin_1(s2DataSet, userDimension, userModel);
} catch (Exception e) {
log.error("Failed to add model demo data", e);
}
}
private void addTagObjectUser() throws Exception {
private TagObjectResp addTagObjectUser(DomainResp s2Domain) throws Exception {
TagObjectReq tagObjectReq = new TagObjectReq();
tagObjectReq.setDomainId(1L);
tagObjectReq.setDomainId(s2Domain.getId());
tagObjectReq.setName("用户");
tagObjectReq.setBizName("user");
User user = User.getFakeUser();
tagObjectService.create(tagObjectReq, user);
return tagObjectService.create(tagObjectReq, user);
}
private void addTagObjectSinger() throws Exception {
private TagObjectResp addTagObjectSinger(DomainResp singerDomain) throws Exception {
TagObjectReq tagObjectReq = new TagObjectReq();
tagObjectReq.setDomainId(2L);
tagObjectReq.setDomainId(singerDomain.getId());
tagObjectReq.setName("艺人");
tagObjectReq.setBizName("singer");
User user = User.getFakeUser();
tagObjectService.create(tagObjectReq, user);
return tagObjectService.create(tagObjectReq, user);
}
public void addDatabase() {
public DatabaseResp addDatabase() {
String url = dataSourceProperties.getUrl();
DatabaseReq databaseReq = new DatabaseReq();
databaseReq.setName("数据实例");
@@ -165,10 +188,10 @@ public class ModelDemoDataLoader {
databaseReq.setUrl(url);
databaseReq.setUsername(dataSourceProperties.getUsername());
databaseReq.setPassword(dataSourceProperties.getPassword());
databaseService.createOrUpdateDatabase(databaseReq, user);
return databaseService.createOrUpdateDatabase(databaseReq, user);
}
public void addDomain() {
public DomainResp addDomain() {
DomainReq domainReq = new DomainReq();
domainReq.setName("超音数");
domainReq.setBizName("supersonic");
@@ -178,17 +201,18 @@ public class ModelDemoDataLoader {
domainReq.setViewOrgs(Collections.singletonList("1"));
domainReq.setAdmins(Arrays.asList("admin", "jack"));
domainReq.setAdminOrgs(Collections.emptyList());
domainService.createDomain(domainReq, user);
return domainService.createDomain(domainReq, user);
}
public void addModel_1() throws Exception {
public ModelResp addModel_1(DomainResp s2Domain, DatabaseResp s2Database,
TagObjectResp s2TagObject) throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("用户部门");
modelReq.setBizName("user_department");
modelReq.setDescription("用户部门信息");
modelReq.setDatabaseId(1L);
modelReq.setDomainId(1L);
modelReq.setTagObjectId(1L);
modelReq.setDatabaseId(s2Database.getId());
modelReq.setDomainId(s2Domain.getId());
modelReq.setTagObjectId(s2TagObject.getId());
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("1"));
modelReq.setAdmins(Arrays.asList("admin", "alice"));
@@ -210,16 +234,16 @@ public class ModelDemoDataLoader {
modelDetail.setQueryType("sql_query");
modelDetail.setSqlQuery("select user_name,department from s2_user_department");
modelReq.setModelDetail(modelDetail);
modelReq.setDomainId(1L);
modelService.createModel(modelReq, user);
return modelService.createModel(modelReq, user);
}
public void addModel_2() throws Exception {
public ModelResp addModel_2(DomainResp s2Domain, DatabaseResp s2Database) throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("PVUV统计");
modelReq.setBizName("s2_pv_uv_statis");
modelReq.setDescription("PVUV统计");
modelReq.setDatabaseId(1L);
modelReq.setDatabaseId(s2Database.getId());
modelReq.setDomainId(s2Domain.getId());
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("1"));
modelReq.setAdmins(Collections.singletonList("admin"));
@@ -253,17 +277,17 @@ public class ModelDemoDataLoader {
modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, "
+ "user_name as user_id FROM s2_pv_uv_statis");
modelDetail.setQueryType("sql_query");
modelReq.setDomainId(1L);
modelReq.setModelDetail(modelDetail);
modelService.createModel(modelReq, user);
return modelService.createModel(modelReq, user);
}
public void addModel_3() throws Exception {
public ModelResp addModel_3(DomainResp s2Domain, DatabaseResp s2Database) throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("停留时长统计");
modelReq.setBizName("s2_stay_time_statis");
modelReq.setDescription("停留时长统计");
modelReq.setDatabaseId(1L);
modelReq.setDomainId(s2Domain.getId());
modelReq.setDatabaseId(s2Database.getId());
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("1"));
modelReq.setAdmins(Collections.singletonList("admin"));
@@ -294,36 +318,35 @@ public class ModelDemoDataLoader {
modelDetail.setFields(fields);
modelDetail.setSqlQuery("select imp_date,user_name,stay_hours,page from s2_stay_time_statis");
modelDetail.setQueryType("sql_query");
modelReq.setDomainId(1L);
modelReq.setModelDetail(modelDetail);
modelService.createModel(modelReq, user);
return modelService.createModel(modelReq, user);
}
public void addModelRela_1() {
public void addModelRela_1(DomainResp s2Domain, ModelResp userDepartmentModel, ModelResp pvUvModel) {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("user_name", "user_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(1L);
modelRelaReq.setFromModelId(1L);
modelRelaReq.setToModelId(2L);
modelRelaReq.setDomainId(s2Domain.getId());
modelRelaReq.setFromModelId(userDepartmentModel.getId());
modelRelaReq.setToModelId(pvUvModel.getId());
modelRelaReq.setJoinType("left join");
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
public void addModelRela_2() {
public void addModelRela_2(DomainResp s2Domain, ModelResp userDepartmentModel, ModelResp stayTimeModel) {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("user_name", "user_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(1L);
modelRelaReq.setFromModelId(1L);
modelRelaReq.setToModelId(3L);
modelRelaReq.setDomainId(s2Domain.getId());
modelRelaReq.setFromModelId(userDepartmentModel.getId());
modelRelaReq.setToModelId(stayTimeModel.getId());
modelRelaReq.setJoinType("left join");
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
public void addDomain_2() {
public DomainResp addDomain_2() {
DomainReq domainReq = new DomainReq();
domainReq.setName("艺人库");
domainReq.setBizName("supersonic");
@@ -333,17 +356,18 @@ public class ModelDemoDataLoader {
domainReq.setViewOrgs(Collections.singletonList("1"));
domainReq.setAdmins(Arrays.asList("admin", "alice"));
domainReq.setAdminOrgs(Collections.emptyList());
domainService.createDomain(domainReq, user);
return domainService.createDomain(domainReq, user);
}
public void addModel_4() throws Exception {
public ModelResp addModel_4(DomainResp singerDomain,
DatabaseResp s2Database, TagObjectResp singerTagObject) throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("艺人库");
modelReq.setBizName("singer");
modelReq.setDescription("艺人库");
modelReq.setDatabaseId(1L);
modelReq.setDomainId(2L);
modelReq.setTagObjectId(2L);
modelReq.setDatabaseId(s2Database.getId());
modelReq.setDomainId(singerDomain.getId());
modelReq.setTagObjectId(singerTagObject.getId());
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("1"));
modelReq.setAdmins(Collections.singletonList("admin"));
@@ -375,16 +399,20 @@ public class ModelDemoDataLoader {
modelDetail.setSqlQuery("select imp_date, singer_name, act_area, song_name, genre, "
+ "js_play_cnt, down_cnt, favor_cnt from singer");
modelReq.setModelDetail(modelDetail);
modelService.createModel(modelReq, user);
return modelService.createModel(modelReq, user);
}
private void addTags() {
addTag(1L, TagDefineType.DIMENSION);
addTag(4L, TagDefineType.DIMENSION);
addTag(5L, TagDefineType.DIMENSION);
addTag(6L, TagDefineType.DIMENSION);
addTag(7L, TagDefineType.DIMENSION);
addTag(5L, TagDefineType.METRIC);
private void addTags(ModelResp singerModel) {
addTag(dimensionService.getDimension("act_area", singerModel.getId()).getId(),
TagDefineType.DIMENSION);
addTag(dimensionService.getDimension("song_name", singerModel.getId()).getId(),
TagDefineType.DIMENSION);
addTag(dimensionService.getDimension("genre", singerModel.getId()).getId(),
TagDefineType.DIMENSION);
addTag(dimensionService.getDimension("singer_name", singerModel.getId()).getId(),
TagDefineType.DIMENSION);
addTag(metricService.getMetric(singerModel.getId(), "js_play_cnt").getId(),
TagDefineType.METRIC);
}
private void addTag(Long itemId, TagDefineType tagDefineType) {
@@ -394,26 +422,29 @@ public class ModelDemoDataLoader {
tagMetaService.create(tagReq, User.getFakeUser());
}
public void updateDimension() throws Exception {
public void updateDimension(ModelResp stayTimeModel, DimensionResp pageDimension) throws Exception {
DimensionReq dimensionReq = new DimensionReq();
dimensionReq.setType(DimensionType.categorical.name());
dimensionReq.setId(3L);
dimensionReq.setId(pageDimension.getId());
dimensionReq.setName("页面");
dimensionReq.setBizName("page");
dimensionReq.setModelId(3L);
dimensionReq.setModelId(stayTimeModel.getId());
dimensionReq.setAlias("page");
dimensionReq.setSemanticType(SemanticType.CATEGORY.name());
dimensionReq.setSensitiveLevel(2);
dimensionReq.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
dimensionReq.setDescription("页面");
dimensionReq.setExpr("page");
dimensionReq.setDimValueMaps(Collections.emptyList());
dimensionService.updateDimension(dimensionReq, user);
}
public void updateMetric() throws Exception {
public void updateMetric(ModelResp stayTimeModel, DimensionResp departmentDimension,
DimensionResp userDimension) throws Exception {
MetricResp stayHoursMetric =
metricService.getMetric(stayTimeModel.getId(), "stay_hours");
MetricReq metricReq = new MetricReq();
metricReq.setModelId(3L);
metricReq.setId(4L);
metricReq.setModelId(stayTimeModel.getId());
metricReq.setId(stayHoursMetric.getId());
metricReq.setName("停留时长");
metricReq.setBizName("stay_hours");
metricReq.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
@@ -429,14 +460,16 @@ public class ModelDemoDataLoader {
metricTypeParams.setMeasures(measures);
metricReq.setMetricDefineByMeasureParams(metricTypeParams);
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
metricReq.setRelateDimension(getRelateDimension(Lists.newArrayList(1L, 2L)));
metricReq.setRelateDimension(getRelateDimension(
Lists.newArrayList(departmentDimension.getId(), userDimension.getId())));
metricService.updateMetric(metricReq, user);
}
public void updateMetric_pv() throws Exception {
public void updateMetric_pv(ModelResp pvUvModel, DimensionResp departmentDimension,
DimensionResp userDimension, MetricResp metricPv) throws Exception {
MetricReq metricReq = new MetricReq();
metricReq.setModelId(2L);
metricReq.setId(1L);
metricReq.setModelId(pvUvModel.getId());
metricReq.setId(metricPv.getId());
metricReq.setName("访问次数");
metricReq.setBizName("pv");
metricReq.setDescription("一段时间内用户的访问次数");
@@ -449,13 +482,14 @@ public class ModelDemoDataLoader {
metricTypeParams.setMeasures(measures);
metricReq.setMetricDefineByMeasureParams(metricTypeParams);
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
metricReq.setRelateDimension(getRelateDimension(Lists.newArrayList(1L, 2L)));
metricReq.setRelateDimension(getRelateDimension(
Lists.newArrayList(departmentDimension.getId(), userDimension.getId())));
metricService.updateMetric(metricReq, user);
}
public void addMetric_uv() throws Exception {
public MetricResp addMetric_uv(ModelResp uvModel, DimensionResp departmentDimension) throws Exception {
MetricReq metricReq = new MetricReq();
metricReq.setModelId(2L);
metricReq.setModelId(uvModel.getId());
metricReq.setName("访问用户数");
metricReq.setBizName("uv");
metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
@@ -466,19 +500,17 @@ public class ModelDemoDataLoader {
List<FieldParam> fieldParams = new ArrayList<>();
fieldParams.add(new FieldParam("user_id"));
metricTypeParams.setFields(fieldParams);
RelateDimension relateDimension = new RelateDimension();
relateDimension.setDrillDownDimensions(Lists.newArrayList(
new DrillDownDimension(1L)));
metricReq.setRelateDimension(relateDimension);
metricReq.setMetricDefineByFieldParams(metricTypeParams);
metricReq.setMetricDefineType(MetricDefineType.FIELD);
metricReq.setRelateDimension(getRelateDimension(Lists.newArrayList(1L)));
metricService.createMetric(metricReq, user);
metricReq.setRelateDimension(getRelateDimension(
Lists.newArrayList(departmentDimension.getId())));
return metricService.createMetric(metricReq, user);
}
public void addMetric_pv_avg() throws Exception {
public MetricResp addMetric_pv_avg(MetricResp metricPv, MetricResp metricUv,
DimensionResp departmentDimension, ModelResp pvModel) throws Exception {
MetricReq metricReq = new MetricReq();
metricReq.setModelId(2L);
metricReq.setModelId(pvModel.getId());
metricReq.setName("人均访问次数");
metricReq.setBizName("pv_avg");
metricReq.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
@@ -488,29 +520,25 @@ public class ModelDemoDataLoader {
MetricDefineByMetricParams metricTypeParams = new MetricDefineByMetricParams();
metricTypeParams.setExpr("pv/uv");
List<MetricParam> metrics = new ArrayList<>();
MetricParam metricPv = new MetricParam(1L, "pv");
MetricParam metricUv = new MetricParam(2L, "uv");
metrics.add(metricPv);
metrics.add(metricUv);
MetricParam pv = new MetricParam(metricPv.getId(), metricPv.getBizName());
MetricParam uv = new MetricParam(metricUv.getId(), metricUv.getBizName());
metrics.add(pv);
metrics.add(uv);
metricTypeParams.setMetrics(metrics);
metricReq.setMetricDefineByMetricParams(metricTypeParams);
metricReq.setMetricDefineType(MetricDefineType.METRIC);
metricReq.setRelateDimension(getRelateDimension(Lists.newArrayList(1L)));
metricService.createMetric(metricReq, user);
metricReq.setRelateDimension(getRelateDimension(Lists.newArrayList(departmentDimension.getId())));
return metricService.createMetric(metricReq, user);
}
public void addDataSet_1() {
public DataSetResp addDataSet_1(DomainResp s2Domain) {
DataSetReq dataSetReq = new DataSetReq();
dataSetReq.setName("超音数");
dataSetReq.setBizName("s2");
dataSetReq.setDomainId(1L);
dataSetReq.setDescription("包含超音数访问统计相关的指标和维度等");
dataSetReq.setAdmins(Lists.newArrayList("admin"));
List<DataSetModelConfig> dataSetModelConfigs = Lists.newArrayList(
new DataSetModelConfig(1L, Lists.newArrayList(1L, 2L), Lists.newArrayList()),
new DataSetModelConfig(2L, Lists.newArrayList(), Lists.newArrayList(1L, 2L, 3L)),
new DataSetModelConfig(3L, Lists.newArrayList(3L), Lists.newArrayList(4L)));
List<DataSetModelConfig> dataSetModelConfigs = getDataSetModelConfigs(s2Domain.getId());
DataSetDetail dataSetDetail = new DataSetDetail();
dataSetDetail.setDataSetModelConfigs(dataSetModelConfigs);
dataSetReq.setDataSetDetail(dataSetDetail);
@@ -523,19 +551,17 @@ public class ModelDemoDataLoader {
metricTypeDefaultConfig.setTimeDefaultConfig(timeDefaultConfig);
queryConfig.setMetricTypeDefaultConfig(metricTypeDefaultConfig);
dataSetReq.setQueryConfig(queryConfig);
dataSetService.save(dataSetReq, User.getFakeUser());
return dataSetService.save(dataSetReq, User.getFakeUser());
}
public void addDataSet_2() {
public void addDataSet_2(DomainResp singerDomain, ModelResp singerModel) {
DataSetReq dataSetReq = new DataSetReq();
dataSetReq.setName("艺人库");
dataSetReq.setBizName("singer");
dataSetReq.setDomainId(2L);
dataSetReq.setDomainId(singerDomain.getId());
dataSetReq.setDescription("包含艺人相关标签和指标信息");
dataSetReq.setAdmins(Lists.newArrayList("admin", "jack"));
List<DataSetModelConfig> dataSetModelConfigs = Lists.newArrayList(
new DataSetModelConfig(4L, Lists.newArrayList(4L, 5L, 6L, 7L), Lists.newArrayList(5L, 6L, 7L))
);
List<DataSetModelConfig> dataSetModelConfigs = getDataSetModelConfigs(singerDomain.getId());
DataSetDetail dataSetDetail = new DataSetDetail();
dataSetDetail.setDataSetModelConfigs(dataSetModelConfigs);
dataSetReq.setDataSetDetail(dataSetDetail);
@@ -547,8 +573,9 @@ public class ModelDemoDataLoader {
tagTimeDefaultConfig.setUnit(7);
tagTypeDefaultConfig.setTimeDefaultConfig(tagTimeDefaultConfig);
DefaultDisplayInfo defaultDisplayInfo = new DefaultDisplayInfo();
defaultDisplayInfo.setDimensionIds(Lists.newArrayList(4L, 5L, 6L, 7L));
defaultDisplayInfo.setMetricIds(Lists.newArrayList(5L));
defaultDisplayInfo.setDimensionIds(dataSetModelConfigs.get(0).getDimensions());
MetricResp jsPlayCntMetric = getMetric("js_play_cnt", singerModel);
defaultDisplayInfo.setMetricIds(Lists.newArrayList(jsPlayCntMetric.getId()));
tagTypeDefaultConfig.setDefaultDisplayInfo(defaultDisplayInfo);
MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig();
TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig();
@@ -561,27 +588,27 @@ public class ModelDemoDataLoader {
dataSetService.save(dataSetReq, User.getFakeUser());
}
public void addTerm() {
public void addTerm(DomainResp s2Domain) {
TermReq termReq = new TermReq();
termReq.setName("近期");
termReq.setDescription("指近10天");
termReq.setAlias(Lists.newArrayList("近一段时间"));
termReq.setDomainId(1L);
termReq.setDomainId(s2Domain.getId());
termService.saveOrUpdate(termReq, User.getFakeUser());
}
public void addTerm_1() {
public void addTerm_1(DomainResp domainResp) {
TermReq termReq = new TermReq();
termReq.setName("核心用户");
termReq.setDescription("核心用户指tom和lucy");
termReq.setAlias(Lists.newArrayList("VIP用户"));
termReq.setDomainId(1L);
termReq.setDomainId(domainResp.getId());
termService.saveOrUpdate(termReq, User.getFakeUser());
}
public void addAuthGroup_1() {
public void addAuthGroup_1(ModelResp stayTimeModel) {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId(3L);
authGroupReq.setModelId(stayTimeModel.getId());
authGroupReq.setName("admin-permission");
List<AuthRule> authRules = new ArrayList<>();
@@ -596,9 +623,9 @@ public class ModelDemoDataLoader {
authService.addOrUpdateAuthGroup(authGroupReq);
}
public void addAuthGroup_2() {
public void addAuthGroup_2(ModelResp stayTimeModel) {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId(3L);
authGroupReq.setModelId(stayTimeModel.getId());
authGroupReq.setName("tom_row_permission");
List<AuthRule> authRules = new ArrayList<>();
@@ -610,6 +637,31 @@ public class ModelDemoDataLoader {
authService.addOrUpdateAuthGroup(authGroupReq);
}
private void addPlugin_1(DataSetResp s2DataSet, DimensionResp userDimension,
ModelResp userModel) {
Plugin plugin1 = new Plugin();
plugin1.setType("WEB_PAGE");
plugin1.setDataSetList(Arrays.asList(s2DataSet.getId()));
plugin1.setPattern("用于分析超音数的流量概况包含UV、PV等核心指标的追踪。P.S. 仅作为示例展示,无实际看板");
plugin1.setName("超音数流量分析看板");
PluginParseConfig pluginParseConfig = new PluginParseConfig();
pluginParseConfig.setDescription(plugin1.getPattern());
pluginParseConfig.setName(plugin1.getName());
pluginParseConfig.setExamples(Lists.newArrayList("tom最近访问超音数情况怎么样"));
plugin1.setParseModeConfig(JSONObject.toJSONString(pluginParseConfig));
WebBase webBase = new WebBase();
webBase.setUrl("www.yourbi.com");
ParamOption paramOption = new ParamOption();
paramOption.setKey("name");
paramOption.setParamType(ParamOption.ParamType.SEMANTIC);
paramOption.setElementId(userDimension.getId());
paramOption.setModelId(userModel.getId());
List<ParamOption> paramOptions = Arrays.asList(paramOption);
webBase.setParamOptions(paramOptions);
plugin1.setConfig(JsonUtil.toString(webBase));
pluginService.createPlugin(plugin1, user);
}
private RelateDimension getRelateDimension(List<Long> dimensionIds) {
RelateDimension relateDimension = new RelateDimension();
for (Long id : dimensionIds) {
@@ -618,4 +670,34 @@ public class ModelDemoDataLoader {
return relateDimension;
}
private DimensionResp getDimension(String bizName, ModelResp model) {
return dimensionService.getDimension(bizName, model.getId());
}
private MetricResp getMetric(String bizName, ModelResp model) {
return metricService.getMetric(model.getId(), bizName);
}
private List<DataSetModelConfig> getDataSetModelConfigs(Long domainId) {
List<DataSetModelConfig> dataSetModelConfigs = Lists.newArrayList();
List<ModelResp> modelByDomainIds =
modelService.getModelByDomainIds(Lists.newArrayList(domainId));
for (ModelResp modelResp : modelByDomainIds) {
DataSetModelConfig dataSetModelConfig = new DataSetModelConfig();
dataSetModelConfig.setId(modelResp.getId());
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelResp.getId()));
List<Long> metrics = metricService.getMetrics(metaFilter)
.stream().map(MetricResp::getId).collect(Collectors.toList());
dataSetModelConfig.setMetrics(metrics);
List<Long> dimensions = dimensionService.getDimensions(metaFilter)
.stream().map(DimensionResp::getId).collect(Collectors.toList());
dataSetModelConfig.setMetrics(metrics);
dataSetModelConfig.setDimensions(dimensions);
dataSetModelConfigs.add(dataSetModelConfig);
}
return dataSetModelConfigs;
}
}

View File

@@ -43,10 +43,14 @@ s2:
chat-model:
provider: open_ai
openai:
api-key: api_key
model-name: gpt-3.5-turbo-16k
temperature: 0.0
timeout: PT60S
# Replace with your LLM configs
# Note: The default API key `demo` is provided by langchain4j community
# which limits 1000 tokens per request.
base-url: ${OPENAI_API_BASE:https://api.openai.com/v1}
api-key: ${OPENAI_API_KEY:demo}
model-name: ${OPENAI_MODEL_NAME:gpt-3.5-turbo}
temperature: ${OPENAI_TEMPERATURE:0.0}
timeout: ${OPENAI_TIMEOUT:PT60S}
#2.embedding-model
#2.1 in_memory(default)
embedding-model: