(fix)(headless)fix the issue that the time schema in LllM is not matched with the sql which modifiedy by sqlAdaptor when the dateformat updated;also fix the issue that the model update is not effected on dimension(#1720)(1719) (#1725)

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java

@@ -41,6 +41,7 @@ public class SchemaElement implements Serializable {
     private String description;
     private boolean descriptionMapped;
     @Builder.Default private Map<String, Object> extInfo = new HashMap<>();
+    private DimensionTimeTypeParams typeParams;
 
     @Override
     public boolean equals(Object o) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DimensionReq.java

@@ -4,6 +4,7 @@ import javax.validation.constraints.NotNull;
 
 import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
 import com.tencent.supersonic.headless.api.pojo.DimValueMap;
+import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import lombok.Data;
 
@@ -34,4 +35,6 @@ public class DimensionReq extends SchemaItem {
     private int isTag;
 
     private Map<String, Object> ext;
+
+    private DimensionTimeTypeParams typeParams;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.api.pojo.response;
 
 import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
 import com.tencent.supersonic.headless.api.pojo.DimValueMap;
+import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
 import lombok.Data;
@@ -39,6 +40,8 @@ public class DimensionResp extends SchemaItem {
 
     private int isTag;
 
+    private DimensionTimeTypeParams typeParams;
+
     private Map<String, Object> ext = new HashMap<>();
 
     public boolean isTimeDimension() {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java

@@ -45,17 +45,14 @@ import com.tencent.supersonic.headless.server.utils.DimensionConverter;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 
 @Service
@@ -121,13 +118,32 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
         Map<String, DimensionResp> nameMap =
                 dimensionResps.stream()
                         .collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
-        List<DimensionReq> dimensionToInsert =
+
+        List<DimensionReq> dimensionToInsert = Lists.newArrayList();
         dimensionReqs.stream()
-                        .filter(
+                .forEach(
-                                dimension ->
+                        dimension -> {
-                                        !bizNameMap.containsKey(dimension.getBizName())
+                            if (!bizNameMap.containsKey(dimension.getBizName())
-                                                && !nameMap.containsKey(dimension.getName()))
+                                    && !nameMap.containsKey(dimension.getName())) {
-                        .collect(Collectors.toList());
+                                dimensionToInsert.add(dimension);
+                            } else {
+                                DimensionResp dimensionRespByBizName =
+                                        bizNameMap.get(dimension.getBizName());
+                                DimensionResp dimensionRespByName =
+                                        nameMap.get(dimension.getName());
+                                if (null != dimensionRespByBizName
+                                        && isChange(dimension, dimensionRespByBizName)) {
+                                    dimension.setId(dimensionRespByBizName.getId());
+                                    this.updateDimension(dimension, user);
+                                } else {
+                                    if (null != dimensionRespByName
+                                            && isChange(dimension, dimensionRespByName)) {
+                                        dimension.setId(dimensionRespByName.getId());
+                                        this.updateDimension(dimension, user);
+                                    }
+                                }
+                            }
+                        });
         if (CollectionUtils.isEmpty(dimensionToInsert)) {
             return;
         }
@@ -481,4 +497,14 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
     private void sendEvent(DataItem dataItem, EventType eventType) {
         eventPublisher.publishEvent(new DataEvent(this, Lists.newArrayList(dataItem), eventType));
     }
+
+    private boolean isChange(DimensionReq dimensionReq, DimensionResp dimensionResp) {
+        boolean isExtChange =
+                !new EqualsBuilder()
+                        .append(dimensionReq.getExt(), dimensionResp.getExt())
+                        .isEquals();
+        boolean isTypeParamChange =
+                !Objects.equals(dimensionReq.getTypeParams(), dimensionResp.getTypeParams());
+        return isExtChange || isTypeParamChange;
+    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java

@@ -2,14 +2,9 @@ package com.tencent.supersonic.headless.server.utils;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.pojo.DimensionConstants;
-import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
+import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.headless.api.pojo.DimValueMap;
+import com.tencent.supersonic.common.util.DateUtils;
-import com.tencent.supersonic.headless.api.pojo.RelateDimension;
+import com.tencent.supersonic.headless.api.pojo.*;
-import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
-import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
-import com.tencent.supersonic.headless.api.pojo.SchemaItem;
-import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
 import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
@@ -179,7 +174,7 @@ public class DataSetSchemaBuilder {
             if (dim.isTimeDimension()) {
                 String timeFormat =
                         String.valueOf(dim.getExt().get(DimensionConstants.DIMENSION_TIME_FORMAT));
-                dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
+                setDefaultTimeFormat(dimToAdd, dim.getTypeParams(), timeFormat);
             }
             dimensions.add(dimToAdd);
         }
@@ -288,4 +283,19 @@ public class DataSetSchemaBuilder {
                         })
                 .collect(Collectors.toList());
     }
+
+    private static void setDefaultTimeFormat(
+            SchemaElement dimToAdd,
+            DimensionTimeTypeParams dimensionTimeTypeParams,
+            String timeFormat) {
+        if (null != dimensionTimeTypeParams
+                && TimeDimensionEnum.DAY
+                        .name()
+                        .equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
+            dimToAdd.getExtInfo()
+                    .put(DimensionConstants.DIMENSION_TIME_FORMAT, DateUtils.DEFAULT_DATE_FORMAT);
+        } else {
+            dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
+        }
+    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java

@@ -7,6 +7,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.DimValueMap;
+import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
 import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
 import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
@@ -43,6 +44,9 @@ public class DimensionConverter {
         if (dimensionReq.getExt() != null) {
             dimensionDO.setExt(JSONObject.toJSONString(dimensionReq.getExt()));
         }
+        if (Objects.nonNull(dimensionReq.getTypeParams())) {
+            dimensionDO.setTypeParams(JSONObject.toJSONString(dimensionReq.getTypeParams()));
+        }
         return dimensionDO;
     }
 
@@ -93,6 +97,11 @@ public class DimensionConverter {
         if (dimensionDO.getExt() != null) {
             dimensionResp.setExt(JSONObject.parseObject(dimensionDO.getExt(), Map.class));
         }
+        if (StringUtils.isNoneBlank(dimensionDO.getTypeParams())) {
+            dimensionResp.setTypeParams(
+                    JSONObject.parseObject(
+                            dimensionDO.getTypeParams(), DimensionTimeTypeParams.class));
+        }
         dimensionResp.setType(getType(dimensionDO.getType()));
         dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
         dimensionResp.setIsTag(dimensionDO.getIsTag());

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -132,6 +132,7 @@ public class ModelConverter {
         dimensionReq.setDescription(
                 Objects.isNull(dim.getDescription()) ? "" : dim.getDescription());
         dimensionReq.setIsTag(dim.getIsTag());
+        dimensionReq.setTypeParams(dim.getTypeParams());
         return dimensionReq;
     }