refactor globalvar (DataLinkDC#2625)

* refactor-globalvar * Spotless Apply * refactor-globalvar * fix CatalogueServiceImpl delete npe bug * fix CatalogueServiceImpl delete npe bug --------- Co-authored-by: Zzm0809 <[email protected]>
aiwenmo · Dec 12, 2023 · 435bde7 · 435bde7
1 parent 8079fd6
commit 435bde7
Show file tree

Hide file tree

Showing 10 changed files with 165 additions and 64 deletions.
diff --git a/dinky-admin/pom.xml b/dinky-admin/pom.xml
@@ -400,11 +400,11 @@
                 <configuration>
                     <classesDirectory>target/classes/</classesDirectory>
                     <excludes>
-                        <!--注意这玩意默认从编译结果目录开始算目录结构-->
                         <exclude>/application*.yml</exclude>
                         <exclude>/mybatis*.xml</exclude>
                         <exclude>/DinkyFlinkDockerfile</exclude>
-                        <exclude>/FlinkConfClass</exclude>
+                        <exclude>/dinky-loader/FlinkConfClass</exclude>
+                        <exclude>/dinky-loader/ExpressionVariableClass</exclude>
                     </excludes>
                 </configuration>
                 <executions>

diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/CatalogueServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/CatalogueServiceImpl.java
@@ -45,7 +45,6 @@
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
@@ -100,8 +99,7 @@ public List<Catalogue> buildCatalogueTree(List<Catalogue> catalogueList) {
         }
 
         List<Catalogue> returnList = new ArrayList<>();
-        for (Iterator<Catalogue> iterator = catalogueList.iterator(); iterator.hasNext(); ) {
-            Catalogue catalogue = iterator.next();
+        for (Catalogue catalogue : catalogueList) {
             //  get all child catalogue of parent catalogue id , the 0 is root catalogue
             if (catalogue.getParentId() == 0) {
                 recursionBuildCatalogueAndChildren(catalogueList, catalogue);
@@ -147,7 +145,7 @@ private void recursionBuildCatalogueAndChildren(List<Catalogue> list, Catalogue
      * @return
      */
     private boolean hasChild(List<Catalogue> list, Catalogue catalogue) {
-        return getChildList(list, catalogue).size() > 0;
+        return !getChildList(list, catalogue).isEmpty();
     }
 
     /**
@@ -341,7 +339,7 @@ public void traverseFile(String sourcePath, Catalogue catalog) {
         File file = new File(sourcePath);
         File[] fs = file.listFiles();
         if (fs == null) {
-            throw new RuntimeException("目录层级有误");
+            throw new RuntimeException("the dir is error");
         }
         for (File fl : fs) {
             if (fl.isFile()) {
@@ -373,7 +371,7 @@ private String getFileText(File sourceFile) {
                 }
             }
         } catch (Exception e) {
-            e.printStackTrace();
+            log.error("read file error, {} ", e);
         }
         return sb.toString();
     }
@@ -416,10 +414,12 @@ public Result<Void> deleteCatalogueById(Integer catalogueId) {
             List<History> historyList = historyService.list(
                     new LambdaQueryWrapper<History>().eq(History::getTaskId, catalogue.getTaskId()));
             historyList.forEach(history -> {
-                // 查询 job history 表中的作业 通过 id 关联查询
+                // 查询 job history 表中的作业 通过 id 关联查询 // TODO npe
                 JobHistory historyServiceById = jobHistoryService.getById(history.getId());
-                // 删除 job history 表中的作业
-                jobHistoryService.removeById(historyServiceById.getId());
+                if (historyServiceById != null) {
+                    // 删除 job history 表中的作业
+                    jobHistoryService.removeById(historyServiceById.getId());
+                }
                 // 删除 history 表中的作业
                 historyService.removeById(history.getId());
             });
@@ -446,6 +446,7 @@ public Result<Void> deleteCatalogueById(Integer catalogueId) {
      * @return
      */
     @Override
+    @Transactional(rollbackFor = Exception.class)
     public Boolean saveOrUpdateOrRename(Catalogue catalogue) {
         if (taskService.getById(catalogue.getTaskId()) != null) {
             toRename(catalogue);

diff --git a/dinky-admin/src/main/java/org/dinky/utils/FlinkConfigOptionsUtils.java b/dinky-admin/src/main/java/org/dinky/utils/FlinkConfigOptionsUtils.java
@@ -91,6 +91,8 @@ public static String parsedBinlogGroup(String name) {
     }
 
     public static String[] getConfigOptionsClass() {
-        return ResourceUtil.readUtf8Str("FlinkConfClass").replace("\r", "").split("\n");
+        return ResourceUtil.readUtf8Str("dinky-loader/FlinkConfClass")
+                .replace("\r", "")
+                .split("\n");
     }
 }
diff --git a/dinky-admin/src/main/resources/dinky-loader/ExpressionVariableClass b/dinky-admin/src/main/resources/dinky-loader/ExpressionVariableClass
@@ -0,0 +1,3 @@
+cn.hutool.core.date.DateUtil
+cn.hutool.core.util.IdUtil
+cn.hutool.core.util.RandomUtil
diff --git a/...y-admin/src/main/resources/FlinkConfClass → ...ain/resources/dinky-loader/FlinkConfClass b/...y-admin/src/main/resources/FlinkConfClass → ...ain/resources/dinky-loader/FlinkConfClass
diff --git a/dinky-assembly/src/main/assembly/package.xml b/dinky-assembly/src/main/assembly/package.xml
@@ -41,7 +41,15 @@
                 <include>**/*.yaml</include>
                 <include>**/log4j2.xml</include>
                 <include>**/DinkyFlinkDockerfile</include>
-                <include>**/FlinkConfClass</include>
+
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/dinky-admin/target/classes</directory>
+            <outputDirectory>dinky-loader</outputDirectory>
+            <includes>
+                <include>**/dinky-loader/FlinkConfClass</include>
+                <include>**/dinky-loader/ExpressionVariableClass</include>
             </includes>
         </fileSet>
         <fileSet>

diff --git a/dinky-core/src/main/java/org/dinky/executor/VariableManager.java b/dinky-core/src/main/java/org/dinky/executor/VariableManager.java
@@ -25,6 +25,8 @@
 
 import org.dinky.assertion.Asserts;
 import org.dinky.constant.FlinkSQLConstant;
+import org.dinky.data.exception.DinkyException;
+import org.dinky.utils.StringUtil;
 
 import org.apache.flink.table.api.DataTypes;
 import org.apache.flink.table.api.Table;
@@ -34,49 +36,71 @@
 import org.apache.flink.util.StringUtils;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.bean.BeanUtil;
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.io.resource.ResourceUtil;
 import cn.hutool.core.lang.Dict;
-import cn.hutool.core.util.IdUtil;
-import cn.hutool.core.util.RandomUtil;
 import cn.hutool.extra.expression.engine.jexl.JexlEngine;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * Flink Sql Variable Manager
  *
  * @since 2021/6/7 22:06
  */
+@Slf4j
 public final class VariableManager {
     public static final String VARIABLE = "variable";
     static final String SHOW_VARIABLES = "SHOW VARIABLES";
     private final Map<String, String> variables;
 
     public static final JexlEngine ENGINE = new JexlEngine();
 
+    public static final Dict ENGINE_CONTEXT = Dict.create();
+
     /**
-     * <p>
-     *      engine , key is variable name , value is class .
-     *      for example:
-     *          random -> RandomUtil -> about random operation
-     *          date -> DateUtil -> about date operation
-     *          id -> IdUtil  -> to generate random uuid
-     *          ...
+     * load expression variable class
      */
-    public static final Dict ENGINE_CONTEXT = Dict.create()
-            .set("random", RandomUtil.class)
-            .set("date", DateUtil.class)
-            .set("id", IdUtil.class);
+    private static void loadExpressionVariableClass() {
+        List<String> classLoaderVariableJexlClass = getClassLoaderVariableJexlClass();
+        if (CollUtil.isEmpty(classLoaderVariableJexlClass)) {
+            return;
+        }
+        classLoaderVariableJexlClass.forEach(fullClassName -> {
+            try {
+                String classSimpleName =
+                        BeanUtil.getBeanDesc(Class.forName(fullClassName)).getSimpleName();
+                String snakeCaseClassName = StringUtil.toSnakeCase(true, classSimpleName);
+                ENGINE_CONTEXT.set(snakeCaseClassName, Class.forName(fullClassName));
+                log.info("load class : {}", fullClassName);
+            } catch (ClassNotFoundException e) {
+                log.error(
+                        "The class [{}] that needs to be loaded may not be loaded by dinky or there is no jar file of this class under dinky's lib/plugins. Please check, and try again. {}",
+                        fullClassName,
+                        e.getMessage(),
+                        e);
+            }
+        });
+    }
 
     public VariableManager() {
         variables = new HashMap<>();
     }
 
+    public static List<String> getClassLoaderVariableJexlClass() {
+        return Arrays.asList(ResourceUtil.readUtf8Str("dinky-loader/ExpressionVariableClass")
+                .replace("\r", "")
+                .split("\n"));
+    }
+
     /**
      * Get names of sql variables loaded.
      *
@@ -140,15 +164,18 @@ public void unregisterVariable(String variableName, boolean ignoreIfNotExists) {
      */
     public Object getVariable(String variableName) {
         checkArgument(
-                !StringUtils.isNullOrWhitespaceOnly(variableName), "sql variableName name cannot be null or empty.");
+                !StringUtils.isNullOrWhitespaceOnly(variableName),
+                "sql variable name or jexl key cannot be null or empty.");
         try {
             if (variables.containsKey(variableName)) {
                 return variables.get(variableName);
             }
+            // load expression variable class
+            loadExpressionVariableClass();
             // use jexl to parse variable value
             return ENGINE.eval(variableName, ENGINE_CONTEXT, null);
         } catch (Exception e) {
-            throw new CatalogException(format("The variable of sql %s does not exist.", variableName));
+            throw new DinkyException(format("The variable name or jexl key of sql %s does not exist.", variableName));
         }
     }
 

diff --git a/dinky-core/src/main/java/org/dinky/utils/StringUtil.java b/dinky-core/src/main/java/org/dinky/utils/StringUtil.java
@@ -0,0 +1,72 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.utils;
+
+import cn.hutool.core.util.StrUtil;
+
+public class StringUtil {
+    /**
+     * 驼峰 -> 蛇形命名
+     *
+     * @param content      内容
+     * @param isFirstUpper 是否首字母大写
+     * @param isFirstLower 是否首字母小写
+     * @return 转换后的内容
+     */
+    public static String toSnakeCase(String content, boolean isFirstUpper, boolean isFirstLower) {
+        if (StrUtil.isEmpty(content)) {
+            return content;
+        }
+        StringBuilder sb = new StringBuilder(content.length());
+        for (int i = 0; i < content.length(); i++) {
+            char c = content.charAt(i);
+            if (isFirstUpper && i == 0) {
+                sb.append(Character.toUpperCase(c));
+            } else if (isFirstLower && i == 0) {
+                sb.append(Character.toLowerCase(c));
+            } else {
+                sb.append(c);
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     * 蛇形命名转换 -> 小驼峰
+     *
+     * @param content      内容
+     * @param isFirstUpper 是否首字母大写
+     * @return 转换后的内容
+     */
+    public static String toSnakeCase(String content, boolean isFirstUpper) {
+        return toSnakeCase(content, isFirstUpper, false);
+    }
+
+    /**
+     * 蛇形命名转换 -> 小驼峰
+     *
+     * @param isFirstLower 是否首字母小写
+     * @param content      内容
+     * @return 转换后的内容
+     */
+    public static String toSnakeCase(boolean isFirstLower, String content) {
+        return toSnakeCase(content, false, isFirstLower);
+    }
+}
diff --git a/docs/docs/administrator_guide/register_center/global_var.md b/docs/docs/administrator_guide/register_center/global_var.md
@@ -111,23 +111,24 @@ SHOW FRAGMENT var1;
 
 如您升级到 `1.0.0` 及以上版本, 请修改引用 `${_CURRENT_DATE_}` `${_CURRENT_TIMESTAMP_}` 变量的相关作业
 
-新的表达式变量无需声明, 已由程序启动时自动加载初始化 ,直接使用即可
+新的表达式变量无需声明 ,直接使用即可,
+请注意: 表达式变量调用时请使用小驼峰命名法
 :::
 
 ### 日期时间类表达式变量
 
-> 日期相关: `date` 为实例名用来调用方法`(必须固定)`,使用 [DateUtil](https://doc.hutool.cn/pages/DateUtil/)工具类实现,且支持此工具类的所有方法调用
+> 日期相关: `dateUtil` 为实例名用来调用方法`(必须固定)`,使用 [DateUtil](https://doc.hutool.cn/pages/DateUtil/)工具类实现,且支持此工具类的所有方法调用
 
 
 ```sql
 -- 获取当前秒
-select '${date.currentSeconds()}';
+select '${dateUtil.currentSeconds()}';
 
 # 获取日期 减去 10 天
-select '${date.offsetDay(date.date(), -10)}';
+select '${dateUtil.offsetDay(date.date(), -10)}';
 
 # 获取当前日期 标准格式 yyyy-MM-dd HH:mm:ss
-select '${date.now()}';
+select '${dateUtil.now()}';
 
 # etc .....
 
@@ -138,30 +139,30 @@ select '${date.now()}';
 
 ### 随机串相关表达式变量
 
-> `random` 为实例名用来调用方法`(必须固定)`, 使用 [RandomUtil](https://doc.hutool.cn/pages/RandomUtil/)工具类实现,且支持此工具类的所有方法调用
+> `randomUtil` 为实例名用来调用方法`(必须固定)`, 使用 [RandomUtil](https://doc.hutool.cn/pages/RandomUtil/)工具类实现,且支持此工具类的所有方法调用
 
 ```sql
 # 产生一个[10, 100)的随机数
-select '${random.randomInt(10, 100)}';
+select '${randomUtil.randomInt(10, 100)}';
 
 # 随机字符串(只包含数字和字符）
-select '${random.randomString()}';
+select '${randomUtil.randomString()}';
 
 # 获得一个只包含数字的字符串
-select '${random.randomNumbers()}';
+select '${randomUtil.randomNumbers()}';
 
 ```
 
 ### 唯一 ID 相关表达式变量
 
-> id 为实例名用来调用方法`(必须固定)` , 使用 [IdUtil](https://doc.hutool.cn/pages/IdUtil/) 工具类实现
+> idUtil 为实例名用来调用方法`(必须固定)` , 使用 [IdUtil](https://doc.hutool.cn/pages/IdUtil/) 工具类实现
 
 ```sql
 # 生成的UUID是带-的字符串，类似于：a5c8a5e8-df2b-4706-bea4-08d0939410e3
-select '${id.randomUUID()}';
+select '${idUtil.randomUUID()}';
 
 # 生成的是不带-的字符串，类似于：b17f24ff026d40949c85a24f4f375d42
-select '${id.simpleUUID()}';
+select '${idUtil.simpleUUID()}';
 ```
 
 :::tip 扩展