[branch-2.0](timezone) make TimeUtils formatter use correct time_zone (…

…#37465) (#37653)
apache · Aug 1, 2024 · c3be8ec · c3be8ec
1 parent f49a731
commit c3be8ec
Show file tree

Hide file tree

Showing 57 changed files with 213 additions and 146 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -648,7 +648,7 @@ public static TQueryGlobals createQueryGlobals() {
         Calendar currentDate = Calendar.getInstance();
         LocalDateTime localDateTime = LocalDateTime.ofInstant(currentDate.toInstant(),
                 currentDate.getTimeZone().toZoneId());
-        String nowStr = localDateTime.format(TimeUtils.DATETIME_NS_FORMAT);
+        String nowStr = localDateTime.format(TimeUtils.getDatetimeNsFormatWithTimeZone());
         queryGlobals.setNowString(nowStr);
         queryGlobals.setNanoSeconds(LocalDateTime.now().getNano());
         return queryGlobals;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -452,7 +452,7 @@ private void restore(Repository repository, Database db, RestoreStmt stmt) throw
                 throw new DdlException("read backup meta failed", e);
             }
             String backupTimestamp = TimeUtils.longToTimeString(
-                    jobInfo.getBackupTime(), TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
+                    jobInfo.getBackupTime(), TimeUtils.getDatetimeFormatWithHyphenWithTimeZone());
             restoreJob = new RestoreJob(stmt.getLabel(), backupTimestamp,
                     db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(),
                     stmt.getTimeoutMs(), metaVersion, stmt.reserveReplica(),

diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -673,7 +673,8 @@ private void waitingAllUploadingFinished() {
     }
 
     private void saveMetaInfo() {
-        String createTimeStr = TimeUtils.longToTimeString(createTime, TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
+        String createTimeStr = TimeUtils.longToTimeString(createTime,
+                TimeUtils.getDatetimeFormatWithHyphenWithTimeZone());
         // local job dir: backup/repo__repo_id/label__createtime/
         // Add repo_id to isolate jobs from different repos.
         localJobDirPath = Paths.get(BackupHandler.BACKUP_ROOT_DIR.toString(),

diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
@@ -141,7 +141,7 @@ private static String jobInfoFileNameWithTimestamp(long createTime) {
             return PREFIX_JOB_INFO;
         } else {
             return PREFIX_JOB_INFO
-                    + TimeUtils.longToTimeString(createTime, TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
+                    + TimeUtils.longToTimeString(createTime, TimeUtils.getDatetimeFormatWithHyphenWithTimeZone());
         }
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -400,9 +400,9 @@ private static void checkStorageMedium(String storageMedium) throws DdlException
 
     private static DateTimeFormatter getDateTimeFormatter(String timeUnit) {
         if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
-            return TimeUtils.DATETIME_FORMAT;
+            return TimeUtils.getDatetimeFormatWithTimeZone();
         } else {
-            return TimeUtils.DATE_FORMAT;
+            return TimeUtils.getDateFormatWithTimeZone();
         }
     }
 
@@ -806,9 +806,9 @@ public static String getHistoryPartitionRangeString(DynamicPartitionProperty dyn
 
     private static LocalDateTime getDateTimeByTimeUnit(String time, String timeUnit) {
         if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
-            return LocalDateTime.parse(time, TimeUtils.DATETIME_FORMAT);
+            return LocalDateTime.parse(time, TimeUtils.getDatetimeFormatWithTimeZone());
         } else {
-            return LocalDate.from(TimeUtils.DATE_FORMAT.parse(time)).atStartOfDay();
+            return LocalDate.from(TimeUtils.getDateFormatWithTimeZone().parse(time)).atStartOfDay();
         }
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
@@ -29,7 +29,6 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -47,6 +46,7 @@
 import java.util.Date;
 import java.util.Map;
 import java.util.TimeZone;
+import java.util.TreeMap;
 import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -55,13 +55,7 @@
 public class TimeUtils {
     public static final String UTC_TIME_ZONE = "Europe/London"; // This is just a Country to represent UTC offset +00:00
     public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai";
-    public static final ZoneId TIME_ZONE;
     public static final ImmutableMap<String, String> timeZoneAliasMap;
-    // NOTICE: Date formats are not synchronized.
-    // it must be used as synchronized externally.
-    public static final DateTimeFormatter DATE_FORMAT;
-    public static final DateTimeFormatter DATETIME_FORMAT;
-    public static final DateTimeFormatter TIME_FORMAT;
     public static final Pattern DATETIME_FORMAT_REG =
             Pattern.compile("^((\\d{2}(([02468][048])|([13579][26]))[\\-\\/\\s]?((((0?[13578])|(1[02]))[\\-\\/\\s]?"
                     + "((0?[1-9])|([1-2][0-9])|(3[01])))|(((0?[469])|(11))[\\-\\/\\s]?"
@@ -70,25 +64,49 @@ public class TimeUtils {
                     + "[\\-\\/\\s]?((0?[1-9])|([1-2][0-9])|(3[01])))|(((0?[469])|(11))[\\-\\/\\s]?"
                     + "((0?[1-9])|([1-2][0-9])|(30)))|(0?2[\\-\\/\\s]?((0?[1-9])|(1[0-9])|(2[0-8]))))))"
                     + "(\\s(((0?[0-9])|([1][0-9])|([2][0-3]))\\:([0-5]?[0-9])((\\s)|(\\:([0-5]?[0-9])))))?$");
-    public static final DateTimeFormatter DATETIME_MS_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
-            .withZone(ZoneId.systemDefault());
-    public static final DateTimeFormatter DATETIME_NS_FORMAT = DateTimeFormatter.ofPattern(
-                    "yyyy-MM-dd HH:mm:ss.SSSSSSSSS")
-            .withZone(ZoneId.systemDefault());
-    public static final DateTimeFormatter DATETIME_FORMAT_WITH_HYPHEN = DateTimeFormatter.ofPattern(
-                    "yyyy-MM-dd-HH-mm-ss")
-            .withZone(ZoneId.systemDefault());
+
+    // these formatters must be visited by getter to make sure they have right
+    // timezone.
+    // NOTICE: Date formats are not synchronized.
+    // it must be used as synchronized externally.
+    private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+    private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+    private static final DateTimeFormatter TIME_FORMAT = DateTimeFormatter.ofPattern("HH");
+    private static final DateTimeFormatter DATETIME_MS_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
+    private static final DateTimeFormatter DATETIME_NS_FORMAT = DateTimeFormatter.ofPattern(
+            "yyyy-MM-dd HH:mm:ss.SSSSSSSSS");
+    private static final DateTimeFormatter DATETIME_FORMAT_WITH_HYPHEN = DateTimeFormatter.ofPattern(
+            "yyyy-MM-dd-HH-mm-ss");
+
+    public static DateTimeFormatter getDateFormatWithTimeZone() {
+        return DATE_FORMAT.withZone(getDorisZoneId());
+    }
+
+    public static DateTimeFormatter getDatetimeFormatWithTimeZone() {
+        return DATETIME_FORMAT.withZone(getDorisZoneId());
+    }
+
+    public static DateTimeFormatter getTimeFormatWithTimeZone() {
+        return TIME_FORMAT.withZone(getDorisZoneId());
+    }
+
+    public static DateTimeFormatter getDatetimeMsFormatWithTimeZone() {
+        return DATETIME_MS_FORMAT.withZone(getDorisZoneId());
+    }
+
+    public static DateTimeFormatter getDatetimeNsFormatWithTimeZone() {
+        return DATETIME_NS_FORMAT.withZone(getDorisZoneId());
+    }
+
+    public static DateTimeFormatter getDatetimeFormatWithHyphenWithTimeZone() {
+        return DATETIME_FORMAT_WITH_HYPHEN.withZone(getDorisZoneId());
+    }
+
     private static final Logger LOG = LogManager.getLogger(TimeUtils.class);
     private static final Pattern TIMEZONE_OFFSET_FORMAT_REG = Pattern.compile("^[+-]?\\d{1,2}:\\d{2}$");
-    public static Date MIN_DATE = null;
-    public static Date MAX_DATE = null;
-    public static Date MIN_DATETIME = null;
-    public static Date MAX_DATETIME = null;
 
     static {
-        TIME_ZONE = ZoneId.of("UTC+8");
-
-        Map<String, String> timeZoneMap = Maps.newHashMap();
+        Map<String, String> timeZoneMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
         timeZoneMap.putAll(ZoneId.SHORT_IDS);
 
         // set CST to +08:00 instead of America/Chicago
@@ -98,32 +116,6 @@ public class TimeUtils {
         timeZoneMap.put("GMT", UTC_TIME_ZONE);
 
         timeZoneAliasMap = ImmutableMap.copyOf(timeZoneMap);
-
-        DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd");
-        DATE_FORMAT.withZone(TIME_ZONE);
-
-        DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
-        DATETIME_FORMAT.withZone(TIME_ZONE);
-
-        TIME_FORMAT = DateTimeFormatter.ofPattern("HH");
-        TIME_FORMAT.withZone(TIME_ZONE);
-
-        try {
-
-            MIN_DATE = Date.from(
-                    LocalDate.parse("0001-01-01", DATE_FORMAT).atStartOfDay().atZone(TIME_ZONE).toInstant());
-            MAX_DATE = Date.from(
-                    LocalDate.parse("9999-12-31", DATE_FORMAT).atStartOfDay().atZone(TIME_ZONE).toInstant());
-
-            MIN_DATETIME = Date.from(
-                    LocalDateTime.parse("0001-01-01 00:00:00", DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
-            MAX_DATETIME = Date.from(
-                    LocalDateTime.parse("9999-12-31 23:59:59", DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
-
-        } catch (DateTimeParseException e) {
-            LOG.error("invalid date format", e);
-            System.exit(-1);
-        }
     }
 
     public static long getStartTimeMs() {
@@ -135,7 +127,7 @@ public static long getElapsedTimeMs(long startTime) {
     }
 
     public static String getCurrentFormatTime() {
-        return LocalDateTime.now().format(DATETIME_FORMAT);
+        return LocalDateTime.now().format(getDatetimeFormatWithTimeZone());
     }
 
     public static TimeZone getTimeZone() {
@@ -148,9 +140,13 @@ public static TimeZone getTimeZone() {
         return TimeZone.getTimeZone(ZoneId.of(timezone, timeZoneAliasMap));
     }
 
+    public static ZoneId getDorisZoneId() {
+        return getTimeZone().toZoneId();
+    }
+
     // return the time zone of current system
     public static TimeZone getSystemTimeZone() {
-        return TimeZone.getTimeZone(ZoneId.of(ZoneId.systemDefault().getId(), timeZoneAliasMap));
+        return TimeZone.getTimeZone(ZoneId.of(TimeZone.getDefault().getID(), timeZoneAliasMap));
     }
 
     // get time zone of given zone name, or return system time zone if name is null.
@@ -165,7 +161,7 @@ public static String longToTimeString(long timeStamp, DateTimeFormatter dateForm
         if (timeStamp <= 0L) {
             return FeConstants.null_string;
         }
-        return dateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeStamp), ZoneId.systemDefault()));
+        return dateFormat.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeStamp), getDorisZoneId()));
     }
 
     public static String longToTimeStringWithFormat(long timeStamp, DateTimeFormatter datetimeFormatTimeZone) {
@@ -174,12 +170,12 @@ public static String longToTimeStringWithFormat(long timeStamp, DateTimeFormatte
         return longToTimeString(timeStamp, datetimeFormatTimeZone);
     }
 
-    public static String longToTimeString(long timeStamp) {
-        return longToTimeStringWithFormat(timeStamp, DATETIME_FORMAT);
+    public static String longToTimeString(Long timeStamp) {
+        return longToTimeStringWithFormat(timeStamp, getDatetimeFormatWithTimeZone());
     }
 
-    public static String longToTimeStringWithms(long timeStamp) {
-        return longToTimeStringWithFormat(timeStamp, DATETIME_MS_FORMAT);
+    public static String longToTimeStringWithms(Long timeStamp) {
+        return longToTimeStringWithFormat(timeStamp, getDatetimeMsFormatWithTimeZone());
     }
 
     public static Date getHourAsDate(String hour) {
@@ -189,7 +185,8 @@ public static Date getHourAsDate(String hour) {
         }
         try {
             return Date.from(
-                    LocalTime.parse(fullHour, TIME_FORMAT).atDate(LocalDate.now()).atZone(TIME_ZONE).toInstant());
+                    LocalTime.parse(fullHour, getTimeFormatWithTimeZone()).atDate(LocalDate.now())
+                            .atZone(getDorisZoneId()).toInstant());
         } catch (DateTimeParseException e) {
             LOG.warn("invalid time format: {}", fullHour);
             return null;
@@ -206,13 +203,15 @@ public static Date parseDate(String dateStr, PrimitiveType type) throws Analysis
         if (type == PrimitiveType.DATE) {
             ParsePosition pos = new ParsePosition(0);
             date = Date.from(
-                    LocalDate.from(DATE_FORMAT.parse(dateStr, pos)).atStartOfDay().atZone(TIME_ZONE).toInstant());
+                    LocalDate.from(getDateFormatWithTimeZone().parse(dateStr, pos)).atStartOfDay()
+                            .atZone(getDorisZoneId()).toInstant());
             if (pos.getIndex() != dateStr.length() || date == null) {
                 throw new AnalysisException("Invalid date string: " + dateStr);
             }
         } else if (type == PrimitiveType.DATETIME) {
             try {
-                date = Date.from(LocalDateTime.parse(dateStr, DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
+                date = Date.from(LocalDateTime.parse(dateStr, getDatetimeFormatWithTimeZone())
+                        .atZone(getDorisZoneId()).toInstant());
             } catch (DateTimeParseException e) {
                 throw new AnalysisException("Invalid date string: " + dateStr);
             }
@@ -229,9 +228,11 @@ public static Date parseDate(String dateStr, Type type) throws AnalysisException
 
     public static String format(Date date, PrimitiveType type) {
         if (type == PrimitiveType.DATE) {
-            return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()).format(DATE_FORMAT);
+            return LocalDateTime.ofInstant(date.toInstant(), getDorisZoneId())
+                    .format(getDateFormatWithTimeZone());
         } else if (type == PrimitiveType.DATETIME) {
-            return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()).format(DATETIME_FORMAT);
+            return LocalDateTime.ofInstant(date.toInstant(), getDorisZoneId())
+                    .format(getDatetimeFormatWithTimeZone());
         } else {
             return "INVALID";
         }
@@ -244,15 +245,16 @@ public static String format(Date date, Type type) {
     public static long timeStringToLong(String timeStr) {
         Date d;
         try {
-            d = Date.from(LocalDateTime.parse(timeStr, DATETIME_FORMAT).atZone(TIME_ZONE).toInstant());
+            d = Date.from(LocalDateTime.parse(timeStr, getDatetimeFormatWithTimeZone())
+                    .atZone(getDorisZoneId()).toInstant());
         } catch (DateTimeParseException e) {
             return -1;
         }
         return d.getTime();
     }
 
     public static long timeStringToLong(String timeStr, TimeZone timeZone) {
-        DateTimeFormatter dateFormatTimeZone = DATETIME_FORMAT;
+        DateTimeFormatter dateFormatTimeZone = getDatetimeFormatWithTimeZone();
         dateFormatTimeZone.withZone(timeZone.toZoneId());
         LocalDateTime d;
         try {

diff --git a/...e-core/src/main/java/org/apache/doris/external/iceberg/IcebergTableCreationRecordMgr.java b/...e-core/src/main/java/org/apache/doris/external/iceberg/IcebergTableCreationRecordMgr.java
@@ -132,7 +132,7 @@ private void removeDuplicateTables() {
     @Override
     protected void runAfterCatalogReady() {
         PropertySchema.DateProperty prop =
-                new PropertySchema.DateProperty("key", TimeUtils.DATETIME_FORMAT);
+                new PropertySchema.DateProperty("key", TimeUtils.getDatetimeFormatWithTimeZone());
         // list iceberg tables in dbs
         // When listing table is done, remove database from icebergDbs.
         for (Iterator<Map.Entry<Long, Database>> it = icebergDbs.entrySet().iterator(); it.hasNext(); it.remove()) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java
@@ -249,9 +249,9 @@ protected void runAfterCatalogReady() {
                 for (Map.Entry<String, TStreamLoadRecord> entry : streamLoadRecordBatch.entrySet()) {
                     TStreamLoadRecord streamLoadItem = entry.getValue();
                     String startTime = TimeUtils.longToTimeString(streamLoadItem.getStartTime(),
-                            TimeUtils.DATETIME_MS_FORMAT);
+                            TimeUtils.getDatetimeMsFormatWithTimeZone());
                     String finishTime = TimeUtils.longToTimeString(streamLoadItem.getFinishTime(),
-                            TimeUtils.DATETIME_MS_FORMAT);
+                            TimeUtils.getDatetimeMsFormatWithTimeZone());
                     if (LOG.isDebugEnabled()) {
                         LOG.debug("receive stream load record info from backend: {}."
                                         + " label: {}, db: {}, tbl: {}, user: {}, user_ip: {},"