[Fix](statistics)Fix auto job start time incorrect bug (apache#27402)

Before, the auto analyze job start time was the job creation time, not the start to execute time, which is inaccurate. This pr is to change the start time to the first task start to execute time.
airborne12 · Nov 22, 2023 · 19c36dc · 19c36dc
1 parent 3e1a5b6
commit 19c36dc
Show file tree

Hide file tree

Showing 4 changed files with 67 additions and 2 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2662,7 +2662,7 @@ private void handleShowAnalyze() {
             }
             row.add(analysisInfo.scheduleType.toString());
             LocalDateTime startTime =
-                    LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.createTime),
+                    LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.startTime),
                             java.time.ZoneId.systemDefault());
             LocalDateTime endTime =
                     LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.endTime),

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -185,6 +185,9 @@ public enum ScheduleType {
     @SerializedName("createTime")
     public final long createTime = System.currentTimeMillis();
 
+    @SerializedName("startTime")
+    public long startTime;
+
     @SerializedName("endTime")
     public long endTime;
     /**
@@ -330,6 +333,10 @@ public static AnalysisInfo read(DataInput dataInput) throws IOException {
         return analysisInfo;
     }
 
+    public void markStartTime(long startTime) {
+        this.startTime = startTime;
+    }
+
     public void markFinished() {
         state = AnalysisState.FINISHED;
         endTime = System.currentTimeMillis();

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -158,6 +158,7 @@ public class AnalysisManager implements Writable {
             // Set the job state to RUNNING when its first task becomes RUNNING.
             if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) {
                 job.state = AnalysisState.RUNNING;
+                job.markStartTime(System.currentTimeMillis());
                 replayCreateAnalysisJob(job);
             }
             boolean allFinished = true;
@@ -200,6 +201,13 @@ public class AnalysisManager implements Writable {
         if (job == null) {
             return null;
         }
+        synchronized (job) {
+            // Set the job state to RUNNING when its first task becomes RUNNING.
+            if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) {
+                job.state = AnalysisState.RUNNING;
+                job.markStartTime(System.currentTimeMillis());
+            }
+        }
         int failedCount = 0;
         StringJoiner reason = new StringJoiner(", ");
         Map<Long, BaseAnalysisTask> taskMap = analysisJobIdToTaskMap.get(info.jobId);
@@ -1002,7 +1010,6 @@ public void logCreateTableStats(TableStatsMeta tableStats) {
     }
 
     public void registerSysJob(AnalysisInfo jobInfo, Map<Long, BaseAnalysisTask> taskInfos) {
-        jobInfo.state = AnalysisState.RUNNING;
         systemJobInfoMap.put(jobInfo.jobId, jobInfo);
         analysisJobIdToTaskMap.put(jobInfo.jobId, taskInfos);
     }

diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -322,6 +322,57 @@ protected void logAutoJob(AnalysisInfo autoJob) {
         Assertions.assertTrue(analysisManager.systemJobInfoMap.isEmpty());
     }
 
+    @Test
+    public void testSystemJobStartTime() {
+        new MockUp<BaseAnalysisTask>() {
+
+            @Mock
+            protected void init(AnalysisInfo info) {
+
+            }
+        };
+
+        new MockUp<AnalysisManager>() {
+            @Mock
+            public void updateTableStats(AnalysisInfo jobInfo) {
+            }
+
+            @Mock
+            protected void logAutoJob(AnalysisInfo autoJob) {
+
+            }
+        };
+
+        AnalysisManager analysisManager = new AnalysisManager();
+        AnalysisInfo job = new AnalysisInfoBuilder()
+            .setJobId(0)
+            .setColName("col1, col2").build();
+        analysisManager.systemJobInfoMap.put(job.jobId, job);
+        AnalysisInfo task1 = new AnalysisInfoBuilder()
+            .setJobId(0)
+            .setTaskId(1)
+            .setState(AnalysisState.PENDING)
+            .setColName("col1").build();
+        AnalysisInfo task2 = new AnalysisInfoBuilder()
+            .setJobId(0)
+            .setTaskId(1)
+            .setState(AnalysisState.PENDING)
+            .setColName("col2").build();
+        OlapAnalysisTask ot1 = new OlapAnalysisTask(task1);
+        OlapAnalysisTask ot2 = new OlapAnalysisTask(task2);
+        Map<Long, BaseAnalysisTask> taskMap = new HashMap<>();
+        taskMap.put(ot1.info.taskId, ot1);
+        taskMap.put(ot2.info.taskId, ot2);
+        analysisManager.analysisJobIdToTaskMap.put(job.jobId, taskMap);
+
+        job.state = AnalysisState.PENDING;
+        long l = System.currentTimeMillis();
+        analysisManager.systemJobInfoMap.put(job.jobId, job);
+        analysisManager.systemJobStatusUpdater.apply(new TaskStatusWrapper(task1,
+            AnalysisState.RUNNING, "", 0));
+        Assertions.assertTrue(job.startTime >= l);
+    }
+
     @Test
     public void testReAnalyze() {
         new MockUp<OlapTable>() {