diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java index 495311bc087d5b..d60006af709014 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java @@ -119,7 +119,7 @@ public long fetchRowCount() { params.put("tblName", name); switch (((JdbcExternalCatalog) catalog).getDatabaseTypeName()) { case JdbcResource.MYSQL: - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false, false)) { + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false)) { StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String sql = stringSubstitutor.replace(MYSQL_ROW_COUNT_SQL); StmtExecutor stmtExecutor = new StmtExecutor(r.connectContext, sql); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java index a2a094cbecacd5..acfd457d8a2108 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java @@ -126,7 +126,7 @@ protected void flushBuffer() { values.add(data.toSQL(true)); } insertStmt += values.toString(); - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false)) { stmtExecutor = new StmtExecutor(r.connectContext, insertStmt); executeWithExceptionOnFail(stmtExecutor); } catch (Exception t) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java index 91cddb333f4274..7d637d97e0638d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java @@ -481,7 +481,7 @@ protected String castToNumeric(String colName) { protected void runQuery(String sql) { long startTime = System.currentTimeMillis(); String queryId = ""; - try (AutoCloseConnectContext a = StatisticsUtil.buildConnectContext()) { + try (AutoCloseConnectContext a = StatisticsUtil.buildConnectContext(false)) { stmtExecutor = new StmtExecutor(a.connectContext, sql); ColStatsData colStatsData = new ColStatsData(stmtExecutor.executeInternalQuery().get(0)); // Update index row count after analyze. @@ -509,7 +509,7 @@ protected void runQuery(String sql) { } protected void runInsert(String sql) throws Exception { - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false)) { stmtExecutor = new StmtExecutor(r.connectContext, sql); try { stmtExecutor.execute(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index 5abf6120f01746..99a29c601dbbb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -32,7 +32,6 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.qe.AutoCloseConnectContext; import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.statistics.AnalysisInfo.JobType; import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.annotations.VisibleForTesting; @@ -113,63 +112,60 @@ protected void doSample() { String tabletStr = tabletIds.stream() .map(Object::toString) .collect(Collectors.joining(", ")); - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext( - info.jobType.equals(JobType.SYSTEM), false)) { - // Get basic stats, including min and max. - ResultRow basicStats = collectBasicStat(r); - String min = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 0 - ? basicStats.get(0) : null); - String max = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 1 - ? basicStats.get(1) : null); + // Get basic stats, including min and max. + ResultRow basicStats = collectBasicStat(); + String min = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 0 + ? basicStats.get(0) : null); + String max = StatisticsUtil.escapeSQL(basicStats != null && basicStats.getValues().size() > 1 + ? basicStats.get(1) : null); - boolean limitFlag = false; - long rowsToSample = pair.second; - Map params = buildSqlParams(); - params.put("scaleFactor", String.valueOf(scaleFactor)); - params.put("sampleHints", tabletStr.isEmpty() ? "" : String.format("TABLET(%s)", tabletStr)); - params.put("ndvFunction", getNdvFunction(String.valueOf(totalRowCount))); - params.put("min", StatisticsUtil.quote(min)); - params.put("max", StatisticsUtil.quote(max)); - params.put("rowCount", String.valueOf(totalRowCount)); - params.put("type", col.getType().toString()); - params.put("limit", ""); - if (needLimit()) { - // If the tablets to be sampled are too large, use limit to control the rows to read, and re-calculate - // the scaleFactor. - rowsToSample = Math.min(getSampleRows(), pair.second); - // Empty table doesn't need to limit. - if (rowsToSample > 0) { - limitFlag = true; - params.put("limit", "limit " + rowsToSample); - params.put("scaleFactor", String.valueOf(scaleFactor * (double) pair.second / rowsToSample)); - } + boolean limitFlag = false; + long rowsToSample = pair.second; + Map params = buildSqlParams(); + params.put("scaleFactor", String.valueOf(scaleFactor)); + params.put("sampleHints", tabletStr.isEmpty() ? "" : String.format("TABLET(%s)", tabletStr)); + params.put("ndvFunction", getNdvFunction(String.valueOf(totalRowCount))); + params.put("min", StatisticsUtil.quote(min)); + params.put("max", StatisticsUtil.quote(max)); + params.put("rowCount", String.valueOf(totalRowCount)); + params.put("type", col.getType().toString()); + params.put("limit", ""); + if (needLimit()) { + // If the tablets to be sampled are too large, use limit to control the rows to read, and re-calculate + // the scaleFactor. + rowsToSample = Math.min(getSampleRows(), pair.second); + // Empty table doesn't need to limit. + if (rowsToSample > 0) { + limitFlag = true; + params.put("limit", "limit " + rowsToSample); + params.put("scaleFactor", String.valueOf(scaleFactor * (double) pair.second / rowsToSample)); } - StringSubstitutor stringSubstitutor = new StringSubstitutor(params); - String sql; - if (useLinearAnalyzeTemplate()) { - // For single unique key, use count as ndv. - if (isSingleUniqueKey()) { - params.put("ndvFunction", String.valueOf(totalRowCount)); - } else { - params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})"); - } - sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE); + } + StringSubstitutor stringSubstitutor = new StringSubstitutor(params); + String sql; + if (useLinearAnalyzeTemplate()) { + // For single unique key, use count as ndv. + if (isSingleUniqueKey()) { + params.put("ndvFunction", String.valueOf(totalRowCount)); } else { - params.put("dataSizeFunction", getDataSizeFunction(col, true)); - params.put("subStringColName", getStringTypeColName(col)); - sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE); + params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})"); } - LOG.info("Sample for column [{}]. Total rows [{}], rows to sample [{}], scale factor [{}], " - + "limited [{}], distribute column [{}], partition column [{}], key column [{}], " - + "is single unique key [{}]", - col.getName(), params.get("rowCount"), rowsToSample, params.get("scaleFactor"), - limitFlag, tbl.isDistributionColumn(col.getName()), - tbl.isPartitionColumn(col.getName()), col.isKey(), isSingleUniqueKey()); - runQuery(sql); + sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE); + } else { + params.put("dataSizeFunction", getDataSizeFunction(col, true)); + params.put("subStringColName", getStringTypeColName(col)); + sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE); } + LOG.info("Sample for column [{}]. Total rows [{}], rows to sample [{}], scale factor [{}], " + + "limited [{}], distribute column [{}], partition column [{}], key column [{}], " + + "is single unique key [{}]", + col.getName(), params.get("rowCount"), rowsToSample, params.get("scaleFactor"), + limitFlag, tbl.isDistributionColumn(col.getName()), + tbl.isPartitionColumn(col.getName()), col.isKey(), isSingleUniqueKey()); + runQuery(sql); } - protected ResultRow collectBasicStat(AutoCloseConnectContext context) { + protected ResultRow collectBasicStat() { // Agg table value columns has no zone map. // For these columns, skip collecting min and max value to avoid scan whole table. if (((OlapTable) tbl).getKeysType().equals(KeysType.AGG_KEYS) && !col.isKey()) { @@ -181,14 +177,17 @@ protected ResultRow collectBasicStat(AutoCloseConnectContext context) { Map params = buildSqlParams(); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String sql = stringSubstitutor.replace(BASIC_STATS_TEMPLATE); - stmtExecutor = new StmtExecutor(context.connectContext, sql); - ResultRow resultRow = stmtExecutor.executeInternalQuery().get(0); - if (LOG.isDebugEnabled()) { - LOG.debug("Cost time in millisec: " + (System.currentTimeMillis() - startTime) - + " Min max SQL: " + sql + " QueryId: " + DebugUtil.printId(stmtExecutor.getContext().queryId())); + ResultRow resultRow = null; + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false)) { + stmtExecutor = new StmtExecutor(r.connectContext, sql); + resultRow = stmtExecutor.executeInternalQuery().get(0); + if (LOG.isDebugEnabled()) { + LOG.debug("Cost time in millisec: " + (System.currentTimeMillis() - startTime) + " Min max SQL: " + + sql + " QueryId: " + DebugUtil.printId(stmtExecutor.getContext().queryId())); + } + // Release the reference to stmtExecutor, reduce memory usage. + stmtExecutor = null; } - // Release the reference to stmtExecutor, reduce memory usage. - stmtExecutor = null; return resultRow; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 00df73c75707a8..d51281eb0e667c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -147,7 +147,7 @@ public static List execStatisticQuery(String sql, boolean enableFileC return Collections.emptyList(); } boolean useFileCacheForStat = (enableFileCache && Config.allow_analyze_statistics_info_polluting_file_cache); - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false, useFileCacheForStat)) { + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(useFileCacheForStat)) { if (Config.isCloudMode()) { try { r.connectContext.getCloudCluster(); @@ -164,7 +164,7 @@ public static List execStatisticQuery(String sql, boolean enableFileC public static QueryState execUpdate(String sql) throws Exception { StmtExecutor stmtExecutor = null; - AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(); + AutoCloseConnectContext r = StatisticsUtil.buildConnectContext(false); try { stmtExecutor = new StmtExecutor(r.connectContext, sql); r.connectContext.setExecutor(stmtExecutor); @@ -202,11 +202,7 @@ public static PartitionColumnStatistic deserializeToPartitionStatistics(List