diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index dd5d68dfbf09fc..399975f9791103 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -199,10 +199,8 @@ DEFINE_mInt32(download_low_speed_limit_kbps, "50"); // download low speed time(seconds) DEFINE_mInt32(download_low_speed_time, "300"); -// log dir -DEFINE_String(sys_log_dir, "${DORIS_HOME}/log"); +DEFINE_String(sys_log_dir, ""); DEFINE_String(user_function_dir, "${DORIS_HOME}/lib/udf"); -DEFINE_String(pipeline_tracing_log_dir, "${DORIS_HOME}/log/tracing"); // INFO, WARNING, ERROR, FATAL DEFINE_mString(sys_log_level, "INFO"); // TIME-DAY, TIME-HOUR, SIZE-MB-nnn diff --git a/be/src/common/config.h b/be/src/common/config.h index d8c77bf94964d3..59e7def3103cfc 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -247,10 +247,10 @@ DECLARE_mInt32(download_low_speed_limit_kbps); // download low speed time(seconds) DECLARE_mInt32(download_low_speed_time); -// log dir +// deprecated, use env var LOG_DIR in be.conf DECLARE_String(sys_log_dir); +// for udf DECLARE_String(user_function_dir); -DECLARE_String(pipeline_tracing_log_dir); // INFO, WARNING, ERROR, FATAL DECLARE_String(sys_log_level); // TIME-DAY, TIME-HOUR, SIZE-MB-nnn diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp index bae037066aea82..bf408272776984 100644 --- a/be/src/common/logconfig.cpp +++ b/be/src/common/logconfig.cpp @@ -63,7 +63,12 @@ bool init_glog(const char* basename) { // so fatal log can output to be.out . FLAGS_stderrthreshold = google::FATAL; // set glog log dir - FLAGS_log_dir = config::sys_log_dir; + // ATTN: sys_log_dir is deprecated, this is just for compatibility + std::string log_dir = config::sys_log_dir; + if (log_dir == "") { + log_dir = getenv("LOG_DIR"); + } + FLAGS_log_dir = log_dir; // 0 means buffer INFO only FLAGS_logbuflevel = 0; // buffer log messages for at most this many seconds diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 09e128d933bbfe..19b4d03c346d66 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -489,7 +489,7 @@ Status Compaction::do_compaction_impl(int64_t permits) { const std::string& file_name) { io::FileWriterPtr file_writer; std::string file_path = - fmt::format("{}/{}.json", config::sys_log_dir, file_name); + fmt::format("{}/{}.json", std::string(getenv("LOG_DIR")), file_name); RETURN_IF_ERROR( io::global_local_filesystem()->create_file(file_path, &file_writer)); RETURN_IF_ERROR(file_writer->append(json_obj.dump())); diff --git a/be/src/pipeline/pipeline_tracing.cpp b/be/src/pipeline/pipeline_tracing.cpp index f635f49682fca4..efe9667f2b57dc 100644 --- a/be/src/pipeline/pipeline_tracing.cpp +++ b/be/src/pipeline/pipeline_tracing.cpp @@ -95,10 +95,11 @@ void PipelineTracerContext::_dump(TUniqueId query_id) { return; } + std::filesystem::path log_dir = fmt::format("{}/pipe_tracing", getenv("LOG_DIR")); //TODO: when dump, now could append records but can't add new query. try use better grained locks. std::unique_lock l(_data_lock); // can't rehash if (_dump_type == RecordType::PerQuery) { - auto path = _dir / fmt::format("query{}", to_string(query_id)); + auto path = log_dir / fmt::format("query{}", to_string(query_id)); int fd = ::open( path.c_str(), O_CREAT | O_WRONLY | O_TRUNC, S_ISGID | S_ISUID | S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP | S_IWOTH | S_IROTH); @@ -123,8 +124,9 @@ void PipelineTracerContext::_dump(TUniqueId query_id) { THROW_IF_ERROR(writer.finalize()); THROW_IF_ERROR(writer.close()); } else if (_dump_type == RecordType::Periodic) { - auto path = _dir / fmt::format("until{}", - std::chrono::steady_clock::now().time_since_epoch().count()); + auto path = + log_dir / + fmt::format("until{}", std::chrono::steady_clock::now().time_since_epoch().count()); int fd = ::open( path.c_str(), O_CREAT | O_WRONLY | O_TRUNC, S_ISGID | S_ISUID | S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP | S_IWOTH | S_IROTH); diff --git a/be/src/pipeline/pipeline_tracing.h b/be/src/pipeline/pipeline_tracing.h index adebd84b05fc6c..e4e4d4e4e7d2ec 100644 --- a/be/src/pipeline/pipeline_tracing.h +++ b/be/src/pipeline/pipeline_tracing.h @@ -52,6 +52,7 @@ struct ScheduleRecord { using OneQueryTraces = moodycamel::ConcurrentQueue; // belongs to exec_env, for all query, if enable +// curl http://{host}:{web_server_port}/api/running_pipeline_tasks class PipelineTracerContext { public: enum class RecordType { @@ -75,7 +76,6 @@ class PipelineTracerContext { phmap::flat_hash_map _id_to_workload_group; RecordType _dump_type = RecordType::None; - std::filesystem::path _dir = config::pipeline_tracing_log_dir; decltype(MonotonicSeconds()) _last_dump_time; decltype(MonotonicSeconds()) _dump_interval_s = 60; // effective iff Periodic mode. 1 minute default. diff --git a/bin/start_be.sh b/bin/start_be.sh index a551d295b4c1b7..06679eb1b375d5 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -208,6 +208,8 @@ if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then source "${DORIS_HOME}/bin/palo_env.sh" fi +export PPROF_TMPDIR="${LOG_DIR}" + if [[ -z "${JAVA_HOME}" ]]; then echo "The JAVA_HOME environment variable is not defined correctly" echo "This environment variable is needed to run this program" diff --git a/conf/be.conf b/conf/be.conf index 4d8677aece7096..cd66596501f21a 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -17,15 +17,16 @@ CUR_DATE=`date +%Y%m%d-%H%M%S` -PPROF_TMPDIR="$DORIS_HOME/log/" +# Log dir +LOG_DIR="${DORIS_HOME}/log/" -JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives" +JAVA_OPTS="-Xmx1024m -DlogPath=$LOG_DIR/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives" # For jdk 9+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives" +JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$LOG_DIR/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives" # For jdk 17+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives --add-opens=java.base/java.net=ALL-UNNAMED" +JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$LOG_DIR/jni.log -Xlog:gc:$LOG_DIR/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives --add-opens=java.base/java.net=ALL-UNNAMED" # since 1.2, the JAVA_HOME need to be set to run BE process. # JAVA_HOME=/path/to/jdk/ @@ -35,9 +36,6 @@ JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORI JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false" JEMALLOC_PROF_PRFIX="" -# INFO, WARNING, ERROR, FATAL -sys_log_level = INFO - # ports for admin, web, heartbeat service be_port = 9060 webserver_port = 8040 @@ -77,7 +75,8 @@ ssl_private_key_path = "$DORIS_HOME/conf/key.pem" # jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers # Advanced configurations -# sys_log_dir = ${DORIS_HOME}/log +# INFO, WARNING, ERROR, FATAL +sys_log_level = INFO # sys_log_roll_mode = SIZE-MB-1024 # sys_log_roll_num = 10 # sys_log_verbose_modules = * diff --git a/conf/fe.conf b/conf/fe.conf index 66794ddf3fa7ad..845dce87cdb6b4 100644 --- a/conf/fe.conf +++ b/conf/fe.conf @@ -23,34 +23,28 @@ CUR_DATE=`date +%Y%m%d-%H%M%S` -# the output dir of stderr and stdout +# Log dir LOG_DIR = ${DORIS_HOME}/log # CMS JAVA OPTS # JAVA_OPTS="-Dsun.security.krb5.debug=true -Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE" # G1 JAVA OPTS -JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE -Dlog4j2.formatMsgNoLookups=true" +JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:$LOG_DIR/fe.gc.log.$CUR_DATE -Dlog4j2.formatMsgNoLookups=true" # For jdk 9+, this JAVA_OPTS_FOR_JDK_9 will be used as default CMS JVM options # JAVA_OPTS_FOR_JDK_9="-Dsun.security.krb5.debug=true -Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time" # For jdk 9+, this JAVA_OPTS_FOR_JDK_9 will be used as default G1 JVM options -JAVA_OPTS_FOR_JDK_9="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time -Dlog4j2.formatMsgNoLookups=true" +JAVA_OPTS_FOR_JDK_9="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time -Dlog4j2.formatMsgNoLookups=true" # For jdk 17+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false -XX:+UseZGC -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$DORIS_HOME/log/ -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time" +JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false -XX:+UseZGC -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR/ -Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time" ## ## the lowercase properties are read by main program. ## -# INFO, WARN, ERROR, FATAL -sys_log_level = INFO - -# NORMAL, BRIEF, ASYNC -sys_log_mode = NORMAL - # store metadata, must be created before start FE. # Default value is ${DORIS_HOME}/doris-meta # meta_dir = ${DORIS_HOME}/doris-meta @@ -73,10 +67,13 @@ arrow_flight_sql_port = -1 # Advanced configurations # log_roll_size_mb = 1024 -# sys_log_dir = ${DORIS_HOME}/log +# INFO, WARN, ERROR, FATAL +sys_log_level = INFO +# NORMAL, BRIEF, ASYNC +sys_log_mode = NORMAL # sys_log_roll_num = 10 # sys_log_verbose_modules = org.apache.doris -# audit_log_dir = ${DORIS_HOME}/log +# audit_log_dir = $LOG_DIR # audit_log_modules = slow_query, query # audit_log_roll_num = 10 # meta_delay_toleration_second = 10 diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 7fc14ecff4c414..9da0e0a45ea4e5 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -62,9 +62,10 @@ public class Config extends ConfigBase { * sys_log_enable_compress: * default is false. if true, will compress fe.log & fe.warn.log by gzip */ + @Deprecated // use env var LOG_DIR instead @ConfField(description = {"FE 日志文件的存放路径,用于存放 fe.log。", "The path of the FE log file, used to store fe.log"}) - public static String sys_log_dir = System.getenv("DORIS_HOME") + "/log"; + public static String sys_log_dir = ""; @ConfField(description = {"FE 日志的级别", "The level of FE log"}, options = {"INFO", "WARN", "ERROR", "FATAL"}) public static String sys_log_level = "INFO"; @@ -101,7 +102,7 @@ public class Config extends ConfigBase { @ConfField(description = {"FE 审计日志文件的存放路径,用于存放 fe.audit.log。", "The path of the FE audit log file, used to store fe.audit.log"}) - public static String audit_log_dir = System.getenv("DORIS_HOME") + "/log"; + public static String audit_log_dir = System.getenv("LOG_DIR"); @ConfField(description = {"FE 审计日志文件的最大数量。超过这个数量后,最老的日志文件会被删除", "The maximum number of FE audit log files. " + "After exceeding this number, the oldest log file will be deleted"}) @@ -570,7 +571,7 @@ public class Config extends ConfigBase { public static String spark_resource_path = ""; @ConfField(description = {"Spark launcher 日志路径", "Spark launcher log dir"}) - public static String spark_launcher_log_dir = sys_log_dir + "/spark_launcher_log"; + public static String spark_launcher_log_dir = System.getenv("LOG_DIR") + "/spark_launcher_log"; @ConfField(description = {"Yarn client 的路径", "Yarn client path"}) public static String yarn_client_path = System.getenv("DORIS_HOME") + "/lib/yarn-client/hadoop/bin/yarn"; @@ -2468,7 +2469,7 @@ public class Config extends ConfigBase { @ConfField(description = {"nereids trace文件的存放路径。", "The path of the nereids trace file."}) - public static String nereids_trace_log_dir = System.getenv("DORIS_HOME") + "/log/nereids_trace"; + public static String nereids_trace_log_dir = System.getenv("LOG_DIR") + "/nereids_trace"; @ConfField(mutable = true, masterOnly = true, description = { "备份过程中,分配给每个be的upload任务最大个数,默认值为3个。", diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/io/DiskUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/DiskUtils.java index 4861f40fe8a67c..9e0a449c2ac386 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/io/DiskUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/DiskUtils.java @@ -44,7 +44,6 @@ public static Df df(String dir) { return df; } - Process process; try { process = Runtime.getRuntime().exec("df -k " + dir); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 47d96765a9c30c..5178a2bcb8ed1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -19,6 +19,7 @@ import org.apache.doris.httpv2.config.SpringLog4j2Config; +import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; @@ -130,7 +131,9 @@ private static void reconfig() throws IOException { String newXmlConfTemplate = xmlConfTemplate; // sys log config - String sysLogDir = Config.sys_log_dir; + // ATTN, sys_log_dir is deprecated, use LOG_DIR instead + String sysLogDir = Strings.isNullOrEmpty(Config.sys_log_dir) ? System.getenv("LOG_DIR") : + Config.sys_log_dir; String sysRollNum = String.valueOf(Config.sys_log_roll_num); String sysDeleteAge = String.valueOf(Config.sys_log_delete_age); boolean compressSysLog = Config.sys_log_enable_compress; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java index 35656f5c4fb06b..d9d3dadfdf9bf8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java @@ -109,7 +109,8 @@ private void appendLogConf(Map> content, String addV private void appendLogInfo(Map> content) { Map map = new HashMap<>(); - final String logPath = Config.sys_log_dir + "/fe.warn.log"; + String logDir = Strings.isNullOrEmpty(Config.sys_log_dir) ? System.getenv("LOG_DIR") : Config.sys_log_dir; + final String logPath = logDir + "/fe.warn.log"; map.put("logPath", logPath); RandomAccessFile raf = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/ExecuteEnv.java b/fe/fe-core/src/main/java/org/apache/doris/service/ExecuteEnv.java index ed1ae82a0325a8..2064bfb267fe39 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/service/ExecuteEnv.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/ExecuteEnv.java @@ -22,6 +22,8 @@ import org.apache.doris.qe.ConnectScheduler; import org.apache.doris.qe.MultiLoadMgr; +import com.google.common.base.Strings; + import java.util.ArrayList; import java.util.List; @@ -40,9 +42,11 @@ private ExecuteEnv() { scheduler = new ConnectScheduler(Config.qe_max_connection); startupTime = System.currentTimeMillis(); processUUID = System.currentTimeMillis(); + String logDir = Strings.isNullOrEmpty(Config.sys_log_dir) ? System.getenv("LOG_DIR") : + Config.sys_log_dir; diskInfos = new ArrayList() {{ add(new FeDiskInfo("meta", Config.meta_dir, DiskUtils.df(Config.meta_dir))); - add(new FeDiskInfo("log", Config.sys_log_dir, DiskUtils.df(Config.sys_log_dir))); + add(new FeDiskInfo("log", logDir, DiskUtils.df(logDir))); add(new FeDiskInfo("audit-log", Config.audit_log_dir, DiskUtils.df(Config.audit_log_dir))); add(new FeDiskInfo("temp", Config.tmp_dir, DiskUtils.df(Config.tmp_dir))); add(new FeDiskInfo("deploy", System.getenv("DORIS_HOME"), DiskUtils.df(System.getenv("DORIS_HOME"))));