diff --git a/cpueff-goweb/spark/cron4cpueff_goweb.sh b/cpueff-goweb/spark/cron4cpueff_goweb.sh index 45df2c80..f46691bd 100755 --- a/cpueff-goweb/spark/cron4cpueff_goweb.sh +++ b/cpueff-goweb/spark/cron4cpueff_goweb.sh @@ -80,7 +80,7 @@ function run_spark() { --conf spark.shuffle.useOldFetchProtocol=true --conf "spark.driver.bindAddress=0.0.0.0" --conf "spark.driver.host=${K8SHOST}" --conf "spark.driver.port=${PORT1}" --conf "spark.driver.blockManager.port=${PORT2}" - --driver-memory=8g --executor-memory=8g --packages org.apache.spark:spark-avro_2.12:3.4.0 + --driver-memory=16g --executor-memory=16g --packages org.apache.spark:spark-avro_2.12:3.4.0 ) py_input_args=(--hdfs_out_dir "$hdfs_out_dir" --start_date "$start_date" --end_date "$end_date") diff --git a/cpueff-goweb/spark/utils.sh b/cpueff-goweb/spark/utils.sh index 77b02134..8fd2a197 100755 --- a/cpueff-goweb/spark/utils.sh +++ b/cpueff-goweb/spark/utils.sh @@ -158,13 +158,8 @@ function util_kerberos_auth_with_keytab() { # setup hadoop and spark in k8s ####################################### function util_setup_spark_k8s() { - # check hava home - util_set_java_home - - hadoop-set-default-conf.sh analytix 'hadoop spark' 3.2 - source hadoop-setconf.sh analytix 3.2 spark3 - export SPARK_LOCAL_IP=127.0.0.1 - export PYSPARK_PYTHON=/cvmfs/sft.cern.ch/lcg/releases/Python/3.9.6-b0f98/x86_64-centos7-gcc8-opt/bin/python3 + hadoop-set-default-conf.sh analytix + source hadoop-setconf.sh analytix 3.3 spark3 # until IT changes this setting, we need to turn off info logs in this way. Don't try spark.sparkContext.setLogLevel('WARN'), doesn't work, since they are not spark logs but spark-submit logs. sed -i 's/rootLogger.level = info/rootLogger.level = warn/g' "$SPARK_CONF_DIR"/log4j2.properties }