Merge tag 'cpueff-0.0.28'

increase spark memory for cpueff-spark
dmwm · Aug 19, 2024 · a4ebc48 · a4ebc48
2 parents a4792c5 + bba8fa3
commit a4ebc48
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 8 deletions.
diff --git a/cpueff-goweb/spark/cron4cpueff_goweb.sh b/cpueff-goweb/spark/cron4cpueff_goweb.sh
@@ -80,7 +80,7 @@ function run_spark() {
         --conf spark.shuffle.useOldFetchProtocol=true
         --conf "spark.driver.bindAddress=0.0.0.0" --conf "spark.driver.host=${K8SHOST}"
         --conf "spark.driver.port=${PORT1}" --conf "spark.driver.blockManager.port=${PORT2}"
-        --driver-memory=8g --executor-memory=8g --packages org.apache.spark:spark-avro_2.12:3.4.0
+        --driver-memory=16g --executor-memory=16g --packages org.apache.spark:spark-avro_2.12:3.4.0
     )
 
     py_input_args=(--hdfs_out_dir "$hdfs_out_dir" --start_date "$start_date" --end_date "$end_date")

diff --git a/cpueff-goweb/spark/utils.sh b/cpueff-goweb/spark/utils.sh
@@ -158,13 +158,8 @@ function util_kerberos_auth_with_keytab() {
 # setup hadoop and spark in k8s
 #######################################
 function util_setup_spark_k8s() {
-    # check hava home
-    util_set_java_home
-
-    hadoop-set-default-conf.sh analytix 'hadoop spark' 3.2
-    source hadoop-setconf.sh analytix 3.2 spark3
-    export SPARK_LOCAL_IP=127.0.0.1
-    export PYSPARK_PYTHON=/cvmfs/sft.cern.ch/lcg/releases/Python/3.9.6-b0f98/x86_64-centos7-gcc8-opt/bin/python3
+    hadoop-set-default-conf.sh analytix
+    source hadoop-setconf.sh analytix 3.3 spark3
     # until IT changes this setting, we need to turn off info logs in this way. Don't try spark.sparkContext.setLogLevel('WARN'), doesn't work, since they are not spark logs but spark-submit logs.
     sed -i 's/rootLogger.level = info/rootLogger.level = warn/g' "$SPARK_CONF_DIR"/log4j2.properties
 }