[#445] feat(lakehouse-iceberg): support load HDFS config files for Ic…

…eberg REST server (#468)

### What changes were proposed in this pull request?
support load HDFS config files for Iceberg REST server, support multiple
classpath for `graviton.auxService.iceberg-rest.classpath`, such as
`catalogs/lakehouse-iceberg/libs, catalogs/lakehouse-iceberg/conf`

### Why are the changes needed?
HDFS configs are complicated, we should put `core-site.xml` and
`hdfs-site.xml` to the classpath

Fix: #445 

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
1. setup local HDFS cluster.
2. start the graviton server with HDFS config files.

catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java

@@ -33,6 +33,7 @@ private static InMemoryCatalog loadMemoryCatalog(Map<String, String> properties)
 
   private static HiveCatalog loadHiveCatalog(Map<String, String> properties) {
     HiveCatalog hiveCatalog = new HiveCatalog();
+    hiveCatalog.setConf(new HdfsConfiguration());
     hiveCatalog.initialize("hive", properties);
     return hiveCatalog;
   }
@@ -59,6 +60,7 @@ public static Catalog loadCatalogBackend(String catalogType, Map<String, String>
     // TODO Organize the configuration properties and adapt them to the lower layer, and map some
     // graviton configuration keys.
     LOG.info("Load catalog backend of {}", catalogType);
+
     switch (catalogType.toLowerCase(Locale.ENGLISH)) {
       case "memory":
         return loadMemoryCatalog(properties);

conf/graviton.conf.template

@@ -40,8 +40,9 @@ graviton.catalog.cache.evictionIntervalMs = 3600000
 # Auxiliary service names, separate by ','
 graviton.auxService.names = iceberg-rest
 # Iceberg REST service classpath
-graviton.auxService.iceberg-rest.classpath = catalogs/lakehouse-iceberg/libs
+graviton.auxService.iceberg-rest.classpath = catalogs/lakehouse-iceberg/libs, catalogs/lakehouse-iceberg/conf
 # Iceberg REST service host
 graviton.auxService.iceberg-rest.host = 127.0.0.1
 # Iceberg REST service http port
 graviton.auxService.iceberg-rest.httpPort = 9001
+

core/src/main/java/com/datastrato/graviton/aux/AuxiliaryServiceManager.java

@@ -12,7 +12,6 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
 import com.google.common.collect.Streams;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -86,12 +85,6 @@ public IsolatedClassLoader getIsolatedClassLoader(List<String> classPaths) {
 
   @VisibleForTesting
   static String getValidPath(String auxServiceName, String pathString) {
-    Preconditions.checkArgument(
-        StringUtils.isNoneBlank(pathString),
-        String.format(
-            "AuxService:%s, %s%s.%s is not set in configuration",
-            auxServiceName, GRAVITON_AUX_SERVICE_PREFIX, auxServiceName, AUX_SERVICE_CLASSPATH));
-
     Path path = Paths.get(pathString);
     if (Files.exists(path)) {
       return path.toAbsolutePath().toString();
@@ -112,11 +105,24 @@ static String getValidPath(String auxServiceName, String pathString) {
   }
 
   private void registerAuxService(String auxServiceName, Map<String, String> config) {
-    String classPath = config.get(AUX_SERVICE_CLASSPATH);
-    classPath = getValidPath(auxServiceName, classPath);
-    LOG.info("AuxService name:{}, config:{}, classpath:{}", auxServiceName, config, classPath);
+    String classpath = config.get(AUX_SERVICE_CLASSPATH);
+    Preconditions.checkArgument(
+        StringUtils.isNoneBlank(classpath),
+        String.format(
+            "AuxService:%s, %s%s.%s is not set in configuration",
+            auxServiceName, GRAVITON_AUX_SERVICE_PREFIX, auxServiceName, AUX_SERVICE_CLASSPATH));
+
+    List<String> validPaths =
+        splitter
+            .trimResults()
+            .omitEmptyStrings()
+            .splitToStream(classpath)
+            .map(path -> getValidPath(auxServiceName, path))
+            .collect(Collectors.toList());
+    LOG.info(
+        "AuxService name:{}, config:{}, valid classpath:{}", auxServiceName, config, validPaths);
 
-    IsolatedClassLoader isolatedClassLoader = getIsolatedClassLoader(Lists.newArrayList(classPath));
+    IsolatedClassLoader isolatedClassLoader = getIsolatedClassLoader(validPaths);
     try {
       GravitonAuxiliaryService gravitonAuxiliaryService =
           loadAuxService(auxServiceName, isolatedClassLoader);

integration-test/src/main/java/com/datastrato/graviton/integration/test/MiniGraviton.java

@@ -169,14 +169,16 @@ private void customizeConfigFile(String configTempFileName, String configFileNam
             + AuxiliaryServiceManager.AUX_SERVICE_NAMES,
         IcebergRESTService.SERVICE_NAME);
 
-    String icebergClassPath =
+    String icebergJarPath =
         Paths.get("catalogs", "catalog-lakehouse-iceberg", "build", "libs").toString();
+    String icebergConfigPath =
+        Paths.get("catalogs", "catalog-lakehouse-iceberg", "src", "main", "resources").toString();
     configMap.put(
         AuxiliaryServiceManager.GRAVITON_AUX_SERVICE_PREFIX
             + IcebergRESTService.SERVICE_NAME
             + "."
             + AuxiliaryServiceManager.AUX_SERVICE_CLASSPATH,
-        icebergClassPath);
+        String.join(",", icebergJarPath, icebergConfigPath));
     configMap.put(
         AuxiliaryServiceManager.GRAVITON_AUX_SERVICE_PREFIX
             + IcebergRESTService.SERVICE_NAME