[#177] feat(CI): Support publish Gravitino related jars and docs to OSSRH

.github/workflows/publish-gravitino.yml

@@ -0,0 +1,39 @@
+name: Publish Gravitino to OSSRH
+
+on:
+  # Triggers the workflow on push events only for the release branches(branch-*) and test branches(test*)
+  push:
+    branches: [ "branch-*", "test*" ]
+
+jobs:
+  publish-gravitino:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    env:
+      SONATYPE_USER: ${{ secrets.OSSRH_USERNAME }}
+      SONATYPE_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+      GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+      GPG_ID: ${{ secrets.GPG_ID }}
+      GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+
+      - name: Check publish username and password
+        run: |
+          if [ "${SONATYPE_USER}" = "" ] || [ "${SONATYPE_PASSWORD}" = "" ]; then
+            echo "Username or password is empty. Please try to set github action secret OSSRH_USERNAME and OSSRH_PASSWORD. "
+            exit 1
+          fi
+
+          if [ "${GPG_PRIVATE_KEY}" = "" ] || [ "${GPG_ID}" = "" ] || [ "${GPG_PASSPHRASE}" = "" ]; then
+            echo "GPG private key, id or passphrase is empty. Please try to set github action secret GPG_PRIVATE_KEY, GPG_ID and GPG_PASSPHRASE. "
+            exit 1
+          fi 
+
+      - name: Clean, build and publish the release(branch-*) branch
+        run: ./gradlew clean build publish

build.gradle.kts

@@ -22,6 +22,7 @@ plugins {
   alias(libs.plugins.gradle.extensions)
   alias(libs.plugins.spotless)
   alias(libs.plugins.publish)
+  id("org.jetbrains.dokka") version "1.9.10"
   // Apply one top level rat plugin to perform any required license enforcement analysis
   alias(libs.plugins.rat)
   id("com.github.jk1.dependency-license-report") version "2.5"
@@ -41,18 +42,110 @@ java {
   }
 }
 
+allprojects {
+  repositories {
+    mavenCentral()
+    mavenLocal()
+  }
+
+  tasks.withType<org.jetbrains.dokka.gradle.DokkaTask>().configureEach {
+    dokkaSourceSets {
+      configureEach {
+        reportUndocumented.set(false)
+      }
+    }
+  }
+}
+
+apply(plugin = "io.github.gradle-nexus.publish-plugin")
+
+nexusPublishing {
+  repositories {
+    sonatype {
+      nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))
+      snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))
+
+      val sonatypeUser =
+              System.getenv("SONATYPE_USER").takeUnless { it.isNullOrEmpty() }
+                      ?: extra["SONATYPE_USER"].toString()
+      val sonatypePassword =
+              System.getenv("SONATYPE_PASSWORD").takeUnless { it.isNullOrEmpty() }
+                      ?: extra["SONATYPE_PASSWORD"].toString()
+
+      username.set(sonatypeUser)
+      password.set(sonatypePassword)
+    }
+  }
+
+  packageGroup.set("com.datastrato.gravitino")
+}
+
 dependencies {
   testImplementation(libs.testng)
 }
 
 subprojects {
   apply(plugin = "jacoco")
+  apply(plugin = "maven-publish")
+  apply(plugin = "java")
 
   repositories {
     mavenCentral()
     mavenLocal()
   }
 
+  val sourcesJar by tasks.registering(Jar::class) {
+    from(sourceSets.named("main").get().allSource)
+    archiveClassifier.set("sources")
+  }
+
+  val javadocJar by tasks.registering(Jar::class) {
+    archiveClassifier.set("javadoc")
+    from(tasks["javadoc"])
+  }
+
+  apply(plugin = "signing")
+  publishing {
+    publications {
+      create<MavenPublication>("MavenJava") {
+        from(components["java"])
+        artifact(sourcesJar)
+        artifact(javadocJar)
+
+        pom {
+          name.set("Gravitino")
+          description.set("Gravitino is a high-performance, geo-distributed and federated metadata lake.")
+          url.set("https://datastrato.ai")
+          licenses {
+            license {
+              name.set("The Apache Software License, Version 2.0")
+              url.set("http://www.apache.org/licenses/LICENSE-2.0.txt")
+            }
+          }
+          developers {
+            developer {
+              id.set("The maintainers of Gravitino")
+              name.set("support")
+              email.set("[email protected]")
+            }
+          }
+          scm {
+            url.set("https://github.com/datastrato/gravitino")
+            connection.set("scm:git:git://github.com/datastrato/gravitino.git")
+          }
+        }
+      }
+    }
+  }
+
+  configure<SigningExtension> {
+    val gpgId = System.getenv("GPG_ID")
+    val gpgSecretKey = System.getenv("GPG_PRIVATE_KEY")
+    val gpgKeyPassword = System.getenv("GPG_PASSPHRASE")
+    useInMemoryPgpKeys(gpgId, gpgSecretKey, gpgKeyPassword)
+    sign(publishing.publications)
+  }
+
   tasks.configureEach<Test> {
     val skipTests = project.hasProperty("skipTests")
     if (!skipTests) {
@@ -114,24 +207,6 @@ subprojects {
   }
 }
 
-nexusPublishing {
-  repositories {
-    create("sonatype") {
-      val sonatypeUser =
-        System.getenv("SONATYPE_USER").takeUnless { it.isNullOrEmpty() }
-          ?: extra["SONATYPE_USER"].toString()
-      val sonatypePassword =
-        System.getenv("SONATYPE_PASSWORD").takeUnless { it.isNullOrEmpty() }
-          ?: extra["SONATYPE_PASSWORD"].toString()
-      nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))
-
-      snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))
-      username.set(sonatypeUser)
-      password.set(sonatypePassword)
-    }
-  }
-}
-
 tasks.rat {
   substringMatcher("DS", "Datastrato", "Copyright 2023 Datastrato.")
   approvedLicense("Datastrato")

core/src/main/java/com/datastrato/gravitino/storage/EntityKeyEncoder.java

@@ -17,7 +17,7 @@ public interface EntityKeyEncoder<T> {
    * @param ident entity identifier to encode
    * @param type entity type to encode
    * @return encoded key for key-value stored
-   * @throws IOException, Exception if error occurs
+   * @throws IOException Exception if error occurs
    */
   default T encode(NameIdentifier ident, EntityType type) throws IOException {
     return encode(ident, type, false);
@@ -28,7 +28,7 @@ default T encode(NameIdentifier ident, EntityType type) throws IOException {
    *
    * @param nullIfMissing return null if the specific entity no found
    * @return encoded key for key-value stored
-   * @throws IOException, Exception if error occurs
+   * @throws IOException Exception if error occurs
    */
   T encode(NameIdentifier ident, EntityType type, boolean nullIfMissing) throws IOException;
 }

core/src/main/java/com/datastrato/gravitino/storage/NameMappingService.java

@@ -42,12 +42,12 @@ public interface NameMappingService extends AutoCloseable {
    *
    * <pre>
    * Before:
-   *   oldname -> 1
-   *   1       -> oldname
+   *   oldName ------ 1
+   *   1       ------ oldName
    *
    * After:
-   *  newname -> 1
-   *  1       -> newname
+   *  newName ---- 1
+   *  1       ---- newName
    * </pre>
    *
    * @param oldName name to be updated
@@ -60,7 +60,7 @@ public interface NameMappingService extends AutoCloseable {
   /**
    * Unbind id-name mapping. Ignore if the name does not exist.
    *
-   * @param name name to be unbined
+   * @param name name to be unbind
    * @return true if the name exists and is deleted successfully, false if the name does not exist
    * @throws IOException if the underlying storage failed
    */

core/src/main/java/com/datastrato/gravitino/storage/kv/BinaryEntityKeyEncoder.java

@@ -34,19 +34,19 @@
  *
  * <pre>
  *     Key                                Value
- * ml_{ml_id}               ----->    matalake info
- * ml_{ml_id}               ----->    matalake info
- * ca_{ml_id}_{ca_id}       ----->    catalog_info
- * ca_{ml_id}_{ca_id}       ----->    catalog_info
- * sc_{ml_id}_{ca_id}_{sc_id} --->    schema_info
- * sc_{ml_id}_{ca_id}_{sc_id} --->    schema_info
- * br_{ml_id}_{ca_id}_{br_id} --->    broker_info
- * br_{ml_id}_{ca_id}_{br_id} --->    broker_info
+ * ml_{ml_id}               -----    metalake info
+ * ml_{ml_id}               -----    metalake info
+ * ca_{ml_id}_{ca_id}       -----    catalog_info
+ * ca_{ml_id}_{ca_id}       -----    catalog_info
+ * sc_{ml_id}_{ca_id}_{sc_id} ---    schema_info
+ * sc_{ml_id}_{ca_id}_{sc_id} ---    schema_info
+ * br_{ml_id}_{ca_id}_{br_id} ---    broker_info
+ * br_{ml_id}_{ca_id}_{br_id} ---    broker_info
  *
- * ta_{ml_id}_{ca_id}_{sc_id}_{table_id}    ----->    table_info
- * ta_{ml_id}_{ca_id}_{sc_id}_{table_id}    ----->    table_info
- * to_{ml_id}_{ca_id}_{br_id}_{to_id}       ----->    topic_info
- * to_{ml_id}_{ca_id}_{br_id}_{to_id}       ----->    topic_info
+ * ta_{ml_id}_{ca_id}_{sc_id}_{table_id}    -----    table_info
+ * ta_{ml_id}_{ca_id}_{sc_id}_{table_id}    -----    table_info
+ * to_{ml_id}_{ca_id}_{br_id}_{to_id}       -----    topic_info
+ * to_{ml_id}_{ca_id}_{br_id}_{to_id}       -----    topic_info
  * </pre>
  */
 public class BinaryEntityKeyEncoder implements EntityKeyEncoder<byte[]> {

core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java

@@ -174,28 +174,28 @@ private String concatIdAndName(long[] namespaceIds, String name) {
    *
    * <pre>
    *   Assume we have the following entities:
-   *   metalake: a1        ----> 1
-   *   catalog : a1.b1     ----> 2
-   *   schema  : a1.b1.c   ----> 3
+   *   metalake: a1        ---- 1
+   *   catalog : a1.b1     ---- 2
+   *   schema  : a1.b1.c   ---- 3
    *
-   *   metalake: a2        ----> 4
-   *   catalog : a2.b2     ----> 5
-   *   schema  : a2.b2.c   ----> 6
-   *   schema  : a2.b2.c1  ----> 7
+   *   metalake: a2        ---- 4
+   *   catalog : a2.b2     ---- 5
+   *   schema  : a2.b2.c   ---- 6
+   *   schema  : a2.b2.c1  ---- 7
    *
-   *   metalake: a1        ----> 1 means the name of metalake is a1 and the corresponding id is 1
+   *   metalake: a1        ---- 1 means the name of metalake is a1 and the corresponding id is 1
    * </pre>
    *
    * Then we will store the name to id mapping as follows
    *
    * <pre>
-   *  a1         --> 1
-   * 	1/b1       --> 2
-   * 	1/2/c      --> 3
-   * 	a2         --> 4
-   * 	4/b2       --> 5
-   * 	4/5/c      --> 6
-   * 	4/5/c1     --> 7
+   *  a1         -- 1
+   * 	1/b1       -- 2
+   * 	1/2/c      -- 3
+   * 	a2         -- 4
+   * 	4/b2       -- 5
+   * 	4/5/c      -- 6
+   * 	4/5/c1     -- 7
    * </pre>
    *
    * @param nameIdentifier name of a specific entity

docs/how-to-build.md

@@ -37,15 +37,17 @@ This software is licensed under the Apache License version 2."
     ```
     ├── ...
     └── distribution/package
-        ├── bin/gravitino.sh          # Gravitino Server Launching scripts
+        ├── bin/gravitino.sh            # Gravitino Server Launching scripts.
         ├── catalogs
-        │   └── hive/libs/           # Hive catalog dependencies
-        ├── conf/                    # All configuration for Gravitino
-        |   ├── gravitino.conf        # Gravitino Server configuration
-        |   ├── gravitino-env.sh      # Environment variables, etc., JAVA_HOME, GRAVITINO_HOME, and more.
-        |   └── log4j2.properties    # log4j configuration for Gravitino Server.
-        ├── libs/                    # Gravitino Server dependencies lib
-        └── logs/                    # Gravitino Server logs
+        │   └── hive/                   # Hive catalog dependencies and configuratons.
+        │   └── lakehouse-iceberg/      # Iceberg catalog dependencies and configuratons.
+        ├── conf/                       # All configuration for Gravitino.
+        |   ├── gravitino.conf          # Gravitino Server configuration.
+        |   ├── gravitino-env.sh        # Environment variables, etc., JAVA_HOME, GRAVITINO_HOME, and more.
+        |   └── log4j2.properties       # log4j configuration for Gravitino Server.
+        ├── libs/                       # Gravitino Server dependencies libraries.
+        └── logs/                       # Gravitino Server logs.
+        └── data/                       # Default directory for Gravitino Server to store data.
     ```
    > Note: The `./gradlew clean` command will delete the `distribution` directory.
 

gradle/libs.versions.toml

@@ -36,7 +36,7 @@ testng = "7.7.1"
 protobuf-plugin = "0.9.2"
 spotless-plugin = '6.11.0'
 gradle-extensions-plugin = '1.74'
-publish-plugin = '1.1.0'
+publish-plugin = '1.2.0'
 rat-plugin = '0.8.0'
 shadow-plugin = "8.1.1"
 

integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java

@@ -312,7 +312,7 @@ void testOperationIcebergSchema() {
 
   @Test
   void testCreateAndLoadIcebergTable() {
-    // Create table from Graviton API
+    // Create table from Gravitino API
     ColumnDTO[] columns = createColumns();
 
     NameIdentifier tableIdentifier =

trino-connector/build.gradle.kts

@@ -42,6 +42,10 @@ java {
   }
 }
 
+tasks.named("generateMetadataFileForMavenJavaPublication") {
+    dependsOn(":trino-connector:copyDepends")
+}
+
 tasks {
     val copyDepends by registering(Copy::class) {
         from(configurations.runtimeClasspath)

trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java

@@ -92,7 +92,7 @@ private static Class getClass(ClassLoader classLoader, String className)
    *  ----nodeManager (DiscoveryNodeManager)
    *  ------nodeManager (DiscoveryNodeManager)
    *  ------allCatalogsOnAllNodes (boolean)
-   *  ------activeNodesByCatalogHandle (Optional<SetMultimap>)
+   *  ------activeNodesByCatalogHandle (SetMultimap)
    *  --metadataProvider(InternalMetadataProvider)
    *  ----metadata (TracingMetadata)
    *  ------delegate (MetadataManager)