Skip to content

Commit

Permalink
[#177] feat(CI): Support publish Gravitino related jars and docs to O…
Browse files Browse the repository at this point in the history
…SSRH (#543)

### What changes were proposed in this pull request?

- Add the Kotlin script in the root `build.gradle.kts` to publish
Gravitino to maven central repo
- Fix several Java doc issues.

### Why are the changes needed?

Fix: #177 

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

N/A
  • Loading branch information
yuqi1129 authored Oct 25, 2023
1 parent a5e91a3 commit c7318af
Show file tree
Hide file tree
Showing 10 changed files with 133 additions and 63 deletions.
100 changes: 82 additions & 18 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -41,18 +41,100 @@ java {
}
}

allprojects {
repositories {
mavenCentral()
mavenLocal()
}
}

nexusPublishing {
repositories {
sonatype {
nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))
snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))

val sonatypeUser =
System.getenv("SONATYPE_USER").takeUnless { it.isNullOrEmpty() }
?: extra["SONATYPE_USER"].toString()
val sonatypePassword =
System.getenv("SONATYPE_PASSWORD").takeUnless { it.isNullOrEmpty() }
?: extra["SONATYPE_PASSWORD"].toString()

username.set(sonatypeUser)
password.set(sonatypePassword)
}
}

packageGroup.set("com.datastrato.gravitino")
}

dependencies {
testImplementation(libs.testng)
}

subprojects {
apply(plugin = "jacoco")
apply(plugin = "maven-publish")
apply(plugin = "java")

repositories {
mavenCentral()
mavenLocal()
}

val sourcesJar by tasks.registering(Jar::class) {
from(sourceSets.named("main").get().allSource)
archiveClassifier.set("sources")
}

val javadocJar by tasks.registering(Jar::class) {
archiveClassifier.set("javadoc")
from(tasks["javadoc"])
}

apply(plugin = "signing")
publishing {
publications {
create<MavenPublication>("MavenJava") {
from(components["java"])
artifact(sourcesJar)
artifact(javadocJar)

pom {
name.set("Gravitino")
description.set("Gravitino is a high-performance, geo-distributed and federated metadata lake.")
url.set("https://datastrato.ai")
licenses {
license {
name.set("The Apache Software License, Version 2.0")
url.set("http://www.apache.org/licenses/LICENSE-2.0.txt")
}
}
developers {
developer {
id.set("The maintainers of Gravitino")
name.set("support")
email.set("[email protected]")
}
}
scm {
url.set("https://github.com/datastrato/gravitino")
connection.set("scm:git:git://github.com/datastrato/gravitino.git")
}
}
}
}
}

configure<SigningExtension> {
val gpgId = System.getenv("GPG_ID")
val gpgSecretKey = System.getenv("GPG_PRIVATE_KEY")
val gpgKeyPassword = System.getenv("GPG_PASSPHRASE")
useInMemoryPgpKeys(gpgId, gpgSecretKey, gpgKeyPassword)
sign(publishing.publications)
}

tasks.configureEach<Test> {
val skipTests = project.hasProperty("skipTests")
if (!skipTests) {
Expand Down Expand Up @@ -114,24 +196,6 @@ subprojects {
}
}

nexusPublishing {
repositories {
create("sonatype") {
val sonatypeUser =
System.getenv("SONATYPE_USER").takeUnless { it.isNullOrEmpty() }
?: extra["SONATYPE_USER"].toString()
val sonatypePassword =
System.getenv("SONATYPE_PASSWORD").takeUnless { it.isNullOrEmpty() }
?: extra["SONATYPE_PASSWORD"].toString()
nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))

snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))
username.set(sonatypeUser)
password.set(sonatypePassword)
}
}
}

tasks.rat {
substringMatcher("DS", "Datastrato", "Copyright 2023 Datastrato.")
approvedLicense("Datastrato")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public interface EntityKeyEncoder<T> {
* @param ident entity identifier to encode
* @param type entity type to encode
* @return encoded key for key-value stored
* @throws IOException, Exception if error occurs
* @throws IOException Exception if error occurs
*/
default T encode(NameIdentifier ident, EntityType type) throws IOException {
return encode(ident, type, false);
Expand All @@ -28,7 +28,7 @@ default T encode(NameIdentifier ident, EntityType type) throws IOException {
*
* @param nullIfMissing return null if the specific entity no found
* @return encoded key for key-value stored
* @throws IOException, Exception if error occurs
* @throws IOException Exception if error occurs
*/
T encode(NameIdentifier ident, EntityType type, boolean nullIfMissing) throws IOException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,12 @@ public interface NameMappingService extends AutoCloseable {
*
* <pre>
* Before:
* oldname -> 1
* 1 -> oldname
* oldName ------ 1
* 1 ------ oldName
*
* After:
* newname -> 1
* 1 -> newname
* newName ---- 1
* 1 ---- newName
* </pre>
*
* @param oldName name to be updated
Expand All @@ -60,7 +60,7 @@ public interface NameMappingService extends AutoCloseable {
/**
* Unbind id-name mapping. Ignore if the name does not exist.
*
* @param name name to be unbined
* @param name name to be unbind
* @return true if the name exists and is deleted successfully, false if the name does not exist
* @throws IOException if the underlying storage failed
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,19 @@
*
* <pre>
* Key Value
* ml_{ml_id} -----> matalake info
* ml_{ml_id} -----> matalake info
* ca_{ml_id}_{ca_id} -----> catalog_info
* ca_{ml_id}_{ca_id} -----> catalog_info
* sc_{ml_id}_{ca_id}_{sc_id} ---> schema_info
* sc_{ml_id}_{ca_id}_{sc_id} ---> schema_info
* br_{ml_id}_{ca_id}_{br_id} ---> broker_info
* br_{ml_id}_{ca_id}_{br_id} ---> broker_info
* ml_{ml_id} ----- metalake info
* ml_{ml_id} ----- metalake info
* ca_{ml_id}_{ca_id} ----- catalog_info
* ca_{ml_id}_{ca_id} ----- catalog_info
* sc_{ml_id}_{ca_id}_{sc_id} --- schema_info
* sc_{ml_id}_{ca_id}_{sc_id} --- schema_info
* br_{ml_id}_{ca_id}_{br_id} --- broker_info
* br_{ml_id}_{ca_id}_{br_id} --- broker_info
*
* ta_{ml_id}_{ca_id}_{sc_id}_{table_id} -----> table_info
* ta_{ml_id}_{ca_id}_{sc_id}_{table_id} -----> table_info
* to_{ml_id}_{ca_id}_{br_id}_{to_id} -----> topic_info
* to_{ml_id}_{ca_id}_{br_id}_{to_id} -----> topic_info
* ta_{ml_id}_{ca_id}_{sc_id}_{table_id} ----- table_info
* ta_{ml_id}_{ca_id}_{sc_id}_{table_id} ----- table_info
* to_{ml_id}_{ca_id}_{br_id}_{to_id} ----- topic_info
* to_{ml_id}_{ca_id}_{br_id}_{to_id} ----- topic_info
* </pre>
*/
public class BinaryEntityKeyEncoder implements EntityKeyEncoder<byte[]> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,28 +174,28 @@ private String concatIdAndName(long[] namespaceIds, String name) {
*
* <pre>
* Assume we have the following entities:
* metalake: a1 ----> 1
* catalog : a1.b1 ----> 2
* schema : a1.b1.c ----> 3
* metalake: a1 ---- 1
* catalog : a1.b1 ---- 2
* schema : a1.b1.c ---- 3
*
* metalake: a2 ----> 4
* catalog : a2.b2 ----> 5
* schema : a2.b2.c ----> 6
* schema : a2.b2.c1 ----> 7
* metalake: a2 ---- 4
* catalog : a2.b2 ---- 5
* schema : a2.b2.c ---- 6
* schema : a2.b2.c1 ---- 7
*
* metalake: a1 ----> 1 means the name of metalake is a1 and the corresponding id is 1
* metalake: a1 ---- 1 means the name of metalake is a1 and the corresponding id is 1
* </pre>
*
* Then we will store the name to id mapping as follows
*
* <pre>
* a1 --> 1
* 1/b1 --> 2
* 1/2/c --> 3
* a2 --> 4
* 4/b2 --> 5
* 4/5/c --> 6
* 4/5/c1 --> 7
* a1 -- 1
* 1/b1 -- 2
* 1/2/c -- 3
* a2 -- 4
* 4/b2 -- 5
* 4/5/c -- 6
* 4/5/c1 -- 7
* </pre>
*
* @param nameIdentifier name of a specific entity
Expand Down
18 changes: 10 additions & 8 deletions docs/how-to-build.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,17 @@ This software is licensed under the Apache License version 2."
```text
├── ...
└── distribution/package
├── bin/gravitino.sh # Gravitino Server Launching scripts
├── bin/gravitino.sh # Gravitino Server Launching scripts.
├── catalogs
│ └── hive/libs/ # Hive catalog dependencies
├── conf/ # All configuration for Gravitino
| ├── gravitino.conf # Gravitino Server configuration
| ├── gravitino-env.sh # Environment variables, etc., JAVA_HOME, GRAVITINO_HOME, and more.
| └── log4j2.properties # log4j configuration for Gravitino Server.
├── libs/ # Gravitino Server dependencies lib
└── logs/ # Gravitino Server logs
│ └── hive/ # Hive catalog dependencies and configuratons.
│ └── lakehouse-iceberg/ # Iceberg catalog dependencies and configuratons.
├── conf/ # All configuration for Gravitino.
| ├── gravitino.conf # Gravitino Server configuration.
| ├── gravitino-env.sh # Environment variables, etc., JAVA_HOME, GRAVITINO_HOME, and more.
| └── log4j2.properties # log4j configuration for Gravitino Server.
├── libs/ # Gravitino Server dependencies libraries.
└── logs/ # Gravitino Server logs.
└── data/ # Default directory for Gravitino Server to store data.
```

> Note: The `./gradlew clean` command will delete the `distribution` directory.
Expand Down
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ testng = "7.7.1"
protobuf-plugin = "0.9.2"
spotless-plugin = '6.11.0'
gradle-extensions-plugin = '1.74'
publish-plugin = '1.1.0'
publish-plugin = '1.2.0'
rat-plugin = '0.8.0'
shadow-plugin = "8.1.1"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ void testOperationIcebergSchema() {

@Test
void testCreateAndLoadIcebergTable() {
// Create table from Graviton API
// Create table from Gravitino API
ColumnDTO[] columns = createColumns();

NameIdentifier tableIdentifier =
Expand Down
4 changes: 4 additions & 0 deletions trino-connector/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ java {
}
}

tasks.named("generateMetadataFileForMavenJavaPublication") {
dependsOn(":trino-connector:copyDepends")
}

tasks {
val copyDepends by registering(Copy::class) {
from(configurations.runtimeClasspath)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ private static Class getClass(ClassLoader classLoader, String className)
* ----nodeManager (DiscoveryNodeManager)
* ------nodeManager (DiscoveryNodeManager)
* ------allCatalogsOnAllNodes (boolean)
* ------activeNodesByCatalogHandle (Optional<SetMultimap>)
* ------activeNodesByCatalogHandle (SetMultimap)
* --metadataProvider(InternalMetadataProvider)
* ----metadata (TracingMetadata)
* ------delegate (MetadataManager)
Expand Down

0 comments on commit c7318af

Please sign in to comment.