Squashed commit of the following:

commit 8deadba Author: Marcos Lopez Gonzalez <[email protected]> Date: Thu Sep 14 09:20:34 2023 +0200 minor improvements commit a2f4ea5 Author: Marcos Lopez Gonzalez <[email protected]> Date: Wed Sep 13 10:21:52 2023 +0200 excluded dependencies commit 5161078 Author: Marcos Lopez Gonzalez <[email protected]> Date: Wed Sep 13 09:51:22 2023 +0200 excluded dependencies commit 73e9a79 Merge: 0903c65 164ba15 Author: Marcos Lopez Gonzalez <[email protected]> Date: Wed Sep 13 09:50:30 2023 +0200 Merge branch 'dev' into ids_example_pipelines_issues commit 0903c65 Author: Marcos Lopez Gonzalez <[email protected]> Date: Tue Sep 12 16:15:38 2023 +0200 added ids samples to ids validation pipelines issues
gbif · Sep 14, 2023 · 6881979 · 6881979
1 parent 164ba15
commit 6881979
Show file tree

Hide file tree

Showing 5 changed files with 245 additions and 40 deletions.
diff --git a/pom.xml b/pom.xml
@@ -129,6 +129,11 @@
     <super-csv.version>2.4.0</super-csv.version>
     <opencsv.version>5.7.1</opencsv.version>
     <geojson.version>1.14</geojson.version>
+    <avro.version>1.8.2</avro.version>
+
+    <!-- Hadoop -->
+    <hadoop.version>2.6.0-cdh5.16.2</hadoop.version>
+    <hadoop-core.version>2.6.0-mr1-cdh5.16.2</hadoop-core.version>
 
     <!-- Test -->
     <junit.version>5.9.1</junit.version>
@@ -608,6 +613,45 @@
         <version>${elasticsearch.version}</version>
       </dependency>
 
+      <!-- Hadoop -->
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs</artifactId>
+        <version>${hadoop.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>xerces</groupId>
+            <artifactId>xercesImpl</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-core</artifactId>
+        <version>${hadoop-core.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-common</artifactId>
+        <version>${hadoop.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>*</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
+      <dependency>
+        <groupId>org.apache.avro</groupId>
+        <artifactId>avro</artifactId>
+        <version>${avro.version}</version>
+      </dependency>
+
       <!-- CSV exports -->
       <dependency>
         <groupId>net.sf.supercsv</groupId>

diff --git a/registry-pipelines/pom.xml b/registry-pipelines/pom.xml
@@ -51,6 +51,10 @@
       <groupId>org.gbif.metrics</groupId>
       <artifactId>metrics-ws-client</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.gbif.occurrence</groupId>
+      <artifactId>occurrence-ws-client</artifactId>
+    </dependency>
 
     <!-- okhttp client -->
     <dependency>
@@ -70,6 +74,25 @@
       <artifactId>okio</artifactId>
     </dependency>
 
+    <!-- Hadoop -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.avro</groupId>
+      <artifactId>avro</artifactId>
+    </dependency>
+
     <!-- Test dependencies -->
     <dependency>
       <groupId>org.junit.jupiter</groupId>

diff --git a/registry-pipelines/src/main/java/org/gbif/registry/pipelines/issues/GithubClientConfig.java b/registry-pipelines/src/main/java/org/gbif/registry/pipelines/issues/GithubClientConfig.java
@@ -13,19 +13,11 @@
  */
 package org.gbif.registry.pipelines.issues;
 
-import javax.validation.constraints.NotEmpty;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.stereotype.Component;
-
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
-
-import lombok.Getter;
-import lombok.Setter;
 import okhttp3.OkHttpClient;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
 import retrofit2.Retrofit;
 import retrofit2.converter.jackson.JacksonConverterFactory;
 
@@ -34,32 +26,23 @@
 public class GithubClientConfig {
 
   @Bean
-  public GithubApiService githubApiService(Config config) {
+  public GithubApiService githubApiService(IssuesConfig config) {
     ObjectMapper mapper =
         new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
     OkHttpClient okHttpClient =
         new OkHttpClient.Builder()
             .cache(null)
-            .addInterceptor(new BasicAuthInterceptor(config.githubUser, config.githubPassword))
+            .addInterceptor(
+                new BasicAuthInterceptor(config.getGithubUser(), config.getGithubPassword()))
             .build();
 
     Retrofit retrofit =
         new Retrofit.Builder()
             .client(okHttpClient)
-            .baseUrl(config.githubWsUrl)
+            .baseUrl(config.getGithubWsUrl())
             .addConverterFactory(JacksonConverterFactory.create(mapper))
             .build();
     return retrofit.create(GithubApiService.class);
   }
-
-  @Component
-  @Getter
-  @Setter
-  @ConfigurationProperties(prefix = "pipelines.issues")
-  public static class Config {
-    @NotEmpty private String githubWsUrl;
-    @NotEmpty private String githubUser;
-    @NotEmpty private String githubPassword;
-  }
 }