Skip to content

Commit

Permalink
Merge branch 'main' into es-8.2
Browse files Browse the repository at this point in the history
  • Loading branch information
yurishkuro authored Mar 6, 2024
2 parents c81c12b + e6055bf commit d7331eb
Show file tree
Hide file tree
Showing 10 changed files with 170 additions and 22 deletions.
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,15 @@ Example usage:
$ STORAGE=elasticsearch ES_NODES=http://localhost:9200 java -jar jaeger-spark-dependencies.jar
```

## Design

At a high-level, this job does the following:
* read lots of spans from a time period
* group them by traceId
* construct a graph using parent-child relationships expressed in span references
* for each edge `(parent span, child span)` output `(parent service, child service, count)`
* write the results to the database (e.g. `dependencies_v2` table in [Cassandra](https://github.com/jaegertracing/jaeger/blob/12e44faabf10cdd866391b78933eec5d6ac50fa9/plugin/storage/cassandra/schema/v004.cql.tmpl#L186))

## Building locally
To build the job locally and run tests:
```bash
Expand Down
29 changes: 29 additions & 0 deletions jaeger-spark-dependencies-cassandra/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,30 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${version.scala.binary}</artifactId>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector-unshaded_${version.scala.binary}</artifactId>
<version>${spark-cassandra-connector.version}</version>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.26.0</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-xml</artifactId>
<version>10.0.16</version>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
Expand All @@ -55,5 +72,17 @@
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>3.0.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@
import java.util.Optional;
import java.util.Set;

import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.FlatMapFunction;

/**
* @author Pavol Loffay
*/
public class SpansToDependencyLinks implements Function<Iterable<Span>, Iterable<Dependency>>{
public class SpansToDependencyLinks implements FlatMapFunction<Iterable<Span>, Dependency>{

/**
* Derives dependency links based on supplied spans.
Expand All @@ -48,7 +48,7 @@ public SpansToDependencyLinks(String peerServiceTag){
}

@Override
public Iterable<Dependency> call(Iterable<Span> trace) {
public java.util.Iterator<Dependency> call(Iterable<Span> trace) {
Map<Long, Set<Span>> spanMap = new LinkedHashMap<>();
Map<Long, Set<Span>> spanChildrenMap = new LinkedHashMap<>();
for (Span span: trace) {
Expand Down Expand Up @@ -111,7 +111,7 @@ public Iterable<Dependency> call(Iterable<Span> trace) {
}
}
}
return result;
return result.iterator();
}

static Optional<Span> serverSpan(Set<Span> sharedSpans) {
Expand Down
6 changes: 6 additions & 0 deletions jaeger-spark-dependencies-elasticsearch/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jaeger-spark-dependencies-common</artifactId>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.thrift.transport.TTransportException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
Expand All @@ -48,7 +49,7 @@ public static void beforeClass() {
}

@Before
public void before() {
public void before() throws TTransportException {
JaegerTracer initStorageTracer = TracersGenerator.createJaeger(UUID.randomUUID().toString(), collectorUrl).getA();
initStorageTracer.buildSpan(UUID.randomUUID().toString()).withTag("foo", "bar").start().finish();
initStorageTracer.close();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public class JaegerElasticsearchEnvironment {

public static String elasticsearchVersion() {
String version = System.getProperty("elasticsearch.version", System.getenv("ELASTICSEARCH_VERSION"));
return version != null ? version : "5.6.9";
return version != null ? version : "7.17.10";
}

public void start(Map<String, String> jaegerEnvs, String jaegerVersion, String elasticsearchVersion) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.thrift.transport.TTransportException;
import zipkin.Span;
import zipkin.reporter.AsyncReporter;
import zipkin.reporter.Encoding;
Expand Down Expand Up @@ -89,7 +90,7 @@ public Flushable flushable() {
}
}

public static List<TracerHolder<JaegerTracer>> generateJaeger(int number, String collectorUrl) {
public static List<TracerHolder<JaegerTracer>> generateJaeger(int number, String collectorUrl) throws TTransportException {
List<TracerHolder<JaegerTracer>> tracers = new ArrayList<>(number);
for (int i = 0; i < number; i++) {
String serviceName = serviceName();
Expand All @@ -99,7 +100,7 @@ public static List<TracerHolder<JaegerTracer>> generateJaeger(int number, String
return tracers;
}

public static Tuple<JaegerTracer, Flushable> createJaeger(String serviceName, String collectorUrl) {
public static Tuple<JaegerTracer, Flushable> createJaeger(String serviceName, String collectorUrl) throws TTransportException {
HttpSender sender = new HttpSender.Builder(collectorUrl + "/api/traces").build();
RemoteReporter reporter = new RemoteReporter.Builder()
.withSender(sender)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.jaegertracing.spark.dependencies.test.tree.TracingWrapper.ZipkinWrapper;
import java.util.ArrayList;
import java.util.List;
import org.apache.thrift.transport.TTransportException;
import org.junit.Test;

/**
Expand All @@ -30,7 +31,7 @@
public class TreeGeneratorTest {

@Test
public void testGenerateOne() {
public void testGenerateOne() throws TTransportException {
Node<JaegerWrapper> root = new TreeGenerator(TracersGenerator.generateJaeger(1, "http://localhost"))
.generateTree(1, 3);
assertEquals(0, root.getDescendants().size());
Expand All @@ -40,7 +41,7 @@ public void testGenerateOne() {
}

@Test
public void testBranchingFactorOne() {
public void testBranchingFactorOne() throws TTransportException {
Node<JaegerWrapper> root = new TreeGenerator(TracersGenerator.generateJaeger(1, "http://localhost"))
.generateTree(16, 3);
List<Node> nodes = new ArrayList<>();
Expand Down
6 changes: 6 additions & 0 deletions jaeger-spark-dependencies/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@
</filter>
<filter>
<artifact>log4j:log4j</artifact>
<includes>
<include>org/apache/log4j/spi/LoggingEvent.class</include>
</includes>
</filter>
<filter>
<artifact>org.apache.logging.log4j:log4j-*</artifact>
<includes>
<include>**</include>
</includes>
Expand Down
119 changes: 107 additions & 12 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -37,23 +37,25 @@
<project.build.resourceEncoding>UTF-8</project.build.resourceEncoding>

<version.scala.binary>2.12</version.scala.binary>
<version.org.apache.spark>2.4.8</version.org.apache.spark>
<version.io.jaegertracing>0.34.0</version.io.jaegertracing>
<version.org.apache.spark>3.5.1</version.org.apache.spark>
<version.io.jaegertracing>0.34.3</version.io.jaegertracing>
<version.io.opentracing>0.31.0</version.io.opentracing>
<version.io.zipkin.brave-brave>4.6.0</version.io.zipkin.brave-brave>
<version.io.zipkin.brave-brave>4.19.3</version.io.zipkin.brave-brave>
<version.io.zipkin.reporter-zipkin-sender-okhttp3>1.0.2</version.io.zipkin.reporter-zipkin-sender-okhttp3>
<version.junit>4.13.2</version.junit>
<version.org.assertj>3.22.0</version.org.assertj>
<version.org.testcontainers>1.14.3</version.org.testcontainers>
<version.org.assertj>3.24.2</version.org.assertj>
<version.org.testcontainers>1.18.1</version.org.testcontainers>
<version.com.squareup.okhttp3-okhttp>3.14.9</version.com.squareup.okhttp3-okhttp>
<version.org.awaitility-awaitility>4.1.1</version.org.awaitility-awaitility>
<version.org.awaitility-awaitility>4.2.0</version.org.awaitility-awaitility>

<version.maven-license-plugin>3.0</version.maven-license-plugin>
<version.maven-compiler-plugin>3.6.1</version.maven-compiler-plugin>
<version.maven-install-plugin>2.5.2</version.maven-install-plugin>
<version.maven-source-plugin>3.0.1</version.maven-source-plugin>
<version.maven-compiler-plugin>3.11.0</version.maven-compiler-plugin>
<version.maven-install-plugin>3.1.1</version.maven-install-plugin>
<version.maven-source-plugin>3.2.1</version.maven-source-plugin>
<version.maven-plugin>0.3.4</version.maven-plugin>
<version.maven-shade-plugin>3.1.0</version.maven-shade-plugin>
<version.maven-shade-plugin>3.5.2</version.maven-shade-plugin>
<version.jackson>2.15.3</version.jackson>
<version.hadoop.client>3.3.6</version.hadoop.client>
</properties>

<inceptionYear>2017</inceptionYear>
Expand Down Expand Up @@ -107,6 +109,26 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${version.scala.binary}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand All @@ -115,13 +137,17 @@
<artifactId>testcontainers</artifactId>
<version>${version.org.testcontainers}</version>
</dependency>

<!-- Forcibly bump Commons Collection version to avoid CVE-2015-7501 -->
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.4</version>
</dependency>
</dependencies>
</dependencyManagement>

Expand All @@ -136,8 +162,77 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${version.scala.binary}</artifactId>
<version>${version.org.apache.spark}</version>
<exclusions>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${version.scala.binary}</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- BEGIN dependecy override versions -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.25.3</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.7.2</version>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${version.jackson}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${version.jackson}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${version.scala.binary}</artifactId>
<version>${version.jackson}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-base</artifactId>
<version>${version.jackson}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>${version.jackson}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${version.hadoop.client}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-protobuf_3_7</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.100.Final</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.11.3</version>
</dependency>
<!-- END dependency override versions -->

<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Expand Down

0 comments on commit d7331eb

Please sign in to comment.