updated CI and minor progress

.github/workflows/changes.yml

@@ -104,6 +104,8 @@ on:
         value: ${{ jobs.int_tests.outputs.nginx }}
       opentelemetry:
         value: ${{ jobs.int_tests.outputs.opentelemetry }}
+      openobserve:
+        value: ${{ jobs.int_tests.outputs.openobserve }}
       postgres:
         value: ${{ jobs.int_tests.outputs.postgres }}
       prometheus:
@@ -230,6 +232,7 @@ jobs:
       nats: ${{ steps.filter.outputs.nats }}
       nginx: ${{ steps.filter.outputs.nginx }}
       opentelemetry: ${{ steps.filter.outputs.opentelemetry }}
+      openobserve: ${{ steps.filter.outputs.openobserve }}
       postgres: ${{ steps.filter.outputs.postgres }}
       prometheus: ${{ steps.filter.outputs.prometheus }}
       pulsar: ${{ steps.filter.outputs.pulsar }}

.github/workflows/ci-integration-review.yml

@@ -387,6 +387,16 @@ jobs:
           timeout_minutes: 30
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int opentelemetry
+
+      - name: openobserve
+        if: ${{ startsWith(github.event.review.body, '/ci-run-integration-openobserve')
+          || startsWith(github.event.review.body, '/ci-run-integration-all')
+          || startsWith(github.event.review.body, '/ci-run-all') }}
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          command: bash scripts/ci-int-e2e-test.sh int openobserve
 
       - name: postgres
         if: ${{ startsWith(github.event.review.body, '/ci-run-integration-postgres')

.github/workflows/integration.yml

@@ -81,6 +81,7 @@ jobs:
             || needs.changes.outputs.nats == 'true'
             || needs.changes.outputs.nginx == 'true'
             || needs.changes.outputs.opentelemetry == 'true'
+            || needs.changes.outputs.openobserve == 'true'
             || needs.changes.outputs.postgres == 'true'
             || needs.changes.outputs.prometheus == 'true'
             || needs.changes.outputs.pulsar == 'true'
@@ -354,6 +355,15 @@ jobs:
           timeout_minutes: 30
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  opentelemetry
+
+      - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.openobserve == 'true') &&
+          (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
+        name: openobserve
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          command: bash scripts/ci-int-e2e-test.sh int  openobserve
 
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.postgres == 'true' }}
         name: postgres

Cargo.toml

@@ -772,6 +772,7 @@ sinks-mqtt = ["dep:rumqttc"]
 sinks-nats = ["dep:async-nats", "dep:nkeys"]
 sinks-new_relic_logs = ["sinks-http"]
 sinks-new_relic = []
+sinks-openobserve = ["sinks-http"]
 sinks-papertrail = ["dep:syslog"]
 sinks-prometheus = ["dep:base64", "dep:prost", "vector-lib/prometheus"]
 sinks-pulsar = ["dep:apache-avro", "dep:pulsar", "dep:lru"]
@@ -821,6 +822,7 @@ all-integration-tests = [
   "nats-integration-tests",
   "nginx-integration-tests",
   "opentelemetry-integration-tests",
+  "openobserve-integration-tests",
   "postgresql_metrics-integration-tests",
   "prometheus-integration-tests",
   "pulsar-integration-tests",
@@ -885,6 +887,7 @@ mqtt-integration-tests = ["sinks-mqtt"]
 nats-integration-tests = ["sinks-nats", "sources-nats"]
 nginx-integration-tests = ["sources-nginx_metrics"]
 opentelemetry-integration-tests = ["sources-opentelemetry", "dep:prost"]
+openobserve-integration-tests = ["sinks-openobserve"]
 postgresql_metrics-integration-tests = ["sources-postgresql_metrics"]
 prometheus-integration-tests = ["sinks-prometheus", "sources-prometheus", "sinks-influxdb"]
 pulsar-integration-tests = ["sinks-pulsar", "sources-pulsar"]

Makefile

@@ -367,7 +367,7 @@ test-integration: ## Runs all integration tests
 test-integration: test-integration-amqp test-integration-appsignal test-integration-aws test-integration-axiom test-integration-azure test-integration-chronicle test-integration-clickhouse
 test-integration: test-integration-databend test-integration-docker-logs test-integration-elasticsearch
 test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-greptimedb test-integration-humio test-integration-http-client test-integration-influxdb
-test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats
+test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats test-integration-openobserve
 test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar
 test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-e2e-datadog-logs
 test-integration: test-integration-datadog-traces test-integration-shutdown

src/sinks/mod.rs

@@ -80,6 +80,8 @@ pub mod mqtt;
 pub mod nats;
 #[cfg(feature = "sinks-new_relic")]
 pub mod new_relic;
+#[cfg(feature = "sinks-openobserve")]
+pub mod openobserve;
 #[cfg(feature = "sinks-webhdfs")]
 pub mod opendal_common;
 #[cfg(feature = "sinks-papertrail")]

src/sinks/openobserve.rs

@@ -0,0 +1,53 @@
+use vector_lib::configurable::configurable_component;
+
+use crate::{
+    codecs::{EncodingConfigWithFraming, Transformer},
+    config::{AcknowledgementsConfig, GenerateConfig, Input, SinkConfig, SinkContext},
+    http::Auth as HttpAuthConfig,
+    sinks::{
+        http::config::{HttpMethod, HttpSinkConfig},
+        util::{
+            http::RequestConfig, BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings,
+        },
+        Healthcheck, VectorSink,
+    },
+    tls::TlsConfig,
+};
+
+/// Configuration for the `openobserve` sink.
+#[configurable_component(sink("openobserve", "Deliver log events to OpenObserve."))]
+#[derive(Clone, Debug)]
+pub struct OpenObserveConfig {
+    /// Wrap the HTTP sink configuration.
+    pub http: HttpSinkConfig,
+}
+
+#[async_trait::async_trait]
+#[typetag::serde(name = "openobserve")]
+impl SinkConfig for OpenObserveConfig {
+    async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> {
+        let http = self.http.clone();
+        let sink = HttpSink::new(http, cx)?;
+        Ok((sink, sink.healthcheck()))
+    }
+
+    fn input_type(&self) -> DataType {
+        DataType::Log
+    }
+
+    fn sink_type(&self) -> &'static str {
+        "openobserve"
+    }
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn generate_config() {
+        crate::test_util::test_generate_config::<super::OpenObserveConfig>();
+    }
+}
+
+#[cfg(feature = "openobserve-integration-tests")]
+#[cfg(test)]
+mod integration_tests {}

website/data/redirects.yaml

@@ -33,6 +33,7 @@ sinks:
 - nats
 - new_relic_logs
 - new_relic
+- openobserve
 - papertrail
 - prometheus_exporter
 - prometheus_remote_write