From 7e0188b102ce1760dc74fa5d1cd5735606859b8c Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 11 Dec 2023 18:20:59 +0200 Subject: [PATCH 01/35] Added support for control the app config from the UI using Polling and Kafka event listeners --- go.mod | 10 +- go.sum | 109 ++++++++++++++-- main.go | 51 ++++++-- pkg/config/config.go | 43 +------ pkg/event_listener/consumer/consumer.go | 78 +++++++++++ pkg/event_listener/event_listener_handler.go | 129 +++++++++++++++++++ pkg/event_listener/polling/polling.go | 58 +++++++++ pkg/goutils/env.go | 36 ++++++ pkg/handlers/controllers.go | 48 ++++--- pkg/k8s/controller.go | 7 +- pkg/port/cli/client.go | 5 + pkg/port/cli/integration.go | 15 ++- pkg/port/cli/kafka_crednetials.go | 20 +++ pkg/port/cli/org_details.go | 25 ++++ pkg/port/integration/integration.go | 24 +++- pkg/port/kafka_credentials/credentials.go | 28 ++++ pkg/port/models.go | 77 +++++++++-- pkg/port/org_details/org_details.go | 21 +++ pkg/signal/signal.go | 31 +++-- 19 files changed, 704 insertions(+), 111 deletions(-) create mode 100644 pkg/event_listener/consumer/consumer.go create mode 100644 pkg/event_listener/event_listener_handler.go create mode 100644 pkg/event_listener/polling/polling.go create mode 100644 pkg/goutils/env.go create mode 100644 pkg/port/cli/kafka_crednetials.go create mode 100644 pkg/port/cli/org_details.go create mode 100644 pkg/port/kafka_credentials/credentials.go create mode 100644 pkg/port/org_details/org_details.go diff --git a/go.mod b/go.mod index f023c34..dac6e5c 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/port-labs/port-k8s-exporter go 1.19 require ( + github.com/confluentinc/confluent-kafka-go v1.9.2 github.com/go-resty/resty/v2 v2.7.0 github.com/itchyny/gojq v0.12.9 gopkg.in/yaml.v2 v2.4.0 @@ -37,11 +38,12 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.7.0 // indirect + github.com/stretchr/testify v1.8.2 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/go.sum b/go.sum index 4c5676f..399958e 100644 --- a/go.sum +++ b/go.sum @@ -37,12 +37,24 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/actgardner/gogen-avro/v10 v10.1.0/go.mod h1:o+ybmVjEa27AAr35FRqU98DJu1fXES56uXniYFv4yDA= +github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= +github.com/actgardner/gogen-avro/v9 v9.1.0/go.mod h1:nyTj6wPqDJoxM3qdnjcLv+EnMDSDFqE0qDpva2QRmKc= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/confluentinc/confluent-kafka-go v1.9.2 h1:gV/GxhMBUb03tFWkN+7kdhg+zf+QUM+wVkI9zwh770Q= +github.com/confluentinc/confluent-kafka-go v1.9.2/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -53,9 +65,17 @@ github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= +github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -99,14 +119,19 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -114,6 +139,7 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -128,33 +154,55 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/heetch/avro v0.3.1/go.mod h1:4xn38Oz/+hiEUTpbVfGVLfvOg0yKLlRP7Q9+gJJILgA= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/invopop/jsonschema v0.4.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= github.com/itchyny/gojq v0.12.9 h1:biKpbKwMxVYhCU1d6mR7qMr3f0Hn9F5k5YykCVb3gmM= github.com/itchyny/gojq v0.12.9/go.mod h1:T4Ip7AETUXeGpD+436m+UEl3m3tokRgajd5pRfsR5oE= github.com/itchyny/timefmt-go v0.1.4 h1:hFEfWVdwsEi+CY8xY2FtgWHGQaBaC3JeHd+cve0ynVM= github.com/itchyny/timefmt-go v0.1.4/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= +github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= +github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= +github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/juju/qthttptest v0.1.1/go.mod h1:aTlAv8TYaflIiTDIQYzxnl1QdPjAg8Q8qJMErpKy6A4= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= +github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/linkedin/goavro/v2 v2.10.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= @@ -162,30 +210,46 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g= github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -195,6 +259,7 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -251,6 +316,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -258,9 +324,10 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -277,6 +344,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -304,20 +372,25 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -357,6 +430,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -416,6 +490,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -423,6 +498,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -435,6 +511,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -447,22 +527,31 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqEtWPM= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index bbf44a1..5859659 100644 --- a/main.go +++ b/main.go @@ -3,20 +3,23 @@ package main import ( "flag" "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/goutils" + "github.com/port-labs/port-k8s-exporter/pkg/port" "os" "github.com/port-labs/port-k8s-exporter/pkg/config" + "github.com/port-labs/port-k8s-exporter/pkg/event_listener" "github.com/port-labs/port-k8s-exporter/pkg/handlers" "github.com/port-labs/port-k8s-exporter/pkg/k8s" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" "github.com/port-labs/port-k8s-exporter/pkg/port/integration" - "github.com/port-labs/port-k8s-exporter/pkg/signal" "k8s.io/klog/v2" ) var ( configFilePath string resyncInterval uint + eventListenerType string stateKey string deleteDependents bool createMissingRelatedEntities bool @@ -25,17 +28,22 @@ var ( portClientSecret string ) +func InitiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portClient *cli.PortClient) (*handlers.ControllersHandler, error) { + apiConfig, err := integration.GetIntegrationConfig(portClient, stateKey) + if err != nil { + klog.Fatalf("Error getting K8s integration config: %s", err.Error()) + } + + newHandler := handlers.NewControllersHandler(exporterConfig, apiConfig, k8sClient, portClient) + newHandler.Handle() + + return newHandler, nil +} + func main() { klog.InitFlags(nil) flag.Parse() - stopCh := signal.SetupSignalHandler() - - exporterConfig, err := config.New(configFilePath, resyncInterval, stateKey) - if err != nil { - klog.Fatalf("Error building Port K8s Exporter config: %s", err.Error()) - } - k8sConfig := k8s.NewKubeConfig() clientConfig, err := k8sConfig.ClientConfig() @@ -49,7 +57,6 @@ func main() { } portClient, err := cli.New(portBaseURL, - cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", exporterConfig.StateKey)), cli.WithClientID(portClientId), cli.WithClientSecret(portClientSecret), cli.WithDeleteDependents(deleteDependents), cli.WithCreateMissingRelatedEntities(createMissingRelatedEntities), ) @@ -58,14 +65,30 @@ func main() { klog.Fatalf("Error building Port client: %s", err.Error()) } - err = integration.NewIntegration(portClient, stateKey) + exporterConfig, err := config.New(configFilePath, resyncInterval, stateKey, eventListenerType) if err != nil { - klog.Fatalf("Error creating K8s integration: %s", err.Error()) + klog.Fatalf("Error building Port K8s Exporter config: %s", err.Error()) } + _, err = integration.GetIntegrationConfig(portClient, stateKey) + if err != nil { + err = integration.NewIntegration(portClient, stateKey, exporterConfig) + if err != nil { + klog.Fatalf("Error creating K8s integration: %s", err.Error()) + } + } + cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", exporterConfig.StateKey))(portClient) + klog.Info("Starting controllers handler") - controllersHandler := handlers.NewControllersHandler(exporterConfig, k8sClient, portClient) - controllersHandler.Handle(stopCh) + handler, _ := InitiateHandler(exporterConfig, k8sClient, portClient) + eventListener := event_listener.NewEventListener(stateKey, eventListenerType, handler, portClient) + err = eventListener.Start(func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { + handler.Stop() + return InitiateHandler(exporterConfig, k8sClient, portClient) + }) + if err != nil { + klog.Fatalf("Error starting event listener: %s", err.Error()) + } klog.Info("Started controllers handler") } @@ -78,4 +101,6 @@ func init() { flag.StringVar(&portBaseURL, "port-base-url", "https://api.getport.io", "Port base URL. Optional.") portClientId = os.Getenv("PORT_CLIENT_ID") portClientSecret = os.Getenv("PORT_CLIENT_SECRET") + + eventListenerType = goutils.GetEnvOrDefault("EVENT_LISTENER__TYPE", "POLLING") } diff --git a/pkg/config/config.go b/pkg/config/config.go index 9909dd7..5f0356b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,44 +7,11 @@ import ( "gopkg.in/yaml.v2" ) -type Entity struct { - Mappings []port.EntityMapping -} - -type Port struct { - Entity Entity -} - -type Selector struct { - Query string -} - -type Resource struct { - Kind string - Selector Selector - Port Port -} - -type Config struct { - Resources []Resource - ResyncInterval uint - StateKey string -} - -type KindConfig struct { - Selector Selector - Port Port -} - -type AggregatedResource struct { - Kind string - KindConfigs []KindConfig -} - -func New(filepath string, resyncInterval uint, stateKey string) (*Config, error) { - c := &Config{ - ResyncInterval: resyncInterval, - StateKey: stateKey, +func New(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { + c := &port.Config{ + ResyncInterval: resyncInterval, + StateKey: stateKey, + EventListenerType: eventListenerType, } config, err := os.ReadFile(filepath) if err != nil { diff --git a/pkg/event_listener/consumer/consumer.go b/pkg/event_listener/consumer/consumer.go new file mode 100644 index 0000000..c2b9537 --- /dev/null +++ b/pkg/event_listener/consumer/consumer.go @@ -0,0 +1,78 @@ +package consumer + +import ( + "github.com/confluentinc/confluent-kafka-go/kafka" + "k8s.io/klog/v2" + "os" + "os/signal" + "syscall" +) + +type Consumer struct { + client *kafka.Consumer +} + +type KafkaConfiguration struct { + Brokers string + SecurityProtocol string + GroupID string + AuthenticationMechanism string + Username string + Password string + KafkaSecurityEnabled bool +} + +type JsonHandler func(value []byte) + +func NewConsumer(config *KafkaConfiguration) (*Consumer, error) { + c, err := kafka.NewConsumer(&kafka.ConfigMap{ + "bootstrap.servers": config.Brokers, + "group.id": config.GroupID, + "security.protocol": config.SecurityProtocol, + "sasl.mechanism": config.AuthenticationMechanism, + "sasl.username": config.Username, + "sasl.password": config.Password, + "auto.offset.reset": "earliest", + }) + + if err != nil { + return nil, err + } + + return &Consumer{client: c}, nil +} + +func (c *Consumer) Consume(topic string, handler JsonHandler) { + topics := []string{topic} + _ = c.client.SubscribeTopics(topics, nil) + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + + run := true + for run { + select { + case sig := <-sigchan: + klog.Infof("Caught signal %v: terminating\n", sig) + run = false + default: + ev := c.client.Poll(100) + if ev == nil { + continue + } + + switch e := ev.(type) { + case *kafka.Message: + klog.Infof("%% Message on %s:\n%s\n", + e.TopicPartition, string(e.Value)) + + handler(e.Value) + case *kafka.AssignedPartitions: + klog.Infof("AssignedPartitions: %v\n", e) + case kafka.Error: + klog.Infof("%% Error: %v\n", e) + default: + klog.Infof("Ignored %v\n", e) + } + } + } +} diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go new file mode 100644 index 0000000..fa63334 --- /dev/null +++ b/pkg/event_listener/event_listener_handler.go @@ -0,0 +1,129 @@ +package event_listener + +import ( + "encoding/json" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/event_listener/consumer" + "github.com/port-labs/port-k8s-exporter/pkg/event_listener/polling" + "github.com/port-labs/port-k8s-exporter/pkg/goutils" + "github.com/port-labs/port-k8s-exporter/pkg/handlers" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/kafka_credentials" + "github.com/port-labs/port-k8s-exporter/pkg/port/org_details" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" +) + +type IncomingMessage struct { + Diff *struct { + After *struct { + Identifier string `json:"installationId"` + } `json:"after"` + } `json:"diff"` +} + +type EventListener struct { + settings port.EventListenerSettings + stateKey string + controllerHandler *handlers.ControllersHandler + portClient *cli.PortClient +} + +func shouldResync(stateKey string, message *IncomingMessage) bool { + return message.Diff != nil && + message.Diff.After != nil && + message.Diff.After.Identifier != "" && + message.Diff.After.Identifier == stateKey +} + +func NewEventListener(stateKey string, eventListenerType string, controllerHandler *handlers.ControllersHandler, client *cli.PortClient) *EventListener { + eventListener := &EventListener{ + settings: port.EventListenerSettings{ + Type: eventListenerType, + }, + stateKey: stateKey, + controllerHandler: controllerHandler, + portClient: client, + } + + return eventListener +} + +func startKafkaEventListener(l *EventListener, resync func()) error { + klog.Infof("Starting Kafka event listener") + klog.Infof("Geting Consumer Information") + credentials, err := kafka_credentials.GetKafkaCredentials(l.portClient) + if err != nil { + return err + } + orgId, err := org_details.GetOrgId(l.portClient) + if err != nil { + return err + } + config := &consumer.KafkaConfiguration{ + Brokers: goutils.GetEnvOrDefault("EVENT_LISTENER__BROKERS", "b-1-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-2-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-3-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196"), + SecurityProtocol: goutils.GetEnvOrDefault("EVENT_LISTENER__SECURITY_PROTOCOL", "SASL_SSL"), + AuthenticationMechanism: goutils.GetEnvOrDefault("EVENT_LISTENER__AUTHENTICATION_MECHANISM", "SCRAM-SHA-512"), + KafkaSecurityEnabled: goutils.GetBooleanEnvOrDefault("EVENT_LISTENER__KAFKA_SECURITY_ENABLED", true), + Username: credentials.Username, + Password: credentials.Password, + GroupID: orgId + ".k8s." + l.stateKey, + } + + topic := orgId + ".change.log" + instance, err := consumer.NewConsumer(config) + + if err != nil { + return err + } + + klog.Infof("Starting consumer for topic %s and groupId %s", topic, config.GroupID) + instance.Consume(topic, func(value []byte) { + incomingMessage := &IncomingMessage{} + parsingError := json.Unmarshal(value, &incomingMessage) + if shouldResync(l.stateKey, incomingMessage) { + klog.Infof("Changes detected. Resyncing...") + resync() + } + if parsingError != nil { + utilruntime.HandleError(fmt.Errorf("error handling message: %s", parsingError.Error())) + } + }) + + return nil +} + +func startPollingEventListener(l *EventListener, resync func()) { + klog.Infof("Starting polling event listener") + pollingRate := goutils.GetIntEnvOrDefault("EVENT_LISTENER__POLLING_RATE", 60) + klog.Infof("Polling rate set to %d seconds", pollingRate) + pollingHandler := polling.NewPollingHandler(pollingRate, l.stateKey, l.portClient) + pollingHandler.Run(resync) +} + +func (l *EventListener) Start(resync func(*handlers.ControllersHandler) (*handlers.ControllersHandler, error)) error { + wrappedResync := func() { + klog.Infof("Resync request received. Recreating controllers for the new port configuration") + newController, resyncErr := resync(l.controllerHandler) + l.controllerHandler = newController + + if resyncErr != nil { + utilruntime.HandleError(fmt.Errorf("error resyncing: %s", resyncErr.Error())) + } + } + klog.Infof("Received event listener type: %s", l.settings.Type) + switch l.settings.Type { + case "KAFKA": + err := startKafkaEventListener(l, wrappedResync) + if err != nil { + return err + } + case "POLLING": + startPollingEventListener(l, wrappedResync) + default: + return fmt.Errorf("unknown event listener type: %s", l.settings.Type) + } + + return nil +} diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_listener/polling/polling.go new file mode 100644 index 0000000..9ded8f2 --- /dev/null +++ b/pkg/event_listener/polling/polling.go @@ -0,0 +1,58 @@ +package polling + +import ( + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + "k8s.io/klog/v2" + "os" + "os/signal" + "reflect" + "syscall" + "time" +) + +type PollingHandler struct { + ticker *time.Ticker + stateKey string + portClient *cli.PortClient +} + +func NewPollingHandler(pollingRate int, stateKey string, portClient *cli.PortClient) *PollingHandler { + rv := &PollingHandler{ + ticker: time.NewTicker(time.Second * time.Duration(pollingRate)), + stateKey: stateKey, + portClient: portClient, + } + return rv +} + +func (h *PollingHandler) Run(resync func()) { + klog.Infof("Starting polling handler") + currentState := &port.Config{} + + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + + klog.Infof("Polling handler started") + run := true + for run { + select { + case sig := <-sigchan: + klog.Infof("Received signal %v: terminating\n", sig) + run = false + case <-h.ticker.C: + klog.Infof("Polling event listener iteration after %d seconds. Checking for changes...", h.ticker.C) + configuration, err := integration.GetIntegrationConfig(h.portClient, h.stateKey) + if err != nil { + klog.Errorf("error resyncing: %s", err.Error()) + } + + if reflect.DeepEqual(currentState, configuration) != true { + klog.Infof("Changes detected. Resyncing...") + currentState = configuration + resync() + } + } + } +} diff --git a/pkg/goutils/env.go b/pkg/goutils/env.go new file mode 100644 index 0000000..e328c14 --- /dev/null +++ b/pkg/goutils/env.go @@ -0,0 +1,36 @@ +package goutils + +import ( + "fmt" + "os" + "strconv" +) + +func GetEnvOrDefault(key string, defaultValue string) string { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + return value +} + +func GetBooleanEnvOrDefault(key string, defaultValue bool) bool { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + return value == "true" +} + +func GetIntEnvOrDefault(key string, defaultValue int) int { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + result, err := strconv.Atoi(value) + if err != nil { + fmt.Printf("Using default value "+strconv.Itoa(defaultValue)+" for "+key+". error parsing env variable %s: %s", key, err.Error()) + return defaultValue + } + return result +} diff --git a/pkg/handlers/controllers.go b/pkg/handlers/controllers.go index 57c8604..c665615 100644 --- a/pkg/handlers/controllers.go +++ b/pkg/handlers/controllers.go @@ -2,10 +2,11 @@ package handlers import ( "context" - "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/goutils" "github.com/port-labs/port-k8s-exporter/pkg/k8s" + "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/signal" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/klog/v2" @@ -15,25 +16,26 @@ import ( type ControllersHandler struct { controllers []*k8s.Controller informersFactory dynamicinformer.DynamicSharedInformerFactory - exporterConfig *config.Config + exporterConfig *port.Config portClient *cli.PortClient + stopCh chan struct{} } -func NewControllersHandler(exporterConfig *config.Config, k8sClient *k8s.Client, portClient *cli.PortClient) *ControllersHandler { +func NewControllersHandler(exporterConfig *port.Config, portConfig *port.AppConfig, k8sClient *k8s.Client, portClient *cli.PortClient) *ControllersHandler { resync := time.Minute * time.Duration(exporterConfig.ResyncInterval) informersFactory := dynamicinformer.NewDynamicSharedInformerFactory(k8sClient.DynamicClient, resync) - aggResources := make(map[string][]config.KindConfig) - for _, resource := range exporterConfig.Resources { - kindConfig := config.KindConfig{Selector: resource.Selector, Port: resource.Port} + aggResources := make(map[string][]port.KindConfig) + for _, resource := range portConfig.Resources { + kindConfig := port.KindConfig{Selector: resource.Selector, Port: resource.Port} if _, ok := aggResources[resource.Kind]; ok { aggResources[resource.Kind] = append(aggResources[resource.Kind], kindConfig) } else { - aggResources[resource.Kind] = []config.KindConfig{kindConfig} + aggResources[resource.Kind] = []port.KindConfig{kindConfig} } } - controllers := make([]*k8s.Controller, 0, len(exporterConfig.Resources)) + controllers := make([]*k8s.Controller, 0, len(portConfig.Resources)) for kind, kindConfigs := range aggResources { var gvr schema.GroupVersionResource @@ -44,7 +46,7 @@ func NewControllersHandler(exporterConfig *config.Config, k8sClient *k8s.Client, } informer := informersFactory.ForResource(gvr) - controller := k8s.NewController(config.AggregatedResource{Kind: kind, KindConfigs: kindConfigs}, portClient, informer) + controller := k8s.NewController(port.AggregatedResource{Kind: kind, KindConfigs: kindConfigs}, portClient, informer) controllers = append(controllers, controller) } @@ -57,17 +59,18 @@ func NewControllersHandler(exporterConfig *config.Config, k8sClient *k8s.Client, informersFactory: informersFactory, exporterConfig: exporterConfig, portClient: portClient, + stopCh: signal.SetupSignalHandler(), } return controllersHandler } -func (c *ControllersHandler) Handle(stopCh <-chan struct{}) { +func (c *ControllersHandler) Handle() { klog.Info("Starting informers") - c.informersFactory.Start(stopCh) + c.informersFactory.Start(c.stopCh) klog.Info("Waiting for informers cache sync") for _, controller := range c.controllers { - if err := controller.WaitForCacheSync(stopCh); err != nil { + if err := controller.WaitForCacheSync(c.stopCh); err != nil { klog.Fatalf("Error while waiting for informer cache sync: %s", err.Error()) } } @@ -75,15 +78,17 @@ func (c *ControllersHandler) Handle(stopCh <-chan struct{}) { c.RunDeleteStaleEntities() klog.Info("Starting controllers") for _, controller := range c.controllers { - controller.Run(1, stopCh) + controller.Run(1, c.stopCh) } - <-stopCh - klog.Info("Shutting down controllers") - for _, controller := range c.controllers { - controller.Shutdown() - } - klog.Info("Exporter exiting") + go func() { + <-c.stopCh + klog.Info("Shutting down controllers") + for _, controller := range c.controllers { + controller.Shutdown() + } + klog.Info("Exporter exiting") + }() } func (c *ControllersHandler) RunDeleteStaleEntities() { @@ -107,3 +112,8 @@ func (c *ControllersHandler) RunDeleteStaleEntities() { } klog.Info("Done deleting stale entities") } + +func (c *ControllersHandler) Stop() { + klog.Info("Stopping controllers") + close(c.stopCh) +} diff --git a/pkg/k8s/controller.go b/pkg/k8s/controller.go index fc6f20a..2d290fd 100644 --- a/pkg/k8s/controller.go +++ b/pkg/k8s/controller.go @@ -3,7 +3,6 @@ package k8s import ( "context" "fmt" - "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/jq" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" @@ -36,14 +35,14 @@ type EventItem struct { } type Controller struct { - resource config.AggregatedResource + resource port.AggregatedResource portClient *cli.PortClient informer cache.SharedIndexInformer lister cache.GenericLister workqueue workqueue.RateLimitingInterface } -func NewController(resource config.AggregatedResource, portClient *cli.PortClient, informer informers.GenericInformer) *Controller { +func NewController(resource port.AggregatedResource, portClient *cli.PortClient, informer informers.GenericInformer) *Controller { controller := &Controller{ resource: resource, portClient: portClient, @@ -205,7 +204,7 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error { return nil } -func (c *Controller) getObjectEntities(obj interface{}, selector config.Selector, mappings []port.EntityMapping) ([]port.Entity, error) { +func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping) ([]port.Entity, error) { unstructuredObj, ok := obj.(*unstructured.Unstructured) if !ok { return nil, fmt.Errorf("error casting to unstructured") diff --git a/pkg/port/cli/client.go b/pkg/port/cli/client.go index 98c930f..ea98e70 100644 --- a/pkg/port/cli/client.go +++ b/pkg/port/cli/client.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "fmt" "strings" "github.com/go-resty/resty/v2" @@ -57,6 +58,10 @@ func (c *PortClient) Authenticate(ctx context.Context, clientID, clientSecret st if err != nil { return "", err } + if resp.StatusCode() != 200 { + return "", fmt.Errorf("failed to authenticate, got: %s", resp.Body()) + } + var tokenResp port.AccessTokenResponse err = json.Unmarshal(resp.Body(), &tokenResp) if err != nil { diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index df81acf..1507e80 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -2,7 +2,6 @@ package cli import ( "fmt" - "github.com/port-labs/port-k8s-exporter/pkg/port" ) @@ -21,3 +20,17 @@ func (c *PortClient) CreateIntegration(i *port.Integration) (*port.Integration, } return &pb.Integration, nil } + +func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.AppConfig, error) { + pb := &port.ResponseBody{} + resp, err := c.Client.R(). + SetResult(&pb). + Get(fmt.Sprintf("v1/integration/%s", stateKey)) + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to get integration config, got: %s", resp.Body()) + } + return pb.Integration.Config, nil +} diff --git a/pkg/port/cli/kafka_crednetials.go b/pkg/port/cli/kafka_crednetials.go new file mode 100644 index 0000000..37ca655 --- /dev/null +++ b/pkg/port/cli/kafka_crednetials.go @@ -0,0 +1,20 @@ +package cli + +import ( + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" +) + +func (c *PortClient) GetKafkaCredentials() (*port.OrgKafkaCredentials, error) { + pb := &port.ResponseBody{} + resp, err := c.Client.R(). + SetResult(&pb). + Get("v1/kafka-credentials") + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to create integration, got: %s", resp.Body()) + } + return &pb.KafkaCredentials, nil +} diff --git a/pkg/port/cli/org_details.go b/pkg/port/cli/org_details.go new file mode 100644 index 0000000..77f9245 --- /dev/null +++ b/pkg/port/cli/org_details.go @@ -0,0 +1,25 @@ +package cli + +import ( + "encoding/json" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" +) + +func (c *PortClient) GetOrgId() (string, error) { + pb := &port.ResponseBody{} + resp, err := c.Client.R(). + SetResult(&pb). + Get("v1/organization") + + z := &struct { + }{} + _ = json.Unmarshal(resp.Body(), &z) + if err != nil { + return "", err + } + if !pb.OK { + return "", fmt.Errorf("failed to create integration, got: %s", resp.Body()) + } + return pb.OrgDetails.OrgId, nil +} diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index a863e7d..3ccc8e7 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -3,17 +3,22 @@ package integration import ( "context" "fmt" - "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" ) -func NewIntegration(portClient *cli.PortClient, stateKey string) error { +func NewIntegration(portClient *cli.PortClient, stateKey string, exporterConfig *port.Config) error { integration := &port.Integration{ Title: stateKey, InstallationAppType: "kubernetes", InstallationId: stateKey, + EventListener: port.EventListenerSettings{ + Type: exporterConfig.EventListenerType, + }, + Config: &port.AppConfig{ + Resources: exporterConfig.Resources, + }, } _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { @@ -26,3 +31,18 @@ func NewIntegration(portClient *cli.PortClient, stateKey string) error { } return nil } + +func GetIntegrationConfig(portClient *cli.PortClient, stateKey string) (*port.AppConfig, error) { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return nil, fmt.Errorf("error authenticating with Port: %v", err) + } + + apiConfig, err := portClient.GetIntegrationConfig(stateKey) + if err != nil { + return nil, fmt.Errorf("error getting Port integration config: %v", err) + } + + return apiConfig, nil + +} diff --git a/pkg/port/kafka_credentials/credentials.go b/pkg/port/kafka_credentials/credentials.go new file mode 100644 index 0000000..4cc49d9 --- /dev/null +++ b/pkg/port/kafka_credentials/credentials.go @@ -0,0 +1,28 @@ +package kafka_credentials + +import ( + "context" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "k8s.io/klog/v2" +) + +func GetKafkaCredentials(portClient *cli.PortClient) (*port.OrgKafkaCredentials, error) { + klog.Infof("Getting Port kafka credentials") + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return nil, fmt.Errorf("error authenticating with Port: %v", err) + } + + r, err := portClient.GetKafkaCredentials() + if err != nil { + return nil, fmt.Errorf("error getting Port org credentials: %v", err) + } + + if r.Username == "" || r.Password == "" { + return nil, fmt.Errorf("error getting Port org credentials: username or password is empty") + } + + return r, nil +} diff --git a/pkg/port/models.go b/pkg/port/models.go index 68c2a66..547d9f5 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -28,10 +28,12 @@ type ( } Integration struct { - InstallationId string `json:"installationId"` - Title string `json:"title,omitempty"` - Version string `json:"version,omitempty"` - InstallationAppType string `json:"installationAppType,omitempty"` + InstallationId string `json:"installationId"` + Title string `json:"title,omitempty"` + Version string `json:"version,omitempty"` + InstallationAppType string `json:"installationAppType,omitempty"` + EventListener EventListenerSettings `json:"changelogDestination,omitempty"` + Config *AppConfig `json:"config,omitempty"` } BlueprintProperty struct { @@ -114,6 +116,15 @@ type ( Operator string `json:"operator"` Value interface{} `json:"value"` } + + OrgKafkaCredentials struct { + Username string `json:"username"` + Password string `json:"password"` + } + + OrgDetails struct { + OrgId string `json:"id"` + } ) type SearchBody struct { @@ -122,12 +133,14 @@ type SearchBody struct { } type ResponseBody struct { - OK bool `json:"ok"` - Entity Entity `json:"entity"` - Blueprint Blueprint `json:"blueprint"` - Action Action `json:"action"` - Entities []Entity `json:"entities"` - Integration Integration `json:"integration"` + OK bool `json:"ok"` + Entity Entity `json:"entity"` + Blueprint Blueprint `json:"blueprint"` + Action Action `json:"action"` + Entities []Entity `json:"entities"` + Integration Integration `json:"integration"` + KafkaCredentials OrgKafkaCredentials `json:"credentials"` + OrgDetails OrgDetails `json:"organization"` } type EntityMapping struct { @@ -138,3 +151,47 @@ type EntityMapping struct { Properties map[string]string Relations map[string]string } + +type EntityMappings struct { + Mappings []EntityMapping +} + +type Port struct { + Entity EntityMappings +} + +type Selector struct { + Query string +} + +type Resource struct { + Kind string + Selector Selector + Port Port +} + +type EventListenerSettings struct { + Type string `json:"type"` +} + +type KindConfig struct { + Selector Selector + Port Port +} + +type AggregatedResource struct { + Kind string + KindConfigs []KindConfig +} + +type AppConfig struct { + Resources []Resource `json:"resources"` +} + +type Config struct { + ResyncInterval uint + StateKey string + EventListenerType string + // Deprecated: use AppConfig instead. Used for updating the Port integration config on startup. + Resources []Resource +} diff --git a/pkg/port/org_details/org_details.go b/pkg/port/org_details/org_details.go new file mode 100644 index 0000000..40f68cf --- /dev/null +++ b/pkg/port/org_details/org_details.go @@ -0,0 +1,21 @@ +package org_details + +import ( + "context" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" +) + +func GetOrgId(portClient *cli.PortClient) (string, error) { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return "", fmt.Errorf("error authenticating with Port: %v", err) + } + + r, err := portClient.GetOrgId() + if err != nil { + return "", fmt.Errorf("error getting Port org credentials: %v", err) + } + + return r, nil +} diff --git a/pkg/signal/signal.go b/pkg/signal/signal.go index fc8d0e0..7b6ba80 100644 --- a/pkg/signal/signal.go +++ b/pkg/signal/signal.go @@ -1,24 +1,35 @@ package signal import ( + "fmt" "os" "os/signal" "syscall" ) -var onlyOneSignalHandler = make(chan struct{}) - -func SetupSignalHandler() (stopCh <-chan struct{}) { - close(onlyOneSignalHandler) +func SetupSignalHandler() (stopCh chan struct{}) { stop := make(chan struct{}) - signalCh := make(chan os.Signal, 2) - signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM) + gracefulStop := false + shutdownCh := make(chan os.Signal, 2) + signal.Notify(shutdownCh, os.Interrupt, syscall.SIGTERM) + go func() { + <-shutdownCh + if gracefulStop == false { + fmt.Fprint(os.Stderr, "Received SIGTERM, exiting gracefully...\n") + close(stop) + } + <-shutdownCh + if gracefulStop == false { + fmt.Fprint(os.Stderr, "Received SIGTERM again, exiting forcefully...\n") + os.Exit(1) + } + }() + go func() { - <-signalCh - close(stop) - <-signalCh - os.Exit(1) + <-stop + gracefulStop = true + close(shutdownCh) }() return stop From e7079db328f3786d7c57661bf7c94aeddd963ec5 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 11 Dec 2023 18:21:40 +0200 Subject: [PATCH 02/35] fixed miss type --- pkg/event_listener/polling/polling.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_listener/polling/polling.go index 9ded8f2..baaae0a 100644 --- a/pkg/event_listener/polling/polling.go +++ b/pkg/event_listener/polling/polling.go @@ -29,7 +29,7 @@ func NewPollingHandler(pollingRate int, stateKey string, portClient *cli.PortCli func (h *PollingHandler) Run(resync func()) { klog.Infof("Starting polling handler") - currentState := &port.Config{} + currentState := &port.AppConfig{} sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) From 311502d92e614e29f6a630670340de979c2efd0d Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 11 Dec 2023 18:21:56 +0200 Subject: [PATCH 03/35] Renaming --- pkg/event_listener/polling/polling.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_listener/polling/polling.go index baaae0a..d4ec0a2 100644 --- a/pkg/event_listener/polling/polling.go +++ b/pkg/event_listener/polling/polling.go @@ -31,14 +31,14 @@ func (h *PollingHandler) Run(resync func()) { klog.Infof("Starting polling handler") currentState := &port.AppConfig{} - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) klog.Infof("Polling handler started") run := true for run { select { - case sig := <-sigchan: + case sig := <-sigChan: klog.Infof("Received signal %v: terminating\n", sig) run = false case <-h.ticker.C: From d84ad22e0f4cb45e91c4f24c763181b24a51db4b Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 11 Dec 2023 18:23:14 +0200 Subject: [PATCH 04/35] updated typing --- pkg/k8s/controller_test.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/k8s/controller_test.go b/pkg/k8s/controller_test.go index b3f40e8..f6dac62 100644 --- a/pkg/k8s/controller_test.go +++ b/pkg/k8s/controller_test.go @@ -1,7 +1,6 @@ package k8s import ( - "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -30,7 +29,7 @@ type fixture struct { controller *Controller } -func newFixture(t *testing.T, portClientId string, portClientSecret string, resource config.Resource, objects []runtime.Object) *fixture { +func newFixture(t *testing.T, portClientId string, portClientSecret string, resource port.Resource, objects []runtime.Object) *fixture { kubeclient := k8sfake.NewSimpleDynamicClient(runtime.NewScheme()) if portClientId == "" { @@ -51,14 +50,14 @@ func newFixture(t *testing.T, portClientId string, portClientSecret string, reso } } -func newResource(selectorQuery string, mappings []port.EntityMapping) config.Resource { - return config.Resource{ +func newResource(selectorQuery string, mappings []port.EntityMapping) port.Resource { + return port.Resource{ Kind: "apps/v1/deployments", - Selector: config.Selector{ + Selector: port.Selector{ Query: selectorQuery, }, - Port: config.Port{ - Entity: config.Entity{ + Port: port.Port{ + Entity: port.EntityMappings{ Mappings: mappings, }, }, @@ -103,13 +102,13 @@ func newUnstructured(obj interface{}) *unstructured.Unstructured { return &unstructured.Unstructured{Object: res} } -func newController(resource config.Resource, objects []runtime.Object, portClient *cli.PortClient, kubeclient *k8sfake.FakeDynamicClient) *Controller { +func newController(resource port.Resource, objects []runtime.Object, portClient *cli.PortClient, kubeclient *k8sfake.FakeDynamicClient) *Controller { k8sI := dynamicinformer.NewDynamicSharedInformerFactory(kubeclient, noResyncPeriodFunc()) s := strings.SplitN(resource.Kind, "/", 3) gvr := schema.GroupVersionResource{Group: s[0], Version: s[1], Resource: s[2]} informer := k8sI.ForResource(gvr) - kindConfig := config.KindConfig{Selector: resource.Selector, Port: resource.Port} - c := NewController(config.AggregatedResource{Kind: resource.Kind, KindConfigs: []config.KindConfig{kindConfig}}, portClient, informer) + kindConfig := port.KindConfig{Selector: resource.Selector, Port: resource.Port} + c := NewController(port.AggregatedResource{Kind: resource.Kind, KindConfigs: []port.KindConfig{kindConfig}}, portClient, informer) for _, d := range objects { informer.Informer().GetIndexer().Add(d) From a32b98b15bba51f0b0050e3e65de61380f40ca23 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 11 Dec 2023 18:27:07 +0200 Subject: [PATCH 05/35] at the start of polling getting the current state --- pkg/event_listener/polling/polling.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_listener/polling/polling.go index d4ec0a2..cb32ad8 100644 --- a/pkg/event_listener/polling/polling.go +++ b/pkg/event_listener/polling/polling.go @@ -1,7 +1,6 @@ package polling import ( - "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" "github.com/port-labs/port-k8s-exporter/pkg/port/integration" "k8s.io/klog/v2" @@ -29,7 +28,10 @@ func NewPollingHandler(pollingRate int, stateKey string, portClient *cli.PortCli func (h *PollingHandler) Run(resync func()) { klog.Infof("Starting polling handler") - currentState := &port.AppConfig{} + currentState, err := integration.GetIntegrationConfig(h.portClient, h.stateKey) + if err != nil { + klog.Errorf("Error fetching the first AppConfig state: %s", err.Error()) + } sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) From a04dfd1fc39f92300d81e19b7c4e8291d071e0e2 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 11 Dec 2023 20:55:28 +0200 Subject: [PATCH 06/35] fixed schema & logs --- pkg/port/cli/integration.go | 19 ++++++++++++++++++- pkg/port/cli/kafka_crednetials.go | 2 +- pkg/port/cli/org_details.go | 2 +- pkg/port/models.go | 24 ++++++++++++------------ 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index 1507e80..f6e108e 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -5,10 +5,27 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port" ) +func parseIntegration(i *port.Integration) *port.Integration { + x := &port.Integration{ + Title: i.Title, + InstallationAppType: i.InstallationAppType, + InstallationId: i.InstallationId, + Config: i.Config, + } + + if i.EventListener.Type == "KAFKA" { + x.EventListener = port.EventListenerSettings{ + Type: i.EventListener.Type, + } + } + + return x +} + func (c *PortClient) CreateIntegration(i *port.Integration) (*port.Integration, error) { pb := &port.ResponseBody{} resp, err := c.Client.R(). - SetBody(i). + SetBody(parseIntegration(i)). SetResult(&pb). SetQueryParam("upsert", "true"). Post("v1/integration") diff --git a/pkg/port/cli/kafka_crednetials.go b/pkg/port/cli/kafka_crednetials.go index 37ca655..0343da4 100644 --- a/pkg/port/cli/kafka_crednetials.go +++ b/pkg/port/cli/kafka_crednetials.go @@ -14,7 +14,7 @@ func (c *PortClient) GetKafkaCredentials() (*port.OrgKafkaCredentials, error) { return nil, err } if !pb.OK { - return nil, fmt.Errorf("failed to create integration, got: %s", resp.Body()) + return nil, fmt.Errorf("failed to get kafka crednetials, got: %s", resp.Body()) } return &pb.KafkaCredentials, nil } diff --git a/pkg/port/cli/org_details.go b/pkg/port/cli/org_details.go index 77f9245..c33f6d0 100644 --- a/pkg/port/cli/org_details.go +++ b/pkg/port/cli/org_details.go @@ -19,7 +19,7 @@ func (c *PortClient) GetOrgId() (string, error) { return "", err } if !pb.OK { - return "", fmt.Errorf("failed to create integration, got: %s", resp.Body()) + return "", fmt.Errorf("failed to get orgId, got: %s", resp.Body()) } return pb.OrgDetails.OrgId, nil } diff --git a/pkg/port/models.go b/pkg/port/models.go index 547d9f5..e88adab 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -144,20 +144,20 @@ type ResponseBody struct { } type EntityMapping struct { - Identifier string - Title string - Blueprint string - Team string - Properties map[string]string - Relations map[string]string + Identifier string `json:"identifier"` + Title string `json:"title"` + Blueprint string `json:"blueprint"` + Team string `json:"team"` + Properties map[string]string `json:"properties,omitempty"` + Relations map[string]string `json:"relations,omitempty"` } type EntityMappings struct { - Mappings []EntityMapping + Mappings []EntityMapping `json:"mappings"` } type Port struct { - Entity EntityMappings + Entity EntityMappings `json:"entity"` } type Selector struct { @@ -165,13 +165,13 @@ type Selector struct { } type Resource struct { - Kind string - Selector Selector - Port Port + Kind string `json:"kind"` + Selector Selector `json:"selector,omitempty"` + Port Port `json:"port"` } type EventListenerSettings struct { - Type string `json:"type"` + Type string `json:"type,omitempty"` } type KindConfig struct { From 25254fb0ba1edb0dcaadcf67c0091ff68a5dc576 Mon Sep 17 00:00:00 2001 From: yair Date: Tue, 12 Dec 2023 15:22:30 +0200 Subject: [PATCH 07/35] revert useragent addition --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 5859659..a99962c 100644 --- a/main.go +++ b/main.go @@ -59,6 +59,7 @@ func main() { portClient, err := cli.New(portBaseURL, cli.WithClientID(portClientId), cli.WithClientSecret(portClientSecret), cli.WithDeleteDependents(deleteDependents), cli.WithCreateMissingRelatedEntities(createMissingRelatedEntities), + cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), ) if err != nil { From 1eef152e5b44121fc842a0eb51c08509210fb0b6 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 11:53:00 +0200 Subject: [PATCH 08/35] fixes --- main.go | 6 ++++-- pkg/event_listener/polling/polling.go | 16 +++++++++------- pkg/jq/parser.go | 15 ++++++++++++++- pkg/port/integration/integration.go | 11 +++++------ pkg/port/models.go | 8 ++++---- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/main.go b/main.go index a99962c..65e2abd 100644 --- a/main.go +++ b/main.go @@ -73,12 +73,14 @@ func main() { _, err = integration.GetIntegrationConfig(portClient, stateKey) if err != nil { - err = integration.NewIntegration(portClient, stateKey, exporterConfig) + if exporterConfig == nil { + klog.Fatalf("The integration does not exist and no config file was provided") + } + err = integration.NewIntegration(portClient, exporterConfig, exporterConfig.Resources) if err != nil { klog.Fatalf("Error creating K8s integration: %s", err.Error()) } } - cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", exporterConfig.StateKey))(portClient) klog.Info("Starting controllers handler") handler, _ := InitiateHandler(exporterConfig, k8sClient, portClient) diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_listener/polling/polling.go index cb32ad8..348e43f 100644 --- a/pkg/event_listener/polling/polling.go +++ b/pkg/event_listener/polling/polling.go @@ -12,16 +12,18 @@ import ( ) type PollingHandler struct { - ticker *time.Ticker - stateKey string - portClient *cli.PortClient + ticker *time.Ticker + stateKey string + portClient *cli.PortClient + pollingRate int } func NewPollingHandler(pollingRate int, stateKey string, portClient *cli.PortClient) *PollingHandler { rv := &PollingHandler{ - ticker: time.NewTicker(time.Second * time.Duration(pollingRate)), - stateKey: stateKey, - portClient: portClient, + ticker: time.NewTicker(time.Second * time.Duration(pollingRate)), + stateKey: stateKey, + portClient: portClient, + pollingRate: pollingRate, } return rv } @@ -44,7 +46,7 @@ func (h *PollingHandler) Run(resync func()) { klog.Infof("Received signal %v: terminating\n", sig) run = false case <-h.ticker.C: - klog.Infof("Polling event listener iteration after %d seconds. Checking for changes...", h.ticker.C) + klog.Infof("Polling event listener iteration after %d seconds. Checking for changes...", h.pollingRate) configuration, err := integration.GetIntegrationConfig(h.portClient, h.stateKey) if err != nil { klog.Errorf("error resyncing: %s", err.Error()) diff --git a/pkg/jq/parser.go b/pkg/jq/parser.go index a6ca0ee..8a02b2c 100644 --- a/pkg/jq/parser.go +++ b/pkg/jq/parser.go @@ -3,6 +3,8 @@ package jq import ( "fmt" "github.com/itchyny/gojq" + "k8s.io/klog/v2" + "os" "strings" "sync" ) @@ -12,11 +14,22 @@ var mutex = &sync.Mutex{} func runJQQuery(jqQuery string, obj interface{}) (interface{}, error) { query, err := gojq.Parse(jqQuery) if err != nil { + klog.Warningf("failed to parse jq query: %s", jqQuery) + return nil, err + } + code, err := gojq.Compile( + query, + gojq.WithEnvironLoader(func() []string { + return os.Environ() + }), + ) + if err != nil { + klog.Warningf("failed to compile jq query: %s", jqQuery) return nil, err } mutex.Lock() - queryRes, ok := query.Run(obj).Next() + queryRes, ok := code.Run(obj).Next() mutex.Unlock() if !ok { diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 3ccc8e7..0c39090 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -7,17 +7,16 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port/cli" ) -func NewIntegration(portClient *cli.PortClient, stateKey string, exporterConfig *port.Config) error { - +func NewIntegration(portClient *cli.PortClient, exporterConfig *port.Config, resources []port.Resource) error { integration := &port.Integration{ - Title: stateKey, - InstallationAppType: "kubernetes", - InstallationId: stateKey, + Title: exporterConfig.StateKey, + InstallationAppType: "K8S EXPORTER", + InstallationId: exporterConfig.StateKey, EventListener: port.EventListenerSettings{ Type: exporterConfig.EventListenerType, }, Config: &port.AppConfig{ - Resources: exporterConfig.Resources, + Resources: resources, }, } _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) diff --git a/pkg/port/models.go b/pkg/port/models.go index e88adab..baec613 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -40,7 +40,7 @@ type ( Type string `json:"type,omitempty"` Title string `json:"title,omitempty"` Identifier string `json:"identifier,omitempty"` - Default string `json:"default,omitempty"` + Default any `json:"default,omitempty"` Icon string `json:"icon,omitempty"` Format string `json:"format,omitempty"` Description string `json:"description,omitempty"` @@ -87,9 +87,9 @@ type ( Description string `json:"description"` Schema BlueprintSchema `json:"schema"` FormulaProperties map[string]BlueprintFormulaProperty `json:"formulaProperties"` - MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties"` + MirrorProperties map[string]BlueprintMirrorProperty `json:"mirrorProperties,omitempty"` ChangelogDestination *ChangelogDestination `json:"changelogDestination,omitempty"` - Relations map[string]Relation `json:"relations"` + Relations map[string]Relation `json:"relations,omitempty"` } Action struct { @@ -147,7 +147,7 @@ type EntityMapping struct { Identifier string `json:"identifier"` Title string `json:"title"` Blueprint string `json:"blueprint"` - Team string `json:"team"` + Team string `json:"team,omitempty"` Properties map[string]string `json:"properties,omitempty"` Relations map[string]string `json:"relations,omitempty"` } From 97edf373139b112ee852a9929a408642fc5610af Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 12:01:43 +0200 Subject: [PATCH 09/35] cr requests --- main.go | 6 +++--- pkg/event_listener/consumer/consumer.go | 11 +++++++---- pkg/event_listener/event_listener_handler.go | 9 ++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index 65e2abd..92ca85f 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,7 @@ var ( portClientSecret string ) -func InitiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portClient *cli.PortClient) (*handlers.ControllersHandler, error) { +func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portClient *cli.PortClient) (*handlers.ControllersHandler, error) { apiConfig, err := integration.GetIntegrationConfig(portClient, stateKey) if err != nil { klog.Fatalf("Error getting K8s integration config: %s", err.Error()) @@ -83,11 +83,11 @@ func main() { } klog.Info("Starting controllers handler") - handler, _ := InitiateHandler(exporterConfig, k8sClient, portClient) + handler, _ := initiateHandler(exporterConfig, k8sClient, portClient) eventListener := event_listener.NewEventListener(stateKey, eventListenerType, handler, portClient) err = eventListener.Start(func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { handler.Stop() - return InitiateHandler(exporterConfig, k8sClient, portClient) + return initiateHandler(exporterConfig, k8sClient, portClient) }) if err != nil { klog.Fatalf("Error starting event listener: %s", err.Error()) diff --git a/pkg/event_listener/consumer/consumer.go b/pkg/event_listener/consumer/consumer.go index c2b9537..2ef4f41 100644 --- a/pkg/event_listener/consumer/consumer.go +++ b/pkg/event_listener/consumer/consumer.go @@ -44,14 +44,17 @@ func NewConsumer(config *KafkaConfiguration) (*Consumer, error) { func (c *Consumer) Consume(topic string, handler JsonHandler) { topics := []string{topic} - _ = c.client.SubscribeTopics(topics, nil) - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + err := c.client.SubscribeTopics(topics, nil) + if err != nil { + klog.Fatalf("Error subscribing to topic: %s", err.Error()) + } + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) run := true for run { select { - case sig := <-sigchan: + case sig := <-sigChan: klog.Infof("Caught signal %v: terminating\n", sig) run = false default: diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go index fa63334..a2148e1 100644 --- a/pkg/event_listener/event_listener_handler.go +++ b/pkg/event_listener/event_listener_handler.go @@ -52,7 +52,7 @@ func NewEventListener(stateKey string, eventListenerType string, controllerHandl func startKafkaEventListener(l *EventListener, resync func()) error { klog.Infof("Starting Kafka event listener") - klog.Infof("Geting Consumer Information") + klog.Infof("Getting Consumer Information") credentials, err := kafka_credentials.GetKafkaCredentials(l.portClient) if err != nil { return err @@ -82,12 +82,11 @@ func startKafkaEventListener(l *EventListener, resync func()) error { instance.Consume(topic, func(value []byte) { incomingMessage := &IncomingMessage{} parsingError := json.Unmarshal(value, &incomingMessage) - if shouldResync(l.stateKey, incomingMessage) { - klog.Infof("Changes detected. Resyncing...") - resync() - } if parsingError != nil { utilruntime.HandleError(fmt.Errorf("error handling message: %s", parsingError.Error())) + } else if shouldResync(l.stateKey, incomingMessage) { + klog.Infof("Changes detected. Resyncing...") + resync() } }) From 56bad417b9b7741d12f3b1677c6622e22803f46a Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 12:54:33 +0200 Subject: [PATCH 10/35] better config management --- main.go | 32 +++++++-------- pkg/config/config.go | 43 ++++++++++++++++++-- pkg/event_listener/event_listener_handler.go | 17 ++++---- pkg/event_listener/polling/polling.go | 4 +- pkg/goutils/env.go | 8 ++-- 5 files changed, 71 insertions(+), 33 deletions(-) diff --git a/main.go b/main.go index 92ca85f..a880746 100644 --- a/main.go +++ b/main.go @@ -3,14 +3,11 @@ package main import ( "flag" "fmt" - "github.com/port-labs/port-k8s-exporter/pkg/goutils" - "github.com/port-labs/port-k8s-exporter/pkg/port" - "os" - "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/event_listener" "github.com/port-labs/port-k8s-exporter/pkg/handlers" "github.com/port-labs/port-k8s-exporter/pkg/k8s" + "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" "github.com/port-labs/port-k8s-exporter/pkg/port/integration" "k8s.io/klog/v2" @@ -66,7 +63,7 @@ func main() { klog.Fatalf("Error building Port client: %s", err.Error()) } - exporterConfig, err := config.New(configFilePath, resyncInterval, stateKey, eventListenerType) + exporterConfig, err := config.GetConfigFile(configFilePath, resyncInterval, stateKey, eventListenerType) if err != nil { klog.Fatalf("Error building Port K8s Exporter config: %s", err.Error()) } @@ -92,18 +89,21 @@ func main() { if err != nil { klog.Fatalf("Error starting event listener: %s", err.Error()) } - klog.Info("Started controllers handler") } func init() { - flag.StringVar(&configFilePath, "config", "", "Path to Port K8s Exporter config file. Required.") - flag.StringVar(&stateKey, "state-key", "", "Port K8s Exporter state key id. Required.") - flag.BoolVar(&deleteDependents, "delete-dependents", false, "Flag to enable deletion of dependent Port Entities. Optional.") - flag.BoolVar(&createMissingRelatedEntities, "create-missing-related-entities", false, "Flag to enable creation of missing related Port entities. Optional.") - flag.UintVar(&resyncInterval, "resync-interval", 0, "The re-sync interval in minutes. Optional.") - flag.StringVar(&portBaseURL, "port-base-url", "https://api.getport.io", "Port base URL. Optional.") - portClientId = os.Getenv("PORT_CLIENT_ID") - portClientSecret = os.Getenv("PORT_CLIENT_SECRET") - - eventListenerType = goutils.GetEnvOrDefault("EVENT_LISTENER__TYPE", "POLLING") + configFilePath = config.NewString("config", "", "Path to Port K8s Exporter config file. Required.") + stateKey = config.NewString("state-key", "", "Port K8s Exporter state key id. Required.") + + // change to the app config + //config.NewBoolean("delete-dependents", false, "Flag to enable deletion of dependent Port Entities. Optional.") + //config.NewBoolean("create-missing-related-entities", false, "Flag to enable creation of missing related Port entities. Optional.") + + resyncInterval = config.NewUInt("resync-interval", 0, "The re-sync interval in minutes. Optional.") + portBaseURL = config.NewString("port-base-url", "https://api.getport.io", "Port base URL. Optional.") + + portClientId = config.NewString("port-client-id", "", "Port client id. Required.") + portClientSecret = config.NewString("port-client-secret", "", "Port client secret. Required.") + + config.NewString("event-listener-type", "POLLING", "Event listener type. Optional.") } diff --git a/pkg/config/config.go b/pkg/config/config.go index 5f0356b..6312f67 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,13 +1,50 @@ package config import ( + "flag" + "github.com/port-labs/port-k8s-exporter/pkg/goutils" "github.com/port-labs/port-k8s-exporter/pkg/port" - "os" - "gopkg.in/yaml.v2" + "k8s.io/klog/v2" + "os" + "slices" + "strings" ) -func New(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { +var keys []string + +func prepareEnvKey(key string) string { + newKey := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) + + if slices.Contains(keys, newKey) { + klog.Fatalf("Application Error : Found duplicate config key: %s", newKey) + } + + keys = append(keys, newKey) + return newKey +} + +func NewString(key string, defaultValue string, description string) string { + var value string + flag.StringVar(&value, key, "", description) + if value == "" { + value = goutils.GetStringEnvOrDefault(prepareEnvKey(key), defaultValue) + } + + return value +} + +func NewUInt(key string, defaultValue uint, description string) uint { + var value uint64 + flag.Uint64Var(&value, key, 0, description) + if value == 0 { + value = goutils.GetUintEnvOrDefault(prepareEnvKey(key), uint64(defaultValue)) + } + + return uint(value) +} + +func GetConfigFile(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { c := &port.Config{ ResyncInterval: resyncInterval, StateKey: stateKey, diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go index a2148e1..2c92293 100644 --- a/pkg/event_listener/event_listener_handler.go +++ b/pkg/event_listener/event_listener_handler.go @@ -3,6 +3,7 @@ package event_listener import ( "encoding/json" "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/event_listener/consumer" "github.com/port-labs/port-k8s-exporter/pkg/event_listener/polling" "github.com/port-labs/port-k8s-exporter/pkg/goutils" @@ -61,24 +62,24 @@ func startKafkaEventListener(l *EventListener, resync func()) error { if err != nil { return err } - config := &consumer.KafkaConfiguration{ - Brokers: goutils.GetEnvOrDefault("EVENT_LISTENER__BROKERS", "b-1-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-2-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-3-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196"), - SecurityProtocol: goutils.GetEnvOrDefault("EVENT_LISTENER__SECURITY_PROTOCOL", "SASL_SSL"), - AuthenticationMechanism: goutils.GetEnvOrDefault("EVENT_LISTENER__AUTHENTICATION_MECHANISM", "SCRAM-SHA-512"), - KafkaSecurityEnabled: goutils.GetBooleanEnvOrDefault("EVENT_LISTENER__KAFKA_SECURITY_ENABLED", true), + + c := &consumer.KafkaConfiguration{ + Brokers: config.NewString("event-listener-brokers", "localhost:9092", "Kafka brokers"), + SecurityProtocol: config.NewString("event-listener-security-protocol", "plaintext", "Kafka security protocol"), + AuthenticationMechanism: config.NewString("event-listener-authentication-mechanism", "none", "Kafka authentication mechanism"), Username: credentials.Username, Password: credentials.Password, GroupID: orgId + ".k8s." + l.stateKey, } topic := orgId + ".change.log" - instance, err := consumer.NewConsumer(config) + instance, err := consumer.NewConsumer(c) if err != nil { return err } - klog.Infof("Starting consumer for topic %s and groupId %s", topic, config.GroupID) + klog.Infof("Starting consumer for topic %s and groupId %s", topic, c.GroupID) instance.Consume(topic, func(value []byte) { incomingMessage := &IncomingMessage{} parsingError := json.Unmarshal(value, &incomingMessage) @@ -95,7 +96,7 @@ func startKafkaEventListener(l *EventListener, resync func()) error { func startPollingEventListener(l *EventListener, resync func()) { klog.Infof("Starting polling event listener") - pollingRate := goutils.GetIntEnvOrDefault("EVENT_LISTENER__POLLING_RATE", 60) + pollingRate := goutils.GetUintEnvOrDefault("EVENT_LISTENER__POLLING_RATE", 60) klog.Infof("Polling rate set to %d seconds", pollingRate) pollingHandler := polling.NewPollingHandler(pollingRate, l.stateKey, l.portClient) pollingHandler.Run(resync) diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_listener/polling/polling.go index 348e43f..0c7b90a 100644 --- a/pkg/event_listener/polling/polling.go +++ b/pkg/event_listener/polling/polling.go @@ -15,10 +15,10 @@ type PollingHandler struct { ticker *time.Ticker stateKey string portClient *cli.PortClient - pollingRate int + pollingRate uint64 } -func NewPollingHandler(pollingRate int, stateKey string, portClient *cli.PortClient) *PollingHandler { +func NewPollingHandler(pollingRate uint64, stateKey string, portClient *cli.PortClient) *PollingHandler { rv := &PollingHandler{ ticker: time.NewTicker(time.Second * time.Duration(pollingRate)), stateKey: stateKey, diff --git a/pkg/goutils/env.go b/pkg/goutils/env.go index e328c14..a88fb70 100644 --- a/pkg/goutils/env.go +++ b/pkg/goutils/env.go @@ -6,7 +6,7 @@ import ( "strconv" ) -func GetEnvOrDefault(key string, defaultValue string) string { +func GetStringEnvOrDefault(key string, defaultValue string) string { value := os.Getenv(key) if value == "" { return defaultValue @@ -22,14 +22,14 @@ func GetBooleanEnvOrDefault(key string, defaultValue bool) bool { return value == "true" } -func GetIntEnvOrDefault(key string, defaultValue int) int { +func GetUintEnvOrDefault(key string, defaultValue uint64) uint64 { value := os.Getenv(key) if value == "" { return defaultValue } - result, err := strconv.Atoi(value) + result, err := strconv.ParseUint(value, 10, 32) if err != nil { - fmt.Printf("Using default value "+strconv.Itoa(defaultValue)+" for "+key+". error parsing env variable %s: %s", key, err.Error()) + fmt.Printf("Using default value "+strconv.FormatUint(uint64(defaultValue), 10)+" for "+key+". error parsing env variable %s: %s", key, err.Error()) return defaultValue } return result From eb240617913346288c8fc528aca5e37f5b55d035 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 12:54:52 +0200 Subject: [PATCH 11/35] removed bool config managment --- pkg/goutils/env.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/goutils/env.go b/pkg/goutils/env.go index a88fb70..3526492 100644 --- a/pkg/goutils/env.go +++ b/pkg/goutils/env.go @@ -14,14 +14,6 @@ func GetStringEnvOrDefault(key string, defaultValue string) string { return value } -func GetBooleanEnvOrDefault(key string, defaultValue bool) bool { - value := os.Getenv(key) - if value == "" { - return defaultValue - } - return value == "true" -} - func GetUintEnvOrDefault(key string, defaultValue uint64) uint64 { value := os.Getenv(key) if value == "" { From 296e170e712484719fecaccdac36efd7b70beae9 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 12:56:14 +0200 Subject: [PATCH 12/35] better error for config file --- pkg/config/config.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 6312f67..2798b4c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -44,6 +44,14 @@ func NewUInt(key string, defaultValue uint, description string) uint { return uint(value) } +type FileNotFoundError struct { + s string +} + +func (e *FileNotFoundError) Error() string { + return e.s +} + func GetConfigFile(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { c := &port.Config{ ResyncInterval: resyncInterval, @@ -52,7 +60,7 @@ func GetConfigFile(filepath string, resyncInterval uint, stateKey string, eventL } config, err := os.ReadFile(filepath) if err != nil { - return nil, err + return c, &FileNotFoundError{err.Error()} } err = yaml.Unmarshal(config, c) From 2c21603ad1b53a188d9ab5e0bad3adbece9a87b5 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 14:12:53 +0200 Subject: [PATCH 13/35] moved dependent and create missing to the app config --- main.go | 10 ++++------ pkg/port/models.go | 4 +++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index a880746..854beef 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,9 @@ func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portCli klog.Fatalf("Error getting K8s integration config: %s", err.Error()) } + cli.WithDeleteDependents(deleteDependents)(portClient) + cli.WithCreateMissingRelatedEntities(createMissingRelatedEntities)(portClient) + newHandler := handlers.NewControllersHandler(exporterConfig, apiConfig, k8sClient, portClient) newHandler.Handle() @@ -55,7 +58,6 @@ func main() { portClient, err := cli.New(portBaseURL, cli.WithClientID(portClientId), cli.WithClientSecret(portClientSecret), - cli.WithDeleteDependents(deleteDependents), cli.WithCreateMissingRelatedEntities(createMissingRelatedEntities), cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), ) @@ -95,15 +97,11 @@ func init() { configFilePath = config.NewString("config", "", "Path to Port K8s Exporter config file. Required.") stateKey = config.NewString("state-key", "", "Port K8s Exporter state key id. Required.") - // change to the app config - //config.NewBoolean("delete-dependents", false, "Flag to enable deletion of dependent Port Entities. Optional.") - //config.NewBoolean("create-missing-related-entities", false, "Flag to enable creation of missing related Port entities. Optional.") - resyncInterval = config.NewUInt("resync-interval", 0, "The re-sync interval in minutes. Optional.") portBaseURL = config.NewString("port-base-url", "https://api.getport.io", "Port base URL. Optional.") portClientId = config.NewString("port-client-id", "", "Port client id. Required.") portClientSecret = config.NewString("port-client-secret", "", "Port client secret. Required.") - config.NewString("event-listener-type", "POLLING", "Event listener type. Optional.") + eventListenerType = config.NewString("event-listener-type", "POLLING", "Event listener type. Optional.") } diff --git a/pkg/port/models.go b/pkg/port/models.go index baec613..8b7000c 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -185,7 +185,9 @@ type AggregatedResource struct { } type AppConfig struct { - Resources []Resource `json:"resources"` + DeleteDependents bool `json:"deleteDependents,omitempty"` + CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` + Resources []Resource `json:"resources"` } type Config struct { From 62a731db42d9e8133661f8c96c5cac00ca8d9323 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 14:17:40 +0200 Subject: [PATCH 14/35] starting kafka config once --- pkg/event_listener/event_listener_handler.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go index 2c92293..d806578 100644 --- a/pkg/event_listener/event_listener_handler.go +++ b/pkg/event_listener/event_listener_handler.go @@ -24,6 +24,12 @@ type IncomingMessage struct { } `json:"diff"` } +var kafkaConfig = &consumer.KafkaConfiguration{ + Brokers: config.NewString("event-listener-brokers", "localhost:9092", "Kafka brokers"), + SecurityProtocol: config.NewString("event-listener-security-protocol", "plaintext", "Kafka security protocol"), + AuthenticationMechanism: config.NewString("event-listener-authentication-mechanism", "none", "Kafka authentication mechanism"), +} + type EventListener struct { settings port.EventListenerSettings stateKey string @@ -64,9 +70,9 @@ func startKafkaEventListener(l *EventListener, resync func()) error { } c := &consumer.KafkaConfiguration{ - Brokers: config.NewString("event-listener-brokers", "localhost:9092", "Kafka brokers"), - SecurityProtocol: config.NewString("event-listener-security-protocol", "plaintext", "Kafka security protocol"), - AuthenticationMechanism: config.NewString("event-listener-authentication-mechanism", "none", "Kafka authentication mechanism"), + Brokers: kafkaConfig.Brokers, + SecurityProtocol: kafkaConfig.SecurityProtocol, + AuthenticationMechanism: kafkaConfig.AuthenticationMechanism, Username: credentials.Username, Password: credentials.Password, GroupID: orgId + ".k8s." + l.stateKey, From f1ef258babf25903815bf7cd94f3675a54a3eb07 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 14:20:55 +0200 Subject: [PATCH 15/35] starting kafka config once --- pkg/event_listener/event_listener_handler.go | 19 +++++++++---------- pkg/event_listener/polling/polling.go | 10 +++++----- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go index d806578..2f490e2 100644 --- a/pkg/event_listener/event_listener_handler.go +++ b/pkg/event_listener/event_listener_handler.go @@ -6,7 +6,6 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/event_listener/consumer" "github.com/port-labs/port-k8s-exporter/pkg/event_listener/polling" - "github.com/port-labs/port-k8s-exporter/pkg/goutils" "github.com/port-labs/port-k8s-exporter/pkg/handlers" "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" @@ -24,12 +23,6 @@ type IncomingMessage struct { } `json:"diff"` } -var kafkaConfig = &consumer.KafkaConfiguration{ - Brokers: config.NewString("event-listener-brokers", "localhost:9092", "Kafka brokers"), - SecurityProtocol: config.NewString("event-listener-security-protocol", "plaintext", "Kafka security protocol"), - AuthenticationMechanism: config.NewString("event-listener-authentication-mechanism", "none", "Kafka authentication mechanism"), -} - type EventListener struct { settings port.EventListenerSettings stateKey string @@ -37,6 +30,13 @@ type EventListener struct { portClient *cli.PortClient } +var kafkaConfig = &consumer.KafkaConfiguration{ + Brokers: config.NewString("event-listener-brokers", "localhost:9092", "Kafka brokers"), + SecurityProtocol: config.NewString("event-listener-security-protocol", "plaintext", "Kafka security protocol"), + AuthenticationMechanism: config.NewString("event-listener-authentication-mechanism", "none", "Kafka authentication mechanism"), +} +var pollingListenerRate = config.NewUInt("event-listener-polling-rate", 60, "Polling rate for the polling event listener") + func shouldResync(stateKey string, message *IncomingMessage) bool { return message.Diff != nil && message.Diff.After != nil && @@ -102,9 +102,8 @@ func startKafkaEventListener(l *EventListener, resync func()) error { func startPollingEventListener(l *EventListener, resync func()) { klog.Infof("Starting polling event listener") - pollingRate := goutils.GetUintEnvOrDefault("EVENT_LISTENER__POLLING_RATE", 60) - klog.Infof("Polling rate set to %d seconds", pollingRate) - pollingHandler := polling.NewPollingHandler(pollingRate, l.stateKey, l.portClient) + klog.Infof("Polling rate set to %d seconds", pollingListenerRate) + pollingHandler := polling.NewPollingHandler(pollingListenerRate, l.stateKey, l.portClient) pollingHandler.Run(resync) } diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_listener/polling/polling.go index 0c7b90a..4c93014 100644 --- a/pkg/event_listener/polling/polling.go +++ b/pkg/event_listener/polling/polling.go @@ -11,15 +11,15 @@ import ( "time" ) -type PollingHandler struct { +type HandlerSettings struct { ticker *time.Ticker stateKey string portClient *cli.PortClient - pollingRate uint64 + pollingRate uint } -func NewPollingHandler(pollingRate uint64, stateKey string, portClient *cli.PortClient) *PollingHandler { - rv := &PollingHandler{ +func NewPollingHandler(pollingRate uint, stateKey string, portClient *cli.PortClient) *HandlerSettings { + rv := &HandlerSettings{ ticker: time.NewTicker(time.Second * time.Duration(pollingRate)), stateKey: stateKey, portClient: portClient, @@ -28,7 +28,7 @@ func NewPollingHandler(pollingRate uint64, stateKey string, portClient *cli.Port return rv } -func (h *PollingHandler) Run(resync func()) { +func (h *HandlerSettings) Run(resync func()) { klog.Infof("Starting polling handler") currentState, err := integration.GetIntegrationConfig(h.portClient, h.stateKey) if err != nil { From 7aae88259104df77076ffbe8aaf1d938343ba669 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 14:41:17 +0200 Subject: [PATCH 16/35] unified configuration --- main.go | 39 +++------- pkg/config/config.go | 82 ++++---------------- pkg/config/models.go | 21 +++++ pkg/config/utils.go | 72 +++++++++++++++++ pkg/event_listener/consumer/consumer.go | 13 +--- pkg/event_listener/event_listener_handler.go | 19 ++--- 6 files changed, 124 insertions(+), 122 deletions(-) create mode 100644 pkg/config/models.go create mode 100644 pkg/config/utils.go diff --git a/main.go b/main.go index 854beef..c313998 100644 --- a/main.go +++ b/main.go @@ -13,26 +13,14 @@ import ( "k8s.io/klog/v2" ) -var ( - configFilePath string - resyncInterval uint - eventListenerType string - stateKey string - deleteDependents bool - createMissingRelatedEntities bool - portBaseURL string - portClientId string - portClientSecret string -) - func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portClient *cli.PortClient) (*handlers.ControllersHandler, error) { - apiConfig, err := integration.GetIntegrationConfig(portClient, stateKey) + apiConfig, err := integration.GetIntegrationConfig(portClient, config.ApplicationConfig.StateKey) if err != nil { klog.Fatalf("Error getting K8s integration config: %s", err.Error()) } - cli.WithDeleteDependents(deleteDependents)(portClient) - cli.WithCreateMissingRelatedEntities(createMissingRelatedEntities)(portClient) + cli.WithDeleteDependents(apiConfig.DeleteDependents)(portClient) + cli.WithCreateMissingRelatedEntities(apiConfig.CreateMissingRelatedEntities)(portClient) newHandler := handlers.NewControllersHandler(exporterConfig, apiConfig, k8sClient, portClient) newHandler.Handle() @@ -56,21 +44,21 @@ func main() { klog.Fatalf("Error building K8s client: %s", err.Error()) } - portClient, err := cli.New(portBaseURL, - cli.WithClientID(portClientId), cli.WithClientSecret(portClientSecret), - cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), + portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, + cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret), + cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", config.ApplicationConfig.StateKey)), ) if err != nil { klog.Fatalf("Error building Port client: %s", err.Error()) } - exporterConfig, err := config.GetConfigFile(configFilePath, resyncInterval, stateKey, eventListenerType) + exporterConfig, err := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath, config.ApplicationConfig.ResyncInterval, config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType) if err != nil { klog.Fatalf("Error building Port K8s Exporter config: %s", err.Error()) } - _, err = integration.GetIntegrationConfig(portClient, stateKey) + _, err = integration.GetIntegrationConfig(portClient, config.ApplicationConfig.StateKey) if err != nil { if exporterConfig == nil { klog.Fatalf("The integration does not exist and no config file was provided") @@ -83,7 +71,7 @@ func main() { klog.Info("Starting controllers handler") handler, _ := initiateHandler(exporterConfig, k8sClient, portClient) - eventListener := event_listener.NewEventListener(stateKey, eventListenerType, handler, portClient) + eventListener := event_listener.NewEventListener(config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType, handler, portClient) err = eventListener.Start(func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { handler.Stop() return initiateHandler(exporterConfig, k8sClient, portClient) @@ -94,14 +82,5 @@ func main() { } func init() { - configFilePath = config.NewString("config", "", "Path to Port K8s Exporter config file. Required.") - stateKey = config.NewString("state-key", "", "Port K8s Exporter state key id. Required.") - - resyncInterval = config.NewUInt("resync-interval", 0, "The re-sync interval in minutes. Optional.") - portBaseURL = config.NewString("port-base-url", "https://api.getport.io", "Port base URL. Optional.") - - portClientId = config.NewString("port-client-id", "", "Port client id. Required.") - portClientSecret = config.NewString("port-client-secret", "", "Port client secret. Required.") - eventListenerType = config.NewString("event-listener-type", "POLLING", "Event listener type. Optional.") } diff --git a/pkg/config/config.go b/pkg/config/config.go index 2798b4c..f2e5637 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,72 +1,18 @@ package config -import ( - "flag" - "github.com/port-labs/port-k8s-exporter/pkg/goutils" - "github.com/port-labs/port-k8s-exporter/pkg/port" - "gopkg.in/yaml.v2" - "k8s.io/klog/v2" - "os" - "slices" - "strings" -) - -var keys []string - -func prepareEnvKey(key string) string { - newKey := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) - - if slices.Contains(keys, newKey) { - klog.Fatalf("Application Error : Found duplicate config key: %s", newKey) - } - - keys = append(keys, newKey) - return newKey +var KafkaConfig = &KafkaConfiguration{ + Brokers: NewString("event-listener-brokers", "localhost:9092", "Kafka brokers"), + SecurityProtocol: NewString("event-listener-security-protocol", "plaintext", "Kafka security protocol"), + AuthenticationMechanism: NewString("event-listener-authentication-mechanism", "none", "Kafka authentication mechanism"), } - -func NewString(key string, defaultValue string, description string) string { - var value string - flag.StringVar(&value, key, "", description) - if value == "" { - value = goutils.GetStringEnvOrDefault(prepareEnvKey(key), defaultValue) - } - - return value -} - -func NewUInt(key string, defaultValue uint, description string) uint { - var value uint64 - flag.Uint64Var(&value, key, 0, description) - if value == 0 { - value = goutils.GetUintEnvOrDefault(prepareEnvKey(key), uint64(defaultValue)) - } - - return uint(value) -} - -type FileNotFoundError struct { - s string -} - -func (e *FileNotFoundError) Error() string { - return e.s -} - -func GetConfigFile(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { - c := &port.Config{ - ResyncInterval: resyncInterval, - StateKey: stateKey, - EventListenerType: eventListenerType, - } - config, err := os.ReadFile(filepath) - if err != nil { - return c, &FileNotFoundError{err.Error()} - } - - err = yaml.Unmarshal(config, c) - if err != nil { - return nil, err - } - - return c, nil +var PollingListenerRate = NewUInt("event-listener-polling-rate", 60, "Polling rate for the polling event listener") + +var ApplicationConfig = &ApplicationConfiguration{ + ConfigFilePath: NewString("config", "", "Path to Port K8s Exporter config file. Required."), + StateKey: NewString("state-key", "", "Port K8s Exporter state key id. Required."), + ResyncInterval: NewUInt("resync-interval", 0, "The re-sync interval in minutes. Optional."), + PortBaseURL: NewString("port-base-url", "https://api.getport.io", "Port base URL. Optional."), + PortClientId: NewString("port-client-id", "", "Port client id. Required."), + PortClientSecret: NewString("port-client-secret", "", "Port client secret. Required."), + EventListenerType: NewString("event-listener-type", "POLLING", "Event listener type. Optional."), } diff --git a/pkg/config/models.go b/pkg/config/models.go new file mode 100644 index 0000000..f57eef1 --- /dev/null +++ b/pkg/config/models.go @@ -0,0 +1,21 @@ +package config + +type KafkaConfiguration struct { + Brokers string + SecurityProtocol string + GroupID string + AuthenticationMechanism string + Username string + Password string + KafkaSecurityEnabled bool +} + +type ApplicationConfiguration struct { + ConfigFilePath string + StateKey string + ResyncInterval uint + PortBaseURL string + PortClientId string + PortClientSecret string + EventListenerType string +} diff --git a/pkg/config/utils.go b/pkg/config/utils.go new file mode 100644 index 0000000..2798b4c --- /dev/null +++ b/pkg/config/utils.go @@ -0,0 +1,72 @@ +package config + +import ( + "flag" + "github.com/port-labs/port-k8s-exporter/pkg/goutils" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "gopkg.in/yaml.v2" + "k8s.io/klog/v2" + "os" + "slices" + "strings" +) + +var keys []string + +func prepareEnvKey(key string) string { + newKey := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) + + if slices.Contains(keys, newKey) { + klog.Fatalf("Application Error : Found duplicate config key: %s", newKey) + } + + keys = append(keys, newKey) + return newKey +} + +func NewString(key string, defaultValue string, description string) string { + var value string + flag.StringVar(&value, key, "", description) + if value == "" { + value = goutils.GetStringEnvOrDefault(prepareEnvKey(key), defaultValue) + } + + return value +} + +func NewUInt(key string, defaultValue uint, description string) uint { + var value uint64 + flag.Uint64Var(&value, key, 0, description) + if value == 0 { + value = goutils.GetUintEnvOrDefault(prepareEnvKey(key), uint64(defaultValue)) + } + + return uint(value) +} + +type FileNotFoundError struct { + s string +} + +func (e *FileNotFoundError) Error() string { + return e.s +} + +func GetConfigFile(filepath string, resyncInterval uint, stateKey string, eventListenerType string) (*port.Config, error) { + c := &port.Config{ + ResyncInterval: resyncInterval, + StateKey: stateKey, + EventListenerType: eventListenerType, + } + config, err := os.ReadFile(filepath) + if err != nil { + return c, &FileNotFoundError{err.Error()} + } + + err = yaml.Unmarshal(config, c) + if err != nil { + return nil, err + } + + return c, nil +} diff --git a/pkg/event_listener/consumer/consumer.go b/pkg/event_listener/consumer/consumer.go index 2ef4f41..9a82e15 100644 --- a/pkg/event_listener/consumer/consumer.go +++ b/pkg/event_listener/consumer/consumer.go @@ -2,6 +2,7 @@ package consumer import ( "github.com/confluentinc/confluent-kafka-go/kafka" + "github.com/port-labs/port-k8s-exporter/pkg/config" "k8s.io/klog/v2" "os" "os/signal" @@ -12,19 +13,9 @@ type Consumer struct { client *kafka.Consumer } -type KafkaConfiguration struct { - Brokers string - SecurityProtocol string - GroupID string - AuthenticationMechanism string - Username string - Password string - KafkaSecurityEnabled bool -} - type JsonHandler func(value []byte) -func NewConsumer(config *KafkaConfiguration) (*Consumer, error) { +func NewConsumer(config *config.KafkaConfiguration) (*Consumer, error) { c, err := kafka.NewConsumer(&kafka.ConfigMap{ "bootstrap.servers": config.Brokers, "group.id": config.GroupID, diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go index 2f490e2..1516b08 100644 --- a/pkg/event_listener/event_listener_handler.go +++ b/pkg/event_listener/event_listener_handler.go @@ -30,13 +30,6 @@ type EventListener struct { portClient *cli.PortClient } -var kafkaConfig = &consumer.KafkaConfiguration{ - Brokers: config.NewString("event-listener-brokers", "localhost:9092", "Kafka brokers"), - SecurityProtocol: config.NewString("event-listener-security-protocol", "plaintext", "Kafka security protocol"), - AuthenticationMechanism: config.NewString("event-listener-authentication-mechanism", "none", "Kafka authentication mechanism"), -} -var pollingListenerRate = config.NewUInt("event-listener-polling-rate", 60, "Polling rate for the polling event listener") - func shouldResync(stateKey string, message *IncomingMessage) bool { return message.Diff != nil && message.Diff.After != nil && @@ -69,10 +62,10 @@ func startKafkaEventListener(l *EventListener, resync func()) error { return err } - c := &consumer.KafkaConfiguration{ - Brokers: kafkaConfig.Brokers, - SecurityProtocol: kafkaConfig.SecurityProtocol, - AuthenticationMechanism: kafkaConfig.AuthenticationMechanism, + c := &config.KafkaConfiguration{ + Brokers: config.KafkaConfig.Brokers, + SecurityProtocol: config.KafkaConfig.SecurityProtocol, + AuthenticationMechanism: config.KafkaConfig.AuthenticationMechanism, Username: credentials.Username, Password: credentials.Password, GroupID: orgId + ".k8s." + l.stateKey, @@ -102,8 +95,8 @@ func startKafkaEventListener(l *EventListener, resync func()) error { func startPollingEventListener(l *EventListener, resync func()) { klog.Infof("Starting polling event listener") - klog.Infof("Polling rate set to %d seconds", pollingListenerRate) - pollingHandler := polling.NewPollingHandler(pollingListenerRate, l.stateKey, l.portClient) + klog.Infof("Polling rate set to %d seconds", config.PollingListenerRate) + pollingHandler := polling.NewPollingHandler(config.PollingListenerRate, l.stateKey, l.portClient) pollingHandler.Run(resync) } From 5be6bdbff8c965a8ea6e37b8ebf8ea9289aa381d Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 13 Dec 2023 16:20:34 +0200 Subject: [PATCH 17/35] fixed configuration flag issue --- main.go | 4 ++-- pkg/config/config.go | 35 +++++++++++++++++++++-------------- pkg/config/utils.go | 22 ++++++---------------- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/main.go b/main.go index c313998..b7fc9b7 100644 --- a/main.go +++ b/main.go @@ -30,7 +30,6 @@ func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portCli func main() { klog.InitFlags(nil) - flag.Parse() k8sConfig := k8s.NewKubeConfig() @@ -82,5 +81,6 @@ func main() { } func init() { - + config.Init() + flag.Parse() } diff --git a/pkg/config/config.go b/pkg/config/config.go index f2e5637..fe8ea14 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,18 +1,25 @@ package config -var KafkaConfig = &KafkaConfiguration{ - Brokers: NewString("event-listener-brokers", "localhost:9092", "Kafka brokers"), - SecurityProtocol: NewString("event-listener-security-protocol", "plaintext", "Kafka security protocol"), - AuthenticationMechanism: NewString("event-listener-authentication-mechanism", "none", "Kafka authentication mechanism"), -} -var PollingListenerRate = NewUInt("event-listener-polling-rate", 60, "Polling rate for the polling event listener") +var KafkaConfig = &KafkaConfiguration{} +var PollingListenerRate uint + +var ApplicationConfig = &ApplicationConfiguration{} + +func Init() { + // Kafka listener Configuration + NewString(&KafkaConfig.Brokers, "event-listener-brokers", "localhost:9092", "Kafka event listener brokers") + NewString(&KafkaConfig.SecurityProtocol, "event-listener-security-protocol", "plaintext", "Kafka event listener security protocol") + NewString(&KafkaConfig.AuthenticationMechanism, "event-listener-authentication-mechanism", "none", "Kafka event listener authentication mechanism") + + // Polling listener Configuration + NewUInt(&PollingListenerRate, "event-listener-polling-rate", 60, "Polling event listener polling rate") -var ApplicationConfig = &ApplicationConfiguration{ - ConfigFilePath: NewString("config", "", "Path to Port K8s Exporter config file. Required."), - StateKey: NewString("state-key", "", "Port K8s Exporter state key id. Required."), - ResyncInterval: NewUInt("resync-interval", 0, "The re-sync interval in minutes. Optional."), - PortBaseURL: NewString("port-base-url", "https://api.getport.io", "Port base URL. Optional."), - PortClientId: NewString("port-client-id", "", "Port client id. Required."), - PortClientSecret: NewString("port-client-secret", "", "Port client secret. Required."), - EventListenerType: NewString("event-listener-type", "POLLING", "Event listener type. Optional."), + // Application Configuration + NewString(&ApplicationConfig.ConfigFilePath, "config", "config.yaml", "Path to Port K8s Exporter config file. Required.") + NewString(&ApplicationConfig.StateKey, "state-key", "my-k8s-exporter", "Port K8s Exporter state key id. Required.") + NewUInt(&ApplicationConfig.ResyncInterval, "resync-interval", 0, "The re-sync interval in minutes. Optional.") + NewString(&ApplicationConfig.PortBaseURL, "port-base-url", "https://api.getport.io", "Port base URL. Optional.") + NewString(&ApplicationConfig.PortClientId, "port-client-id", "", "Port client id. Required.") + NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.") + NewString(&ApplicationConfig.EventListenerType, "event-listener-type", "POLLING", "Event listener type, can be either POLLING or KAFKA. Optional.") } diff --git a/pkg/config/utils.go b/pkg/config/utils.go index 2798b4c..f71bee9 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -24,24 +24,14 @@ func prepareEnvKey(key string) string { return newKey } -func NewString(key string, defaultValue string, description string) string { - var value string - flag.StringVar(&value, key, "", description) - if value == "" { - value = goutils.GetStringEnvOrDefault(prepareEnvKey(key), defaultValue) - } - - return value +func NewString(v *string, key string, defaultValue string, description string) { + value := goutils.GetStringEnvOrDefault(prepareEnvKey(key), defaultValue) + flag.StringVar(v, key, value, description) } -func NewUInt(key string, defaultValue uint, description string) uint { - var value uint64 - flag.Uint64Var(&value, key, 0, description) - if value == 0 { - value = goutils.GetUintEnvOrDefault(prepareEnvKey(key), uint64(defaultValue)) - } - - return uint(value) +func NewUInt(v *uint, key string, defaultValue uint, description string) { + value := uint(goutils.GetUintEnvOrDefault(prepareEnvKey(key), uint64(defaultValue))) + flag.UintVar(v, key, value, description) } type FileNotFoundError struct { From a6ced85bae0a54f7b744a21543a5d5874558d645 Mon Sep 17 00:00:00 2001 From: yair Date: Thu, 14 Dec 2023 13:06:39 +0200 Subject: [PATCH 18/35] not failing for non-existing configuration on startup --- main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/main.go b/main.go index b7fc9b7..353a914 100644 --- a/main.go +++ b/main.go @@ -52,10 +52,7 @@ func main() { klog.Fatalf("Error building Port client: %s", err.Error()) } - exporterConfig, err := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath, config.ApplicationConfig.ResyncInterval, config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType) - if err != nil { - klog.Fatalf("Error building Port K8s Exporter config: %s", err.Error()) - } + exporterConfig, _ := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath, config.ApplicationConfig.ResyncInterval, config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType) _, err = integration.GetIntegrationConfig(portClient, config.ApplicationConfig.StateKey) if err != nil { From 1db8dedfc5203152dc6463d56b936ca65c646f9d Mon Sep 17 00:00:00 2001 From: yair Date: Thu, 14 Dec 2023 13:08:57 +0200 Subject: [PATCH 19/35] change imports --- pkg/config/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/utils.go b/pkg/config/utils.go index f71bee9..ba61c39 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -6,8 +6,8 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port" "gopkg.in/yaml.v2" "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" "os" - "slices" "strings" ) From 74989485107c5109a01e8059b496b5a2b157e666 Mon Sep 17 00:00:00 2001 From: yair Date: Thu, 14 Dec 2023 13:38:04 +0200 Subject: [PATCH 20/35] changing offset reset to latest --- pkg/event_listener/consumer/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/event_listener/consumer/consumer.go b/pkg/event_listener/consumer/consumer.go index 9a82e15..2f9a574 100644 --- a/pkg/event_listener/consumer/consumer.go +++ b/pkg/event_listener/consumer/consumer.go @@ -23,7 +23,7 @@ func NewConsumer(config *config.KafkaConfiguration) (*Consumer, error) { "sasl.mechanism": config.AuthenticationMechanism, "sasl.username": config.Username, "sasl.password": config.Password, - "auto.offset.reset": "earliest", + "auto.offset.reset": "latest", }) if err != nil { From f4e8dd5f63a527f353dca11f7ed360bec09ae945 Mon Sep 17 00:00:00 2001 From: yair Date: Thu, 14 Dec 2023 14:04:46 +0200 Subject: [PATCH 21/35] starting resync in goroutine --- main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 353a914..c5c2255 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,6 @@ func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portCli cli.WithCreateMissingRelatedEntities(apiConfig.CreateMissingRelatedEntities)(portClient) newHandler := handlers.NewControllersHandler(exporterConfig, apiConfig, k8sClient, portClient) - newHandler.Handle() return newHandler, nil } @@ -70,7 +69,11 @@ func main() { eventListener := event_listener.NewEventListener(config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType, handler, portClient) err = eventListener.Start(func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { handler.Stop() - return initiateHandler(exporterConfig, k8sClient, portClient) + newHandler, handlerError := initiateHandler(exporterConfig, k8sClient, portClient) + go func() { + newHandler.Handle() + }() + return newHandler, handlerError }) if err != nil { klog.Fatalf("Error starting event listener: %s", err.Error()) From 6326254196017661cde3f22ae43253cd2ddfa157 Mon Sep 17 00:00:00 2001 From: yair Date: Sun, 17 Dec 2023 22:29:12 +0200 Subject: [PATCH 22/35] revert moving the handle --- main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index c5c2255..1b8e562 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portCli cli.WithCreateMissingRelatedEntities(apiConfig.CreateMissingRelatedEntities)(portClient) newHandler := handlers.NewControllersHandler(exporterConfig, apiConfig, k8sClient, portClient) + newHandler.Handle() return newHandler, nil } @@ -70,9 +71,7 @@ func main() { err = eventListener.Start(func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { handler.Stop() newHandler, handlerError := initiateHandler(exporterConfig, k8sClient, portClient) - go func() { - newHandler.Handle() - }() + return newHandler, handlerError }) if err != nil { From bbb39eea29be28d041e500f376adb9b8f87cde4b Mon Sep 17 00:00:00 2001 From: yair Date: Sun, 17 Dec 2023 22:29:31 +0200 Subject: [PATCH 23/35] revert moving the handle --- main.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/main.go b/main.go index 1b8e562..353a914 100644 --- a/main.go +++ b/main.go @@ -70,9 +70,7 @@ func main() { eventListener := event_listener.NewEventListener(config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType, handler, portClient) err = eventListener.Start(func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { handler.Stop() - newHandler, handlerError := initiateHandler(exporterConfig, k8sClient, portClient) - - return newHandler, handlerError + return initiateHandler(exporterConfig, k8sClient, portClient) }) if err != nil { klog.Fatalf("Error starting event listener: %s", err.Error()) From a1e40318a7bc81db87a5c128a66967acf818cba3 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 18 Dec 2023 17:15:23 +0200 Subject: [PATCH 24/35] iconsume and consumer tests --- go.mod | 2 +- pkg/event_listener/consumer/consumer.go | 119 ++++++++++++------- pkg/event_listener/consumer/consumer_test.go | 106 +++++++++++++++++ pkg/event_listener/event_listener_handler.go | 2 +- 4 files changed, 186 insertions(+), 43 deletions(-) create mode 100644 pkg/event_listener/consumer/consumer_test.go diff --git a/go.mod b/go.mod index dac6e5c..152d8fd 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( k8s.io/apimachinery v0.25.2 k8s.io/client-go v0.25.2 k8s.io/klog/v2 v2.80.1 + k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed ) require ( @@ -50,7 +51,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/pkg/event_listener/consumer/consumer.go b/pkg/event_listener/consumer/consumer.go index 2f9a574..3195c59 100644 --- a/pkg/event_listener/consumer/consumer.go +++ b/pkg/event_listener/consumer/consumer.go @@ -9,64 +9,101 @@ import ( "syscall" ) +type IConsume interface { + SubscribeTopics(topics []string, rebalanceCb kafka.RebalanceCb) (err error) + Poll(timeoutMs int) (event kafka.Event) + Commit() (offsets []kafka.TopicPartition, err error) + Close() (err error) +} + type Consumer struct { - client *kafka.Consumer + client IConsume + enabled bool } type JsonHandler func(value []byte) -func NewConsumer(config *config.KafkaConfiguration) (*Consumer, error) { - c, err := kafka.NewConsumer(&kafka.ConfigMap{ - "bootstrap.servers": config.Brokers, - "group.id": config.GroupID, - "security.protocol": config.SecurityProtocol, - "sasl.mechanism": config.AuthenticationMechanism, - "sasl.username": config.Username, - "sasl.password": config.Password, - "auto.offset.reset": "latest", - }) - - if err != nil { - return nil, err +func NewConsumer(config *config.KafkaConfiguration, overrideConsumer IConsume) (*Consumer, error) { + c := overrideConsumer + var err error + if overrideConsumer == nil { + c, err = kafka.NewConsumer(&kafka.ConfigMap{ + "bootstrap.servers": config.Brokers, + "client.id": "port-k8s-exporter", + "group.id": config.GroupID, + "security.protocol": config.SecurityProtocol, + "sasl.mechanism": config.AuthenticationMechanism, + "sasl.username": config.Username, + "sasl.password": config.Password, + "auto.offset.reset": "latest", + }) + + if err != nil { + return nil, err + } } - return &Consumer{client: c}, nil + return &Consumer{client: c, enabled: true}, nil } -func (c *Consumer) Consume(topic string, handler JsonHandler) { +func (c *Consumer) Consume(topic string, handler JsonHandler, readyChan chan bool) { topics := []string{topic} - err := c.client.SubscribeTopics(topics, nil) - if err != nil { + ready := false + + rebalanceCallback := func(c *kafka.Consumer, event kafka.Event) error { + switch ev := event.(type) { + case kafka.AssignedPartitions: + klog.Infof("partition(s) assigned: %v\n", ev.Partitions) + if readyChan != nil && ready == false { + close(readyChan) + ready = true + } + } + + return nil + } + + if err := c.client.SubscribeTopics(topics, rebalanceCallback); err != nil { klog.Fatalf("Error subscribing to topic: %s", err.Error()) } sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - run := true - for run { - select { - case sig := <-sigChan: - klog.Infof("Caught signal %v: terminating\n", sig) - run = false - default: - ev := c.client.Poll(100) - if ev == nil { - continue - } + klog.Infof("Waiting for assigned partitions...") + + go func() { + sig := <-sigChan + klog.Infof("Caught signal %v: terminating\n", sig) + c.Close() + }() + + for c.enabled { + ev := c.client.Poll(100) + if ev == nil { + continue + } - switch e := ev.(type) { - case *kafka.Message: - klog.Infof("%% Message on %s:\n%s\n", - e.TopicPartition, string(e.Value)) - - handler(e.Value) - case *kafka.AssignedPartitions: - klog.Infof("AssignedPartitions: %v\n", e) - case kafka.Error: - klog.Infof("%% Error: %v\n", e) - default: - klog.Infof("Ignored %v\n", e) + switch e := ev.(type) { + case *kafka.Message: + klog.Infof("%% New event\n%s\n", + string(e.Value)) + + handler(e.Value) + if _, err := c.client.Commit(); err != nil { + klog.Error("Error committing offset: %s", err.Error()) } + case kafka.Error: + klog.Infof("%% Error: %v\n", e) + default: + klog.Infof("Ignored %v\n", e) } } + klog.Infof("Closed consumer\n") +} + +func (c *Consumer) Close() { + if err := c.client.Close(); err != nil { + klog.Fatalf("Error closing consumer: %s", err.Error()) + } + c.enabled = false } diff --git a/pkg/event_listener/consumer/consumer_test.go b/pkg/event_listener/consumer/consumer_test.go new file mode 100644 index 0000000..aa39aa1 --- /dev/null +++ b/pkg/event_listener/consumer/consumer_test.go @@ -0,0 +1,106 @@ +package consumer + +import ( + "github.com/confluentinc/confluent-kafka-go/kafka" + "testing" + "time" + + "github.com/port-labs/port-k8s-exporter/pkg/config" +) + +type Fixture struct { + t *testing.T + mockConsumer *MockConsumer + consumer *Consumer + producer *kafka.Producer + topic string +} + +type MockConsumer struct { + pollData kafka.Event +} + +func (m *MockConsumer) SubscribeTopics(topics []string, rebalanceCb kafka.RebalanceCb) (err error) { + _ = rebalanceCb(nil, kafka.AssignedPartitions{ + Partitions: []kafka.TopicPartition{ + { + Topic: &topics[0], + Offset: 0, + Partition: 0, + Metadata: nil, + Error: nil, + }, + }, + }) + return nil +} + +func (m *MockConsumer) Poll(timeoutMs int) (event kafka.Event) { + return m.pollData +} + +func (m *MockConsumer) Commit() (offsets []kafka.TopicPartition, err error) { + return nil, nil +} + +func (m *MockConsumer) Close() (err error) { + return nil +} + +func NewFixture(t *testing.T) *Fixture { + mock := &MockConsumer{} + consumer, err := NewConsumer(&config.KafkaConfiguration{}, mock) + if err != nil { + t.Fatalf("Error creating consumer: %v", err) + } + + producer, err := kafka.NewProducer(&kafka.ConfigMap{ + "bootstrap.servers": "localhost:9092", + }) + if err != nil { + t.Fatalf("Error creating producer: %v", err) + } + + return &Fixture{ + t: t, + mockConsumer: mock, + consumer: consumer, + producer: producer, + topic: "test-topic", + } +} + +func (f *Fixture) Produce(t *testing.T, value []byte) { + f.mockConsumer.pollData = &kafka.Message{ + TopicPartition: kafka.TopicPartition{Topic: &f.topic, Partition: 0}, + Value: value, + } +} + +func (f *Fixture) Consume(handler JsonHandler) { + readyChan := make(chan bool) + go f.consumer.Consume(f.topic, handler, readyChan) + <-readyChan +} + +type MockJsonHandler struct { + CapturedValue []byte +} + +func (m *MockJsonHandler) HandleJson(value []byte) { + m.CapturedValue = value +} + +func TestConsumer_HandleJson(t *testing.T) { + f := NewFixture(t) + mockHandler := &MockJsonHandler{} + + f.Consume(mockHandler.HandleJson) + + f.Produce(t, []byte("test-value")) + time.Sleep(time.Second) + + if len(mockHandler.CapturedValue) == 0 { + t.Error("Handler was not called") + } +} diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go index 1516b08..63d83c2 100644 --- a/pkg/event_listener/event_listener_handler.go +++ b/pkg/event_listener/event_listener_handler.go @@ -88,7 +88,7 @@ func startKafkaEventListener(l *EventListener, resync func()) error { klog.Infof("Changes detected. Resyncing...") resync() } - }) + }, nil) return nil } From 3e3c91c5d11ac9586a88eb488aa73ea5cbffd015 Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 18 Dec 2023 17:16:10 +0200 Subject: [PATCH 25/35] iconsume and consumer tests --- pkg/event_listener/consumer/consumer.go | 6 +++--- pkg/event_listener/event_listener_handler.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/event_listener/consumer/consumer.go b/pkg/event_listener/consumer/consumer.go index 3195c59..6ecc450 100644 --- a/pkg/event_listener/consumer/consumer.go +++ b/pkg/event_listener/consumer/consumer.go @@ -23,10 +23,10 @@ type Consumer struct { type JsonHandler func(value []byte) -func NewConsumer(config *config.KafkaConfiguration, overrideConsumer IConsume) (*Consumer, error) { - c := overrideConsumer +func NewConsumer(config *config.KafkaConfiguration, overrideKafkaConsumer IConsume) (*Consumer, error) { + c := overrideKafkaConsumer var err error - if overrideConsumer == nil { + if overrideKafkaConsumer == nil { c, err = kafka.NewConsumer(&kafka.ConfigMap{ "bootstrap.servers": config.Brokers, "client.id": "port-k8s-exporter", diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go index 63d83c2..55647b6 100644 --- a/pkg/event_listener/event_listener_handler.go +++ b/pkg/event_listener/event_listener_handler.go @@ -72,7 +72,7 @@ func startKafkaEventListener(l *EventListener, resync func()) error { } topic := orgId + ".change.log" - instance, err := consumer.NewConsumer(c) + instance, err := consumer.NewConsumer(c, nil) if err != nil { return err From 80421b10b28b7da2f4c46c32cf55777eaf6804be Mon Sep 17 00:00:00 2001 From: yair Date: Mon, 18 Dec 2023 23:09:12 +0200 Subject: [PATCH 26/35] dependency injection and tests --- go.mod | 4 +- main.go | 24 +++- pkg/event_listener/consumer/event_listener.go | 86 +++++++++++++ pkg/event_listener/event_listener_handler.go | 115 ++---------------- pkg/event_listener/polling/event_listener.go | 28 +++++ pkg/event_listener/polling/polling.go | 38 +++++- pkg/event_listener/polling/polling_test.go | 82 +++++++++++++ pkg/port/cli/integration.go | 46 +++++++ pkg/port/integration/integration.go | 39 ++++++ pkg/port/models.go | 3 +- 10 files changed, 351 insertions(+), 114 deletions(-) create mode 100644 pkg/event_listener/consumer/event_listener.go create mode 100644 pkg/event_listener/polling/event_listener.go create mode 100644 pkg/event_listener/polling/polling_test.go diff --git a/go.mod b/go.mod index 152d8fd..2ce19e9 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.19 require ( github.com/confluentinc/confluent-kafka-go v1.9.2 github.com/go-resty/resty/v2 v2.7.0 + github.com/google/uuid v1.3.0 github.com/itchyny/gojq v0.12.9 + github.com/stretchr/testify v1.8.2 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.25.2 k8s.io/apimachinery v0.25.2 @@ -38,8 +40,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.2 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sys v0.15.0 // indirect diff --git a/main.go b/main.go index 353a914..99698a0 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/port-labs/port-k8s-exporter/pkg/config" "github.com/port-labs/port-k8s-exporter/pkg/event_listener" + "github.com/port-labs/port-k8s-exporter/pkg/event_listener/consumer" + "github.com/port-labs/port-k8s-exporter/pkg/event_listener/polling" "github.com/port-labs/port-k8s-exporter/pkg/handlers" "github.com/port-labs/port-k8s-exporter/pkg/k8s" "github.com/port-labs/port-k8s-exporter/pkg/port" @@ -28,6 +30,19 @@ func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portCli return newHandler, nil } +func CreateEventListener(portClient *cli.PortClient) (event_listener.IEventListener, error) { + klog.Infof("Received event listener type: %s", config.ApplicationConfig.EventListenerType) + switch config.ApplicationConfig.EventListenerType { + case "KAFKA": + return consumer.NewEventListener(portClient) + case "POLLING": + return polling.NewEventListener(portClient), nil + default: + return nil, fmt.Errorf("unknown event listener type: %s", config.ApplicationConfig.EventListenerType) + } + +} + func main() { klog.InitFlags(nil) @@ -65,13 +80,18 @@ func main() { } } + eventListener, err := CreateEventListener(portClient) + if err != nil { + klog.Fatalf("Error creating event listener: %s", err.Error()) + } + klog.Info("Starting controllers handler") handler, _ := initiateHandler(exporterConfig, k8sClient, portClient) - eventListener := event_listener.NewEventListener(config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType, handler, portClient) - err = eventListener.Start(func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { + err = event_listener.StartEventHandler(eventListener, handler, func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { handler.Stop() return initiateHandler(exporterConfig, k8sClient, portClient) }) + if err != nil { klog.Fatalf("Error starting event listener: %s", err.Error()) } diff --git a/pkg/event_listener/consumer/event_listener.go b/pkg/event_listener/consumer/event_listener.go new file mode 100644 index 0000000..26e8cbe --- /dev/null +++ b/pkg/event_listener/consumer/event_listener.go @@ -0,0 +1,86 @@ +package consumer + +import ( + "encoding/json" + "fmt" + "github.com/port-labs/port-k8s-exporter/pkg/config" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/kafka_credentials" + "github.com/port-labs/port-k8s-exporter/pkg/port/org_details" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" +) + +type EventListener struct { + stateKey string + portClient *cli.PortClient + topic string + consumer *Consumer +} + +type IncomingMessage struct { + Diff *struct { + After *struct { + Identifier string `json:"installationId"` + } `json:"after"` + } `json:"diff"` +} + +func NewEventListener(portClient *cli.PortClient) (*EventListener, error) { + klog.Infof("Getting Consumer Information") + credentials, err := kafka_credentials.GetKafkaCredentials(portClient) + if err != nil { + return nil, err + } + orgId, err := org_details.GetOrgId(portClient) + if err != nil { + return nil, err + } + + c := &config.KafkaConfiguration{ + Brokers: config.KafkaConfig.Brokers, + SecurityProtocol: config.KafkaConfig.SecurityProtocol, + AuthenticationMechanism: config.KafkaConfig.AuthenticationMechanism, + Username: credentials.Username, + Password: credentials.Password, + GroupID: orgId + ".k8s." + config.ApplicationConfig.StateKey, + } + + topic := orgId + ".change.log" + instance, err := NewConsumer(c, nil) + if err != nil { + return nil, err + } + + return &EventListener{ + stateKey: config.ApplicationConfig.StateKey, + portClient: portClient, + topic: topic, + consumer: instance, + }, nil +} + +func shouldResync(stateKey string, message *IncomingMessage) bool { + return message.Diff != nil && + message.Diff.After != nil && + message.Diff.After.Identifier != "" && + message.Diff.After.Identifier == stateKey +} + +func (l *EventListener) Run(resync func()) error { + klog.Infof("Starting Kafka event listener") + + klog.Infof("Starting consumer for topic %s", l.topic) + l.consumer.Consume(l.topic, func(value []byte) { + incomingMessage := &IncomingMessage{} + parsingError := json.Unmarshal(value, &incomingMessage) + if parsingError != nil { + utilruntime.HandleError(fmt.Errorf("error handling message: %s", parsingError.Error())) + } else if shouldResync(l.stateKey, incomingMessage) { + klog.Infof("Changes detected. Resyncing...") + resync() + } + }, nil) + + return nil +} diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go index 55647b6..68192f7 100644 --- a/pkg/event_listener/event_listener_handler.go +++ b/pkg/event_listener/event_listener_handler.go @@ -1,126 +1,33 @@ package event_listener import ( - "encoding/json" "fmt" - "github.com/port-labs/port-k8s-exporter/pkg/config" - "github.com/port-labs/port-k8s-exporter/pkg/event_listener/consumer" - "github.com/port-labs/port-k8s-exporter/pkg/event_listener/polling" "github.com/port-labs/port-k8s-exporter/pkg/handlers" - "github.com/port-labs/port-k8s-exporter/pkg/port" - "github.com/port-labs/port-k8s-exporter/pkg/port/cli" - "github.com/port-labs/port-k8s-exporter/pkg/port/kafka_credentials" - "github.com/port-labs/port-k8s-exporter/pkg/port/org_details" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/klog/v2" ) -type IncomingMessage struct { - Diff *struct { - After *struct { - Identifier string `json:"installationId"` - } `json:"after"` - } `json:"diff"` +type IEventListener interface { + Run(resync func()) error } -type EventListener struct { - settings port.EventListenerSettings - stateKey string +type Handler struct { + eventListener IEventListener controllerHandler *handlers.ControllersHandler - portClient *cli.PortClient } -func shouldResync(stateKey string, message *IncomingMessage) bool { - return message.Diff != nil && - message.Diff.After != nil && - message.Diff.After.Identifier != "" && - message.Diff.After.Identifier == stateKey -} - -func NewEventListener(stateKey string, eventListenerType string, controllerHandler *handlers.ControllersHandler, client *cli.PortClient) *EventListener { - eventListener := &EventListener{ - settings: port.EventListenerSettings{ - Type: eventListenerType, - }, - stateKey: stateKey, - controllerHandler: controllerHandler, - portClient: client, - } - - return eventListener -} - -func startKafkaEventListener(l *EventListener, resync func()) error { - klog.Infof("Starting Kafka event listener") - klog.Infof("Getting Consumer Information") - credentials, err := kafka_credentials.GetKafkaCredentials(l.portClient) - if err != nil { - return err - } - orgId, err := org_details.GetOrgId(l.portClient) - if err != nil { - return err - } - - c := &config.KafkaConfiguration{ - Brokers: config.KafkaConfig.Brokers, - SecurityProtocol: config.KafkaConfig.SecurityProtocol, - AuthenticationMechanism: config.KafkaConfig.AuthenticationMechanism, - Username: credentials.Username, - Password: credentials.Password, - GroupID: orgId + ".k8s." + l.stateKey, - } - - topic := orgId + ".change.log" - instance, err := consumer.NewConsumer(c, nil) - - if err != nil { - return err - } - - klog.Infof("Starting consumer for topic %s and groupId %s", topic, c.GroupID) - instance.Consume(topic, func(value []byte) { - incomingMessage := &IncomingMessage{} - parsingError := json.Unmarshal(value, &incomingMessage) - if parsingError != nil { - utilruntime.HandleError(fmt.Errorf("error handling message: %s", parsingError.Error())) - } else if shouldResync(l.stateKey, incomingMessage) { - klog.Infof("Changes detected. Resyncing...") - resync() - } - }, nil) - - return nil -} - -func startPollingEventListener(l *EventListener, resync func()) { - klog.Infof("Starting polling event listener") - klog.Infof("Polling rate set to %d seconds", config.PollingListenerRate) - pollingHandler := polling.NewPollingHandler(config.PollingListenerRate, l.stateKey, l.portClient) - pollingHandler.Run(resync) -} - -func (l *EventListener) Start(resync func(*handlers.ControllersHandler) (*handlers.ControllersHandler, error)) error { - wrappedResync := func() { +func StartEventHandler(eventListener IEventListener, controllerHandler *handlers.ControllersHandler, resync func(*handlers.ControllersHandler) (*handlers.ControllersHandler, error)) error { + err := eventListener.Run(func() { klog.Infof("Resync request received. Recreating controllers for the new port configuration") - newController, resyncErr := resync(l.controllerHandler) - l.controllerHandler = newController + newController, resyncErr := resync(controllerHandler) + controllerHandler = newController if resyncErr != nil { utilruntime.HandleError(fmt.Errorf("error resyncing: %s", resyncErr.Error())) } - } - klog.Infof("Received event listener type: %s", l.settings.Type) - switch l.settings.Type { - case "KAFKA": - err := startKafkaEventListener(l, wrappedResync) - if err != nil { - return err - } - case "POLLING": - startPollingEventListener(l, wrappedResync) - default: - return fmt.Errorf("unknown event listener type: %s", l.settings.Type) + }) + if err != nil { + return err } return nil diff --git a/pkg/event_listener/polling/event_listener.go b/pkg/event_listener/polling/event_listener.go new file mode 100644 index 0000000..546f052 --- /dev/null +++ b/pkg/event_listener/polling/event_listener.go @@ -0,0 +1,28 @@ +package polling + +import ( + "github.com/port-labs/port-k8s-exporter/pkg/config" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "k8s.io/klog/v2" +) + +type EventListener struct { + stateKey string + portClient *cli.PortClient +} + +func NewEventListener(portClient *cli.PortClient) *EventListener { + return &EventListener{ + stateKey: config.ApplicationConfig.StateKey, + portClient: portClient, + } +} + +func (l *EventListener) Run(resync func()) error { + klog.Infof("Starting polling event listener") + klog.Infof("Polling rate set to %d seconds", config.PollingListenerRate) + pollingHandler := NewPollingHandler(config.PollingListenerRate, l.stateKey, l.portClient, nil) + pollingHandler.Run(resync) + + return nil +} diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_listener/polling/polling.go index 4c93014..24c5b38 100644 --- a/pkg/event_listener/polling/polling.go +++ b/pkg/event_listener/polling/polling.go @@ -11,16 +11,42 @@ import ( "time" ) +type ITicker interface { + GetC() <-chan time.Time +} + +type Ticker struct { + ticker *time.Ticker +} + +func NewTicker(d time.Duration) *Ticker { + return &Ticker{ + ticker: time.NewTicker(d), + } +} + +func (t *Ticker) Stop() { + t.ticker.Stop() +} + +func (t *Ticker) GetC() <-chan time.Time { + return t.ticker.C +} + type HandlerSettings struct { - ticker *time.Ticker + ticker ITicker stateKey string portClient *cli.PortClient pollingRate uint } -func NewPollingHandler(pollingRate uint, stateKey string, portClient *cli.PortClient) *HandlerSettings { +func NewPollingHandler(pollingRate uint, stateKey string, portClient *cli.PortClient, tickerOverride ITicker) *HandlerSettings { + ticker := tickerOverride + if ticker == nil { + ticker = NewTicker(time.Second * time.Duration(pollingRate)) + } rv := &HandlerSettings{ - ticker: time.NewTicker(time.Second * time.Duration(pollingRate)), + ticker: ticker, stateKey: stateKey, portClient: portClient, pollingRate: pollingRate, @@ -30,7 +56,7 @@ func NewPollingHandler(pollingRate uint, stateKey string, portClient *cli.PortCl func (h *HandlerSettings) Run(resync func()) { klog.Infof("Starting polling handler") - currentState, err := integration.GetIntegrationConfig(h.portClient, h.stateKey) + currentState, err := integration.GetIntegration(h.portClient, h.stateKey) if err != nil { klog.Errorf("Error fetching the first AppConfig state: %s", err.Error()) } @@ -45,9 +71,9 @@ func (h *HandlerSettings) Run(resync func()) { case sig := <-sigChan: klog.Infof("Received signal %v: terminating\n", sig) run = false - case <-h.ticker.C: + case <-h.ticker.GetC(): klog.Infof("Polling event listener iteration after %d seconds. Checking for changes...", h.pollingRate) - configuration, err := integration.GetIntegrationConfig(h.portClient, h.stateKey) + configuration, err := integration.GetIntegration(h.portClient, h.stateKey) if err != nil { klog.Errorf("error resyncing: %s", err.Error()) } diff --git a/pkg/event_listener/polling/polling_test.go b/pkg/event_listener/polling/polling_test.go new file mode 100644 index 0000000..6476c3b --- /dev/null +++ b/pkg/event_listener/polling/polling_test.go @@ -0,0 +1,82 @@ +package polling + +import ( + "flag" + "fmt" + guuid "github.com/google/uuid" + "github.com/port-labs/port-k8s-exporter/pkg/config" + "github.com/port-labs/port-k8s-exporter/pkg/port" + "github.com/port-labs/port-k8s-exporter/pkg/port/cli" + "github.com/port-labs/port-k8s-exporter/pkg/port/integration" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +type Fixture struct { + t *testing.T + ticker MockTicker + portClient *cli.PortClient + stateKey string +} + +type MockTicker struct { + c chan time.Time +} + +func (m *MockTicker) GetC() <-chan time.Time { + return m.c +} + +func NewFixture(t *testing.T, c chan time.Time) *Fixture { + config.Init() + flag.Parse() + stateKey := guuid.NewString() + portClient, err := cli.New("https://api.getport.io", cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", stateKey)), + cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret)) + if err != nil { + t.Errorf("Error building Port client: %s", err.Error()) + } + + _ = integration.DeleteIntegration(portClient, stateKey) + err = integration.NewIntegration(portClient, &port.Config{ + StateKey: stateKey, + }, []port.Resource{}) + if err != nil { + t.Errorf("Error creating Port integration: %s", err.Error()) + } + return &Fixture{ + t: t, + ticker: MockTicker{c: c}, + portClient: portClient, + stateKey: stateKey, + } +} + +func (f *Fixture) CleanIntegration() { + _ = integration.DeleteIntegration(f.portClient, f.stateKey) +} + +func TestPolling_DifferentConfiguration(t *testing.T) { + called := false + c := make(chan time.Time) + fixture := NewFixture(t, c) + defer fixture.CleanIntegration() + handler := NewPollingHandler(uint(1), fixture.stateKey, fixture.portClient, &fixture.ticker) + go handler.Run(func() { + called = true + }) + + c <- time.Now() + time.Sleep(time.Millisecond * 500) + assert.False(t, called) + + _ = integration.UpdateIntegrationConfig(fixture.portClient, fixture.stateKey, &port.AppConfig{ + Resources: []port.Resource{}, + }) + + c <- time.Now() + time.Sleep(time.Millisecond * 500) + + assert.True(t, called) +} diff --git a/pkg/port/cli/integration.go b/pkg/port/cli/integration.go index f6e108e..df38d9a 100644 --- a/pkg/port/cli/integration.go +++ b/pkg/port/cli/integration.go @@ -38,6 +38,20 @@ func (c *PortClient) CreateIntegration(i *port.Integration) (*port.Integration, return &pb.Integration, nil } +func (c *PortClient) GetIntegration(stateKey string) (*port.Integration, error) { + pb := &port.ResponseBody{} + resp, err := c.Client.R(). + SetResult(&pb). + Get(fmt.Sprintf("v1/integration/%s", stateKey)) + if err != nil { + return nil, err + } + if !pb.OK { + return nil, fmt.Errorf("failed to get integration, got: %s", resp.Body()) + } + return &pb.Integration, nil +} + func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.AppConfig, error) { pb := &port.ResponseBody{} resp, err := c.Client.R(). @@ -51,3 +65,35 @@ func (c *PortClient) GetIntegrationConfig(stateKey string) (*port.AppConfig, err } return pb.Integration.Config, nil } + +func (c *PortClient) DeleteIntegration(stateKey string) error { + resp, err := c.Client.R(). + Delete(fmt.Sprintf("v1/integration/%s", stateKey)) + if err != nil { + return err + } + if resp.StatusCode() != 200 { + return fmt.Errorf("failed to delete integration, got: %s", resp.Body()) + } + return nil +} + +func (c *PortClient) UpdateConfig(stateKey string, config *port.AppConfig) error { + type Config struct { + Config *port.AppConfig `json:"config"` + } + pb := &port.ResponseBody{} + resp, err := c.Client.R(). + SetBody(&Config{ + Config: config, + }). + SetResult(&pb). + Patch(fmt.Sprintf("v1/integration/%s/config", stateKey)) + if err != nil { + return err + } + if !pb.OK { + return fmt.Errorf("failed to update config, got: %s", resp.Body()) + } + return nil +} diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 0c39090..06241e6 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -43,5 +43,44 @@ func GetIntegrationConfig(portClient *cli.PortClient, stateKey string) (*port.Ap } return apiConfig, nil +} + +func GetIntegration(portClient *cli.PortClient, stateKey string) (*port.Integration, error) { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return nil, fmt.Errorf("error authenticating with Port: %v", err) + } + apiIntegration, err := portClient.GetIntegration(stateKey) + if err != nil { + return nil, fmt.Errorf("error getting Port integration: %v", err) + } + + return apiIntegration, nil +} + +func DeleteIntegration(portClient *cli.PortClient, stateKey string) error { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return fmt.Errorf("error authenticating with Port: %v", err) + } + + err = portClient.DeleteIntegration(stateKey) + if err != nil { + return fmt.Errorf("error deleting Port integration: %v", err) + } + return nil +} + +func UpdateIntegrationConfig(portClient *cli.PortClient, stateKey string, config *port.AppConfig) error { + _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) + if err != nil { + return fmt.Errorf("error authenticating with Port: %v", err) + } + + err = portClient.UpdateConfig(stateKey, config) + if err != nil { + return fmt.Errorf("error updating Port integration config: %v", err) + } + return nil } diff --git a/pkg/port/models.go b/pkg/port/models.go index 8b7000c..2f06df7 100644 --- a/pkg/port/models.go +++ b/pkg/port/models.go @@ -28,12 +28,13 @@ type ( } Integration struct { - InstallationId string `json:"installationId"` + InstallationId string `json:"installationId,omitempty"` Title string `json:"title,omitempty"` Version string `json:"version,omitempty"` InstallationAppType string `json:"installationAppType,omitempty"` EventListener EventListenerSettings `json:"changelogDestination,omitempty"` Config *AppConfig `json:"config,omitempty"` + UpdatedAt *time.Time `json:"updatedAt,omitempty"` } BlueprintProperty struct { From d0e4f66547a6f6263862b1f8d3fa40fcec8aeda1 Mon Sep 17 00:00:00 2001 From: yair Date: Tue, 19 Dec 2023 01:56:24 +0200 Subject: [PATCH 27/35] dep injection --- main.go | 39 ++++++++++--------- .../consumer/consumer.go | 0 .../consumer/consumer_test.go | 0 .../consumer/event_listener.go | 6 +-- pkg/event_handler/event_handler.go | 33 ++++++++++++++++ pkg/event_handler/handler_test.go | 13 +++++++ .../polling/event_listener.go | 4 +- .../polling/polling.go | 0 .../polling/polling_test.go | 4 +- pkg/event_listener/event_listener_handler.go | 34 ---------------- pkg/handlers/controllers.go | 6 +-- pkg/port/integration/integration.go | 8 ++-- 12 files changed, 79 insertions(+), 68 deletions(-) rename pkg/{event_listener => event_handler}/consumer/consumer.go (100%) rename pkg/{event_listener => event_handler}/consumer/consumer_test.go (100%) rename pkg/{event_listener => event_handler}/consumer/event_listener.go (91%) create mode 100644 pkg/event_handler/event_handler.go create mode 100644 pkg/event_handler/handler_test.go rename pkg/{event_listener => event_handler}/polling/event_listener.go (83%) rename pkg/{event_listener => event_handler}/polling/polling.go (100%) rename pkg/{event_listener => event_handler}/polling/polling_test.go (95%) delete mode 100644 pkg/event_listener/event_listener_handler.go diff --git a/main.go b/main.go index 99698a0..f3020a4 100644 --- a/main.go +++ b/main.go @@ -4,9 +4,9 @@ import ( "flag" "fmt" "github.com/port-labs/port-k8s-exporter/pkg/config" - "github.com/port-labs/port-k8s-exporter/pkg/event_listener" - "github.com/port-labs/port-k8s-exporter/pkg/event_listener/consumer" - "github.com/port-labs/port-k8s-exporter/pkg/event_listener/polling" + "github.com/port-labs/port-k8s-exporter/pkg/event_handler" + "github.com/port-labs/port-k8s-exporter/pkg/event_handler/consumer" + "github.com/port-labs/port-k8s-exporter/pkg/event_handler/polling" "github.com/port-labs/port-k8s-exporter/pkg/handlers" "github.com/port-labs/port-k8s-exporter/pkg/k8s" "github.com/port-labs/port-k8s-exporter/pkg/port" @@ -16,7 +16,7 @@ import ( ) func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portClient *cli.PortClient) (*handlers.ControllersHandler, error) { - apiConfig, err := integration.GetIntegrationConfig(portClient, config.ApplicationConfig.StateKey) + apiConfig, err := integration.GetIntegrationConfig(portClient, exporterConfig.StateKey) if err != nil { klog.Fatalf("Error getting K8s integration config: %s", err.Error()) } @@ -30,24 +30,29 @@ func initiateHandler(exporterConfig *port.Config, k8sClient *k8s.Client, portCli return newHandler, nil } -func CreateEventListener(portClient *cli.PortClient) (event_listener.IEventListener, error) { - klog.Infof("Received event listener type: %s", config.ApplicationConfig.EventListenerType) - switch config.ApplicationConfig.EventListenerType { +func createEventListener(stateKey string, eventListenerType string, portClient *cli.PortClient) (event_handler.IListener, error) { + klog.Infof("Received event listener type: %s", eventListenerType) + switch eventListenerType { case "KAFKA": - return consumer.NewEventListener(portClient) + return consumer.NewEventListener(stateKey, portClient) case "POLLING": - return polling.NewEventListener(portClient), nil + return polling.NewEventListener(stateKey, portClient), nil default: - return nil, fmt.Errorf("unknown event listener type: %s", config.ApplicationConfig.EventListenerType) + return nil, fmt.Errorf("unknown event listener type: %s", eventListenerType) } } +func initIntegrationConfig() { + +} + func main() { klog.InitFlags(nil) k8sConfig := k8s.NewKubeConfig() + exporterConfig, _ := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath, config.ApplicationConfig.ResyncInterval, config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType) clientConfig, err := k8sConfig.ClientConfig() if err != nil { klog.Fatalf("Error getting K8s client config: %s", err.Error()) @@ -60,35 +65,31 @@ func main() { portClient, err := cli.New(config.ApplicationConfig.PortBaseURL, cli.WithClientID(config.ApplicationConfig.PortClientId), cli.WithClientSecret(config.ApplicationConfig.PortClientSecret), - cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", config.ApplicationConfig.StateKey)), + cli.WithHeader("User-Agent", fmt.Sprintf("port-k8s-exporter/0.1 (statekey/%s)", exporterConfig.StateKey)), ) if err != nil { klog.Fatalf("Error building Port client: %s", err.Error()) } - exporterConfig, _ := config.GetConfigFile(config.ApplicationConfig.ConfigFilePath, config.ApplicationConfig.ResyncInterval, config.ApplicationConfig.StateKey, config.ApplicationConfig.EventListenerType) - - _, err = integration.GetIntegrationConfig(portClient, config.ApplicationConfig.StateKey) + _, err = integration.GetIntegrationConfig(portClient, exporterConfig.StateKey) if err != nil { if exporterConfig == nil { klog.Fatalf("The integration does not exist and no config file was provided") } - err = integration.NewIntegration(portClient, exporterConfig, exporterConfig.Resources) + err = integration.NewIntegration(portClient, exporterConfig.StateKey, exporterConfig.EventListenerType, exporterConfig.Resources) if err != nil { klog.Fatalf("Error creating K8s integration: %s", err.Error()) } } - eventListener, err := CreateEventListener(portClient) + eventListener, err := createEventListener(exporterConfig.StateKey, exporterConfig.EventListenerType, portClient) if err != nil { klog.Fatalf("Error creating event listener: %s", err.Error()) } klog.Info("Starting controllers handler") - handler, _ := initiateHandler(exporterConfig, k8sClient, portClient) - err = event_listener.StartEventHandler(eventListener, handler, func(handler *handlers.ControllersHandler) (*handlers.ControllersHandler, error) { - handler.Stop() + err = event_handler.StartEventHandler(eventListener, func() (event_handler.IStoppableRsync, error) { return initiateHandler(exporterConfig, k8sClient, portClient) }) diff --git a/pkg/event_listener/consumer/consumer.go b/pkg/event_handler/consumer/consumer.go similarity index 100% rename from pkg/event_listener/consumer/consumer.go rename to pkg/event_handler/consumer/consumer.go diff --git a/pkg/event_listener/consumer/consumer_test.go b/pkg/event_handler/consumer/consumer_test.go similarity index 100% rename from pkg/event_listener/consumer/consumer_test.go rename to pkg/event_handler/consumer/consumer_test.go diff --git a/pkg/event_listener/consumer/event_listener.go b/pkg/event_handler/consumer/event_listener.go similarity index 91% rename from pkg/event_listener/consumer/event_listener.go rename to pkg/event_handler/consumer/event_listener.go index 26e8cbe..49ec773 100644 --- a/pkg/event_listener/consumer/event_listener.go +++ b/pkg/event_handler/consumer/event_listener.go @@ -26,7 +26,7 @@ type IncomingMessage struct { } `json:"diff"` } -func NewEventListener(portClient *cli.PortClient) (*EventListener, error) { +func NewEventListener(stateKey string, portClient *cli.PortClient) (*EventListener, error) { klog.Infof("Getting Consumer Information") credentials, err := kafka_credentials.GetKafkaCredentials(portClient) if err != nil { @@ -43,7 +43,7 @@ func NewEventListener(portClient *cli.PortClient) (*EventListener, error) { AuthenticationMechanism: config.KafkaConfig.AuthenticationMechanism, Username: credentials.Username, Password: credentials.Password, - GroupID: orgId + ".k8s." + config.ApplicationConfig.StateKey, + GroupID: orgId + ".k8s." + stateKey, } topic := orgId + ".change.log" @@ -53,7 +53,7 @@ func NewEventListener(portClient *cli.PortClient) (*EventListener, error) { } return &EventListener{ - stateKey: config.ApplicationConfig.StateKey, + stateKey: stateKey, portClient: portClient, topic: topic, consumer: instance, diff --git a/pkg/event_handler/event_handler.go b/pkg/event_handler/event_handler.go new file mode 100644 index 0000000..b2a247a --- /dev/null +++ b/pkg/event_handler/event_handler.go @@ -0,0 +1,33 @@ +package event_handler + +import ( + "fmt" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" +) + +type IListener interface { + Run(resync func()) error +} + +type IStoppableRsync interface { + Stop() +} + +func StartEventHandler(eventListener IListener, controllerHandlerFactory func() (IStoppableRsync, error)) error { + controllerHandler, err := controllerHandlerFactory() + if err != nil { + return err + } + + return eventListener.Run(func() { + klog.Infof("Resync request received. Recreating controllers for the new port configuration") + controllerHandler.Stop() + newController, resyncErr := controllerHandlerFactory() + controllerHandler = newController + + if resyncErr != nil { + utilruntime.HandleError(fmt.Errorf("error resyncing: %s", resyncErr.Error())) + } + }) +} diff --git a/pkg/event_handler/handler_test.go b/pkg/event_handler/handler_test.go new file mode 100644 index 0000000..f027d9b --- /dev/null +++ b/pkg/event_handler/handler_test.go @@ -0,0 +1,13 @@ +package event_handler + +import ( + "testing" +) + +type fixture struct { + t *testing.T +} + +func TestStartKafkaEventListener(t *testing.T) { + +} diff --git a/pkg/event_listener/polling/event_listener.go b/pkg/event_handler/polling/event_listener.go similarity index 83% rename from pkg/event_listener/polling/event_listener.go rename to pkg/event_handler/polling/event_listener.go index 546f052..8856ca8 100644 --- a/pkg/event_listener/polling/event_listener.go +++ b/pkg/event_handler/polling/event_listener.go @@ -11,9 +11,9 @@ type EventListener struct { portClient *cli.PortClient } -func NewEventListener(portClient *cli.PortClient) *EventListener { +func NewEventListener(stateKey string, portClient *cli.PortClient) *EventListener { return &EventListener{ - stateKey: config.ApplicationConfig.StateKey, + stateKey: stateKey, portClient: portClient, } } diff --git a/pkg/event_listener/polling/polling.go b/pkg/event_handler/polling/polling.go similarity index 100% rename from pkg/event_listener/polling/polling.go rename to pkg/event_handler/polling/polling.go diff --git a/pkg/event_listener/polling/polling_test.go b/pkg/event_handler/polling/polling_test.go similarity index 95% rename from pkg/event_listener/polling/polling_test.go rename to pkg/event_handler/polling/polling_test.go index 6476c3b..759e84d 100644 --- a/pkg/event_listener/polling/polling_test.go +++ b/pkg/event_handler/polling/polling_test.go @@ -39,9 +39,7 @@ func NewFixture(t *testing.T, c chan time.Time) *Fixture { } _ = integration.DeleteIntegration(portClient, stateKey) - err = integration.NewIntegration(portClient, &port.Config{ - StateKey: stateKey, - }, []port.Resource{}) + err = integration.NewIntegration(portClient, stateKey, "", []port.Resource{}) if err != nil { t.Errorf("Error creating Port integration: %s", err.Error()) } diff --git a/pkg/event_listener/event_listener_handler.go b/pkg/event_listener/event_listener_handler.go deleted file mode 100644 index 68192f7..0000000 --- a/pkg/event_listener/event_listener_handler.go +++ /dev/null @@ -1,34 +0,0 @@ -package event_listener - -import ( - "fmt" - "github.com/port-labs/port-k8s-exporter/pkg/handlers" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/klog/v2" -) - -type IEventListener interface { - Run(resync func()) error -} - -type Handler struct { - eventListener IEventListener - controllerHandler *handlers.ControllersHandler -} - -func StartEventHandler(eventListener IEventListener, controllerHandler *handlers.ControllersHandler, resync func(*handlers.ControllersHandler) (*handlers.ControllersHandler, error)) error { - err := eventListener.Run(func() { - klog.Infof("Resync request received. Recreating controllers for the new port configuration") - newController, resyncErr := resync(controllerHandler) - controllerHandler = newController - - if resyncErr != nil { - utilruntime.HandleError(fmt.Errorf("error resyncing: %s", resyncErr.Error())) - } - }) - if err != nil { - return err - } - - return nil -} diff --git a/pkg/handlers/controllers.go b/pkg/handlers/controllers.go index c665615..385aff3 100644 --- a/pkg/handlers/controllers.go +++ b/pkg/handlers/controllers.go @@ -16,7 +16,7 @@ import ( type ControllersHandler struct { controllers []*k8s.Controller informersFactory dynamicinformer.DynamicSharedInformerFactory - exporterConfig *port.Config + stateKey string portClient *cli.PortClient stopCh chan struct{} } @@ -57,7 +57,7 @@ func NewControllersHandler(exporterConfig *port.Config, portConfig *port.AppConf controllersHandler := &ControllersHandler{ controllers: controllers, informersFactory: informersFactory, - exporterConfig: exporterConfig, + stateKey: exporterConfig.StateKey, portClient: portClient, stopCh: signal.SetupSignalHandler(), } @@ -106,7 +106,7 @@ func (c *ControllersHandler) RunDeleteStaleEntities() { klog.Errorf("error authenticating with Port: %v", err) } - err = c.portClient.DeleteStaleEntities(context.Background(), c.exporterConfig.StateKey, goutils.MergeMaps(currentEntitiesSet...)) + err = c.portClient.DeleteStaleEntities(context.Background(), c.stateKey, goutils.MergeMaps(currentEntitiesSet...)) if err != nil { klog.Errorf("error deleting stale entities: %s", err.Error()) } diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 06241e6..1e92d18 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -7,13 +7,13 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port/cli" ) -func NewIntegration(portClient *cli.PortClient, exporterConfig *port.Config, resources []port.Resource) error { +func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerType string, resources []port.Resource) error { integration := &port.Integration{ - Title: exporterConfig.StateKey, + Title: stateKey, InstallationAppType: "K8S EXPORTER", - InstallationId: exporterConfig.StateKey, + InstallationId: stateKey, EventListener: port.EventListenerSettings{ - Type: exporterConfig.EventListenerType, + Type: eventListenerType, }, Config: &port.AppConfig{ Resources: resources, From a522369146ca6e6b4945c0cea45c77158cc29c41 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 09:32:23 +0200 Subject: [PATCH 28/35] event handler tests --- pkg/event_handler/handler_test.go | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/pkg/event_handler/handler_test.go b/pkg/event_handler/handler_test.go index f027d9b..481f33b 100644 --- a/pkg/event_handler/handler_test.go +++ b/pkg/event_handler/handler_test.go @@ -1,6 +1,7 @@ package event_handler import ( + "github.com/stretchr/testify/assert" "testing" ) @@ -8,6 +9,46 @@ type fixture struct { t *testing.T } +type ControllerHandlerMock struct { + stopped bool +} + +func (c *ControllerHandlerMock) Stop() { + c.stopped = true +} + +type EventListenerMock struct { +} + +func (e *EventListenerMock) Run(resync func()) error { + resync() + resync() + return nil +} + func TestStartKafkaEventListener(t *testing.T) { + eventListenerMock := &EventListenerMock{} + firstResponse := &ControllerHandlerMock{} + secondResponse := &ControllerHandlerMock{} + thirdResponse := &ControllerHandlerMock{} + responses := []*ControllerHandlerMock{ + firstResponse, + secondResponse, + thirdResponse, + } + + err := StartEventHandler(eventListenerMock, func() (IStoppableRsync, error) { + r := responses[0] + responses = responses[1:] + + return r, nil + }) + + if err != nil { + t.Errorf("Expected no error, got %s", err.Error()) + } + assert.True(t, firstResponse.stopped) + assert.True(t, secondResponse.stopped) + assert.False(t, thirdResponse.stopped) } From 1838961e5357003229445d4297e4b96a6f120a58 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 09:32:37 +0200 Subject: [PATCH 29/35] file renaming --- pkg/event_handler/{handler_test.go => event_handler_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/event_handler/{handler_test.go => event_handler_test.go} (100%) diff --git a/pkg/event_handler/handler_test.go b/pkg/event_handler/event_handler_test.go similarity index 100% rename from pkg/event_handler/handler_test.go rename to pkg/event_handler/event_handler_test.go From 9879466cf640f2dfd73d196025b560b33e4f35be Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 12:53:27 +0200 Subject: [PATCH 30/35] removed function --- main.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/main.go b/main.go index f3020a4..d38627e 100644 --- a/main.go +++ b/main.go @@ -43,10 +43,6 @@ func createEventListener(stateKey string, eventListenerType string, portClient * } -func initIntegrationConfig() { - -} - func main() { klog.InitFlags(nil) From 2cf5c27fcf74de9071e0cf38f2614d45238b68c6 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 15:59:35 +0200 Subject: [PATCH 31/35] cr --- main.go | 2 +- pkg/event_handler/event_handler.go | 6 +++--- pkg/event_handler/event_handler_test.go | 5 ++++- pkg/event_handler/polling/event_listener.go | 7 +++---- pkg/event_handler/polling/polling.go | 8 ++++---- pkg/port/integration/integration.go | 1 + 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index d38627e..7d19fab 100644 --- a/main.go +++ b/main.go @@ -85,7 +85,7 @@ func main() { } klog.Info("Starting controllers handler") - err = event_handler.StartEventHandler(eventListener, func() (event_handler.IStoppableRsync, error) { + err = event_handler.Start(eventListener, func() (event_handler.IStoppableRsync, error) { return initiateHandler(exporterConfig, k8sClient, portClient) }) diff --git a/pkg/event_handler/event_handler.go b/pkg/event_handler/event_handler.go index b2a247a..b75e394 100644 --- a/pkg/event_handler/event_handler.go +++ b/pkg/event_handler/event_handler.go @@ -14,8 +14,8 @@ type IStoppableRsync interface { Stop() } -func StartEventHandler(eventListener IListener, controllerHandlerFactory func() (IStoppableRsync, error)) error { - controllerHandler, err := controllerHandlerFactory() +func Start(eventListener IListener, initControllerHandler func() (IStoppableRsync, error)) error { + controllerHandler, err := initControllerHandler() if err != nil { return err } @@ -23,7 +23,7 @@ func StartEventHandler(eventListener IListener, controllerHandlerFactory func() return eventListener.Run(func() { klog.Infof("Resync request received. Recreating controllers for the new port configuration") controllerHandler.Stop() - newController, resyncErr := controllerHandlerFactory() + newController, resyncErr := initControllerHandler() controllerHandler = newController if resyncErr != nil { diff --git a/pkg/event_handler/event_handler_test.go b/pkg/event_handler/event_handler_test.go index 481f33b..4f0ca4f 100644 --- a/pkg/event_handler/event_handler_test.go +++ b/pkg/event_handler/event_handler_test.go @@ -27,6 +27,9 @@ func (e *EventListenerMock) Run(resync func()) error { } func TestStartKafkaEventListener(t *testing.T) { + // Test should get a new controller handler on each call to the passed function and stop the previous one + // The flow for this test will be: create controller handler -> resync and stop the controller handler & create a + // new controller handler X 2 which will result the last controller handler to not be stopped eventListenerMock := &EventListenerMock{} firstResponse := &ControllerHandlerMock{} secondResponse := &ControllerHandlerMock{} @@ -37,7 +40,7 @@ func TestStartKafkaEventListener(t *testing.T) { thirdResponse, } - err := StartEventHandler(eventListenerMock, func() (IStoppableRsync, error) { + err := Start(eventListenerMock, func() (IStoppableRsync, error) { r := responses[0] responses = responses[1:] diff --git a/pkg/event_handler/polling/event_listener.go b/pkg/event_handler/polling/event_listener.go index 8856ca8..fbdf9ce 100644 --- a/pkg/event_handler/polling/event_listener.go +++ b/pkg/event_handler/polling/event_listener.go @@ -9,20 +9,19 @@ import ( type EventListener struct { stateKey string portClient *cli.PortClient + handler *Handler } func NewEventListener(stateKey string, portClient *cli.PortClient) *EventListener { return &EventListener{ stateKey: stateKey, portClient: portClient, + handler: NewPollingHandler(config.PollingListenerRate, stateKey, portClient, nil), } } func (l *EventListener) Run(resync func()) error { klog.Infof("Starting polling event listener") klog.Infof("Polling rate set to %d seconds", config.PollingListenerRate) - pollingHandler := NewPollingHandler(config.PollingListenerRate, l.stateKey, l.portClient, nil) - pollingHandler.Run(resync) - - return nil + return l.handler.Run(resync) } diff --git a/pkg/event_handler/polling/polling.go b/pkg/event_handler/polling/polling.go index 24c5b38..8a7bdc6 100644 --- a/pkg/event_handler/polling/polling.go +++ b/pkg/event_handler/polling/polling.go @@ -33,19 +33,19 @@ func (t *Ticker) GetC() <-chan time.Time { return t.ticker.C } -type HandlerSettings struct { +type Handler struct { ticker ITicker stateKey string portClient *cli.PortClient pollingRate uint } -func NewPollingHandler(pollingRate uint, stateKey string, portClient *cli.PortClient, tickerOverride ITicker) *HandlerSettings { +func NewPollingHandler(pollingRate uint, stateKey string, portClient *cli.PortClient, tickerOverride ITicker) *Handler { ticker := tickerOverride if ticker == nil { ticker = NewTicker(time.Second * time.Duration(pollingRate)) } - rv := &HandlerSettings{ + rv := &Handler{ ticker: ticker, stateKey: stateKey, portClient: portClient, @@ -54,7 +54,7 @@ func NewPollingHandler(pollingRate uint, stateKey string, portClient *cli.PortCl return rv } -func (h *HandlerSettings) Run(resync func()) { +func (h *Handler) Run(resync func()) { klog.Infof("Starting polling handler") currentState, err := integration.GetIntegration(h.portClient, h.stateKey) if err != nil { diff --git a/pkg/port/integration/integration.go b/pkg/port/integration/integration.go index 1e92d18..4a5c227 100644 --- a/pkg/port/integration/integration.go +++ b/pkg/port/integration/integration.go @@ -31,6 +31,7 @@ func NewIntegration(portClient *cli.PortClient, stateKey string, eventListenerTy return nil } +// ToDo: remove this function func GetIntegrationConfig(portClient *cli.PortClient, stateKey string) (*port.AppConfig, error) { _, err := portClient.Authenticate(context.Background(), portClient.ClientID, portClient.ClientSecret) if err != nil { From d4816cdd7c78f1f22dabf4ed058fdde0efc3fe78 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 16:00:00 +0200 Subject: [PATCH 32/35] cr --- pkg/event_handler/polling/event_listener.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/event_handler/polling/event_listener.go b/pkg/event_handler/polling/event_listener.go index fbdf9ce..b3af4a8 100644 --- a/pkg/event_handler/polling/event_listener.go +++ b/pkg/event_handler/polling/event_listener.go @@ -23,5 +23,7 @@ func NewEventListener(stateKey string, portClient *cli.PortClient) *EventListene func (l *EventListener) Run(resync func()) error { klog.Infof("Starting polling event listener") klog.Infof("Polling rate set to %d seconds", config.PollingListenerRate) - return l.handler.Run(resync) + l.handler.Run(resync) + + return nil } From ae6ccaddb059266ee9d8a6789fbe5dfb08425430 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 16:07:34 +0200 Subject: [PATCH 33/35] closing after one message consumer --- pkg/event_handler/consumer/consumer_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/event_handler/consumer/consumer_test.go b/pkg/event_handler/consumer/consumer_test.go index aa39aa1..8b8a4f0 100644 --- a/pkg/event_handler/consumer/consumer_test.go +++ b/pkg/event_handler/consumer/consumer_test.go @@ -18,6 +18,7 @@ type Fixture struct { type MockConsumer struct { pollData kafka.Event + close func() } func (m *MockConsumer) SubscribeTopics(topics []string, rebalanceCb kafka.RebalanceCb) (err error) { @@ -36,6 +37,9 @@ func (m *MockConsumer) SubscribeTopics(topics []string, rebalanceCb kafka.Rebala } func (m *MockConsumer) Poll(timeoutMs int) (event kafka.Event) { + defer func() { + m.close() + }() return m.pollData } @@ -50,6 +54,7 @@ func (m *MockConsumer) Close() (err error) { func NewFixture(t *testing.T) *Fixture { mock := &MockConsumer{} consumer, err := NewConsumer(&config.KafkaConfiguration{}, mock) + mock.close = consumer.Close if err != nil { t.Fatalf("Error creating consumer: %v", err) } From 06451f1cf5df88bd4f43c57d24a2707d96b8d404 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 16:08:08 +0200 Subject: [PATCH 34/35] closing after one message consumer --- pkg/event_handler/consumer/consumer_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/event_handler/consumer/consumer_test.go b/pkg/event_handler/consumer/consumer_test.go index 8b8a4f0..77a3a2b 100644 --- a/pkg/event_handler/consumer/consumer_test.go +++ b/pkg/event_handler/consumer/consumer_test.go @@ -37,6 +37,7 @@ func (m *MockConsumer) SubscribeTopics(topics []string, rebalanceCb kafka.Rebala } func (m *MockConsumer) Poll(timeoutMs int) (event kafka.Event) { + // The consumer will poll this in while true loop so we need to close it inorder not to spam the logs defer func() { m.close() }() From 86ec4e71ab4edc03da613d7782dba96cf03b06e3 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 20 Dec 2023 16:47:26 +0200 Subject: [PATCH 35/35] cr --- pkg/event_handler/consumer/consumer_test.go | 27 +++++++-------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/pkg/event_handler/consumer/consumer_test.go b/pkg/event_handler/consumer/consumer_test.go index 77a3a2b..81ae13a 100644 --- a/pkg/event_handler/consumer/consumer_test.go +++ b/pkg/event_handler/consumer/consumer_test.go @@ -9,11 +9,10 @@ import ( ) type Fixture struct { - t *testing.T - mockConsumer *MockConsumer - consumer *Consumer - producer *kafka.Producer - topic string + t *testing.T + mockKafkaConsumer *MockConsumer + consumer *Consumer + topic string } type MockConsumer struct { @@ -60,24 +59,16 @@ func NewFixture(t *testing.T) *Fixture { t.Fatalf("Error creating consumer: %v", err) } - producer, err := kafka.NewProducer(&kafka.ConfigMap{ - "bootstrap.servers": "localhost:9092", - }) - if err != nil { - t.Fatalf("Error creating producer: %v", err) - } - return &Fixture{ - t: t, - mockConsumer: mock, - consumer: consumer, - producer: producer, - topic: "test-topic", + t: t, + mockKafkaConsumer: mock, + consumer: consumer, + topic: "test-topic", } } func (f *Fixture) Produce(t *testing.T, value []byte) { - f.mockConsumer.pollData = &kafka.Message{ + f.mockKafkaConsumer.pollData = &kafka.Message{ TopicPartition: kafka.TopicPartition{Topic: &f.topic, Partition: 0}, Value: value, }