integration tests upgrade (#1401)

* fix .env with newer docker-compose
* fix basic auth integration test
* protoc test fix with newer Python and protobuf versions

tests/README.md

@@ -60,7 +60,7 @@ The easiest way to arrange for this is:
 
 And also:
 
-    source ./tests/docker/.env
+    source ./tests/docker/.env.sh
 
 which sets environment variables referenced by `./tests/integration/testconf.json`.
 

tests/docker/bin/certify.sh

@@ -5,7 +5,7 @@ set -eu
 PY_DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 export PASS="abcdefgh"
 
-source ${PY_DOCKER_BIN}/../.env
+source ${PY_DOCKER_BIN}/../.env.sh
 
 mkdir -p ${TLS}
 

tests/docker/bin/cluster_down.sh

@@ -3,7 +3,7 @@
 set -eu
 
 PY_DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source ${PY_DOCKER_BIN}/../.env
+source ${PY_DOCKER_BIN}/../.env.sh
 
 echo "Destroying cluster.."
 docker-compose -f $PY_DOCKER_COMPOSE_FILE down -v --remove-orphans

tests/docker/bin/cluster_up.sh

@@ -3,7 +3,7 @@
 set -eu
 
 PY_DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source ${PY_DOCKER_BIN}/../.env
+source ${PY_DOCKER_BIN}/../.env.sh
 
 # Wait for http service listener to come up and start serving
 # $1 http service name
@@ -28,7 +28,7 @@ await_http() {
 }
 
 echo "Configuring Environment..."
-source ${PY_DOCKER_SOURCE}/.env
+source ${PY_DOCKER_SOURCE}/.env.sh
 
 echo "Generating SSL certs..."
 ${PY_DOCKER_BIN}/certify.sh

tests/docker/docker-compose.yaml

@@ -52,9 +52,9 @@ services:
     volumes:
       - ./conf:/conf
     environment:
-      SCHEMA_REGISTRY_HOST_NAME: schema-registry2
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry-basic-auth
       SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas2
-      SCHEMA_REGISTRY_SCHEMA_REGISTRY_ZK_NAMESPACE: schema_registry2
+      SCHEMA_REGISTRY_SCHEMA_REGISTRY_GROUP_ID: schema-registry-basic-auth
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8083, https://0.0.0.0:8084
       SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: https

tests/integration/integration_test.py

@@ -28,6 +28,7 @@
 import json
 import gc
 import struct
+import re
 
 try:
     # Memory tracker
@@ -795,13 +796,17 @@ def verify_avro_basic_auth(mode_conf):
     if mode_conf is None:
         abort_on_missing_configuration('avro-basic-auth')
 
-    url = {
-        'schema.registry.basic.auth.credentials.source': 'URL'
+    url = mode_conf.get('schema.registry.url')
+    credentials = mode_conf.get('schema.registry.basic.auth.user.info')
+
+    url_conf = {
+        'schema.registry.basic.auth.credentials.source': 'URL',
+        'schema.registry.url': str(re.sub("(^https?://)", f"\\1{credentials}@", url))
     }
 
     user_info = {
         'schema.registry.basic.auth.credentials.source': 'USER_INFO',
-        'schema.registry.basic.auth.user.info': mode_conf.get('schema.registry.basic.auth.user.info')
+        'schema.registry.basic.auth.user.info': credentials
     }
 
     sasl_inherit = {
@@ -813,7 +818,7 @@ def verify_avro_basic_auth(mode_conf):
     base_conf = {
         'bootstrap.servers': bootstrap_servers,
         'error_cb': error_cb,
-        'schema.registry.url': schema_registry_url
+        'schema.registry.url': url
     }
 
     consumer_conf = dict({'group.id': generate_group_id(),
@@ -829,7 +834,7 @@ def verify_avro_basic_auth(mode_conf):
     run_avro_loop(dict(base_conf, **sasl_inherit), dict(consumer_conf, **sasl_inherit))
 
     print('-' * 10, 'Verifying basic auth source URL', '-' * 10)
-    run_avro_loop(dict(base_conf, **url), dict(consumer_conf, **url))
+    run_avro_loop(dict(base_conf, **url_conf), dict(consumer_conf, **url_conf))
 
 
 def run_avro_loop(producer_conf, consumer_conf):

tests/integration/schema_registry/data/DependencyTestProto.proto → tests/integration/schema_registry/data/proto/DependencyTestProto.proto

@@ -1,7 +1,7 @@
 syntax = "proto3";
 
-import "NestedTestProto.proto";
-import "PublicTestProto.proto";
+import "tests/integration/schema_registry/data/proto/NestedTestProto.proto";
+import "tests/integration/schema_registry/data/proto/PublicTestProto.proto";
 
 package tests.integration.serialization.data;
 

tests/integration/schema_registry/data/Makefile → tests/integration/schema_registry/data/proto/Makefile

@@ -1,16 +1,15 @@
 WORK_DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
 PROTO_HOME=/usr/local/opt/include
-SRC_DIR=$(WORK_DIR)
-TARGET_DIR=$(WORK_DIR)/../gen
+SRC_DIR=$(realpath $(WORK_DIR)/../../../../..)
 
 PROTOS := common_proto.proto DependencyTestProto.proto exampleProtoCriteo.proto $\
          metadata_proto.proto NestedTestProto.proto PublicTestProto.proto $\
          SInt32Value.proto SInt64Value.proto TestProto.proto
 
 compile: $(PROTOS)
 	for proto in $(PROTOS); do \
-		protoc -I=$(PROTO_HOME) -I=$(SRC_DIR) --python_out=$(TARGET_DIR) $$proto ; \
+		(cd $(SRC_DIR) && protoc -I=$(PROTO_HOME) -I=$(SRC_DIR) --python_out=$(SRC_DIR) tests/integration/schema_registry/data/proto/$$proto ;) \
 	done
 
 clean:
-	rm -f $(TARGET_DIR)/*_pb2.py
+	rm -f *_pb2.py

tests/integration/schema_registry/data/proto/PublicTestProto.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+import public "tests/integration/schema_registry/data/proto/TestProto.proto";
+
+package tests.integration.serialization.data;
+

tests/integration/schema_registry/data/common_proto.proto → tests/integration/schema_registry/data/proto/common_proto.proto

@@ -3,7 +3,7 @@ syntax = "proto3";
 package Criteo.Glup;
 option java_package = "com.criteo.glup";
 
-import "metadata_proto.proto";
+import "tests/integration/schema_registry/data/proto/metadata_proto.proto";
 
 /* Describes if an event or campaign is an appinstall or normal
  * retargeting one