ingress gateways

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt

@@ -29,8 +29,10 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.NoopSnapshotChangeAudit
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotChangeAuditor
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotUpdater
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotsVersions
+import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.IngressGatewayPortMappingsCache
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.endpoints.EnvoyEndpointsFactory
+import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.EnvoyIngressGatewayListenersFactory
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.EnvoyListenersFactory
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters.EnvoyHttpFilters
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.EnvoyEgressRoutesFactory
@@ -97,6 +99,7 @@ class ControlPlane private constructor(
         var groupSnapshotParallelExecutorSupplier: () -> Executor? = { null }
         var metrics: EnvoyControlMetrics = DefaultEnvoyControlMetrics(meterRegistry = meterRegistry)
         var envoyHttpFilters: EnvoyHttpFilters = EnvoyHttpFilters.emptyFilters
+        var ingressGatewayPortMappingsCache: IngressGatewayPortMappingsCache? = null
         var snapshotChangeAuditor: SnapshotChangeAuditor = NoopSnapshotChangeAuditor
 
         var nodeGroup: NodeGroup<Group> = MetadataNodeGroup(
@@ -138,6 +141,10 @@ class ControlPlane private constructor(
                 )
             }
 
+            if (ingressGatewayPortMappingsCache == null) {
+                ingressGatewayPortMappingsCache = IngressGatewayPortMappingsCache()
+            }
+
             val groupSnapshotProperties = properties.server.groupSnapshotUpdateScheduler
 
             val groupSnapshotScheduler = buildGroupSnapshotScheduler(groupSnapshotProperties)
@@ -176,6 +183,9 @@ class ControlPlane private constructor(
                     snapshotProperties,
                     envoyHttpFilters
                 ),
+                ingressGatewayListenersFactory = EnvoyIngressGatewayListenersFactory(
+                  mappingsCache = ingressGatewayPortMappingsCache!!
+                ),
                 // Remember when LDS change we have to send RDS again
                 snapshotsVersions = snapshotsVersions,
                 properties = snapshotProperties,
@@ -352,6 +362,10 @@ class ControlPlane private constructor(
             this.envoyHttpFilters = envoyHttpFilters
             return this
         }
+        fun withIngressGatewayPortMappingsCache(mappingsCache: IngressGatewayPortMappingsCache): ControlPlaneBuilder {
+            this.ingressGatewayPortMappingsCache = mappingsCache
+            return this
+        }
 
         private fun NettyServerBuilder.withEnvoyServices(discoveryServer: V3DiscoveryServer): NettyServerBuilder {
             return this.addService(discoveryServer.aggregatedDiscoveryServiceImpl)

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/Groups.kt

@@ -8,21 +8,40 @@ sealed class Group {
     abstract val listenersConfig: ListenersConfig?
 }
 
+sealed class IngressGatewayGroup: Group()
+sealed class SidecarGroup: Group()
+
 data class ServicesGroup(
     override val communicationMode: CommunicationMode,
     override val serviceName: String = "",
     override val discoveryServiceName: String? = null,
     override val proxySettings: ProxySettings = ProxySettings(),
     override val listenersConfig: ListenersConfig? = null
-) : Group()
+) : SidecarGroup()
 
 data class AllServicesGroup(
     override val communicationMode: CommunicationMode,
     override val serviceName: String = "",
     override val discoveryServiceName: String? = null,
     override val proxySettings: ProxySettings = ProxySettings(),
     override val listenersConfig: ListenersConfig? = null
-) : Group()
+) : SidecarGroup()
+
+data class ServicesIngressGatewayGroup(
+    override val communicationMode: CommunicationMode,
+    override val serviceName: String = "",
+    override val discoveryServiceName: String? = null,
+    override val proxySettings: ProxySettings = ProxySettings(),
+    override val listenersConfig: ListenersConfig? = null
+) : IngressGatewayGroup()
+
+data class AllServicesIngressGatewayGroup(
+    override val communicationMode: CommunicationMode,
+    override val serviceName: String = "",
+    override val discoveryServiceName: String? = null,
+    override val proxySettings: ProxySettings = ProxySettings(),
+    override val listenersConfig: ListenersConfig? = null
+) : IngressGatewayGroup()
 
 data class ListenersConfig(
     val ingressHost: String,

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroup.kt

@@ -131,27 +131,47 @@ class MetadataNodeGroup(
     private fun createV3Group(node: NodeV3): Group {
         val nodeMetadata = NodeMetadata(node.metadata, properties)
         val serviceName = serviceName(nodeMetadata)
-        val discoveryServiceName = nodeMetadata.discoveryServiceName
+        val discoveryServiceName = nodeMetadata.discoveryServiceName ?: serviceName
         val proxySettings = proxySettings(nodeMetadata)
         val listenersConfig = createListenersConfig(node.id, node.metadata)
 
-        return when {
-            hasAllServicesDependencies(nodeMetadata) ->
-                AllServicesGroup(
-                    nodeMetadata.communicationMode,
-                    serviceName,
-                    discoveryServiceName,
-                    proxySettings,
-                    listenersConfig
-                )
-            else ->
-                ServicesGroup(
-                    nodeMetadata.communicationMode,
-                    serviceName,
-                    discoveryServiceName,
-                    proxySettings,
-                    listenersConfig
-                )
+        return when (nodeMetadata.mode) {
+            Mode.SERVICE -> when {
+                hasAllServicesDependencies(nodeMetadata) ->
+                    AllServicesGroup(
+                        nodeMetadata.communicationMode,
+                        serviceName,
+                        discoveryServiceName,
+                        proxySettings,
+                        listenersConfig
+                    )
+                else ->
+                    ServicesGroup(
+                        nodeMetadata.communicationMode,
+                        serviceName,
+                        discoveryServiceName,
+                        proxySettings,
+                        listenersConfig
+                    )
+            }
+            Mode.INGRESS_GATEWAY -> when {
+                hasAllServicesDependencies(nodeMetadata) ->
+                    AllServicesIngressGatewayGroup(
+                        nodeMetadata.communicationMode,
+                        serviceName,
+                        discoveryServiceName,
+                        proxySettings,
+                        listenersConfig
+                    )
+                else ->
+                    ServicesIngressGatewayGroup(
+                        nodeMetadata.communicationMode,
+                        serviceName,
+                        discoveryServiceName,
+                        proxySettings,
+                        listenersConfig
+                    )
+            }
         }
     }
 

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadata.kt

@@ -28,6 +28,7 @@ class NodeMetadata(metadata: Struct, properties: SnapshotProperties) {
         .fieldsMap["discovery_service_name"]
         ?.stringValue
 
+    val mode = metadata.fieldsMap["mode"]?.stringValue?.let { Mode.valueOf(it) } ?: Mode.SERVICE
     val communicationMode = getCommunicationMode(metadata.fieldsMap["ads"])
 
     val proxySettings: ProxySettings = ProxySettings(metadata.fieldsMap["proxy_settings"], properties)
@@ -677,6 +678,10 @@ enum class CommunicationMode {
     ADS, XDS
 }
 
+enum class Mode {
+    SERVICE, INGRESS_GATEWAY
+}
+
 data class OAuth(
     val provider: String = "",
     val verification: Verification = Verification.OFFLINE,

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt

@@ -9,27 +9,34 @@ import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret
 import io.micrometer.core.instrument.MeterRegistry
 import io.micrometer.core.instrument.Timer
 import pl.allegro.tech.servicemesh.envoycontrol.groups.AllServicesGroup
+import pl.allegro.tech.servicemesh.envoycontrol.groups.AllServicesIngressGatewayGroup
 import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode
 import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings
 import pl.allegro.tech.servicemesh.envoycontrol.groups.Group
 import pl.allegro.tech.servicemesh.envoycontrol.groups.IncomingRateLimitEndpoint
+import pl.allegro.tech.servicemesh.envoycontrol.groups.ServicesIngressGatewayGroup
+import pl.allegro.tech.servicemesh.envoycontrol.groups.IngressGatewayGroup
 import pl.allegro.tech.servicemesh.envoycontrol.groups.ServicesGroup
+import pl.allegro.tech.servicemesh.envoycontrol.groups.SidecarGroup
 import pl.allegro.tech.servicemesh.envoycontrol.services.MultiClusterState
 import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceInstance
 import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceInstances
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.endpoints.EnvoyEndpointsFactory
+import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.EnvoyIngressGatewayListenersFactory
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.EnvoyListenersFactory
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.EnvoyEgressRoutesFactory
 import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.EnvoyIngressRoutesFactory
 import java.util.SortedMap
 
+@Suppress("LongParameterList")
 class EnvoySnapshotFactory(
     private val ingressRoutesFactory: EnvoyIngressRoutesFactory,
     private val egressRoutesFactory: EnvoyEgressRoutesFactory,
     private val clustersFactory: EnvoyClustersFactory,
     private val endpointsFactory: EnvoyEndpointsFactory,
     private val listenersFactory: EnvoyListenersFactory,
+    private val ingressGatewayListenersFactory: EnvoyIngressGatewayListenersFactory,
     private val snapshotsVersions: SnapshotsVersions,
     private val properties: SnapshotProperties,
     private val meterRegistry: MeterRegistry
@@ -150,7 +157,10 @@ class EnvoySnapshotFactory(
     fun getSnapshotForGroup(group: Group, globalSnapshot: GlobalSnapshot): Snapshot {
         val groupSample = Timer.start(meterRegistry)
 
-        val newSnapshotForGroup = newSnapshotForGroup(group, globalSnapshot)
+        val newSnapshotForGroup = when (group) {
+            is IngressGatewayGroup -> newSnapshotForIngressGroup(group, globalSnapshot)
+            is SidecarGroup -> newSnapshotForGroup(group, globalSnapshot)
+        }
         groupSample.stop(meterRegistry.timer("snapshot-factory.get-snapshot-for-group.time"))
         return newSnapshotForGroup
     }
@@ -188,10 +198,10 @@ class EnvoySnapshotFactory(
             )
         }
         return when (group) {
-            is ServicesGroup -> {
+            is ServicesGroup, is ServicesIngressGatewayGroup -> {
                 definedServicesRoutes
             }
-            is AllServicesGroup -> {
+            is AllServicesGroup, is AllServicesIngressGatewayGroup -> {
                 val servicesNames = group.proxySettings.outgoing.getServiceDependencies().map { it.service }.toSet()
                 val allServicesRoutes = globalSnapshot.allServicesNames.subtract(servicesNames).map {
                     RouteSpecification(
@@ -227,7 +237,7 @@ class EnvoySnapshotFactory(
     }
 
     private fun newSnapshotForGroup(
-        group: Group,
+        group: SidecarGroup,
         globalSnapshot: GlobalSnapshot
     ): Snapshot {
 
@@ -268,8 +278,44 @@ class EnvoySnapshotFactory(
 
         // TODO(dj): endpoints depends on prerequisite of routes -> but only to extract clusterName,
         // which is present only in services (not domains) so it could be implemented differently.
-        val endpoints = getServicesEndpointsForGroup(group.proxySettings.incoming.rateLimitEndpoints, globalSnapshot,
-                            egressRouteSpecification)
+        val endpoints = getServicesEndpointsForGroup(
+            group.proxySettings.incoming.rateLimitEndpoints, globalSnapshot,
+            egressRouteSpecification
+        )
+
+        val version = snapshotsVersions.version(group, clusters, endpoints, listeners, routes)
+
+        return createSnapshot(
+            clusters = clusters,
+            clustersVersion = version.clusters,
+            endpoints = endpoints,
+            endpointsVersions = version.endpoints,
+            listeners = listeners,
+            // TODO: java-control-plane: https://github.com/envoyproxy/java-control-plane/issues/134
+            listenersVersion = version.listeners,
+            routes = routes,
+            routesVersion = version.routes
+        )
+    }
+
+    private fun newSnapshotForIngressGroup(
+        group: IngressGatewayGroup,
+        globalSnapshot: GlobalSnapshot
+    ): Snapshot {
+
+        val clusters: List<Cluster> =
+            clustersFactory.getClustersForGroup(group, globalSnapshot)
+
+        val serviceRouteSpecification = getServiceRouteSpecifications(group, globalSnapshot)
+
+        val routes = emptyList<RouteConfiguration>()
+
+        val listeners = ingressGatewayListenersFactory.createListeners(group, globalSnapshot)
+
+        val endpoints = getServicesEndpointsForGroup(
+            group.proxySettings.incoming.rateLimitEndpoints, globalSnapshot,
+            serviceRouteSpecification
+        )
 
         val version = snapshotsVersions.version(group, clusters, endpoints, listeners, routes)
 

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt

@@ -36,6 +36,7 @@ class SnapshotProperties {
     var deltaXdsEnabled = false
     var retryPolicy = RetryPolicyProperties()
     var tcpDumpsEnabled: Boolean = true
+    var dcIngressGatewayService = "envoy-ingress-gateway"
 }
 
 class MetricsProperties {

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/IngressGatewayPortMappingsCache.kt

@@ -0,0 +1,29 @@
+package pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource
+
+import pl.allegro.tech.servicemesh.envoycontrol.groups.AllServicesIngressGatewayGroup
+import pl.allegro.tech.servicemesh.envoycontrol.groups.IngressGatewayGroup
+import pl.allegro.tech.servicemesh.envoycontrol.groups.ServicesIngressGatewayGroup
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ConcurrentMap
+
+typealias GatewayName = String
+typealias Port = Int
+typealias Cluster = String
+
+class IngressGatewayPortMappingsCache {
+    private val serviceIngressGatewaysMappings: ConcurrentMap<GatewayName, Map<Cluster, Port>> = ConcurrentHashMap()
+    private val allServicesIngressGatewayMappings: ConcurrentMap<GatewayName, Map<Cluster, Port>> = ConcurrentHashMap()
+
+    fun addMapping(group: IngressGatewayGroup, mapping: Map<Cluster, Port>) {
+        when (group) {
+            is AllServicesIngressGatewayGroup -> allServicesIngressGatewayMappings[group.discoveryServiceName] = mapping
+            is ServicesIngressGatewayGroup -> serviceIngressGatewaysMappings[group.discoveryServiceName] = mapping
+        }
+    }
+
+    fun ingressGatewayMapping(name: GatewayName): Map<Cluster, Port> = serviceIngressGatewaysMappings[name] ?: mapOf()
+    fun dcIngressGatewayMapping(name: GatewayName): Map<Cluster, Port> =
+        allServicesIngressGatewayMappings[name] ?: mapOf()
+
+    fun dcIngressGatewayNames() = allServicesIngressGatewayMappings.keys
+}

envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt

@@ -36,12 +36,15 @@ import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.SdsSecretConfig
 import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.TlsParameters
 import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
 import pl.allegro.tech.servicemesh.envoycontrol.groups.AllServicesGroup
+import pl.allegro.tech.servicemesh.envoycontrol.groups.AllServicesIngressGatewayGroup
 import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode
 import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.ADS
 import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.XDS
 import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings
 import pl.allegro.tech.servicemesh.envoycontrol.groups.DomainDependency
 import pl.allegro.tech.servicemesh.envoycontrol.groups.Group
+import pl.allegro.tech.servicemesh.envoycontrol.groups.IngressGatewayGroup
+import pl.allegro.tech.servicemesh.envoycontrol.groups.ServicesIngressGatewayGroup
 import pl.allegro.tech.servicemesh.envoycontrol.groups.ServicesGroup
 import pl.allegro.tech.servicemesh.envoycontrol.groups.containsGlobalRateLimits
 import pl.allegro.tech.servicemesh.envoycontrol.logger
@@ -184,7 +187,7 @@ class EnvoyClustersFactory(
     }
 
     private fun getEdsClustersForGroup(group: Group, globalSnapshot: GlobalSnapshot): List<Cluster> {
-        val clusters: Map<String, Cluster> = if (enableTlsForGroup(group)) {
+        val clusters: Map<String, Cluster> = if (enableTlsForGroup(group) && group !is IngressGatewayGroup) {
             globalSnapshot.securedClusters
         } else {
             globalSnapshot.clusters
@@ -193,10 +196,10 @@ class EnvoyClustersFactory(
         val serviceDependencies = group.proxySettings.outgoing.getServiceDependencies().associateBy { it.service }
 
         val clustersForGroup = when (group) {
-            is ServicesGroup -> serviceDependencies.mapNotNull {
+            is ServicesGroup, is ServicesIngressGatewayGroup -> serviceDependencies.mapNotNull {
                 createClusterForGroup(it.value.settings, clusters[it.key])
             }
-            is AllServicesGroup -> {
+            is AllServicesGroup, is AllServicesIngressGatewayGroup -> {
                 globalSnapshot.allServicesNames.mapNotNull {
                     val dependency = serviceDependencies[it]
                     if (dependency != null && dependency.settings.timeoutPolicy.connectionIdleTimeout != null) {
@@ -242,10 +245,12 @@ class EnvoyClustersFactory(
         .setName(tlsProperties.validationContextSecretName).build()
 
     private val tlsCertificateSecretConfig = SdsSecretConfig.newBuilder()
-        .setName(tlsProperties.tlsCertificateSecretName).build()
+        .setName(tlsProperties.tlsCertificateSecretName)
+        .build()
 
     private fun createTlsContextWithSdsSecretConfig(serviceName: String): UpstreamTlsContext {
         val sanMatch = sanUriMatcher.createSanUriMatcher(serviceName)
+        val gatewaySanMatch = sanUriMatcher.createSanUriMatcher(properties.dcIngressGatewayService)
         return UpstreamTlsContext.newBuilder()
             .setCommonTlsContext(
                 CommonTlsContext.newBuilder()
@@ -254,7 +259,7 @@ class EnvoyClustersFactory(
                         CommonTlsContext.CombinedCertificateValidationContext.newBuilder()
                             .setDefaultValidationContext(
                                 CertificateValidationContext.newBuilder()
-                                    .addAllMatchSubjectAltNames(listOf(sanMatch))
+                                    .addAllMatchSubjectAltNames(listOf(sanMatch, gatewaySanMatch))
                                     .build()
                             )
                             .setValidationContextSdsSecretConfig(validationContextSecretConfig)