From b3021149943c54f22f32d183f16113b9f3ae640a Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Wed, 22 May 2024 11:48:02 -0700 Subject: [PATCH] [computes] add k8s label selector filter for service discovery In some cases, there might be multiple clusters running in the same k8s environment. In such cases, we can pass a label selector to limit the k8s namespaces that hue service-discovery looks at. Change-Id: Ib98aeeafc7a19313abc3d7bc81bc17e4b646dd43 --- .../management/commands/sync_warehouses.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/desktop/core/src/desktop/management/commands/sync_warehouses.py b/desktop/core/src/desktop/management/commands/sync_warehouses.py index d5e7e50237..432b9f018c 100644 --- a/desktop/core/src/desktop/management/commands/sync_warehouses.py +++ b/desktop/core/src/desktop/management/commands/sync_warehouses.py @@ -17,16 +17,18 @@ # HUE_CONF_DIR=/etc/hue/conf HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1 /opt/hive/build/env/bin/hue sync_warehouses -from beeswax import models -from beeswax.conf import AUTH_USERNAME, AUTH_PASSWORD -from django.core.management.base import BaseCommand -from hadoop import confparse -import json -from kubernetes import client, config -import logging import os import re import sys +import json +import logging + +from django.core.management.base import BaseCommand +from kubernetes import client, config + +from beeswax import models +from beeswax.conf import AUTH_PASSWORD, AUTH_USERNAME +from hadoop import confparse LOG = logging.getLogger() @@ -95,7 +97,9 @@ def get_computes_from_k8s(): catalogs = [] computes = [] - for n in core_v1.list_namespace().items: + label_selector = os.environ.get("K8S_LABEL_SELECTOR") # clusterid=env-urmgt6-env + + for n in core_v1.list_namespace(label_selector=label_selector).items: try: namespace = n.metadata.name LOG.info('Getting details for ns: %s' % namespace) @@ -103,7 +107,7 @@ def get_computes_from_k8s(): 'name': n.metadata.labels.get('displayname'), 'description': '%s (%s)' % (n.metadata.labels.get('displayname'), n.metadata.name), 'external_id': namespace, - #'creation_timestamp': n.metadata.labels.get('creation_timestamp'), + # 'creation_timestamp': n.metadata.labels.get('creation_timestamp'), } if namespace.startswith('warehouse-'): @@ -119,6 +123,7 @@ def get_computes_from_k8s(): return computes + def update_hive_configs(namespace, hive, host, port=80): hs2_stfs = apps_v1.read_namespaced_stateful_set('hiveserver2', namespace) @@ -187,6 +192,7 @@ def populate_impala(namespace, impala): impala['api_port'] = next((p.port for p in ports if p.name == 'web'), 25000) update_impala_configs(namespace, impala, 'coordinator.%s.svc.cluster.local' % namespace) + def update_impala_configs(namespace, impala, host): hive_configs = core_v1.read_namespaced_config_map('impala-coordinator-hive-conf', namespace) hive_site_data = confparse.ConfParse(hive_configs.data['hive-site.xml'])