values.yaml

# Default values for archery.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

replicaCount: 1

image:
  repository: hhyo/archery
  tag: 1.7.0
  pullPolicy: IfNotPresent

nameOverride: ""
fullnameOverride: ""

service:
  type: NodePort
  port: 9123
  targetPort: 9123

ingress:
  enabled: false
  annotations: {}
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  paths: []
  hosts:
    - chart-example.local
  tls: []
  #  - secretName: chart-example-tls
  #    hosts:
  #      - chart-example.local

# subcharts redis
redis:
  usePassword: "123456"
  persistence:
    enabled: false
# subchats mysql
mysql:
  mysqlRootPassword: archery
  mysqlDatabase: archery
  persistence:
    enabled: false
    accessMode: ReadWriteOnce
    size: 20Gi
  ## Persist data to a persistent volume
  ## database data Persistent Volume Storage Class
  ## If defined, storageClassName: <storageClass>
  ## If set to "-", storageClassName: "", which disables dynamic provisioning
  ## If undefined (the default) or set to null, no storageClassName spec is
  ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
  ##   GKE, AWS & OpenStack)
  ##
  # storageClass: "-"

  configurationFiles:
    mysql.cnf: |-
       [mysqld_safe]
       nice            = 0
       [mysqld]
       port            = 3306
       skip-external-locking
       lower_case_table_names=1
       default-time_zone = '+8:00'
       innodb_buffer_pool_size = 512M
       server-id              = 100
       log_bin                        = /var/log/mysql/mysql-bin.log
       expire_logs_days        = 1
       max_binlog_size         = 500M
       character-set-server = utf8mb4
       collation-server = utf8mb4_general_ci
       slow_query_log_file = mysql-slow.log
       slow_query_log      = 1
       long_query_time = 1
       [client]
       default-character-set=utf8mb4
       [mysqldump]
       quick
       quote-names
       max_allowed_packet      = 1024M
       !includedir /etc/mysql/conf.d/
    
resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #  cpu: 100m
  #  memory: 128Mi
  # requests:
  #  cpu: 100m
  #  memory: 128Mi
volumeMounts:
  - name: archery-settings
    subPath: settings.py
    mountPath: /opt/archery/archery/settings.py
  - name: archery-settings
    subPath: soar.yaml
    mountPath: /etc/soar.yaml
  - name: archery-settings
    subPath: analysis_slow_query.sh
    mountPath: /opt/archery/src/script/analysis_slow_query.sh
  - name: archery-settings
    subPath: startup.sh
    mountPath: /opt/archery/src/docker/startup.sh
  - name: archery-settings
    subPath: init-archery.sh
    mountPath: /opt/archery/src/docker/init-archery.sh
  - name: archery-settings
    subPath: createsuperuser.py
    mountPath: /opt/archery/src/docker/createsuperuser.py
nodeSelector: {}

tolerations: []

affinity: {}

volumes:
  - name: archery-settings
    configMap:
      name: archery-config
      defaultMode: 420
  - name: archery-download
    hostPath:
      path: /data/archery/downloads
  - name: archery-script
    hostPath:
      path: /data/archery/scripts

configMap:
  enabled: true
  data:
    settings.py: |-
        # -*- coding: UTF-8 -*-
        
        # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
        import os
        
        BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        
        # SECURITY WARNING: keep the secret key used in production secret!
        SECRET_KEY = 'hfusaf2m4ot#7)fkw#di2bu6(cv0@opwmafx5n#6=3d%x^hpl6'
        
        # SECURITY WARNING: don't run with debug turned on in production!
        DEBUG = True
        
        ALLOWED_HOSTS = ['*']
        
        # 解决nginx部署跳转404
        USE_X_FORWARDED_HOST = True
        
        # 请求限制
        DATA_UPLOAD_MAX_MEMORY_SIZE = 15728640
        
        # Application definition
        INSTALLED_APPS = (
            'django.contrib.admin',
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'django.contrib.sessions',
            'django.contrib.messages',
            'django.contrib.staticfiles',
            'django_q',
            'sql',
            'sql_api',
            'common',
        )
        
        MIDDLEWARE = (
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.common.CommonMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.middleware.clickjacking.XFrameOptionsMiddleware',
            'django.middleware.security.SecurityMiddleware',
            'common.middleware.check_login_middleware.CheckLoginMiddleware',
            'common.middleware.exception_logging_middleware.ExceptionLoggingMiddleware',
        )
        
        ROOT_URLCONF = 'archery.urls'
        
        TEMPLATES = [
            {
                'BACKEND': 'django.template.backends.django.DjangoTemplates',
                'DIRS': [os.path.join(BASE_DIR, 'common/templates')],
                'APP_DIRS': True,
                'OPTIONS': {
                    'context_processors': [
                        'django.template.context_processors.debug',
                        'django.template.context_processors.request',
                        'django.contrib.auth.context_processors.auth',
                        'django.contrib.messages.context_processors.messages',
                        'common.utils.global_info.global_info',
                    ],
                },
            },
        ]
        
        WSGI_APPLICATION = 'archery.wsgi.application'
        
        # Internationalization
        LANGUAGE_CODE = 'zh-hans'
        
        TIME_ZONE = 'Asia/Shanghai'
        
        USE_I18N = True
        
        USE_TZ = False
        
        # 时间格式化
        USE_L10N = False
        DATETIME_FORMAT = 'Y-m-d H:i:s'
        DATE_FORMAT = 'Y-m-d'
        
        # Static files (CSS, JavaScript, Images)
        STATIC_URL = '/static/'
        STATIC_ROOT = os.path.join(BASE_DIR, 'static')
        STATICFILES_DIRS = [os.path.join(BASE_DIR, 'common/static'), ]
        
        # 扩展django admin里users字段用到，指定了sql/models.py里的class users
        AUTH_USER_MODEL = "sql.Users"
        
        # 密码校验
        AUTH_PASSWORD_VALIDATORS = [
            {
                'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
            },
            {
                'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
                'OPTIONS': {
                    'min_length': 9,
                }
            },
            {
                'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
            },
            {
                'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
            },
        ]
        
        ############### 以下部分需要用户根据自己环境自行修改 ###################
        
        # SESSION 设置
        SESSION_COOKIE_AGE = 60 * 300  # 300分钟
        SESSION_SAVE_EVERY_REQUEST = True
        SESSION_EXPIRE_AT_BROWSER_CLOSE = True  # 关闭浏览器，则COOKIE失效
        
        # 该项目本身的mysql数据库地址
        DATABASES = {
            'default': {
                'ENGINE': 'django.db.backends.mysql',
                'NAME': 'archery',
                'USER': 'root',
                'PASSWORD': 'archery',
                'HOST': 'archery-mysql',
                'PORT': '3306',
                'OPTIONS': {
                    'init_command': "SET sql_mode='STRICT_TRANS_TABLES'",
                    'charset': 'utf8mb4'
                },
                'TEST': {
                    'NAME': 'test_archery',
                    'CHARSET': 'utf8mb4',
                },
            }
        }
        
        # Django-Q
        Q_CLUSTER = {
            'name': 'archery',
            'workers': 4,
            'recycle': 500,
            'timeout': 60,
            'compress': True,
            'cpu_affinity': 1,
            'save_limit': 0,
            'queue_limit': 50,
            'label': 'Django Q',
            'django_redis': 'default',
            'sync': False  # 本地调试可以修改为True，使用同步模式
        }
        
        # 缓存配置
        CACHES = {
            "default": {
                "BACKEND": "django_redis.cache.RedisCache",
                "LOCATION": "redis://archery-redis:6379/0",
                "OPTIONS": {
                    "CLIENT_CLASS": "django_redis.client.DefaultClient",
                    "PASSWORD": "123456"
                }
            },
            "dingding": {
                "BACKEND": "django_redis.cache.RedisCache",
                "LOCATION": "redis://archery-redis:6379/1",
                "OPTIONS": {
                    "CLIENT_CLASS": "django_redis.client.DefaultClient",
                    "PASSWORD": "123456"
                }
            }
        }
        
        # LDAP
        ENABLE_LDAP = False
        if ENABLE_LDAP:
            import ldap
            from django_auth_ldap.config import LDAPSearch
        
            AUTHENTICATION_BACKENDS = (
                'django_auth_ldap.backend.LDAPBackend',  # 配置为先使用LDAP认证，如通过认证则不再使用后面的认证方式
                'django.contrib.auth.backends.ModelBackend',  # django系统中手动创建的用户也可使用，优先级靠后。注意这2行的顺序
            )
        
            AUTH_LDAP_SERVER_URI = "ldap://xxx"
            AUTH_LDAP_USER_DN_TEMPLATE = "cn=%(user)s,ou=xxx,dc=xxx,dc=xxx"
            AUTH_LDAP_ALWAYS_UPDATE_USER = True  # 每次登录从ldap同步用户信息
            AUTH_LDAP_USER_ATTR_MAP = {  # key为archery.sql_users字段名，value为ldap中字段名，用户同步信息
                "username": "cn",
                "display": "displayname",
                "email": "mail"
            }
        
        # LOG配置
        LOGGING = {
            'version': 1,
            'disable_existing_loggers': False,
            'formatters': {
                'verbose': {
                    'format': '[%(asctime)s][%(threadName)s:%(thread)d][task_id:%(name)s][%(filename)s:%(lineno)d][%(levelname)s]- %(message)s'
                },
            },
            'handlers': {
                'default': {
                    'level': 'DEBUG',
                    'class': 'logging.handlers.RotatingFileHandler',
                    'filename': 'downloads/log/archery.log',
                    'maxBytes': 1024 * 1024 * 100,  # 5 MB
                    'backupCount': 5,
                    'formatter': 'verbose',
                },
                'console': {
                    'level': 'DEBUG',
                    'class': 'logging.StreamHandler',
                    'formatter': 'verbose'
                }
            },
            'loggers': {
                'default': {  # default日志
                    'handlers': ['console', 'default'],
                    'level': 'DEBUG'
                },
                'django-q': {  # django_q模块相关日志
                    'handlers': ['console', 'default'],
                    'level': 'DEBUG',
                    'propagate': False
                },
                'django_auth_ldap': {  # django_auth_ldap模块相关日志
                    'handlers': ['console', 'default'],
                    'level': 'DEBUG',
                    'propagate': False
                },
                # 'django.db': {  # 打印SQL语句，方便开发
                #     'handlers': ['console', 'default'],
                #     'level': 'DEBUG',
                #     'propagate': False
                # },
                'django.request': {  # 打印请求错误堆栈信息，方便开发
                    'handlers': ['console', 'default'],
                    'level': 'DEBUG',
                    'propagate': False
                },
            }
        }
    soar.yaml: |-
        # 是否允许测试环境与线上环境配置相同
        allow-online-as-test: false
        # 是否清理测试时产生的临时文件
        drop-test-temporary: true
        # 语法检查小工具
        only-syntax-check: false
        sampling-data-factor: 100
        sampling: false
        sampling-statistic-target: 100
        profiling: false
        trace: false
        # 日志级别，[0:Emergency, 1:Alert, 2:Critical, 3:Error, 4:Warning, 5:Notice, 6:Informational, 7:Debug]
        log-level: 3
        log-output: /opt/archery/downloads/log/soar.log
        # 优化建议输出格式
        report-type: markdown
        ignore-rules:
        - ""
        # 启发式算法相关配置
        max-join-table-count: 5
        max-group-by-cols-count: 5
        max-distinct-count: 5
        max-index-cols-count: 5
        max-total-rows: 9999999
        spaghetti-query-length: 2048
        allow-drop-index: false
        # EXPLAIN相关配置
        explain-sql-report-type: pretty
        explain-type: extended
        explain-format: traditional
        explain-warn-select-type:
        - ""
        explain-warn-access-type:
        - ALL
        explain-max-keys: 3
        explain-min-keys: 0
        explain-max-rows: 10000
        explain-warn-extra:
        - ""
        explain-max-filtered: 100
        explain-warn-scalability:
        - O(n)
        query: ""
        list-heuristic-rules: false
        list-test-sqls: false
        verbose: true
    analysis_slow_query.sh: |-
        #!/bin/bash
        DIR="$( cd "$( dirname "$0"  )" && pwd  )"
        cd $DIR
        
        #配置archery数据库的连接地址
        monitor_db_host="archery-mysql"
        monitor_db_port=3306
        monitor_db_user="root"
        monitor_db_password="archery"
        monitor_db_database="archery"
        
        #实例慢日志位置
        slowquery_file="/home/mysql/log_slow.log"
        pt_query_digest="/usr/bin/pt-query-digest"
        
        #实例连接信息
        hostname="mysql_host:mysql_port" # 和archery实例配置内容保持一致，用于archery做筛选
        
        #获取上次分析时间，初始化时请删除last_analysis_time_$hostname文件，可分析全部日志数据
        if [ -s last_analysis_time_$hostname ]; then
            last_analysis_time=`cat last_analysis_time_$hostname`
        else
            last_analysis_time='1000-01-01 00:00:00'
        fi
        
        #收集日志
        #RDS需要增加--no-version-check选项
        $pt_query_digest \
        --user=$monitor_db_user --password=$monitor_db_password --port=$monitor_db_port \
        --review h=$monitor_db_host,D=$monitor_db_database,t=mysql_slow_query_review  \
        --history h=$monitor_db_host,D=$monitor_db_database,t=mysql_slow_query_review_history  \
        --no-report --limit=100% --charset=utf8 \
        --since "$last_analysis_time" \
        --filter="\$event->{Bytes} = length(\$event->{arg}) and \$event->{hostname}=\"$hostname\"  and \$event->{client}=\$event->{ip} " \
        $slowquery_file > /tmp/analysis_slow_query.log
        
        echo `date +"%Y-%m-%d %H:%M:%S"`>last_analysis_time_$hostname
    init-archery.sh: |-
      #!/bin/bash
      cd /opt/archery
      echo 切换python运行环境
      source /opt/venv4archery/bin/activate
      
      python3 manage.py makemigrations sql
      python3 manage.py migrate
      #数据初始化
      python3 manage.py loaddata initial_data.json
      #创建管理用户
      #python3 manage.py createsuperuser
      cat /opt/archery/src/docker/createsuperuser.py | python3 manage.py shell
    createsuperuser.py: |-
      #/usr/bin/python3 env
      from django.contrib.auth import get_user_model
      DJANGO_SU_NAME = "admin"
      DJANGO_SU_EMAIL = "ll_archery@gmail.com"
      DJANGO_SU_PASSWORD = "Archery2019"
      DJANGO_DB_NAME = "archery"
      User = get_user_model()
      try:
        superuser=User.objects.create_superuser(
        username=DJANGO_SU_NAME,
        email=DJANGO_SU_EMAIL,
        password=DJANGO_SU_PASSWORD)
      except: 
        print('Error,用户重复创建或创建失败，请登录pod检查!')
        exit(0)
      else:
        superuser.save()
    startup.sh: |-
      #!/bin/bash
      cd /opt/archery
      
      echo 切换python运行环境
      source /opt/venv4archery/bin/activate

      echo 修改重定向端口
      if [[ -z $NGINX_PORT ]]; then
          sed -i "s/:nginx_port//g" /etc/nginx/nginx.conf
      else
          sed -i "s/nginx_port/$NGINX_PORT/g" /etc/nginx/nginx.conf
      fi
      
      echo 启动nginx
      /usr/sbin/nginx
      
      echo 收集所有的静态文件到STATIC_ROOT
      python3 manage.py collectstatic -v0 --noinput
      
      echo 启动Django Q cluster
      supervisord -c qcluster_supervisord.conf
      
      echo 启动服务
      settings=${1:-"archery.settings"}
      ip=${2:-"127.0.0.1"}
      port=${3:-8888}
      
      gunicorn -w 4 --env DJANGO_SETTINGS_MODULE=${settings} --log-level=debug --error-logfile=/tmp/archery.err -b ${ip}:${port} --preload --timeout 600  archery.wsgi:application