Merge pull request #18 from flownative/access-logs

Introduce access log support
flownative · Jan 11, 2024 · 0c191a1 · 0c191a1
2 parents dafb733 + 7adda3c
commit 0c191a1
Show file tree

Hide file tree

Showing 7 changed files with 67 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -38,17 +38,35 @@ redirected to STDERR. That way, you can follow logs by watching
 container logs with `docker logs` or using a similar mechanism in
 Kubernetes or your actual platform.
 
+Additionally, logs are also stored in /opt/flownative/log/nginx-error.log
+and /opt/flownative/log/nginx-access.log. If the log format is "json",
+the access log file is /opt/flownative/log/nginx-access.json.log
+
 The log level for error can be defined via the `NGINX_LOG_LEVEL`
 environment variable. See the
 [Nginx documentation](https://docs.nginx.com/nginx/admin-guide/monitoring/logging/)
 for possible values. The default value is `warn`.
 
+The access log is disabled by default, it can be enabled by setting 
+`NGINX_ACCESS_LOG_ENABLE` to "true". 
+
+The access log's default format is similar to the standard Nginx
+"combined" format with a few additions, so that the IP address of
+the original request is shown since this Nginx is usually operated
+behind a reverse proxy.
+
+Instead of the default format, a JSON format can be used by setting
+`NGINX_ACCESS_LOG_FORMAT` to "json".
+
 ### Environment variables
 
 | Variable Name                            | Type    | Default                               | Description                                                                                                                                                                                                       |
 |:-----------------------------------------|:--------|:--------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | NGINX_BASE_PATH                          | string  | /opt/flownative/nginx                 | Base path for Nginx                                                                                                                                                                                               |
-| NGINX_LOG_LEVEL                          | string  | warn                                  | Nginx log level (see [documentation](https://docs.nginx.com/nginx/admin-guide/monitoring/logging/))                                                                                                               |
+| NGINX_ERROR_LOG_LEVEL                    | string  | warn                                  | Nginx log level (see [documentation](https://docs.nginx.com/nginx/admin-guide/monitoring/logging/))                                                                                                               |
+| NGINX_ACCESS_LOG_ENABLE                  | boolean | no                                    | Nginx log level (see [documentation](https://docs.nginx.com/nginx/admin-guide/monitoring/logging/))                                                                                                               |
+| NGINX_ACCESS_LOG_FORMAT                  | string  | default                               | Format of the access log; possible values are "default" and "json"                                                                                                                                                |
+| NGINX_ACCESS_LOG_MODE                    | string  | dynamic                               | Defines which requests should be logged: "dynamic" only logs dynamic requests to PHP, "all" also includes requests to static files                                                                                |
 | NGINX_CACHE_ENABLE                       | boolean | no                                    | If the FastCGI cache should be enabled; see section about caching                                                                                                                                                 |
 | NGINX_CACHE_NAME                         | string  | application                           | Name of the memory zone Nginx should use for caching                                                                                                                                                              |
 | NGINX_CACHE_DEFAULT_LIFETIME             | string  | 5s                                    | Default cache lifetime to use when caching is enabled                                                                                                                                                             |

diff --git a/root-files/build.sh b/root-files/build.sh
@@ -10,7 +10,6 @@ groupadd --gid 1000 nginx
 mkdir -p \
     "${NGINX_BASE_PATH}/cache" \
     "${NGINX_BASE_PATH}/etc" \
-    "${NGINX_BASE_PATH}/log" \
     "${NGINX_BASE_PATH}/modules" \
     "${NGINX_BASE_PATH}/sbin" \
     "${NGINX_BASE_PATH}/tmp"
@@ -27,7 +26,6 @@ chmod -R g+rwX "${NGINX_BASE_PATH}"
 
 chown -R nginx:nginx \
     "${NGINX_BASE_PATH}/cache" \
-    "${NGINX_BASE_PATH}/log" \
     "${NGINX_BASE_PATH}/tmp"
 
 # Fix ownership of syslog-ng's etc directory because COPY in this Dockerfile

diff --git a/root-files/opt/flownative/lib/nginx-legacy.sh b/root-files/opt/flownative/lib/nginx-legacy.sh
@@ -18,7 +18,7 @@
 # ---------------------------------------------------------------------------------------
 # nginx_legacy_env() - Load global environment variables for configuring Nginx
 #
-# @global NGINX_* The NGINX_ evnironment variables
+# @global NGINX_* The NGINX_ environment variables
 # @return "export" statements which can be passed to eval()
 #
 nginx_legacy_env() {
@@ -146,7 +146,30 @@ EOM
 EOM
     fi
 
+    dynamicAccessLogDirective=""
+    staticAccessLogDirective=""
+
+    if is_boolean_yes "${NGINX_ACCESS_LOG_ENABLE}"; then
+        if [ "${NGINX_ACCESS_LOG_FORMAT}" == "json" ]; then
+            info "Nginx: Enabling access log using format 'json' ..."
+            dynamicAccessLogDirective="    access_log ${FLOWNATIVE_LOG_PATH}/nginx-access.json.log main_json buffer=256k flush=5s;"
+        else
+            info "Nginx: Enabling access log using format 'default' ..."
+            dynamicAccessLogDirective="    access_log ${FLOWNATIVE_LOG_PATH}/nginx-access.log main_ext buffer=256k flush=5s;"
+        fi
+    else
+        info "Nginx: Access log is disabled"
+    fi
+
+    if [ "${NGINX_ACCESS_LOG_MODE}" == "all" ]; then
+            info "Nginx: Enabling access log for all types of requests ..."
+        staticAccessLogDirective=${dynamicAccessLogDirective}
+    fi
+
     cat >>"${NGINX_CONF_PATH}/sites-enabled/site.conf" <<-EOM
+
+    $staticAccessLogDirective
+
     location ~ \\.php\$ {
            include fastcgi_params;
 
@@ -157,7 +180,10 @@ EOM
 
            fastcgi_pass ${BEACH_PHP_FPM_HOST}:${BEACH_PHP_FPM_PORT};
            fastcgi_index index.php;
+
+           $dynamicAccessLogDirective
 EOM
+
     if [ -n "${NGINX_CUSTOM_ERROR_PAGE_TARGET}" ]; then
         info "Nginx: Enabling custom error page pointing to ${BEACH_NGINX_CUSTOM_ERROR_PAGE_TARGET} ..."
         nginx_config_fastcgi_custom_error_page >>"${NGINX_CONF_PATH}/sites-enabled/site.conf"
@@ -219,7 +245,7 @@ EOM
     elif [ -n "${BEACH_PERSISTENT_RESOURCES_FALLBACK_BASE_URI}" ]; then
         cat >>"${NGINX_CONF_PATH}/sites-enabled/site.conf" <<-EOM
     location ~* "^${BEACH_PERSISTENT_RESOURCES_BASE_PATH}(.*)$" {
-        access_log off;
+        ${staticAccessLogDirective}
         expires ${NGINX_STATIC_FILES_LIFETIME};
         add_header Via '\$hostname' always;
         ${addHeaderStrictTransportSecurity}
@@ -237,7 +263,7 @@ EOM
     else
         cat >>"${NGINX_CONF_PATH}/sites-enabled/site.conf" <<-EOM
     location ~* ^/_Resources/Persistent/(.*)$ {
-        access_log off;
+        ${staticAccessLogDirective}
         expires ${NGINX_STATIC_FILES_LIFETIME};
         add_header Via '\$hostname' always;
         ${addHeaderStrictTransportSecurity}
@@ -256,7 +282,7 @@ EOM
     # for all static resources
     location ~ ^/_Resources/Static/ {
         add_header X-Static-Resource '\$hostname' always;
-        access_log off;
+        ${staticAccessLogDirective}
         expires ${NGINX_STATIC_FILES_LIFETIME};
     }
 }

diff --git a/root-files/opt/flownative/lib/nginx.sh b/root-files/opt/flownative/lib/nginx.sh
@@ -24,7 +24,10 @@ export NGINX_BASE_PATH="${NGINX_BASE_PATH}"
 export NGINX_CONF_PATH="${NGINX_BASE_PATH}/etc"
 export NGINX_TMP_PATH="${NGINX_BASE_PATH}/tmp"
 export NGINX_LOG_PATH="${NGINX_BASE_PATH}/log"
-export NGINX_LOG_LEVEL="${NGINX_LOG_LEVEL:-info}"
+export NGINX_ERROR_LOG_LEVEL="${NGINX_ERROR_LOG_LEVEL:-${NGINX_LOG_LEVEL:-warn}}"
+export NGINX_ACCESS_LOG_ENABLE="${NGINX_ACCESS_LOG_ENABLE:-false}"
+export NGINX_ACCESS_LOG_MODE="${NGINX_ACCESS_LOG_MODE:-dynamic}"
+export NGINX_ACCESS_LOG_FORMAT="${NGINX_ACCESS_LOG_FORMAT:-default}"
 
 export NGINX_CACHE_PATH="${NGINX_CACHE_PATH:-${NGINX_BASE_PATH}/cache}"
 export NGINX_CACHE_ENABLE="${NGINX_CACHE_ENABLE:-no}"

diff --git a/root-files/opt/flownative/logrotate/etc/logrotate.d/nginx.conf b/root-files/opt/flownative/logrotate/etc/logrotate.d/nginx.conf
@@ -1,10 +1,11 @@
-/opt/flownative/nginx/log/access.log /opt/flownative/nginx/log/error.log /opt/flownative/nginx/log/error_log.json /opt/flownative/nginx/log/access_log.json {
+/opt/flownative/log/nginx-access.log /opt/flownative/log/nginx-access.json.log /opt/flownative/log/nginx-error.log {
 
     rotate 1
     daily
     missingok
     notifempty
     minsize 100k
+    maxsize 50M
 
     sharedscripts
     postrotate

diff --git a/root-files/opt/flownative/nginx/etc/nginx.conf.template b/root-files/opt/flownative/nginx/etc/nginx.conf.template
@@ -2,7 +2,7 @@ daemon off;
 worker_processes  auto;
 
 pid ${NGINX_TMP_PATH}/nginx.pid;
-error_log ${FLOWNATIVE_LOG_PATH}/nginx_error.log ${NGINX_LOG_LEVEL};
+error_log ${FLOWNATIVE_LOG_PATH}/nginx-error.log ${NGINX_ERROR_LOG_LEVEL};
 
 load_module ${NGINX_BASE_PATH}/modules/ngx_http_headers_more_filter_module.so;
 
@@ -27,14 +27,14 @@ http {
     tcp_nopush on;
     tcp_nodelay on;
 
-    log_format  main_ext  '$remote_addr - $remote_user [$time_local] "$request" '
-                         '$status $body_bytes_sent "$http_referer" '
-                         '"$http_user_agent" "$http_x_forwarded_for" '
-                         '"$host" sn="$server_name" '
-                         'rt=$request_time '
-                         'ua="$upstream_addr" us="$upstream_status" '
-                         'ut="$upstream_response_time" ul="$upstream_response_length" '
-                         'cs=$upstream_cache_status' ;
+    log_format  main_ext  '$$remote_addr - $$remote_user [$$time_local] "$$request" '
+                         '$$status $$body_bytes_sent "$$http_referer" '
+                         '"$$http_user_agent" "$$http_x_forwarded_for" '
+                         '"$$host" sn="$$server_name" '
+                         'rt=$$request_time '
+                         'ua="$$upstream_addr" us="$$upstream_status" '
+                         'ut="$$upstream_response_time" ul="$$upstream_response_length" '
+                         'cs=$$upstream_cache_status' ;
 
     log_format main_json escape=json '{'
       '"msec": "$$msec", '                                               # request unixtime in seconds with a milliseconds resolution
@@ -65,9 +65,6 @@ http {
       '"server_protocol": "$$server_protocol"'                         # request protocol, like HTTP/1.1 or HTTP/2.0
     '}';
 
-#    access_log ${FLOWNATIVE_LOG_PATH}/nginx_access.log main_ext buffer=256k flush=5m;
-#    access_log ${FLOWNATIVE_LOG_PATH}/nginx_access.log.json main_json buffer=256k flush=5m;
-
     client_body_buffer_size 5M;
     client_max_body_size 500M;
 

diff --git a/root-files/opt/flownative/syslog-ng/etc/conf.d/nginx.conf b/root-files/opt/flownative/syslog-ng/etc/conf.d/nginx.conf
@@ -1,5 +1,5 @@
 source s_nginx_access_json {
-    file("`FLOWNATIVE_LOG_PATH`/nginx_access.log.json"
+    file("`FLOWNATIVE_LOG_PATH`/nginx-access.json.log"
         program-override("nginx")
         follow_freq(1)
         default-priority(info)
@@ -9,7 +9,7 @@ source s_nginx_access_json {
 };
 
 source s_nginx_access_common {
-    file("`FLOWNATIVE_LOG_PATH`/nginx_access.log"
+    file("`FLOWNATIVE_LOG_PATH`/nginx-access.log"
         program-override("nginx")
         default-priority(info)
         follow_freq(1)
@@ -18,7 +18,7 @@ source s_nginx_access_common {
 };
 
 source s_nginx_error {
-    file("`FLOWNATIVE_LOG_PATH`/nginx_error.log"
+    file("`FLOWNATIVE_LOG_PATH`/nginx-error.log"
         program-override("nginx")
         default-priority(error)
         follow_freq(1)