diff --git a/CHANGES b/CHANGES index 990caaa..5f3e2e7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,4 @@ +v0.3.7 Added Docker pieces. v0.3.6 Fixed a bug with the slash on the end of the ES url option. v0.3.5 Removed need for trailing slash on ES URL. v0.3.4 Made datastream names consistent with ES expectations if -d is used without an index name. diff --git a/Readme.md b/Readme.md index 64c6715..2bfc4fa 100644 --- a/Readme.md +++ b/Readme.md @@ -7,6 +7,7 @@ logs into [ElasticSearch's bulk load JSON format](https://www.elastic.co/guide/e - [Introduction](#introduction) - [Installation](#installation) - [Elastic v8.0+](#elastic80) + - [Docker](#docker) - [Upgrading zeek2es](#upgradingzeek2es) - [ES Ingest Pipeline](#esingestpipeline) - [Filtering Data](#filteringdata) @@ -98,6 +99,28 @@ You can use `zeek2es.py` with the `--user` and `--passwd` command line options t credentials to ES. You can also supply these options via the extra command line arguments for the helper scripts. +### Docker + +Probably the easiest way to use this code is through Docker. First, you will want to edit the lines +with `CHANGEME!!!` in the `.env` file to fit your environment. You will also need to edit the Elastic +password in `docker/zeek2es/entrypoint.sh` to match. It can be found after the `--passwd` option. +Next, you can change directory into the `docker` directory and type the following commands to bring +up a zeek2es and Elasticsearch cluster: + +``` +docker-compose build +dockr-compose up +``` + +Now you can put logs in the `VOLUME_MOUNT\logs` directory (`VOLUME_MOUNT` you set in the `.env` file). +When logs are CREATED in this directory, zeek2es will begin processing them and pushing them into Elasticsearch. +You can then login to https://localhost:5601 with the username and password you set up in the `.env` file. +By default there is a self signed certificate, but you can change that if you edit the docker compose files. Once inside +Kibana you will go to Stack Management->Data Views and create a data view for `logs*` with the timestamp `@timestamp`. +Now you will be able to go to Discover and start searching your logs! Your data is persistent, in the `VOLUME_MOUNT` directory. +If you would like to remove all data, just `rm -rf VOLUME_MOUNT`, substituting the directory you set into that remove command. +The next time you start your cluster it will be brand new for more data. + ## Upgrading zeek2es Most upgrades should be as simple as copying the newer [zeek2es.py](zeek2es.py) over diff --git a/docker/.env b/docker/.env new file mode 100644 index 0000000..3a21869 --- /dev/null +++ b/docker/.env @@ -0,0 +1,32 @@ +# Password for the 'elastic' user (at least 6 characters) CHANGEME!!! +ELASTIC_PASSWORD=elastic + +# Password for the 'kibana_system' user (at least 6 characters) CHANGEME!!! +KIBANA_PASSWORD=elasticANDkibana + +# Version of Elastic products +STACK_VERSION=8.1.3 + +# Set the cluster name +CLUSTER_NAME=docker-cluster + +# Set to 'basic' or 'trial' to automatically start the 30-day trial +LICENSE=basic +#LICENSE=trial + +# Port to expose Elasticsearch HTTP API to the host +ES_PORT=9200 +#ES_PORT=127.0.0.1:9200 + +# Port to expose Kibana to the host +KIBANA_PORT=5601 +#KIBANA_PORT=80 + +# Increase or decrease based on the available host memory (in bytes) +MEM_LIMIT=1073741824 + +# Project namespace (defaults to the current folder name if not set) +#COMPOSE_PROJECT_NAME=myproject + +# Where the data directory resides for volumes CHANGEME!!! +VOLUME_MOUNT=./ \ No newline at end of file diff --git a/docker/data/.empty b/docker/data/.empty new file mode 100644 index 0000000..e69de29 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..4886f37 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,254 @@ +version: "2.2" + +services: + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: es02\n"\ + " dns:\n"\ + " - es02\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: es03\n"\ + " dns:\n"\ + " - es03\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: kibana\n"\ + " dns:\n"\ + " - kibana\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + + es01: + depends_on: + setup: + condition: service_healthy + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + restart: "unless-stopped" + volumes: + - certs:/usr/share/elasticsearch/config/certs + - ${VOLUME_MOUNT}/data/es01:/usr/share/elasticsearch/data + ports: + - ${ES_PORT}:9200 + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es02,es03 + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.http.ssl.verification_mode=certificate + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + es02: + depends_on: + - es01 + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + restart: "unless-stopped" + volumes: + - certs:/usr/share/elasticsearch/config/certs + - ${VOLUME_MOUNT}/data/es02:/usr/share/elasticsearch/data + environment: + - node.name=es02 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es01,es03 + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es02/es02.key + - xpack.security.http.ssl.certificate=certs/es02/es02.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.http.ssl.verification_mode=certificate + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es02/es02.key + - xpack.security.transport.ssl.certificate=certs/es02/es02.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + es03: + depends_on: + - es02 + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + restart: "unless-stopped" + volumes: + - certs:/usr/share/elasticsearch/config/certs + - ${VOLUME_MOUNT}/data/es03:/usr/share/elasticsearch/data + environment: + - node.name=es03 + - cluster.name=${CLUSTER_NAME} + - cluster.initial_master_nodes=es01,es02,es03 + - discovery.seed_hosts=es01,es02 + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es03/es03.key + - xpack.security.http.ssl.certificate=certs/es03/es03.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.http.ssl.verification_mode=certificate + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es03/es03.key + - xpack.security.transport.ssl.certificate=certs/es03/es03.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + kibana: + depends_on: + es01: + condition: service_healthy + es02: + condition: service_healthy + es03: + condition: service_healthy + image: docker.elastic.co/kibana/kibana:${STACK_VERSION} + restart: "unless-stopped" + volumes: + - certs:/usr/share/kibana/config/certs + - ${VOLUME_MOUNT}/data/kibana:/usr/share/kibana/data + ports: + - ${KIBANA_PORT}:5601 + environment: + - SERVERNAME=kibana + - ELASTICSEARCH_HOSTS=https://es01:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + - SERVER_SSL_ENABLED=true + - SERVER_SSL_KEY=/usr/share/kibana/config/certs/kibana/kibana.key + - SERVER_SSL_CERTIFICATE=/usr/share/kibana/config/certs/kibana/kibana.crt + - SERVER_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt +# - SERVER_SSL_PASSWORD=${KIBANA_CERT_PASSWORD} + mem_limit: ${MEM_LIMIT} + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", + ] + interval: 10s + timeout: 10s + retries: 120 + + zeek2es: + build: + context: ./zeek2es + dockerfile: Dockerfile + restart: "unless-stopped" + depends_on: + es01: + condition: service_healthy + es02: + condition: service_healthy + es03: + condition: service_healthy + volumes: + - ${VOLUME_MOUNT}/data/logs:/logs + entrypoint: /entrypoint.sh + tty: true + +volumes: + certs: + driver: local \ No newline at end of file diff --git a/docker/zeek2es/Dockerfile b/docker/zeek2es/Dockerfile new file mode 100644 index 0000000..b1efe07 --- /dev/null +++ b/docker/zeek2es/Dockerfile @@ -0,0 +1,35 @@ +FROM ubuntu:focal + +RUN apt-get -q update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + curl \ + fswatch \ + geoipupdate \ + git \ + iproute2 \ + jq \ + less \ + net-tools \ + parallel \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + swig \ + tcpdump \ + tcpreplay \ + termshark \ + tshark \ + vim \ + wget && \ + pip3 install --no-cache-dir pre-commit requests + +# Clean up files +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf ~/.cache/pip + +# Install zeek2es +RUN cd / && git clone https://github.com/corelight/zeek2es.git + +COPY entrypoint.sh /entrypoint.sh +RUN chmod 755 /entrypoint.sh \ No newline at end of file diff --git a/docker/zeek2es/entrypoint.sh b/docker/zeek2es/entrypoint.sh new file mode 100644 index 0000000..7308887 --- /dev/null +++ b/docker/zeek2es/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +fswatch -m poll_monitor --event Created -r /logs | parallel -j 3 python3 /zeek2es/zeek2es.py {} --compress -g -l 5000 -d 25 -u https://es01:9200 --user elastic --passwd elastic :::: - \ No newline at end of file