From b138364323b563249564e798e445809a48ef4ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Copin?= Date: Sat, 8 Jul 2023 20:50:49 +0200 Subject: [PATCH 1/5] Add external configuration --- REQUIREMENTS | 1 + bin/import.py | 11 ++++++----- bin/server.py | 6 +++++- config/settings.yaml | 8 ++++++++ 4 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 config/settings.yaml diff --git a/REQUIREMENTS b/REQUIREMENTS index 7656de5..eca36df 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,2 +1,3 @@ redis falcon +dynaconf diff --git a/bin/import.py b/bin/import.py index 2165063..c1bf1f4 100644 --- a/bin/import.py +++ b/bin/import.py @@ -10,14 +10,15 @@ import xml.sax import redis import time +from dynaconf import Dynaconf # Configuration -cpe_path = '../data/official-cpe-dictionary_v2.3.xml' -cpe_source = ( - 'https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz' +settings = Dynaconf( + settings_files=['../config/settings.yaml'] ) -rdb = redis.Redis(host='127.0.0.1', port=6379, db=8) - +cpe_path = settings.cpe.path +cpe_source = (settings.cpe.source) +rdb = redis.Redis(host=settings.redis.host, port=settings.redis.port, db=8) class CPEHandler(xml.sax.ContentHandler): def __init__(self): diff --git a/bin/server.py b/bin/server.py index 9e7cf57..16b49f0 100644 --- a/bin/server.py +++ b/bin/server.py @@ -6,9 +6,13 @@ import falcon from wsgiref.simple_server import make_server import json +from dynaconf import Dynaconf # Configuration -port = 8000 +settings = Dynaconf( + settings_files=['../config/settings.yaml'] +) +port = settings.server.port runPath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(runPath, "..")) diff --git a/config/settings.yaml b/config/settings.yaml new file mode 100644 index 0000000..45ddd82 --- /dev/null +++ b/config/settings.yaml @@ -0,0 +1,8 @@ +server: + port: 8000 +redis: + host: 127.0.0.1 + port: 6379 +cpe: + path: '../data/official-cpe-dictionary_v2.3.xml' + source: 'https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz' \ No newline at end of file From cd17ad64344404e260dfe3f0bdba3bed616bce44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Copin?= Date: Sat, 8 Jul 2023 20:51:16 +0200 Subject: [PATCH 2/5] Add Docker & docker-compose --- Dockerfile | 18 ++++++++++++++++++ docker/docker-compose.yml | 16 ++++++++++++++++ docker/entrypoint.sh | 5 +++++ docker/settings.yaml | 8 ++++++++ 4 files changed, 47 insertions(+) create mode 100644 Dockerfile create mode 100644 docker/docker-compose.yml create mode 100644 docker/entrypoint.sh create mode 100644 docker/settings.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2be6a3e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 + +FROM python:3.8-slim-buster + +WORKDIR /app + +COPY REQUIREMENTS REQUIREMENTS +RUN pip3 install -r REQUIREMENTS + +COPY bin bin +COPY etc /etc +COPY lib lib +COPY docker/entrypoint.sh entrypoint.sh + +RUN mkdir /app/config +RUN chmod u+x entrypoint.sh + +ENTRYPOINT ["/app/entrypoint.sh"] \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..5deb1c7 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3" +services: + server: + container_name: cpe-guesser + image: cpe-guesser:1.0 + build: .. + volumes: + - ../data/:/data/:rw + - ./settings.yaml:/app/config/settings.yaml + ports: + - 8000:8000 + depends_on: + - redis + redis: + container_name: cpe-guesser-db + image: "redis:alpine" \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..9b476b0 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +python3 -u /app/bin/import.py +python3 -u /app/bin/server.py \ No newline at end of file diff --git a/docker/settings.yaml b/docker/settings.yaml new file mode 100644 index 0000000..64eb959 --- /dev/null +++ b/docker/settings.yaml @@ -0,0 +1,8 @@ +server: + port: 8000 +redis: + host: redis + port: 6379 +cpe: + path: '/data/official-cpe-dictionary_v2.3.xml' + source: 'https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz' \ No newline at end of file From 6be6f0210a7d634b36f26143e0d6614f0e4c0623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Copin?= Date: Sat, 8 Jul 2023 20:51:47 +0200 Subject: [PATCH 3/5] Update readme --- README.md | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f941f7a..2a012c8 100644 --- a/README.md +++ b/README.md @@ -10,19 +10,46 @@ be used against [cve-search](https://github.com/cve-search/cve-search) to do act ## Usage -To use CPE guesser, you have to initialise the Redis database with `import.py`. Then you can use -the software with `lookup.py` to find the most probable CPE matching the keywords provided. +To use CPE guesser, you have to initialise the Redis database with `import.py`. + +Then you can use the software with `lookup.py` to find the most probable CPE matching the keywords provided. + +Or by calling the Web server (After running `server.py`), example: `curl -s -X POST http://localhost:8000/search -d "{\"query\": [\"tomcat\"]}" | jq .` ### Installation - `git clone https://github.com/cve-search/cpe-guesser.git` - `cd cpe-guesser/bin` - Download the CPE dictionary & populate the database with `python3 ./import.py`. -- Take a cup of black or green tea. +- Take a cup of black or green tea (). - `python3 cpe-guesser/bin/server.py` to run the local HTTP server. If you don't want to install it locally, there is a public online version. Check below. +### Docker + +#### Single image with existing Redis + +```bash +docker build . -t cpe-guesser:l.0 +# Edit settings.yaml content and/or path +docker run cpe-guesser:l.0 -v $(pwd)/config/settings.yaml:/app/config/settings.yaml +# Please wait for full import +``` + +#### Docker-compose + +```bash +cd docker +# Edit docker/settings.yaml as you want +docker-compose up --build -d +# Please wait for full import +``` + +#### Specific usage + +If you do not want to use the Web server, `lookup.py` can still be used. Example: `docker exec -it cpe-guesser python3 /app/bin/lookup.py tomcat` + ## Public online version [cpe-guesser.cve-search.org](https://cpe-guesser.cve-search.org) is public online version of CPE guesser which can be used via From 8fc6b9c45eb961cc40e390f7f6d9acdf50e1149f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Copin?= Date: Tue, 11 Jul 2023 09:09:16 +0200 Subject: [PATCH 4/5] Add missing configuration in lib cpeguesser.py --- lib/cpeguesser.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/cpeguesser.py b/lib/cpeguesser.py index 45d20e2..cbc9789 100644 --- a/lib/cpeguesser.py +++ b/lib/cpeguesser.py @@ -2,11 +2,16 @@ # -*- coding: utf-8 -*- import redis +from dynaconf import Dynaconf +# Configuration +settings = Dynaconf( + settings_files=['../config/settings.yaml'] +) class CPEGuesser: def __init__(self): - self.rdb = redis.Redis(host='127.0.0.1', port=6379, db=8, decode_responses=True) + self.rdb = redis.Redis(host=settings.redis.host, port=settings.redis.port, db=8, decode_responses=True) def guessCpe(self, words): k = [] From ab6402257eb5bcc7b6542348ba92f5cd859d37ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Copin?= Date: Tue, 11 Jul 2023 09:10:39 +0200 Subject: [PATCH 5/5] No error code if import.py exits with warning It can block app restarting, especially with container usage --- bin/import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/import.py b/bin/import.py index c1bf1f4..8c91e34 100644 --- a/bin/import.py +++ b/bin/import.py @@ -128,7 +128,7 @@ def insert(word=None, cpe=None): if args.replace == 0 and rdb.dbsize() > 0 and not args.update: print(f"Warning! The Redis database already has {rdb.dbsize()} keys.") print("Use --replace if you want to flush the database and repopulate it.") - sys.exit(1) + sys.exit(0) if args.download > 0 or not os.path.isfile(cpe_path): print(f"Downloading CPE data from {cpe_source} ...")