Importing v3

.github/workflows/deploy-staging.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: staging

.github/workflows/deploy.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Deploy backend to heroku

.github/workflows/e2e.yml

@@ -7,30 +7,24 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3.11.4' 
           cache: 'pip'
       - uses: actions/setup-node@v3
         with:
           cache: 'yarn'
-      - name: Install python dependencies
+          node-version: 'v20.12.1'
+      - name: Install dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y python3-setuptools python3-pip python3-virtualenv chromium-browser libgbm1
           make install
       - name: DB setup
         run: |
-            cp cres/db.sqlite standards_cache.sqlite
             make migrate-upgrade
+            python cre.py --upstream_sync
       - name: Run app and e2e tests
         run: |
-          yarn build
-          [ -d "./venv" ] && . ./venv/bin/activate
-          export FLASK_APP=./cre.py
-          export FLASK_CONFIG=development
-          export INSECURE_REQUESTS=1
-          FLASK_CONFIG=development flask run &
-          sleep 20s
-          yarn test:e2e
+            make e2e

.github/workflows/linter.yml

@@ -8,14 +8,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Lint Code Base
         uses: github/super-linter@v5
         env:
-
           VALIDATE_PYTHON_BLACK: true
           VALIDATE_ALL_CODEBASE: false
-          DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish.yml

@@ -0,0 +1,35 @@
+name: Publish
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+permissions:
+  # Grant the ability to checkout the repository
+  contents: write
+  # Grant the ability to push packages
+  packages: write
+
+jobs:
+  publish-docker-images:
+    name: Push Docker images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish Docker images
+        run: |
+            CRE_VERSION_SEMVER=$(sed 's/v//' <<< ${{ github.ref_name }});
+            make docker-prod
+            docker tag opencre:$(git rev-parse HEAD) ghcr.io/owasp/OpenCRE/opencre:${CRE_VERSION_SEMVER}
+            docker tag opencre:$(git rev-parse HEAD) ghcr.io/owasp/OpenCRE/opencre:latest
+            docker push ghcr.io/owasp/OpenCRE/opencre:${CRE_VERSION_SEMVER}
+            docker push ghcr.io/owasp/OpenCRE/opencre:latest

.github/workflows/test.yml

@@ -6,19 +6,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
-      - uses: actions/setup-python@v4
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.11.4' 
-          cache: 'pip'
-      - uses: actions/setup-node@v3
-        with:
-          cache: 'yarn'
+          python-version: '3.12.3'
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+              python3-setuptools \
+              python3-virtualenv \
+              python3-pip \
+              libxml2-dev \
+              libxslt-dev 
       - name: Install python dependencies
         run: |
-         sudo apt-get update
-         sudo apt-get install -y python3-setuptools python3-virtualenv python3-pip
-         pip install --upgrade pip
-         make install-python
+          pip install --upgrade pip
+          pip install --upgrade setuptools
+          make install-python
       - name: Test
         run: make test

.gitignore

@@ -65,4 +65,7 @@ neo4j/
 .neo4j/
 
 .mypy_cache
-tmp/
+tmp/
+
+### CREs dir
+cres/*

Dockerfile

@@ -4,11 +4,14 @@ WORKDIR /code
 COPY . /code
 RUN yarn install && yarn build
 
-FROM python:3.11.0 as run
+FROM python:3.11 as run
 
 COPY --from=build /code /code
 WORKDIR /code
+COPY ./scripts/prod-docker-entrypoint.sh /code
 RUN pip install -r requirements.txt gunicorn
 
-ENTRYPOINT gunicorn
-CMD ["--timeout","800","--workers","8","cre:app"]
+ENV INSECURE_REQUESTS=1
+ENV FLASK_CONFIG="production"
+RUN chmod +x /code/prod-docker-entrypoint.sh
+ENTRYPOINT ["/code/prod-docker-entrypoint.sh"]

Makefile

@@ -1,13 +1,27 @@
-
 .ONESHELL:
 
 .PHONY: run test covers install-deps dev docker lint frontend clean all
 
+prod-run:
+	cp cres/db.sqlite standards_cache.sqlite; gunicorn cre:app --log-file=-
+
+docker-neo4j-rm:
+	docker stop cre-neo4j
+	docker rm -f cre-neo4j
+	docker volume rm cre_neo4j_data
+	docker volume rm cre_neo4j_logs
+	# rm -rf .neo4j
+
 docker-neo4j:
 	docker start cre-neo4j 2>/dev/null   || docker run -d --name cre-neo4j --env NEO4J_PLUGINS='["apoc"]'  --env NEO4J_AUTH=neo4j/password --volume=`pwd`/.neo4j/data:/data --volume=`pwd`/.neo4j/logs:/logs --workdir=/var/lib/neo4j -p 7474:7474 -p 7687:7687 neo4j
 
+docker-redis-rm:
+	docker stop cre-redis-stack
+	docker rm -f cre-redis-stack
+
 docker-redis:
-	docker start redis-stack 2>/dev/null || docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
+	docker start cre-redis-stack 2>/dev/null ||\
+	docker run -d --name cre-redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
 
 start-containers: docker-neo4j docker-redis
 
@@ -49,7 +63,7 @@ install-deps-typescript:
 install-deps: install-deps-python install-deps-typescript
 
 install-python:
-	virtualenv -p python3.11 venv
+	virtualenv -p python3 --system-site-packages venv
 	. ./venv/bin/activate &&\
 	make install-deps-python &&\
 	playwright install
@@ -101,35 +115,17 @@ migrate-downgrade:
 	export FLASK_APP=$(CURDIR)/cre.py
 	flask db downgrade
 
+import-projects:
+	$(shell CRE_SKIP_IMPORT_CORE=1 bash  ./scripts/import-all.sh)
+
 import-all:
-	[ -d "./venv" ] && . ./venv/bin/activate &&\
-	rm -rf standards_cache.sqlite &&\
-	make migrate-upgrade && export FLASK_APP=$(CURDIR)/cre.py &&\
-	python cre.py --add --from_spreadsheet https://docs.google.com/spreadsheets/d/1eZOEYgts7d_-Dr-1oAbogPfzBLh6511b58pX3b59kvg &&\
-	python cre.py --generate_embeddings && \
-	python cre.py --zap_in --cheatsheets_in --github_tools_in  --capec_in --owasp_secure_headers_in --pci_dss_4_in --juiceshop_in --dsomm_in --dsomm_in --cloud_native_security_controls_in &&\
-	python cre.py --generate_embeddings
+	$(shell bash ./scripts/import-all.sh)
 
 import-neo4j:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	export FLASK_APP=$(CURDIR)/cre.py && python cre.py --populate_neo4j_db
 
-preload-map-analysis: 
-	make docker-redis&\
-	make start-worker&\
-	make start-worker&\
-	make  start-worker&\
-	make  start-worker&\
-	make  start-worker&\
-	make start-worker&\
-	make  start-worker&\
-	make  start-worker&\
-	make  start-worker&\
-	make  start-worker&\
-	make dev-flask&
-	sleep 5
-	[ -d "./venv" ] && . ./venv/bin/activate &&\
-	export FLASK_APP=$(CURDIR)/cre.py 
-	python cre.py --preload_map_analysis_target_url 'http://127.0.0.1:5000'	
-	killall python flask
+preload-map-analysis:
+	$(shell RUN_COUNT=5 bash ./scripts/preload_gap_analysis.sh)
+
 all: clean lint test dev dev-run

Procfile

@@ -1,2 +1,2 @@
-web: gunicorn cre:app --log-file=-g
-worker: FLASK_APP=`pwd`/cre.py python cre.py --start_worker
+web: gunicorn cre:app
+worker: FLASK_APP=`pwd`/cre.py python cre.py --start_worker