diff --git a/.github/workflows/generate_tarballs.yml b/.github/workflows/generate_tarballs.yml new file mode 100644 index 00000000..baca0453 --- /dev/null +++ b/.github/workflows/generate_tarballs.yml @@ -0,0 +1,38 @@ +# Github action to generate of the registry database archives +# + +name: Registry Database Archive + +on: + workflow_dispatch: + schedule: + - cron: "0 0 1,15 * *" + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: checkout source + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + + - name: Setup mongo tools - only shell + uses: boly38/action-mongo-tools@stable + with: + mongo-shell: "false" + mongo-tools: "true" + + - name: Install python libraries + run: pip3 install --user -r backend/requirements.txt + + - name: fortran_package + run: cd backend && python generate_tarball.py + + - name: Archives + uses: actions/upload-artifact@v2 + with: + name: registry_database + path: backend/static/ diff --git a/.github/workflows/security_checks.yml b/.github/workflows/security_checks.yml new file mode 100644 index 00000000..028c9df0 --- /dev/null +++ b/.github/workflows/security_checks.yml @@ -0,0 +1,30 @@ +# Github action to build and verify the fortran packages +# + +name: Fortran Package Verification + +on: + workflow_dispatch: + schedule: + - cron: "0 2 * * *" + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: checkout source + uses: actions/checkout@v4 + + - uses: fortran-lang/setup-fpm@v5 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python + uses: actions/setup-python@v5 + + - name: Install python libraries + run: pip3 install --user -r backend/requirements.txt + + - name: fortran_package + run: cd backend && python validate.py diff --git a/backend/validate.py b/backend/validate.py index e1a28a14..cfb3116a 100755 --- a/backend/validate.py +++ b/backend/validate.py @@ -1,26 +1,11 @@ -import os -from flask import request, jsonify -from auth import is_ci from app import app -import logging import subprocess import toml - - -logging.basicConfig( - filename="validate.log", - level=logging.ERROR, - format="%(asctime)s [%(levelname)s] %(message)s", -) - -@app.errorhandler(404) -def page_not_found(e): - return jsonify({"message": "Page not found", "code": 404}) - -@app.errorhandler(500) -def internal_server_error(e): - logging.error("Server Error: %s", str(e)) - return jsonify({"message": "Internal server error", "code": 500}) +from mongo import db +from mongo import file_storage +from bson.objectid import ObjectId +from gridfs.errors import NoFile +import toml def run_command(command): @@ -31,12 +16,15 @@ def run_command(command): return result.stdout if result.stdout else result.stderr def process_package(packagename): + # Create a temp directory + create_dir_command = f'mkdir -p static/temp/{packagename}' + run_command(create_dir_command) # Extract the archive - extract_command = f'tar -xzf static/temp/{packagename}.tar.gz -C static/temp/' + extract_command = f'tar -xzf static/temp/{packagename}.tar.gz -C static/temp/{packagename}/' run_command(extract_command) # Build the package - build_command = f'cd static/temp/{packagename} && home/registry/fpm build' + build_command = f'cd static/temp/{packagename} && fpm build' result = run_command(build_command) # Read fpm.toml @@ -46,7 +34,7 @@ def process_package(packagename): parsed_toml = toml.loads(file_content) # Clean up - cleanup_command = f'rm -rf static/temp/{packagename}' + cleanup_command = f'rm -rf static/temp/{packagename} static/temp/{packagename}.tar.gz' run_command(cleanup_command) if '' in result: @@ -57,30 +45,32 @@ def process_package(packagename): return True, parsed_toml - -@app.route("/", methods=['POST']) def validate(): - print("request received") - package = request.files['package'] - packagename = request.form['packagename'] - with open(f"static/temp/{packagename}.tar.gz", "wb") as f: - f.write(package.read()) - # package.save(f"tmp/{packagename}.tar.gz") - result = process_package(packagename) - print(result) - return result - - -# Log all unhandled exceptions -def log_exception(sender, exception, **extra): - sender.logger.error( - "An exception occurred: %s", str(exception), exc_info=(exception) - ) - - -app.register_error_handler(Exception, log_exception) - -debug = True if is_ci != "true" else False - -if __name__ == "__main__": - app.run(host="0.0.0.0", port=os.environ.get("FLASK_SERVER_PORT", 5000), debug=debug) \ No newline at end of file + packages = db.packages.find({"versions": {"$elemMatch": {"isVerified": False}}}) + packages = list(packages) + for package in packages: + for i in package['versions']: + if 'isVerified' in i.keys() and i['isVerified'] == False: + tarball = file_storage.get(ObjectId(i['oid'])) + packagename = package['name'] + '-' + i['version'] + with open(f"static/temp/{packagename}.tar.gz", "wb") as f: + f.write(tarball.read()) + result = process_package(packagename) + if result[0] == False: + db.packages.update_one({"name": packages['name'],"namespace":package['namespace']}, {"$set": {"versions.$[elem].isVerified": False}}, array_filters=[{"elem.version": i['version']}]) + print("Package build failed for " + packagename) + else: + print("Package build success for " + packagename) + db.packages.update_one({"name": package['name'],"namespace":package['namespace']}, {"$set": {"versions.$[elem].isVerified": True}}, array_filters=[{"elem.version": i['version']}]) + + update_data = {} + + for key in ['repository', 'copyright', 'description']: + if key in result[1] and package[key] == "Package Under Verification": + update_data[key] = result[1][key] + + if update_data: + db.packages.update_one({"name": package['name'],"namespace":package['namespace']}, {"$set": update_data}) + return 0 + +validate()