diff --git a/backend/_unverify.py b/backend/_unverify.py new file mode 100644 index 00000000..bae3b336 --- /dev/null +++ b/backend/_unverify.py @@ -0,0 +1,59 @@ +from mongo import db +import pymongo + +# t = db.packages.update_many({}, {"$set": {"is_verified": False}}) +# print(t.modified_count) +# packages = list(db.packages.find()) + +# # Step 2: Make keywords unique for each package +# for package in packages: +# unique_keywords = list(set(package.get("categories", []))) # Making keywords unique +# package["categories"] = unique_keywords + +# # Step 3: Store the updated packages in a list +# updated_packages = [{"_id": package["_id"], "categories": package["categories"]} for package in packages] + +# # Step 4: Perform bulk update operation +# bulk_operations = [pymongo.UpdateOne({"_id": package["_id"]}, {"$set": {"categories": package["categories"]}}) for package in updated_packages] +# result = db.packages.bulk_write(bulk_operations) + +# print("Number of documents matched:", result.matched_count) +# print("Number of documents modified:", result.modified_count) +# remove key k from all the documents in the collection +t = db.packages.update_many({}, {"$set": {"is_verified": False}}) +# result = db.packages.update_many({}, {"$set": {"unable_to_verify": False}}) +# t = list(db.packages.find()) +# for i in t: + # n = db.namespaces.find_one({"_id": i["namespace"]})['namespace'] + # db.packages.update_one({"_id": i["_id"]}, {"$set": {"description": "Package Under Verification","registry_description": "Package Under Verification"}}) + # print(i["namespace"], n) + +# print("Number of documents matched:", result.matched_count) +# print("Number of documents modified:", result.modified_count) +# t = db.packages.update_many({}, {"$set": {"registry_description": "Package Under Verification"}}) +# print(t.modified_count,t.matched_count) +# t = db.packages.update_many({}, {"$set": {"description": "Package Under Verification"}}) +# print(t.modified_count,t.matched_count) + + +# report check if access_token is not undefined. + +# port from namespace obj id to name # +# optimise package search, and rendering # +# readme rendering in packages bring clarity on how to support README.md # +# registry-description # +# time and speed benchmarks # +# erase GPF # NEW RELASE NEW DB +# keywords, description, categories ( search and sort by categories and keywords https://github.com/Beliavsky?tab=repositories ) # +# bring clarity on multiple namespaces and hierarchy # + +# support catch-22 +# test-drive for module naming +# ability to register programs as well as libraries; where plug-ins and programs could be registered and easily installed. +# fpm new folder_name --namespace n --package p --version v +# DOCS!!!!!!!!!!!!! + + + +# tests, docs, PR reviews, CI , Vercel migration. +# fpm release, fpm pr, (meeting time + discourse support). diff --git a/backend/check_digests.py b/backend/check_digests.py index 0e705ccc..409ce60e 100644 --- a/backend/check_digests.py +++ b/backend/check_digests.py @@ -1,6 +1,7 @@ from typing import Tuple import numpy as np import json +import re def hash(input_string): @@ -28,6 +29,10 @@ def dilate(instr): outstr.append(outstr_c) return outstr +def process(lines): + cleaned = re.sub(r'(?<=\S) +(?=\n)', '', ''.join(lines)) + cleaned = re.sub(r'(?<=\n)\t+(?=\n)', '', cleaned) + return re.sub(r'\n\s+\n', '\n\n\n', cleaned) def check_digests(file_path: str) -> Tuple[int, bool]: """ @@ -65,9 +70,10 @@ def check_digests(file_path: str) -> Tuple[int, bool]: try: # Read the content of the file - with open(f"{file_path}{file_name}", 'r',newline='') as file: - file_content: str = file.read() + with open(f'{file_path}{file_name.replace("./", "")}', 'r',newline='') as file: + file_content: str = process(file.readlines()) except: + print(f'Error reading file content: {file_path}{file_name}') return (-1, "Error reading file content.") # Compute the digest of the file content diff --git a/backend/models/package.py b/backend/models/package.py index 0a76dd09..eaf4aded 100644 --- a/backend/models/package.py +++ b/backend/models/package.py @@ -1,10 +1,12 @@ class Package: - def __init__(self, name, namespace, description, homepage, repository, - copyright, license, created_at, updated_at, author, maintainers, keywords, categories, is_deprecated, versions=[], id=None, - malicious_report={}, is_verified=False, is_malicious=False, security_status="No security issues found", ratings={"users": {}, "avg_ratings": 0}): + def __init__(self, name, namespace, namespace_name, description, homepage, repository, + copyright, license, created_at, updated_at, author, maintainers, keywords, categories, is_deprecated, versions=[], id=None,unable_to_verify=False, + malicious_report={}, registry_description=None,is_verified=False, is_malicious=False, security_status="No security issues found", ratings={"users": {}, "avg_ratings": 0}): self.id = id self.name = name self.namespace = namespace + self.namespace_name = namespace_name + self.registry_description = registry_description self.description = description self.homepage = homepage self.repository = repository @@ -24,6 +26,7 @@ def __init__(self, name, namespace, description, homepage, repository, self.downloads_stats = {} self.ratings = ratings self.categories = categories + self.unable_to_verify = unable_to_verify # Ensure that versions list only contains instances of Version class for v in self.versions: @@ -43,7 +46,9 @@ def to_json(self): "id": str(self.id), "name": self.name, "namespace": self.namespace, + "namespace_name": self.namespace_name, "description": self.description, + "registry_description": self.registry_description, "homepage": self.homepage, "repository": self.repository, "copyright": self.copyright, @@ -61,6 +66,7 @@ def to_json(self): "is_malicious": self.is_malicious, "security_status": self.security_status, "ratings": self.ratings, + "unable_to_verify": self.unable_to_verify, } # Create a from_json method. @@ -76,7 +82,9 @@ def from_json(json_data): id=str(json_data.get("_id")), name=json_data.get("name"), namespace=json_data.get("namespace"), + namespace_name=json_data.get("namespace_name"), description=json_data.get("description"), + registry_description=json_data.get("registry_description"), homepage=json_data.get("homepage"), repository=json_data.get("repository"), copyright=json_data.get("copyright"), @@ -94,6 +102,7 @@ def from_json(json_data): is_malicious=json_data.get("is_malicious"), security_status=json_data.get("security_status"), ratings=json_data.get("ratings"), + unable_to_verify=json_data.get("unable_to_verify"), ) class Version: diff --git a/backend/packages.py b/backend/packages.py index db16aff3..07e39c52 100644 --- a/backend/packages.py +++ b/backend/packages.py @@ -110,6 +110,7 @@ def search_packages(): "_id": 0, "name": 1, "namespace": 1, + "namespace_name": 1, "author": 1, "description": 1, "keywords": 1, @@ -132,24 +133,13 @@ def search_packages(): for i in packages: package_obj = Package.from_json(i) - namespace = db.namespaces.find_one({"_id": package_obj.namespace}) - namespace_obj = Namespace.from_json(namespace) - - author = db.users.find_one({"_id": package_obj.author}) - author_obj = User.from_json(author) - - package_obj.namespace = namespace_obj.namespace - package_obj.author = author_obj.username search_packages.append({ "name": package_obj.name, - "namespace": package_obj.namespace, - "author": package_obj.author, + "namespace": package_obj.namespace_name, "description": package_obj.description, - "keywords": package_obj.keywords, - "categories": package_obj.categories, + "keywords": package_obj.keywords+package_obj.categories, "updated_at": package_obj.updated_at, }) - return ( jsonify( {"code": 200, "packages": search_packages, "total_pages": total_pages} @@ -207,11 +197,8 @@ def upload(): namespace_doc = db.namespaces.find_one( {"upload_tokens": {"$elemMatch": {"token": upload_token}}} ) - package_doc = db.packages.find_one( - {"upload_tokens": {"$elemMatch": {"token": upload_token}}} - ) - if not namespace_doc and not package_doc: + if not namespace_doc: return jsonify({"code": 401, "message": "Invalid upload token"}), 401 if namespace_doc: @@ -225,18 +212,6 @@ def upload(): {"name": package_name, "namespace": namespace_obj.id} ) - elif package_doc: - package_obj = Package.from_json(package_doc) - if package_obj.name != package_name: - return jsonify({"code": 401, "message": "Invalid upload token"}), 401 - - upload_token_doc = next( - item - for item in package_doc["upload_tokens"] - if item["token"] == upload_token - ) - namespace_doc = db.namespaces.find_one({"_id": package_doc["namespace"]}) - # Check if the token is expired. # Expire the token after one week of it's creation. if check_token_expiry(upload_token_created_at=upload_token_doc["createdAt"]): @@ -294,6 +269,7 @@ def upload(): "description": "Package Under Verification", "copyright": "Package Under Verification", "homepage": "Package Under Verification", + "registry_description": "Package Under Verification", } file_object_id = file_storage.put( @@ -306,6 +282,7 @@ def upload(): package_obj = Package( name=package_name, namespace=namespace_obj.id, + namespace_name=namespace_obj.namespace, description=package_data["description"], homepage=package_data["homepage"], repository=package_data["repository"], @@ -478,20 +455,9 @@ def check_version(current_version, new_version): @app.route("/packages//", methods=["GET"]) @swag_from("documentation/get_package.yaml", methods=["GET"]) def get_package(namespace_name, package_name): - # Get namespace from namespace name. - namespace = db.namespaces.find_one({"namespace": namespace_name}) - - # Check if namespace exists. - if not namespace: - return ( - jsonify({"status": "error", "message": "Namespace not found", "code": 404}), - 404, - ) - - namespace_obj = Namespace.from_json(namespace) - # Get package from a package_name and namespace's id. + # Get package from a package_name and namespace's name. package = db.packages.find_one( - {"name": package_name, "namespace": namespace_obj.id} + {"name": package_name, "namespace_name": namespace_name} ) # Check if package is not found. @@ -549,7 +515,7 @@ def get_package(namespace_name, package_name): # Only latest version of the package will be sent as a response. package_response_data = { "name": package_obj.name, - "namespace": namespace_obj.namespace, + "namespace": package_obj.namespace_name, "latest_version_data": latest_version_data, "author": package_author_obj.username, "keywords": package_obj.keywords if package_obj.keywords else [], @@ -559,6 +525,7 @@ def get_package(namespace_name, package_name): "version_history": version_history, "updated_at": package_obj.updated_at, "description": package_obj.description, + "registry_description": package_obj.registry_description, "ratings": ratings, "downloads": downloads_stats, "ratings_count": rating_count @@ -586,7 +553,7 @@ def verify_user_role(namespace_name, package_name): ) package = db.packages.find_one( - {"name": package_name, "namespace": namespace["_id"]} + {"name": package_name, "namespace": namespace_name} ) if not package: @@ -612,20 +579,11 @@ def verify_user_role(namespace_name, package_name): @app.route("/packages///", methods=["GET"]) @swag_from("documentation/get_version.yaml", methods=["GET"]) def get_package_from_version(namespace_name, package_name, version): - # Get namespace from namespace name. - namespace = db.namespaces.find_one({"namespace": namespace_name}) - - # Check if namespace does not exists. - if not namespace: - return jsonify({"message": "Namespace not found", "code": 404}), 404 - - namespace_obj = Namespace.from_json(namespace) - - # Get package from a package_name, namespace's id and version. + # Get package from a package_name, namespace_name and version. package = db.packages.find_one( { "name": package_name, - "namespace": namespace["_id"], + "namespace_name": namespace_name, "versions.version": version, } ) @@ -652,7 +610,7 @@ def get_package_from_version(namespace_name, package_name, version): # Only queried version should be sent as response. package_response_data = { "name": package_obj.name, - "namespace": namespace_obj.namespace, + "namespace": package_obj.namespace_name, "author": package_author_obj.username, "keywords": package_obj.keywords, "categories": package_obj.categories, diff --git a/backend/validate.py b/backend/validate.py index 9dc08ef6..f8cdd5b2 100755 --- a/backend/validate.py +++ b/backend/validate.py @@ -1,15 +1,15 @@ from app import app import subprocess import toml -import html +import os from mongo import db from mongo import file_storage from bson.objectid import ObjectId from gridfs.errors import NoFile import toml from check_digests import check_digests +from bson.objectid import ObjectId from typing import Union,List, Tuple, Dict, Any -import markdown def run_command(command: str) -> Union[str, None]: @@ -29,24 +29,27 @@ def run_command(command: str) -> Union[str, None]: print(result.stderr) return result.stdout if result.stdout else result.stderr -def collect_dependencies(section: str, parsed_toml: Dict[str, List[Dict[str, Any]]]) -> List[Tuple[str, str]]: +def extract_dependencies(parsed_toml: Dict[str, List[Dict[str, Any]]]) -> List[Tuple[str, str, str]]: """ - Collect dependencies from a section in a parsed TOML file. - + Extracts dependencies from a parsed TOML file. + Args: - section (str): The section in the TOML file to collect dependencies from. parsed_toml (Dict[str, List[Dict[str, Any]]]): The parsed TOML file represented as a dictionary. Returns: - List[Tuple[str, str]]: A list of dependency tuples containing (namespace, dependency_name). - + List[Tuple[str, str, str]]: A list of dependency tuples containing (namespace, package_name, version). + """ dependencies = list() - for dependency_dict in parsed_toml.get(section, []): - for dependency_name, dependency_info in dependency_dict.get('dependencies', {}).items(): - dependencies.append((dependency_info['namespace'],dependency_name)) + for dependency_name, dependency_info in parsed_toml.get('dependencies', {}).items(): + dependencies.append((dependency_info['namespace'], dependency_name, dependency_info.get('v', None))) + for section in ['test', 'example', 'executable']: + if section in parsed_toml and 'dependencies' in parsed_toml[section]: + for dependency_name, dependency_info in parsed_toml[section].get('dependencies', {}).items(): + dependencies.append((dependency_info['namespace'], dependency_name, dependency_info.get('v', None))) return dependencies + def process_package(packagename: str) -> Tuple[bool, Union[dict, None], str]: """ This function creates a directory, extracts package contents, reads and parses 'fpm.toml', @@ -79,18 +82,7 @@ def process_package(packagename: str) -> Tuple[bool, Union[dict, None], str]: return False, None,"Error parsing toml file" result = check_digests(f'static/temp/{packagename}/') - - # Clean up - cleanup_command = f'rm -rf static/temp/{packagename} static/temp/{packagename}.tar.gz' - # run_command(cleanup_command) - print(result) - - if 'description' in parsed_toml and parsed_toml['description'] == "README.md": - try: - with open(f'static/temp/{packagename}/README.md', 'r') as file: - parsed_toml['description'] = markdown.markdown(file.read()) # Sanitize HTML content - except: - parsed_toml['description'] = "README.md not found." + # print(result) if result[0]==-1: # Package verification failed @@ -111,7 +103,6 @@ def validate() -> None: Returns: None """ - # packages = db.packages.find({"versions": {"$elemMatch": {"isVerified": False}}}) # find packages with unverified versions packages = db.packages.find({"is_verified": False}) packages = list(packages) for package in packages: @@ -128,43 +119,41 @@ def validate() -> None: result = process_package(packagename) update_data = {} if result[0] == False: - update_data['isVerified'] = False - update_data['unabletoVerify'] = True + update_data['is_verified'] = False + update_data['unable_to_verify'] = True print("Package tests failed for " + packagename) print(result) else: print("Package tests success for " + packagename) - update_data['isVerified'] = True + update_data['is_verified'] = True + update_data['unable_to_verify'] = False if result[2] == "Error parsing toml file": db.packages.update_one({"name": package['name'],"namespace":package['namespace']}, {"$set": update_data}) pass + try: + update_data['registry_description'] = open(f"static/temp/{packagename}/README.md", "r").read() + except: + update_data['registry_description'] = result[1].get('description', "description not provided.") for key in ['repository', 'copyright', 'description',"homepage", 'categories', 'keywords']: if key in result[1] and package[key] != result[1][key]: if key in ['categories', 'keywords']: - update_data[key] = package[key] + result[1][key] + update_data[key] = list(set(package[key] + list(map(str.strip, result[1][key])))) else: update_data[key] = result[1][key] - dependencies = list() - try: - dependencies += [(dependency_info['namespace'],dependency_name) for dependency_name, dependency_info in result[1].get('dependencies', {}).items()] - except: - pass - for section in ['test', 'example', 'executable']: - try: - dependencies += collect_dependencies(section, result[1]) - except: - pass - - update_data['dependencies'] = list(set(dependencies)) + dependencies = extract_dependencies(result[1]) for i in dependencies: - dependency_package = db.packages.find_one({"name": i[1], "namespace": i[0]}) # TODO: enable version checking + namespace = db.namespaces.find_one({"namespace": i[0]}) + query = {"name": i[1], "namespace": ObjectId(str(namespace['_id']))} + if i[2] is not None: + query['versions.version'] = i[2] + dependency_package = db.packages.find_one(query) if dependency_package is None: print(f"Dependency {i[0]}/{i[1]} not found in the database") - update_data['isVerified'] = False # if any dependency is not found, the package is not verified + update_data['is_verified'] = False for k,v in package.items(): if v == "Package Under Verification" and k not in update_data.keys(): @@ -172,6 +161,9 @@ def validate() -> None: db.packages.update_one({"name": package['name'],"namespace":package['namespace']}, {"$set": update_data}) print(f"Package {packagename} verified successfully.") + # Clean up + cleanup_command = f'rm -rf static/temp/{packagename} static/temp/{packagename}.tar.gz' + run_command(cleanup_command) validate() \ No newline at end of file diff --git a/frontend/package.json b/frontend/package.json index a1b351f7..ba0c8855 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,6 +16,7 @@ "react-cookie": "^4.1.1", "react-dom": "^18.2.0", "react-icons": "^4.8.0", + "react-markdown": "^9.0.1", "react-redux": "^8.0.5", "react-router-dom": "^6.11.2", "react-scripts": "5.0.1", diff --git a/frontend/src/components/packageItem.js b/frontend/src/components/packageItem.js index bd3c619a..ac8e1efb 100644 --- a/frontend/src/components/packageItem.js +++ b/frontend/src/components/packageItem.js @@ -1,8 +1,21 @@ import { MDBListGroupItem } from "mdb-react-ui-kit"; import { Row, Col, Image } from "react-bootstrap"; import { Link } from "react-router-dom"; +import { searchPackage, setQuery } from "../store/actions/searchActions"; +import { useDispatch, useSelector } from "react-redux"; +import { useNavigate } from "react-router-dom"; const PackageItem = ({ packageEntity }) => { + const dispatch = useDispatch(); + const query = useSelector((state) => state.search.query); + const navigate = useNavigate(); + const search = () => { + if (query.trim().length !== 0) { + dispatch(searchPackage(query, 0)); + navigate("/search"); + } + }; + function formatDate(timestamp) { const now = new Date(); const date = new Date(timestamp); @@ -32,10 +45,18 @@ const PackageItem = ({ packageEntity }) => { return `${monthName} ${day}, ${year}`; } } + const spanStyle = { + borderRadius: "5px", + backgroundColor: "lavender", + padding: "3px 8px", + margin: "2px", + textDecoration: "none", + color: "grey", + }; return ( - + { height={60} /> - + -
{packageEntity.name}
+
+ {packageEntity.name} +
- -
- - Namespace {packageEntity.namespace} - +
+ + Namespace {packageEntity.namespace} +
- + {packageEntity.keywords.map((keyword, index) => ( + { + dispatch(setQuery(keyword)); + search(); + }} + > + {keyword} + + ))} ); }; diff --git a/frontend/src/pages/namespace.js b/frontend/src/pages/namespace.js index 3307a93e..b325dd36 100644 --- a/frontend/src/pages/namespace.js +++ b/frontend/src/pages/namespace.js @@ -56,7 +56,7 @@ const NamespacePage = () => {

- {" Created " + dateJoined.slice(4, 16)} + {" Created: " + dateJoined.slice(4, 16)} { const [iconsActive, setIconsActive] = useState("readme"); @@ -131,8 +132,7 @@ const PackagePage = () => { - {data.description} - + {data.registry_description} {sideBar(data, setShowReportForm, setRateForm)} diff --git a/frontend/src/store/actions/ratePackageActions.js b/frontend/src/store/actions/ratePackageActions.js index d9ed385a..2aae9c4b 100644 --- a/frontend/src/store/actions/ratePackageActions.js +++ b/frontend/src/store/actions/ratePackageActions.js @@ -11,6 +11,18 @@ export const ratePackage = (data, access_token) => async (dispatch) => { let packageName = data.package; let namespaceName = data.namespace; + if(access_token === null){ + // exit this function early if we don't have an access token + dispatch({ + type: RATE_PACKAGE_FAILURE, + payload: { + message: "Unauthorized to rate packages. Please login to rate packages.", + statuscode: "403", + }, + }); + return; + } + try { dispatch({