Merge pull request #584 from genedna/main

Remove crate version valid function from script

scripts/crates-sync/crates-sync.py

@@ -8,14 +8,14 @@
 from collections import defaultdict
 from packaging import version
 
+# Create a directory if it doesn't exist
 def ensure_directory(path):
-    # Create a directory if it doesn't exist
     if not os.path.exists(path):
         os.makedirs(path)
         print(f"Created directory: {path}")
 
+# Construct the filename and path for the crate
 def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url):
-    # Construct the filename and path for the crate
     crate_filename = f"{crate_name}-{crate_version}.crate"
     crate_path = os.path.join(crates_dir, crate_name, crate_filename)
 
@@ -31,8 +31,8 @@ def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url)
             print(f"Error downloading {crate_filename}: {str(e)}")
     return crate_path
 
+# Run a git command in the specified repository
 def run_git_command(repo_path, command):
-    # Run a git command in the specified repository
     try:
         result = subprocess.run(command, cwd=repo_path, check=True, capture_output=True, text=True)
         return result.stdout.strip()
@@ -41,29 +41,29 @@ def run_git_command(repo_path, command):
         print(f"Command output: {e.output}")
         return None
 
+# Initialize a git repository if it doesn't exist
 def init_git_repo(repo_path):
-    # Initialize a git repository if it doesn't exist
     if not os.path.exists(os.path.join(repo_path, '.git')):
         run_git_command(repo_path, ['git', 'init', '-b', 'main'])
         print(f"Initialized git repository in {repo_path}")
 
 def extract_crate(crate_path, extract_path):
+    # Check if a path is within a directory (for security)
     def is_within_directory(directory, target):
-        # Check if a path is within a directory (for security)
         abs_directory = os.path.abspath(directory)
         abs_target = os.path.abspath(target)
         prefix = os.path.commonprefix([abs_directory, abs_target])
         return prefix == abs_directory
 
+    # Safely extract files from a tar archive
     def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
-        # Safely extract files from a tar archive
         for member in tar.getmembers():
             member_path = os.path.join(path, member.name)
             if not is_within_directory(path, member_path):
                 raise Exception("Attempted Path Traversal in Tar File")
 
+        # Filter function to ensure extracted files are within the target directory
         def filter_member(tarinfo, filterpath):
-            # Filter function to ensure extracted files are within the target directory
             if is_within_directory(path, os.path.join(filterpath, tarinfo.name)):
                 return tarinfo
             else:
@@ -98,8 +98,8 @@ def filter_member(tarinfo, filterpath):
         print(f"Warning: Failed to read crate file {crate_path}. Skipping extraction.")
         return False
 
+# Process a specific version of a crate
 def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_base_url):
-    # Process a specific version of a crate
     repo_path = os.path.join(git_repos_dir, crate_name, version)
     ensure_directory(repo_path)
 
@@ -116,7 +116,7 @@ def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_ba
 
     # Commit changes with updated message format
     commit_message = f"{crate_name} {version}"
-    run_git_command(repo_path, ['git', 'commit', '-m', commit_message])
+    run_git_command(repo_path, ['git', 'commit', '-a', '-s', '-S', '-m', commit_message])
 
     # Add remote and push
     remote_url = f"{git_base_url}/third-part/rust/crates/{crate_name}/{version}.git"
@@ -129,24 +129,9 @@ def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_ba
     else:
         print(f"Successfully pushed {crate_name} version {version} to remote repository.")
 
+# Process all versions of a crate
 def process_crate(crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url):
-    # Process all versions of a crate
-    def version_key(v):
-        # Function to parse version for sorting
-        try:
-            return version.parse(v)
-        except version.InvalidVersion:
-            print(f"Warning: Invalid version '{v}' for crate '{crate_name}'. Skipping this version.")
-            return version.parse("0.0.0")  # Use a default low version number
-
-    versions_to_process = sorted(versions, key=version_key)
-
-    for v in versions_to_process:
-        try:
-            version.parse(v)  # Check again if the version is valid
-        except version.InvalidVersion:
-            continue  # Skip invalid version
-
+    for v in versions:
         repo_path = os.path.join(git_repos_dir, crate_name, v)
         if os.path.exists(repo_path):
             print(f"Repository for {crate_name} version {v} already exists. Skipping.")
@@ -157,8 +142,8 @@ def version_key(v):
 
     print(f"Finished processing {crate_name}")
 
+# Scan the crates.io index and process all crates
 def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url):
-    # Scan the crates.io index and process all crates
     crates = defaultdict(set)
     dl_base_url = None
 
@@ -209,8 +194,8 @@ def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url)
 
     return len(crates)
 
+# Main function to run the script
 def main():
-    # Main function to run the script
     if len(sys.argv) != 5:
         print("Usage: python script.py <path_to_crates.io-index> <path_to_crates_directory> <path_to_git_repos_directory> <git_base_url>")
         sys.exit(1)
@@ -220,5 +205,6 @@ def main():
     total_crates = scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url)
     print(f"\nTotal number of crates processed: {total_crates}")
 
+# Run the main function if this script is executed directly
 if __name__ == "__main__":
-    main()  # Run the main function if this script is executed directly
+    main()