Pulpcore upgrade (Katello#10949)

* Fixes #37301 - Upgrade pulp binding gems

* Refs #37301 - Delete distributions for repo versions

(cherry picked from commit 4b306986bb493e77882b386984abc2850c030463)

* Refs #37301 - Remove distro deletion + Add cleanups to tests + VCRs

* Refs #37301 - Even more teardowns

app/services/katello/pulp3/repository.rb

@@ -210,6 +210,10 @@ def create_publication
         api.publications_api.create(publication_data)
       end
 
+      def delete_publication
+        ignore_404_exception { api.publications_api.delete(repo.publication_href) } if repo.publication_href
+      end
+
       def publication_options(repository_version)
         {
           repository_version: repository_version

app/services/katello/pulp3/smart_proxy_repository.rb

@@ -35,13 +35,11 @@ def current_repositories(environment_id = nil, content_view_id = nil)
 
       def delete_orphan_repository_versions
         tasks = []
-
         orphan_repository_versions.each do |api, version_hrefs|
           tasks << version_hrefs.collect do |href|
             api.repository_versions_api.delete(href)
           end
         end
-
         tasks.flatten
       end
 

katello.gemspec

@@ -53,15 +53,15 @@ Gem::Specification.new do |gem|
   # webrick is no longer a part of Ruby's stdlib: https://bugs.ruby-lang.org/issues/17303
   gem.add_dependency "webrick"
   #pulp3
-  gem.add_dependency "pulpcore_client", ">= 3.39.0", "< 3.40.0"
-  gem.add_dependency "pulp_file_client", ">= 1.15.0", "< 1.16.0"
-  gem.add_dependency "pulp_ansible_client", ">= 0.20.0", "< 0.21.0"
-  gem.add_dependency "pulp_container_client", ">= 2.16.0", "< 2.17.0"
-  gem.add_dependency "pulp_deb_client", ">= 3.0.0", "< 3.1.0"
-  gem.add_dependency "pulp_rpm_client", ">= 3.23.0", "< 3.24.0"
-  gem.add_dependency "pulp_certguard_client", "< 2.0.0"
-  gem.add_dependency "pulp_python_client", ">= 3.10.0", "< 3.11.0"
-  gem.add_dependency "pulp_ostree_client", ">= 2.1.0", "< 2.2.0"
+  gem.add_dependency "pulpcore_client", ">= 3.49.1", "< 3.50.0"
+  gem.add_dependency "pulp_file_client", ">= 3.49.1", "< 3.50.0"
+  gem.add_dependency "pulp_ansible_client", ">= 0.21.3", "< 0.22.0"
+  gem.add_dependency "pulp_container_client", ">= 2.19.2", "< 2.20.0"
+  gem.add_dependency "pulp_deb_client", ">= 3.2.0", "< 3.3.0"
+  gem.add_dependency "pulp_rpm_client", ">= 3.25.2", "< 3.26.0"
+  gem.add_dependency "pulp_certguard_client", ">= 3.49.1", "< 3.50.0"
+  gem.add_dependency "pulp_python_client", ">= 3.11.0", "< 3.12.0"
+  gem.add_dependency "pulp_ostree_client", ">= 2.3.0", "< 2.4.0"
 
   # UI
   gem.add_dependency "deface", '>= 1.0.2', '< 2.0.0'

test/actions/katello/repository/index_content_test.rb

@@ -11,6 +11,10 @@ def setup
       @repo.update(last_contents_changed: @now - 600.seconds)
     end
 
+    def teardown
+      ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+    end
+
     def test_index_not_performed_if_last_contents_changed_older_than_last_indexed
       @repo.update(last_indexed: (@now - 300.seconds).to_datetime)
       indexed_time = @repo.last_indexed

test/actions/pulp3/capsule_content/reclaim_space_test.rb

@@ -18,6 +18,16 @@ def setup
       @repo2.root.update(download_policy: 'on_demand')
     end
 
+    def teardown
+      #destroy the repositories
+      @repo.backend_service(@primary).delete_distributions
+      @repo2.backend_service(@primary).delete_distributions
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo2, @primary)
+    end
+
     def test_pulp_primary_has_space_reclaimed
       task = ForemanTasks.async_task(::Actions::Pulp3::CapsuleContent::ReclaimSpace, @primary)
       # Check that at least repo and repo2 are among the cleaned repositories

test/actions/pulp3/content_guard/secure_repository_create_test.rb

@@ -10,6 +10,13 @@ def setup
       @repo.root.update(:url => 'http://test/test/', :unprotected => false)
     end
 
+    def teardown
+      @repo.backend_service(@primary).delete_distributions
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+      @repo.reload
+    end
+
     def test_create
       content_guard = ::Katello::Pulp3::ContentGuard.first
       assert_nil content_guard

test/actions/pulp3/content_view/delete_repository_references_test.rb

@@ -11,6 +11,10 @@ def setup
       ensure_creatable(@repo, @primary)
     end
 
+    def teardown
+      ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+    end
+
     def test_create
       ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Create, @repo, @primary)
       repo_reference = Katello::Pulp3::RepositoryReference.find_by(:content_view => @content_view, :root_repository_id => @repo.root.id)

test/actions/pulp3/orchestration/ansible_collection_create_test.rb

@@ -11,6 +11,13 @@ def setup
       ensure_creatable(@repo, @primary)
     end
 
+    def teardown
+      @repo.backend_service(@primary).delete_distributions
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+      @repo.reload
+    end
+
     def test_create
       ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Create, @repo, @primary)
       @repo.reload

test/actions/pulp3/orchestration/ansible_collection_update_test.rb

@@ -10,6 +10,12 @@ def setup
       create_repo(@repo, @primary)
     end
 
+    def teardown
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+      @repo.reload
+    end
+
     def test_invalid_update
       ::Katello::Pulp3::Repository::AnsibleCollection.any_instance.stubs(:test_remote_name).returns(:test_remote_name)
 

test/actions/pulp3/orchestration/apt_update_test.rb

@@ -21,6 +21,13 @@ def setup
         ssl_client_key: katello_gpg_keys(:unassigned_gpg_key))
     end
 
+    def teardown
+      @repo.backend_service(@primary).delete_distributions
+      @repo.backend_service(@primary).delete_publication
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+    end
+
     def test_update_http_proxy_with_no_url
       @repo.root.update(url: nil)
       @repo.root.update(http_proxy_policy: ::Katello::RootRepository::USE_SELECTED_HTTP_PROXY)
@@ -68,6 +75,13 @@ def setup
       create_repo(@repo, @primary)
     end
 
+    def teardown
+      @repo.backend_service(@primary).delete_distributions
+      @repo.backend_service(@primary).delete_publication
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+    end
+
     def test_addurl
       ::Katello::Pulp3::Repository.any_instance.stubs(:fail_missing_publication).returns(nil)
       @repo.root.update(url: "http://someotherurl")

test/actions/pulp3/orchestration/copy_all_units_test.rb

@@ -9,6 +9,11 @@ def setup
       @file_clone = katello_repositories(:generic_file_dev)
     end
 
+    def teardown
+      ensure_creatable(@file_repo, @primary)
+      ensure_creatable(@file_clone, @primary)
+    end
+
     def test_file_repo_copy_all_units_uses_same_version_href
       @file_repo.update!(:version_href => "my/custom/path")
       ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::CopyAllUnits, @file_clone, @primary, [@file_repo])
@@ -34,6 +39,11 @@ def setup
       create_repo(@docker_clone, @primary)
     end
 
+    def teardown
+      ensure_creatable(@docker_repo, @primary)
+      ensure_creatable(@docker_clone, @primary)
+    end
+
     def test_inclusion_docker_filters
       sync_args = {:smart_proxy_id => @primary.id, :repo_id => @docker_repo.id}
       ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Sync, @docker_repo, @primary, sync_args)
@@ -58,11 +68,6 @@ def test_inclusion_docker_filters
       refute_nil(@docker_clone.version_href)
       assert_not_equal @docker_repo.version_href, @docker_clone.version_href
       assert_equal @docker_clone.docker_tags.pluck(:name).sort, ["latest", "glibc"].sort
-
-      @file_clone = katello_repositories(:generic_file_dev)
-      @docker_clone = katello_repositories(:busybox_dev)
-      @rule = FactoryBot.build(:katello_content_view_docker_filter_rule)
-      @rule2 = FactoryBot.build(:katello_content_view_docker_filter_rule)
     end
 
     def test_exclusion_docker_filters
@@ -86,7 +91,7 @@ def test_exclusion_docker_filters
       refute_nil(@docker_repo.version_href)
       refute_nil(@docker_clone.version_href)
       assert_not_equal @docker_repo.version_href, @docker_clone.version_href
-      assert_equal @docker_clone.docker_tags.pluck(:name), @docker_repo.docker_tags.pluck(:name) - ["latest"]
+      assert_equal @docker_clone.docker_tags.pluck(:name).sort, @docker_repo.docker_tags.pluck(:name).sort - ["latest"]
     end
   end
 
@@ -114,6 +119,11 @@ def setup
       @repo.reload
     end
 
+    def teardown
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+    end
+
     def test_yum_copy_all_no_filter_rules
       filter = FactoryBot.build(:katello_content_view_package_filter)
       @repo_clone_original_version_href = @repo_clone.version_href
@@ -226,9 +236,10 @@ def test_copy_duplicated_errata
       assert_equal ['armadillo'], @repo_clone.rpms.pluck(:name)
       assert_equal ["KATELLO-RHEA-2010:0001", "KATELLO-RHEA-2010:99143", "KATELLO-RHSA-2010:0858", "RHEA-2021:9999"].sort, @repo_clone.errata.pluck(:pulp_id).sort
     ensure
-      ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
-      ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Delete, repo2, @primary)
-      ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Delete, repo3, @primary)
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+      ensure_creatable(repo2, @primary)
+      ensure_creatable(repo3, @primary)
     end
     # rubocop:enable Metrics/AbcSize
     # rubocop:enable Metrics/MethodLength
@@ -412,6 +423,11 @@ def setup
       @repo.reload
     end
 
+    def teardown
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+    end
+
     def test_all_srpms_copied_despite_filter_rules
       filter = FactoryBot.build(:katello_content_view_package_filter, :inclusion => true)
       FactoryBot.create(:katello_content_view_package_filter_rule, :filter => filter, :name => "kangaroo")
@@ -453,6 +469,11 @@ def setup
       @repo.reload
     end
 
+    def teardown
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+    end
+
     def test_all_errata_copied_if_no_filter_rules
       filter = FactoryBot.build(:katello_content_view_package_filter, :inclusion => true)
 
@@ -543,6 +564,11 @@ def setup
       @repo.reload
     end
 
+    def teardown
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+    end
+
     def test_all_module_streams_copied_if_no_modular_filter_rules
       filter = FactoryBot.build(:katello_content_view_package_filter, :inclusion => true)
 
@@ -651,6 +677,11 @@ def setup
       @repo.reload
     end
 
+    def teardown
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+    end
+
     def test_all_package_groups_copied_with_no_filter_rules
       filter = FactoryBot.build(:katello_content_view_package_filter, :inclusion => true)
 
@@ -709,6 +740,11 @@ def setup
       @repo.reload
     end
 
+    def teardown
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+    end
+
     def test_all_package_environments_are_copied_by_default
       filter = FactoryBot.build(:katello_content_view_package_filter, :inclusion => true)
 
@@ -773,6 +809,11 @@ def setup
       @repo.reload
     end
 
+    def teardown
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+    end
+
     def test_all_modulemd_defaults_are_copied_by_default
       filter = FactoryBot.build(:katello_content_view_package_filter, :inclusion => true)
 
@@ -817,6 +858,11 @@ def setup
       @repo.reload
     end
 
+    def teardown
+      ensure_creatable(@repo, @primary)
+      ensure_creatable(@repo_clone, @primary)
+    end
+
     def test_all_distribution_trees_are_copied_by_default
       filter = FactoryBot.build(:katello_content_view_package_filter, :inclusion => true)
 

test/actions/pulp3/orchestration/docker_create_test.rb

@@ -10,6 +10,12 @@ def setup
       ensure_creatable(@repo, @primary)
     end
 
+    def teardown
+      @repo.backend_service(@primary).delete_distributions
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+    end
+
     def test_create
       ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Create, @repo, @primary)
       @repo.reload

test/actions/pulp3/orchestration/docker_update_test.rb

@@ -22,6 +22,10 @@ def setup
         ssl_client_key: katello_gpg_keys(:unassigned_gpg_key))
     end
 
+    def teardown
+      ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+    end
+
     def test_update_ssl_validation
       refute @repo.root.verify_ssl_on_sync, "Respository verify_ssl_on_sync option was false."
       @repo.root.update(

test/actions/pulp3/orchestration/file_create_test.rb

@@ -11,6 +11,12 @@ def setup
       ensure_creatable(@repo, @primary)
     end
 
+    def teardown
+      @repo.backend_service(@primary).delete_distributions
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+    end
+
     def test_create
       ForemanTasks.sync_task(::Actions::Pulp3::Orchestration::Repository::Create, @repo, @primary)
       @repo.reload

test/actions/pulp3/orchestration/file_delete_test.rb

@@ -19,6 +19,12 @@ def setup
       refute_empty Katello::Pulp3::DistributionReference.where(repository_id: @repo.id)
     end
 
+    def teardown
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+      @repo.reload
+    end
+
     def test_repository_reference_is_deleted
       ForemanTasks.sync_task(
           ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)

test/actions/pulp3/orchestration/file_update_test.rb

@@ -26,6 +26,12 @@ def setup
         ssl_client_key: katello_gpg_keys(:unassigned_gpg_key))
     end
 
+    def teardown
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
+      @repo.reload
+    end
+
     def test_update_ssl_validation
       refute @repo.root.verify_ssl_on_sync, "Respository verify_ssl_on_sync option was false."
       @repo.root.update(

test/actions/pulp3/orchestration/file_upload_test.rb

@@ -15,6 +15,8 @@ def setup
     end
 
     def teardown
+      @repo.backend_service(@primary).delete_distributions
+      @repo.backend_service(@primary).delete_publication
       ForemanTasks.sync_task(
           ::Actions::Pulp3::Orchestration::Repository::Delete, @repo, @primary)
       ForemanTasks.sync_task(

test/actions/pulp3/orchestration/generate_metadata_test.rb

@@ -40,6 +40,8 @@ def test_generate_with_source_repo
       assert_equal @repo.publication_href, @clone.reload.publication_href
       assert_equal 1, Katello::Pulp3::DistributionReference.where(repository_id: @clone.id).count
       assert_equal @clone.smart_proxy_sync_histories.count, 0
+      ForemanTasks.sync_task(
+        ::Actions::Pulp3::Orchestration::Repository::Delete, @clone, @primary)
     end
   end
 end