From 3beb1e5cac7f585efd600d69ef20d26959a54dbd Mon Sep 17 00:00:00 2001
From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com>
Date: Wed, 29 Nov 2023 08:51:41 +0100
Subject: [PATCH] Remove youtube-dl tests from archiver_test.rb

Removing unneeded tests because we are not supporting youtube-dl anymore.
---
 spec/requests/api/base_api_spec.rb |   2 +-
 test/models/archiver_test.rb       | 303 -----------------------------
 2 files changed, 1 insertion(+), 304 deletions(-)

diff --git a/spec/requests/api/base_api_spec.rb b/spec/requests/api/base_api_spec.rb
index 1922b18d..bceb7750 100644
--- a/spec/requests/api/base_api_spec.rb
+++ b/spec/requests/api/base_api_spec.rb
@@ -49,7 +49,7 @@
           expect(data).not_to be_nil
           expect(data['name']).to eq('Keep')
           expect(data['version']).to eq(VERSION)
-          expect(data['archivers']).to eq([{"key"=>"archive_org", "label"=>"Archive.org"}, {"key"=>"perma_cc", "label"=>"Perma.cc"}, {"key"=>"video", "label"=>"Video"}])
+          expect(data['archivers']).to eq([{"key"=>"archive_org", "label"=>"Archive.org"}, {"key"=>"perma_cc", "label"=>"Perma.cc"}])
         end
 
         include_context 'generate examples'
diff --git a/test/models/archiver_test.rb b/test/models/archiver_test.rb
index f51a4e60..5afa1783 100644
--- a/test/models/archiver_test.rb
+++ b/test/models/archiver_test.rb
@@ -564,309 +564,6 @@ def create_api_key_with_webhook_for_perma_cc
     end
   end
 
-  test "should call youtube-dl and call video upload when archive video" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    api_key = create_api_key_with_webhook
-    url = 'https://www.bbc.com/news/av/world-us-canada-57176620'
-
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-    Media.any_instance.unstub(:archive_to_video)
-
-    m = Media.new url: url, key: api_key
-    m.as_json
-
-    Media.stubs(:supported_video?).with(m.url, api_key.id).returns(true)
-    Media.stubs(:notify_video_already_archived).with(m.url, api_key.id).returns(nil)
-
-    Media.stubs(:store_video_folder).returns('store_video_folder')
-    Media.stubs(:system).returns(`(exit 0)`)
-    assert_equal 'store_video_folder', Media.send_to_video_archiver(m.url, api_key.id)
-    assert_nil Media.send_to_video_archiver(m.url, api_key.id, false)
-  ensure
-    WebMock.disable!
-  end
-
-  test "should return false and add error to data when video archiving is not supported" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    api_key = create_api_key_with_webhook
-    
-    Media.unstub(:supported_video?)
-    Media.any_instance.stubs(:parse)
-    Metrics.stubs(:schedule_fetching_metrics_from_facebook)
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-    
-    Media.stubs(:system).returns(`(exit 0)`)
-    url = 'https://www.folha.uol.com.br/'
-    m = create_media url: url
-    m.as_json(archivers: 'none')
-    assert Media.supported_video?(m.url, api_key.id)
-
-    media_data = Pender::Store.current.read(Media.get_id(url), :json)
-    assert_nil media_data.dig('archives', 'video_archiver')
-
-    Media.stubs(:system).returns(`(exit 1)`)
-    url = 'https://www.r7.com/'
-    m = create_media url: url
-    m.as_json(archivers: 'none')
-    assert !Media.supported_video?(m.url, api_key.id)
-
-    media_data = Pender::Store.current.read(Media.get_id(url), :json)
-    assert_equal Lapis::ErrorCodes::const_get('ARCHIVER_NOT_SUPPORTED_MEDIA'), media_data.dig('archives', 'video_archiver', 'error', 'code')
-    assert_equal '1 Unsupported URL', media_data.dig('archives', 'video_archiver', 'error', 'message')
-  ensure
-    WebMock.disable!
-  end
-
-  test "should check if non-ascii URL support video download" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    Media.unstub(:supported_video?)
-    assert !Media.supported_video?('http://example.com/pages/category/Musician-Band/चौधरी-कमला-बाड़मेर-108960273957085')
-  end
-
-  test "should notify if URL was already parsed and has a location on data when archive video" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    url = 'https://www.bbc.com/news/av/world-us-canada-57176620'
-
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-
-    Pender::Store.any_instance.stubs(:read).with(Media.get_id(url), :json).returns(nil)
-    assert_nil Media.notify_video_already_archived(url, nil)
-
-    data = { archives: { video_archiver: { error: 'could not download video data'}}}
-    Pender::Store.any_instance.stubs(:read).with(Media.get_id(url), :json).returns(data)
-    Media.stubs(:notify_webhook).with('video_archiver', url, data, {}).returns('Notify webhook')
-    assert_nil Media.notify_video_already_archived(url, nil)
-
-    data[:archives][:video_archiver] = { location: 'path_to_video' }
-    Pender::Store.any_instance.stubs(:read).with(Media.get_id(url), :json).returns(data)
-    Media.stubs(:notify_webhook).with('video_archiver', url, data, {}).returns('Notify webhook')
-    assert_equal 'Notify webhook', Media.notify_video_already_archived(url, nil)
-  ensure
-    WebMock.disable!
-  end
-
-  # FIXME Mocking Youtube-DL to avoid `HTTP Error 429: Too Many Requests`
-  test "should archive video info subtitles, thumbnails and update cache" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    
-    api_key = create_api_key_with_webhook
-    url = 'https://www.youtube.com/watch?v=1vSJrexmVWU'
-    id = Media.get_id url
-    
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-    Media.stubs(:supported_video?).with(url, api_key.id).returns(true)
-    Media.stubs(:system).returns(`(exit 0)`)
-    local_folder = File.join(Rails.root, 'tmp', 'videos', id)
-    video_files = "#{local_folder}/#{id}/#{id}.es.vtt", "#{local_folder}/#{id}/#{id}.jpg", "#{local_folder}/#{id}/#{id}.vtt", "#{local_folder}/#{id}/#{id}.mp4", "#{local_folder}/#{id}/#{id}.jpg", "#{local_folder}/#{id}/#{id}.info.json"
-    Dir.stubs(:glob).returns(video_files)
-    Pender::Store.any_instance.stubs(:upload_video_folder)
-
-    m = create_media url: url, key: api_key
-    data = m.as_json
-    assert_nil data.dig('archives', 'video_archiver')
-    Media.send_to_video_archiver(url, api_key.id, 20)
-
-    data = m.as_json
-    assert_nil data.dig('archives', 'video_archiver', 'error', 'message')
-
-    folder = File.join(Media.archiving_folder, id)
-    assert_equal ['info', 'location', 'subtitles', 'thumbnails', 'videos'], data.dig('archives', 'video_archiver').keys.sort
-    assert_equal "#{folder}/#{id}.mp4", data.dig('archives', 'video_archiver', 'location')
-    assert_equal "#{folder}/#{id}.info.json", data.dig('archives', 'video_archiver', 'info')
-    assert_equal "#{folder}/#{id}.mp4", data.dig('archives', 'video_archiver', 'videos').first
-    data.dig('archives', 'video_archiver', 'subtitles').each do |sub|
-      assert_match /\A#{folder}\/#{id}/, sub
-    end
-    data.dig('archives', 'video_archiver', 'thumbnails').each do |thumb|
-      assert_match /\A#{folder}\/#{id}.*\.jpg\z/, thumb
-    end
-  ensure
-    WebMock.disable!
-  end
-
-  test "should raise retry error when video archiving fails" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    Sidekiq::Testing.fake!
-    api_key = create_api_key_with_webhook
-    url = 'https://www.wsj.com/'
-
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-
-    Media.stubs(:supported_video?).with(url, api_key.id).returns(true)
-    m = create_media url: url, key: api_key
-    data = m.as_json
-    assert_nil data.dig('archives', 'video_archiver')
-
-    Media.stubs(:system).returns(`(exit 1)`)
-    not_video_url = 'https://www.uol.com.br/'
-    Media.stubs(:supported_video?).with(not_video_url, api_key.id).returns(true)
-    Media.stubs(:notify_video_already_archived).with(not_video_url, api_key.id).returns(nil)
-
-    Media.stubs(:system).returns(`(exit 1)`)
-    assert_raises Pender::Exception::RetryLater do
-      Media.send_to_video_archiver(not_video_url, api_key.id)
-    end
-  ensure
-    WebMock.disable!
-  end
-
-  test "should update media with error when supported video call raises on video archiving" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    Sidekiq::Testing.fake!
-    api_key = create_api_key_with_webhook
-    url = 'https://example.com'
-
-    Media.any_instance.stubs(:follow_redirections)
-    Media.any_instance.stubs(:get_canonical_url).returns(true)
-    Media.any_instance.stubs(:try_https)
-    Media.any_instance.stubs(:parse)
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-
-    assert_raises Pender::Exception::RetryLater do
-      m = Media.new url: url
-      m.as_json
-      assert m.data.dig('archives', 'video_archiver').nil?
-      error = StandardError.new('some error')
-      Media.stubs(:supported_video?).with(url, api_key.id).raises(error)
-      Media.send_to_video_archiver(url, api_key.id, 20)
-      media_data = Pender::Store.current.read(Media.get_id(url), :json)
-      assert_equal Lapis::ErrorCodes::const_get('ARCHIVER_ERROR'), media_data.dig('archives', 'video_archiver', 'error', 'code')
-      assert_equal "#{error.class} #{error.message}", media_data.dig('archives', 'video_archiver', 'error', 'message')
-    end
-  ensure
-    WebMock.disable!
-  end
-
-  test "should update media with error when video download fails when video archiving" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    api_key = create_api_key_with_webhook
-    url = 'https://www.tiktok.com/@scout2015/video/6771039287917038854'
-
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-
-    Media.any_instance.stubs(:follow_redirections)
-    Media.any_instance.stubs(:get_canonical_url).returns(true)
-    Media.any_instance.stubs(:try_https)
-    Media.any_instance.stubs(:parse)
-    Media.stubs(:supported_video?).returns(true)
-    Media.stubs(:system).returns(`(exit 1)`)
-
-    assert_raises Pender::Exception::RetryLater do
-      m = Media.new url: url
-      m.as_json(archivers: 'none')
-      assert_nil m.data.dig('archives', 'video_archiver')
-      Media.send_to_video_archiver(url, api_key.id, 20)
-      media_data = Pender::Store.current.read(Media.get_id(url), :json)
-      assert_equal Lapis::ErrorCodes::const_get('ARCHIVER_FAILURE'), media_data.dig('archives', 'video_archiver', 'error', 'code')
-      assert_match 'not available', media_data.dig('archives', 'video_archiver', 'error', 'message').downcase
-    end
-  ensure
-    WebMock.disable!
-  end
-
-  test "should generate the public archiving folder for videos" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    api_key = create_api_key application_settings: { config: { storage_endpoint: 'http://minio:9000', storage_bucket: 'default-bucket', storage_video_asset_path: nil, storage_video_bucket: nil }}
-    ApiKey.current = api_key
-
-    assert_match /#{PenderConfig.get('storage_endpoint')}\/default-bucket\d*\/video/, Media.archiving_folder
-
-    api_key.application_settings[:config][:storage_video_bucket] = 'bucket-for-videos'; api_key.save
-    ApiKey.current = api_key
-    Pender::Store.current = nil
-    PenderConfig.current = nil
-    assert_match /#{PenderConfig.get('storage_endpoint')}\/bucket-for-videos\d*\/video/, Media.archiving_folder
-
-    api_key.application_settings[:config][:storage_video_asset_path] = 'http://public-storage/my-videos'; api_key.save
-    ApiKey.current = api_key
-    Pender::Store.current = nil
-    PenderConfig.current = nil
-    assert_equal "http://public-storage/my-videos", Media.archiving_folder
-  end
-
-  test "should send to video archiver when call archive to video" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    Media.any_instance.unstub(:archive_to_video)
-    Media.any_instance.stubs(:follow_redirections)
-    Media.any_instance.stubs(:get_canonical_url).returns(true)
-    Media.any_instance.stubs(:try_https)
-
-    Sidekiq::Testing.fake! do
-      url = 'http://example.com'
-      m = Media.new url: url
-      assert_difference 'ArchiverWorker.jobs.size', 1 do
-        m.archive_to_video(m.url, nil)
-      end
-    end
-  end
-
-  test "should get proxy to download video from api key if present" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    api_key = create_api_key application_settings: { 'webhook_url': 'https://example.com/webhook.php', 'webhook_token': 'test' }
-    url = 'https://www.youtube.com/watch?v=unv9aPZYF6E'
-
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-
-    m = Media.new url: url, key: api_key
-
-    assert_nil Media.yt_download_proxy(m.url)
-
-    api_key.application_settings = { config: { ytdl_proxy_host: 'my-proxy.mine', ytdl_proxy_port: '1111', ytdl_proxy_user_prefix: 'my-user-prefix', ytdl_proxy_pass: '12345' }}; api_key.save
-    PenderConfig.current = nil
-    m = Media.new url: url, key: api_key
-    assert_equal 'http://my-user-prefix:12345@my-proxy.mine:1111', Media.yt_download_proxy(m.url)
-  ensure
-    WebMock.disable!
-  end
-
-  test "should use api key config when archiving video if present" do
-    skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
-    WebMock.enable!
-    api_key = create_api_key application_settings: { 'webhook_url': 'https://example.com/webhook.php', 'webhook_token': 'test' }
-    url = 'https://www.youtube.com/watch?v=o1V1LnUU5VM'
-
-    WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
-
-    Media.unstub(:supported_video?)
-    Media.stubs(:system).returns(`(exit 0)`)
-
-    config = {}
-    %w(ytdl_proxy_host ytdl_proxy_port ytdl_proxy_user_prefix ytdl_proxy_pass storage_endpoint storage_access_key storage_secret_key storage_bucket storage_bucket_region storage_video_bucket).each do |config_key|
-      config[config_key] = PenderConfig.get(config_key, "test_#{config_key}")
-    end
-
-    ApiKey.current = PenderConfig.current = Pender::Store.current = nil
-    Media.send_to_video_archiver(url, api_key.id)
-
-    assert_equal api_key, ApiKey.current
-    %w(endpoint access_key secret_key bucket bucket_region medias_asset_path).each do |key|
-      assert !PenderConfig.current("storage_#{key}").blank?
-      assert !Pender::Store.current.instance_variable_get(:@storage)[key].blank?
-    end
-
-    ApiKey.current = PenderConfig.current = Pender::Store.current = nil
-    api_key.application_settings = { config: { ytdl_proxy_host: 'my-proxy.mine', ytdl_proxy_port: '1111', ytdl_proxy_user_prefix: 'my-user-prefix', ytdl_proxy_pass: '12345', storage_endpoint: config['storage_endpoint'], storage_access_key: config['storage_access_key'], storage_secret_key: config['storage_secret_key'], storage_bucket: 'my-bucket', storage_bucket_region: config['storage_bucket_region'], storage_video_bucket: 'video-bucket'}}; api_key.save
-    Media.send_to_video_archiver(url, api_key.id, 20)
-    assert_equal api_key, ApiKey.current
-    %w(host port user_prefix pass).each do |key|
-      assert_equal api_key.settings[:config]["ytdl_proxy_#{key}"], PenderConfig.current("ytdl_proxy_#{key}")
-    end
-    %w(endpoint access_key secret_key bucket bucket_region video_bucket).each do |key|
-      assert_equal api_key.settings[:config]["storage_#{key}"], PenderConfig.current("storage_#{key}")
-      assert_equal api_key.settings[:config]["storage_#{key}"], Pender::Store.current.instance_variable_get(:@storage)[key]
-    end
-  ensure
-    WebMock.disable!
-  end
-
   test "include error on data when cannot use archiver" do
     skip = ENV['archiver_skip_hosts']
     ENV['archiver_skip_hosts'] = 'example.com'