Skip to content

Commit

Permalink
Remove youtube-dl tests from archiver_test.rb
Browse files Browse the repository at this point in the history 
Removing unneeded tests because we are not supporting youtube-dl anymore.
  • Loading branch information
@jayjay-w
jayjay-w committed Nov 29, 2023
1 parent 4a33dc8 commit 3beb1e5
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 304 deletions.
2 changes: 1 addition & 1 deletion spec/requests/api/base_api_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
expect(data).not_to be_nil
expect(data['name']).to eq('Keep')
expect(data['version']).to eq(VERSION)
expect(data['archivers']).to eq([{"key"=>"archive_org", "label"=>"Archive.org"}, {"key"=>"perma_cc", "label"=>"Perma.cc"}, {"key"=>"video", "label"=>"Video"}])
expect(data['archivers']).to eq([{"key"=>"archive_org", "label"=>"Archive.org"}, {"key"=>"perma_cc", "label"=>"Perma.cc"}])
end

include_context 'generate examples'
Expand Down
303 changes: 0 additions & 303 deletions test/models/archiver_test.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,309 +564,6 @@ def create_api_key_with_webhook_for_perma_cc
end
end

test "should call youtube-dl and call video upload when archive video" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!
api_key = create_api_key_with_webhook
url = 'https://www.bbc.com/news/av/world-us-canada-57176620'

WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
Media.any_instance.unstub(:archive_to_video)

m = Media.new url: url, key: api_key
m.as_json

Media.stubs(:supported_video?).with(m.url, api_key.id).returns(true)
Media.stubs(:notify_video_already_archived).with(m.url, api_key.id).returns(nil)

Media.stubs(:store_video_folder).returns('store_video_folder')
Media.stubs(:system).returns(`(exit 0)`)
assert_equal 'store_video_folder', Media.send_to_video_archiver(m.url, api_key.id)
assert_nil Media.send_to_video_archiver(m.url, api_key.id, false)
ensure
WebMock.disable!
end

test "should return false and add error to data when video archiving is not supported" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!
api_key = create_api_key_with_webhook

Media.unstub(:supported_video?)
Media.any_instance.stubs(:parse)
Metrics.stubs(:schedule_fetching_metrics_from_facebook)
WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')

Media.stubs(:system).returns(`(exit 0)`)
url = 'https://www.folha.uol.com.br/'
m = create_media url: url
m.as_json(archivers: 'none')
assert Media.supported_video?(m.url, api_key.id)

media_data = Pender::Store.current.read(Media.get_id(url), :json)
assert_nil media_data.dig('archives', 'video_archiver')

Media.stubs(:system).returns(`(exit 1)`)
url = 'https://www.r7.com/'
m = create_media url: url
m.as_json(archivers: 'none')
assert !Media.supported_video?(m.url, api_key.id)

media_data = Pender::Store.current.read(Media.get_id(url), :json)
assert_equal Lapis::ErrorCodes::const_get('ARCHIVER_NOT_SUPPORTED_MEDIA'), media_data.dig('archives', 'video_archiver', 'error', 'code')
assert_equal '1 Unsupported URL', media_data.dig('archives', 'video_archiver', 'error', 'message')
ensure
WebMock.disable!
end

test "should check if non-ascii URL support video download" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
Media.unstub(:supported_video?)
assert !Media.supported_video?('http://example.com/pages/category/Musician-Band/चौधरी-कमला-बाड़मेर-108960273957085')
end

test "should notify if URL was already parsed and has a location on data when archive video" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!
url = 'https://www.bbc.com/news/av/world-us-canada-57176620'

WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')

Pender::Store.any_instance.stubs(:read).with(Media.get_id(url), :json).returns(nil)
assert_nil Media.notify_video_already_archived(url, nil)

data = { archives: { video_archiver: { error: 'could not download video data'}}}
Pender::Store.any_instance.stubs(:read).with(Media.get_id(url), :json).returns(data)
Media.stubs(:notify_webhook).with('video_archiver', url, data, {}).returns('Notify webhook')
assert_nil Media.notify_video_already_archived(url, nil)

data[:archives][:video_archiver] = { location: 'path_to_video' }
Pender::Store.any_instance.stubs(:read).with(Media.get_id(url), :json).returns(data)
Media.stubs(:notify_webhook).with('video_archiver', url, data, {}).returns('Notify webhook')
assert_equal 'Notify webhook', Media.notify_video_already_archived(url, nil)
ensure
WebMock.disable!
end

# FIXME Mocking Youtube-DL to avoid `HTTP Error 429: Too Many Requests`
test "should archive video info subtitles, thumbnails and update cache" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!

api_key = create_api_key_with_webhook
url = 'https://www.youtube.com/watch?v=1vSJrexmVWU'
id = Media.get_id url

WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')
Media.stubs(:supported_video?).with(url, api_key.id).returns(true)
Media.stubs(:system).returns(`(exit 0)`)
local_folder = File.join(Rails.root, 'tmp', 'videos', id)
video_files = "#{local_folder}/#{id}/#{id}.es.vtt", "#{local_folder}/#{id}/#{id}.jpg", "#{local_folder}/#{id}/#{id}.vtt", "#{local_folder}/#{id}/#{id}.mp4", "#{local_folder}/#{id}/#{id}.jpg", "#{local_folder}/#{id}/#{id}.info.json"
Dir.stubs(:glob).returns(video_files)
Pender::Store.any_instance.stubs(:upload_video_folder)

m = create_media url: url, key: api_key
data = m.as_json
assert_nil data.dig('archives', 'video_archiver')
Media.send_to_video_archiver(url, api_key.id, 20)

data = m.as_json
assert_nil data.dig('archives', 'video_archiver', 'error', 'message')

folder = File.join(Media.archiving_folder, id)
assert_equal ['info', 'location', 'subtitles', 'thumbnails', 'videos'], data.dig('archives', 'video_archiver').keys.sort
assert_equal "#{folder}/#{id}.mp4", data.dig('archives', 'video_archiver', 'location')
assert_equal "#{folder}/#{id}.info.json", data.dig('archives', 'video_archiver', 'info')
assert_equal "#{folder}/#{id}.mp4", data.dig('archives', 'video_archiver', 'videos').first
data.dig('archives', 'video_archiver', 'subtitles').each do |sub|
assert_match /\A#{folder}\/#{id}/, sub
end
data.dig('archives', 'video_archiver', 'thumbnails').each do |thumb|
assert_match /\A#{folder}\/#{id}.*\.jpg\z/, thumb
end
ensure
WebMock.disable!
end

test "should raise retry error when video archiving fails" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!
Sidekiq::Testing.fake!
api_key = create_api_key_with_webhook
url = 'https://www.wsj.com/'

WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')

Media.stubs(:supported_video?).with(url, api_key.id).returns(true)
m = create_media url: url, key: api_key
data = m.as_json
assert_nil data.dig('archives', 'video_archiver')

Media.stubs(:system).returns(`(exit 1)`)
not_video_url = 'https://www.uol.com.br/'
Media.stubs(:supported_video?).with(not_video_url, api_key.id).returns(true)
Media.stubs(:notify_video_already_archived).with(not_video_url, api_key.id).returns(nil)

Media.stubs(:system).returns(`(exit 1)`)
assert_raises Pender::Exception::RetryLater do
Media.send_to_video_archiver(not_video_url, api_key.id)
end
ensure
WebMock.disable!
end

test "should update media with error when supported video call raises on video archiving" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!
Sidekiq::Testing.fake!
api_key = create_api_key_with_webhook
url = 'https://example.com'

Media.any_instance.stubs(:follow_redirections)
Media.any_instance.stubs(:get_canonical_url).returns(true)
Media.any_instance.stubs(:try_https)
Media.any_instance.stubs(:parse)
WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')

assert_raises Pender::Exception::RetryLater do
m = Media.new url: url
m.as_json
assert m.data.dig('archives', 'video_archiver').nil?
error = StandardError.new('some error')
Media.stubs(:supported_video?).with(url, api_key.id).raises(error)
Media.send_to_video_archiver(url, api_key.id, 20)
media_data = Pender::Store.current.read(Media.get_id(url), :json)
assert_equal Lapis::ErrorCodes::const_get('ARCHIVER_ERROR'), media_data.dig('archives', 'video_archiver', 'error', 'code')
assert_equal "#{error.class} #{error.message}", media_data.dig('archives', 'video_archiver', 'error', 'message')
end
ensure
WebMock.disable!
end

test "should update media with error when video download fails when video archiving" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!
api_key = create_api_key_with_webhook
url = 'https://www.tiktok.com/@scout2015/video/6771039287917038854'

WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')

Media.any_instance.stubs(:follow_redirections)
Media.any_instance.stubs(:get_canonical_url).returns(true)
Media.any_instance.stubs(:try_https)
Media.any_instance.stubs(:parse)
Media.stubs(:supported_video?).returns(true)
Media.stubs(:system).returns(`(exit 1)`)

assert_raises Pender::Exception::RetryLater do
m = Media.new url: url
m.as_json(archivers: 'none')
assert_nil m.data.dig('archives', 'video_archiver')
Media.send_to_video_archiver(url, api_key.id, 20)
media_data = Pender::Store.current.read(Media.get_id(url), :json)
assert_equal Lapis::ErrorCodes::const_get('ARCHIVER_FAILURE'), media_data.dig('archives', 'video_archiver', 'error', 'code')
assert_match 'not available', media_data.dig('archives', 'video_archiver', 'error', 'message').downcase
end
ensure
WebMock.disable!
end

test "should generate the public archiving folder for videos" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
api_key = create_api_key application_settings: { config: { storage_endpoint: 'http://minio:9000', storage_bucket: 'default-bucket', storage_video_asset_path: nil, storage_video_bucket: nil }}
ApiKey.current = api_key

assert_match /#{PenderConfig.get('storage_endpoint')}\/default-bucket\d*\/video/, Media.archiving_folder

api_key.application_settings[:config][:storage_video_bucket] = 'bucket-for-videos'; api_key.save
ApiKey.current = api_key
Pender::Store.current = nil
PenderConfig.current = nil
assert_match /#{PenderConfig.get('storage_endpoint')}\/bucket-for-videos\d*\/video/, Media.archiving_folder

api_key.application_settings[:config][:storage_video_asset_path] = 'http://public-storage/my-videos'; api_key.save
ApiKey.current = api_key
Pender::Store.current = nil
PenderConfig.current = nil
assert_equal "http://public-storage/my-videos", Media.archiving_folder
end

test "should send to video archiver when call archive to video" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
Media.any_instance.unstub(:archive_to_video)
Media.any_instance.stubs(:follow_redirections)
Media.any_instance.stubs(:get_canonical_url).returns(true)
Media.any_instance.stubs(:try_https)

Sidekiq::Testing.fake! do
url = 'http://example.com'
m = Media.new url: url
assert_difference 'ArchiverWorker.jobs.size', 1 do
m.archive_to_video(m.url, nil)
end
end
end

test "should get proxy to download video from api key if present" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!
api_key = create_api_key application_settings: { 'webhook_url': 'https://example.com/webhook.php', 'webhook_token': 'test' }
url = 'https://www.youtube.com/watch?v=unv9aPZYF6E'

WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')

m = Media.new url: url, key: api_key

assert_nil Media.yt_download_proxy(m.url)

api_key.application_settings = { config: { ytdl_proxy_host: 'my-proxy.mine', ytdl_proxy_port: '1111', ytdl_proxy_user_prefix: 'my-user-prefix', ytdl_proxy_pass: '12345' }}; api_key.save
PenderConfig.current = nil
m = Media.new url: url, key: api_key
assert_equal 'http://my-user-prefix:[email protected]:1111', Media.yt_download_proxy(m.url)
ensure
WebMock.disable!
end

test "should use api key config when archiving video if present" do
skip('we are not supporting archiving videos with youtube-dl anymore, will remove this on a separate ticket')
WebMock.enable!
api_key = create_api_key application_settings: { 'webhook_url': 'https://example.com/webhook.php', 'webhook_token': 'test' }
url = 'https://www.youtube.com/watch?v=o1V1LnUU5VM'

WebMock.stub_request(:post, /example.com\/webhook/).to_return(status: 200, body: '')

Media.unstub(:supported_video?)
Media.stubs(:system).returns(`(exit 0)`)

config = {}
%w(ytdl_proxy_host ytdl_proxy_port ytdl_proxy_user_prefix ytdl_proxy_pass storage_endpoint storage_access_key storage_secret_key storage_bucket storage_bucket_region storage_video_bucket).each do |config_key|
config[config_key] = PenderConfig.get(config_key, "test_#{config_key}")
end

ApiKey.current = PenderConfig.current = Pender::Store.current = nil
Media.send_to_video_archiver(url, api_key.id)

assert_equal api_key, ApiKey.current
%w(endpoint access_key secret_key bucket bucket_region medias_asset_path).each do |key|
assert !PenderConfig.current("storage_#{key}").blank?
assert !Pender::Store.current.instance_variable_get(:@storage)[key].blank?
end

ApiKey.current = PenderConfig.current = Pender::Store.current = nil
api_key.application_settings = { config: { ytdl_proxy_host: 'my-proxy.mine', ytdl_proxy_port: '1111', ytdl_proxy_user_prefix: 'my-user-prefix', ytdl_proxy_pass: '12345', storage_endpoint: config['storage_endpoint'], storage_access_key: config['storage_access_key'], storage_secret_key: config['storage_secret_key'], storage_bucket: 'my-bucket', storage_bucket_region: config['storage_bucket_region'], storage_video_bucket: 'video-bucket'}}; api_key.save
Media.send_to_video_archiver(url, api_key.id, 20)
assert_equal api_key, ApiKey.current
%w(host port user_prefix pass).each do |key|
assert_equal api_key.settings[:config]["ytdl_proxy_#{key}"], PenderConfig.current("ytdl_proxy_#{key}")
end
%w(endpoint access_key secret_key bucket bucket_region video_bucket).each do |key|
assert_equal api_key.settings[:config]["storage_#{key}"], PenderConfig.current("storage_#{key}")
assert_equal api_key.settings[:config]["storage_#{key}"], Pender::Store.current.instance_variable_get(:@storage)[key]
end
ensure
WebMock.disable!
end

test "include error on data when cannot use archiver" do
skip = ENV['archiver_skip_hosts']
ENV['archiver_skip_hosts'] = 'example.com'
Expand Down

0 comments on commit 3beb1e5

Please sign in to comment.