Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
tastelikefeet committed Nov 13, 2024
1 parent 15f2abf commit 5fd1ff9
Showing 1 changed file with 19 additions and 4 deletions.
23 changes: 19 additions & 4 deletions swift/llm/dataset/media.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,23 @@ def download(media_type_or_url: Union[str, List[str]],
return MediaResource._safe_download(
media_type=media_type_or_url, media_name=local_alias, file_type=file_type)

@staticmethod
def move_directory_contents(src_dir, dst_dir):
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)

for dirpath, dirnames, filenames in os.walk(src_dir):
relative_path = os.path.relpath(dirpath, src_dir)
target_dir = os.path.join(dst_dir, relative_path)

if not os.path.exists(target_dir):
os.makedirs(target_dir)

for file in filenames:
src_file = os.path.join(dirpath, file)
dst_file = os.path.join(target_dir, file)
shutil.move(src_file, dst_file)

@staticmethod
def _safe_download(media_type: Union[str, List[str]],
media_name: Optional[str] = None,
Expand Down Expand Up @@ -102,10 +119,8 @@ def _safe_download(media_type: Union[str, List[str]],
else:
for media_url in media_type:
local_dirs = DownloadManager(download_config=DownloadConfig(
cache_dir=MediaResource.cache_dir)).download(media_url)
local_dirs = DownloadManager(download_config=DownloadConfig(
cache_dir=MediaResource.cache_dir)).extract(local_dirs)
shutil.move(str(local_dirs), final_folder)
cache_dir=MediaResource.cache_dir)).download_and_extract(media_url)
MediaResource.move_directory_contents(str(local_dirs), final_folder)
logger.info('# #################Resource downloading finished#################')
return final_folder

Expand Down

0 comments on commit 5fd1ff9

Please sign in to comment.