Skip to content

Commit

Permalink
Close zip file after reading pytorch params
Browse files Browse the repository at this point in the history
  • Loading branch information
jonatanklosko committed Feb 22, 2024
1 parent beca321 commit a72c5dd
Showing 1 changed file with 37 additions and 32 deletions.
69 changes: 37 additions & 32 deletions lib/bumblebee/conversion/pytorch/loader.ex
Original file line number Diff line number Diff line change
Expand Up @@ -27,39 +27,44 @@ defmodule Bumblebee.Conversion.PyTorch.Loader do

defp load_zip!(path) do
zip_file = Unzip.LocalFile.open(path)
{:ok, unzip} = Unzip.new(zip_file)

contents =
unzip
|> Unzip.list_entries()
|> Map.new(fn %Unzip.Entry{file_name: file_name} ->
content =
unzip
|> Unzip.file_stream!(file_name)
|> Enum.to_list()
|> IO.iodata_to_binary()

# Strip the root dir from the file name
name =
file_name
|> Path.split()
|> Enum.drop(1)
|> Enum.join("/")

{name, content}
end)

{term, ""} =
Unpickler.load!(Map.fetch!(contents, "data.pkl"),
object_resolver: &object_resolver/1,
persistent_id_resolver: fn
{"storage", storage_type, key, _location, _size} ->
binary = Map.fetch!(contents, "data/#{key}")
{:storage, storage_type, binary}
end
)

term
try do
{:ok, unzip} = Unzip.new(zip_file)

contents =
unzip
|> Unzip.list_entries()
|> Map.new(fn %Unzip.Entry{file_name: file_name} ->
content =
unzip
|> Unzip.file_stream!(file_name)
|> Enum.to_list()
|> IO.iodata_to_binary()

# Strip the root dir from the file name
name =
file_name
|> Path.split()
|> Enum.drop(1)
|> Enum.join("/")

{name, content}
end)

{term, ""} =
Unpickler.load!(Map.fetch!(contents, "data.pkl"),
object_resolver: &object_resolver/1,
persistent_id_resolver: fn
{"storage", storage_type, key, _location, _size} ->
binary = Map.fetch!(contents, "data/#{key}")
{:storage, storage_type, binary}
end
)

term
after
Unzip.LocalFile.close(zip_file)
end
end

defp object_resolver(%{constructor: "collections.OrderedDict", set_items: items}) do
Expand Down

0 comments on commit a72c5dd

Please sign in to comment.