Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Attempt to improve runtime #134

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@ step3:
python3 step3_generative_sheet_to_output/build.py

all:
make step1
make step2
make step3
python3 all.py

solana:
cd step2_spritesheet_to_generative_sheet && npm run generate:solana && cd ..
Expand All @@ -37,7 +35,7 @@ update_json_tezos:
cd step2_spritesheet_to_generative_sheet && npm run update_info:tezos && cd ..

all_batch:
python3 batch.py
make all

replace:
cd step2_spritesheet_to_generative_sheet && npm run replace ../ultraRares && cd ..
Expand Down
71 changes: 71 additions & 0 deletions all.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
from step1_layers_to_spritesheet.build import main as step1_main
from step3_generative_sheet_to_output.build import main as step3_main
import subprocess
from utils.file import parse_global_config
import multiprocessing
import time

global_config_json = parse_global_config()
num_total_frames = global_config_json["numberOfFrames"]
use_batching = global_config_json["useBatches"]
num_frames_per_batch = global_config_json["numFramesPerBatch"] if use_batching else num_total_frames
total_supply = global_config_json["totalSupply"]
use_multiprocessing = global_config_json["useMultiprocessing"]
processor_count = global_config_json["processorCount"]
start_index = global_config_json["startIndex"]
height = global_config_json["height"]
width = global_config_json["width"]


def create_from_dna(edition):
subprocess.run(
f"cd step2_spritesheet_to_generative_sheet && npm run create_from_dna {edition}",
shell=True,
)


def create_all_from_dna():
if use_multiprocessing:
if processor_count > multiprocessing.cpu_count():
raise Exception(
f"You are trying to use too many processors, you passed in {processor_count} "
f"but your computer can only handle {multiprocessing.cpu_count()}. Change this value and run make step3 again."
)

args = [
(edition,) for edition in range(start_index, start_index + total_supply)
]
with multiprocessing.Pool(processor_count) as pool:
pool.starmap(
create_from_dna,
args,
)
else:
# Then recreate DNA from the editions
for edition in range(start_index, start_index + total_supply):
create_from_dna(edition)


def main():
start_time = time.time()

# run step 1 with one pixel dimensions to speed up JSON processing and generate hashes
step1_main(0, height=1, width=1)
subprocess.run(
f"cd step2_spritesheet_to_generative_sheet && npm run generate 10 10 && cd ..",
shell=True,
)
for i in range(num_total_frames // num_frames_per_batch):
print(f"*******Starting Batch {i}*******")
step1_main(i)
create_all_from_dna()
# Only generate gif if its the last batch
step3_main(
i,
should_generate_output=i == (num_total_frames // num_frames_per_batch - 1),
)
print("--- %s seconds ---" % (time.time() - start_time))


if __name__ == "__main__":
main()
56 changes: 28 additions & 28 deletions global_config.json
Original file line number Diff line number Diff line change
@@ -1,29 +1,29 @@
{
"totalSupply": 4,
"height": 1000,
"width": 1000,
"framesPerSecond": 12,
"numberOfFrames": 12,
"saveIndividualFrames": false,
"description": "Your description here",
"baseUri": "ipfs://NewUriToReplace",
"startIndex": 0,
"quality": 100,
"layersFolder": "layers",
"gifTool": "gifski",
"mp4Tool": "ffmpeg",
"outputType": "gif",
"loopGif": true,
"numLoopMP4": 0,
"debug": false,
"useBatches": false,
"numFramesPerBatch": 4,
"useMultiprocessing": false,
"processorCount": 1,
"useFileNumbering": true,
"enableAudio": false,
"generateThumbnail": false,
"thumbnailHeight": 350,
"thumbnailWidth": 350,
"thumbnailUri": "ipfs://ThumbnailUriToReplace"
}
"totalSupply": 5000,
"height": 1000,
"width": 1000,
"framesPerSecond": 12,
"numberOfFrames": 12,
"saveIndividualFrames": false,
"description": "Your description here",
"baseUri": "ipfs://NewUriToReplace",
"startIndex": 0,
"quality": 100,
"layersFolder": "layers",
"gifTool": "gifski",
"mp4Tool": "ffmpeg",
"outputType": "gif",
"loopGif": true,
"numLoopMP4": 0,
"debug": false,
"useBatches": false,
"numFramesPerBatch": 4,
"useMultiprocessing": true,
"processorCount": 8,
"useFileNumbering": true,
"enableAudio": false,
"generateThumbnail": false,
"thumbnailHeight": 350,
"thumbnailWidth": 350,
"thumbnailUri": "ipfs://ThumbnailUriToReplace"
}
Binary file added layers/Background/Dark Bluecopy#20/0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Greencopy#20/0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Greycopy#20/0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Light Bluecopy#20/0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/1.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/10.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/11.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/2.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/3.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/4.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/5.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/6.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/7.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/8.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Multicolorcopy#20/9.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Orangecopy#20/0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Pinkcopy#20/0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Background/Yellowcopy#20/0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Hat/Birthdaycopy#20/1.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Hat/Birthdaycopy#20/10.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Hat/Birthdaycopy#20/11.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Hat/Birthdaycopy#20/12.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Hat/Birthdaycopy#20/2.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Hat/Birthdaycopy#20/3.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added layers/Hat/Birthdaycopy#20/4.png
Binary file added layers/Hat/Birthdaycopy#20/5.png
Binary file added layers/Hat/Birthdaycopy#20/6.png
Binary file added layers/Hat/Birthdaycopy#20/7.png
Binary file added layers/Hat/Birthdaycopy#20/8.png
Binary file added layers/Hat/Birthdaycopy#20/9.png
Binary file added layers/Hat/Cowboycopy#20/1.png
Binary file added layers/Hat/Cowboycopy#20/10.png
Binary file added layers/Hat/Cowboycopy#20/11.png
Binary file added layers/Hat/Cowboycopy#20/12.png
Binary file added layers/Hat/Cowboycopy#20/2.png
Binary file added layers/Hat/Cowboycopy#20/3.png
Binary file added layers/Hat/Cowboycopy#20/4.png
Binary file added layers/Hat/Cowboycopy#20/5.png
Binary file added layers/Hat/Cowboycopy#20/6.png
Binary file added layers/Hat/Cowboycopy#20/7.png
Binary file added layers/Hat/Cowboycopy#20/8.png
Binary file added layers/Hat/Cowboycopy#20/9.png
Binary file added layers/Hat/Green Squarecopy#20/1.png
Binary file added layers/Hat/Green Squarecopy#20/10.png
Binary file added layers/Hat/Green Squarecopy#20/11.png
Binary file added layers/Hat/Green Squarecopy#20/12.png
Binary file added layers/Hat/Green Squarecopy#20/2.png
Binary file added layers/Hat/Green Squarecopy#20/3.png
Binary file added layers/Hat/Green Squarecopy#20/4.png
Binary file added layers/Hat/Green Squarecopy#20/5.png
Binary file added layers/Hat/Green Squarecopy#20/6.png
Binary file added layers/Hat/Green Squarecopy#20/7.png
Binary file added layers/Hat/Green Squarecopy#20/8.png
Binary file added layers/Hat/Green Squarecopy#20/9.png
Binary file added layers/Hat/Mini Ballcopy#20/1.png
Binary file added layers/Hat/Mini Ballcopy#20/10.png
Binary file added layers/Hat/Mini Ballcopy#20/11.png
Binary file added layers/Hat/Mini Ballcopy#20/12.png
Binary file added layers/Hat/Mini Ballcopy#20/2.png
Binary file added layers/Hat/Mini Ballcopy#20/3.png
Binary file added layers/Hat/Mini Ballcopy#20/4.png
Binary file added layers/Hat/Mini Ballcopy#20/5.png
Binary file added layers/Hat/Mini Ballcopy#20/6.png
Binary file added layers/Hat/Mini Ballcopy#20/7.png
Binary file added layers/Hat/Mini Ballcopy#20/8.png
Binary file added layers/Hat/Mini Ballcopy#20/9.png
Binary file added layers/Hat/Nonecopy#20/0.png
Binary file added layers/Hat/Top Hatcopy#20/1.png
Binary file added layers/Hat/Top Hatcopy#20/10.png
Binary file added layers/Hat/Top Hatcopy#20/11.png
Binary file added layers/Hat/Top Hatcopy#20/12.png
Binary file added layers/Hat/Top Hatcopy#20/2.png
Binary file added layers/Hat/Top Hatcopy#20/3.png
Binary file added layers/Hat/Top Hatcopy#20/4.png
Binary file added layers/Hat/Top Hatcopy#20/5.png
Binary file added layers/Hat/Top Hatcopy#20/6.png
Binary file added layers/Hat/Top Hatcopy#20/7.png
Binary file added layers/Hat/Top Hatcopy#20/8.png
Binary file added layers/Hat/Top Hatcopy#20/9.png
Binary file added layers/Hat/White Capcopy#20/0.png
Binary file added layers/Hat/White Capcopy#20/1.png
Binary file added layers/Hat/White Capcopy#20/10.png
Binary file added layers/Hat/White Capcopy#20/11.png
Binary file added layers/Hat/White Capcopy#20/2.png
Binary file added layers/Hat/White Capcopy#20/3.png
Binary file added layers/Hat/White Capcopy#20/4.png
Binary file added layers/Hat/White Capcopy#20/5.png
Binary file added layers/Hat/White Capcopy#20/6.png
Binary file added layers/Hat/White Capcopy#20/7.png
Binary file added layers/Hat/White Capcopy#20/8.png
Binary file added layers/Hat/White Capcopy#20/9.png
Binary file added layers/Hat/Wizardcopy#20/1.png
Binary file added layers/Hat/Wizardcopy#20/10.png
Binary file added layers/Hat/Wizardcopy#20/11.png
Binary file added layers/Hat/Wizardcopy#20/12.png
Binary file added layers/Hat/Wizardcopy#20/2.png
Binary file added layers/Hat/Wizardcopy#20/3.png
Binary file added layers/Hat/Wizardcopy#20/4.png
Binary file added layers/Hat/Wizardcopy#20/5.png
Binary file added layers/Hat/Wizardcopy#20/6.png
Binary file added layers/Hat/Wizardcopy#20/7.png
Binary file added layers/Hat/Wizardcopy#20/8.png
Binary file added layers/Hat/Wizardcopy#20/9.png
25 changes: 16 additions & 9 deletions step1_layers_to_spritesheet/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
num_total_frames = global_config_json["numberOfFrames"]
num_batch_frames = global_config_json["numFramesPerBatch"]
use_batches = global_config_json["useBatches"]
width = global_config_json["width"]
height = global_config_json["height"]
use_multiprocessing = global_config_json["useMultiprocessing"]
processor_count = global_config_json["processorCount"]

Expand All @@ -27,7 +25,7 @@
OUTPUT_DIRECTORY = "./step1_layers_to_spritesheet/output"


def combine_images(images: List[Image], batch_number: int) -> Image:
def combine_images(images: List[Image], batch_number: int, height: int, width: int) -> Image:
"""
Combines images horizontally in a new image. This assumes
all images are the same size.
Expand Down Expand Up @@ -79,7 +77,8 @@ def duplicate_images_number_of_frames_times(images: List[Image], num_total_frame


def parse_attributes_into_images(
attribute_folder: str, attribute_path: str, output_path: bool, batch_number: int
attribute_folder: str, attribute_path: str, output_path: bool, batch_number: int,
height: int, width: int
) -> Tuple[List[Image], bool]:
"""
Mutual recursive function that parses the attributes
Expand Down Expand Up @@ -108,7 +107,7 @@ def parse_attributes_into_images(
output_attribute_path = os.path.join(output_path, attribute_folder)
setup_directory(output_attribute_path, delete_if_exists=False)
parse_attribute_folders(
filename, file_path, output_attribute_path, batch_number
filename, file_path, output_attribute_path, batch_number, height, width
)

if len(images) == 0:
Expand All @@ -124,6 +123,8 @@ def parse_attribute_folders(
attribute_path: str,
output_path: str,
batch_number: int,
height: int,
width: int,
) -> None:
"""
Mutually recursive function that parses attribute folders by
Expand All @@ -144,11 +145,13 @@ def parse_attribute_folders(
attribute_path,
output_path=output_path,
batch_number=batch_number,
height=height,
width=width,
)
if len(images) == 0:
return

spritesheet = combine_images(images, batch_number)
spritesheet = combine_images(images, batch_number, height, width)
# If it contains subfolder, that means there is if-then logic and we need to
# place the file in the subfolder
if containsSubFolder:
Expand Down Expand Up @@ -207,7 +210,7 @@ def parse_gifs_into_temp_directory(directory: str, output_directory: str) -> Non
parse_gifs_into_temp_directory(file_path, output_path)


def process_layer_folder(layers_directory, layer_folder, batch_number):
def process_layer_folder(layers_directory, layer_folder, batch_number, height, width):
layer_path = os.path.join(layers_directory, layer_folder)
# hidden files should be ignored
if layer_folder.startswith("."):
Expand All @@ -225,10 +228,12 @@ def process_layer_folder(layers_directory, layer_folder, batch_number):
attribute_path,
output_layer_path,
batch_number,
height,
width
)


def main(batch_number=0):
def main(batch_number=0, height=global_config_json["height"], width=global_config_json["width"]):
print("********Starting step 1: Converting pngs to spritesheets********")

setup_directory(OUTPUT_DIRECTORY)
Expand Down Expand Up @@ -256,6 +261,8 @@ def main(batch_number=0):
layers_directory,
layer_folder,
batch_number,
height,
width
)
for layer_folder in os.listdir(layers_directory)
]
Expand All @@ -266,7 +273,7 @@ def main(batch_number=0):
)
else:
for layer_folder in os.listdir(layers_directory):
process_layer_folder(layers_directory, layer_folder, batch_number)
process_layer_folder(layers_directory, layer_folder, batch_number, height, width)


if __name__ == "__main__":
Expand Down
15 changes: 10 additions & 5 deletions step2_spritesheet_to_generative_sheet/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ const { startCreating, buildSetup } = require(path.join(

program
.name("generate")

.option("-c, --continue <dna>", "Continues generation using a _dna.json file")
.option("--continue <dna>", "Continues generation using a _dna.json file")
.option("--height <height>", "Override height")
.option("--width <width>", "Override width")
.action((options) => {
console.log(chalk.green("generator started"), options.continue);
options.continue
? console.log(
chalk.bgCyanBright("\n continuing generation using _dna.json file \n")
Expand All @@ -32,8 +32,13 @@ program
dna = new Set(storedGenomes);
console.log({ dna });
}
const args = program.args;
if (args.length == 2) {
startCreating(dna, parseInt(args[0]), parseInt(args[1]));
} else {
startCreating(dna, null, null);
}

startCreating(dna);
});

program.parse();
program.parse(process.argv);
27 changes: 14 additions & 13 deletions step2_spritesheet_to_generative_sheet/src/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,6 @@ const {
useRootTraitType,
} = require(path.join(basePath, "/src/config.js"));

const canvas = createCanvas(format.width, format.height);
const ctxMain = canvas.getContext("2d");
ctxMain.imageSmoothingEnabled = format.smoothing;

let dnaSet = new Set();
const DNA_DELIMITER = "*";

Expand Down Expand Up @@ -267,7 +263,7 @@ const layersSetup = (layersOrder) => {
return layers;
};

const saveImage = (_editionCount) => {
const saveImage = (_editionCount, canvas) => {
fs.writeFileSync(
`${outputDir}/images/${_editionCount}.png`,
canvas.toBuffer("image/png")
Expand All @@ -280,7 +276,7 @@ const genColor = () => {
return pastel;
};

const drawBackground = (canvasContext) => {
const drawBackground = (canvasContext, height, width) => {
canvasContext.fillStyle = genColor();
canvasContext.fillRect(0, 0, format.width, format.height);
};
Expand Down Expand Up @@ -706,7 +702,7 @@ function shuffle(array) {
* @param {Object} layerData data passed from the current iteration of the loop or configured dna-set
*
*/
const paintLayers = (canvasContext, renderObjectArray, layerData, attributesList) => {
const paintLayers = (canvasContext, renderObjectArray, layerData, attributesList, height, width) => {
debugLogs ? console.log("\nClearing canvas") : null;
canvasContext.clearRect(0, 0, format.width, format.height);

Expand All @@ -729,7 +725,7 @@ const paintLayers = (canvasContext, renderObjectArray, layerData, attributesList

if (_background.generate) {
canvasContext.globalCompositeOperation = "destination-over";
drawBackground(canvasContext);
drawBackground(canvasContext, height, width);
}
debugLogs
? console.log("Editions left to create: ", abstractedIndexes)
Expand All @@ -756,10 +752,10 @@ const postProcessMetadata = (layerData) => {
};
};

const outputFiles = (abstractedIndexes, layerData, metadataList, attributesList) => {
const outputFiles = (abstractedIndexes, layerData, metadataList, attributesList, canvas) => {
const { newDna, _ } = layerData;
// Save the canvas buffer to file
saveImage(abstractedIndexes[0]);
saveImage(abstractedIndexes[0], canvas);

const { _prefix, _offset } = postProcessMetadata(layerData);

Expand All @@ -778,7 +774,12 @@ const outputFiles = (abstractedIndexes, layerData, metadataList, attributesList)
);
};

const startCreating = async (storedDNA) => {
const startCreating = async (storedDNA, overrideHeight = null, overrideWidth = null) => {
const height = overrideHeight || format.height;
const width = overrideWidth || format.width
const canvas = createCanvas(width, height);
const ctxMain = canvas.getContext("2d");
ctxMain.imageSmoothingEnabled = format.smoothing;
let dnaList = [];
if (storedDNA) {
dnaSet = storedDNA;
Expand Down Expand Up @@ -833,8 +834,8 @@ const startCreating = async (storedDNA) => {
abstractedIndexes,
_background: background,
};
paintLayers(ctxMain, renderObjectArray, layerData, attributesList);
outputFiles(abstractedIndexes, layerData, metadataList, attributesList);
paintLayers(ctxMain, renderObjectArray, layerData, attributesList, height, width);
outputFiles(abstractedIndexes, layerData, metadataList, attributesList, canvas);
});
const filteredDna = filterDNAOptions(newDna);
dnaSet.add(filteredDna);
Expand Down