From 01bacceb326d02adfa95a793711cb60470d82efb Mon Sep 17 00:00:00 2001 From: Luke Bemish Date: Wed, 6 Nov 2024 19:53:09 -0600 Subject: [PATCH] Give content-ordered lists consistent ordering --- .../taskgraphrunner/runtime/TaskInput.java | 22 ++++++++++++++----- .../taskgraphrunner/model/Input.java | 4 ++-- .../taskgraphrunner/model/InputAdapter.java | 12 +++++----- ...ntsHashStrategy.java => ListOrdering.java} | 2 +- .../taskgraphrunner/model/Value.java | 4 ++-- .../taskgraphrunner/model/ValueAdapter.java | 12 +++++----- 6 files changed, 33 insertions(+), 23 deletions(-) rename src/model/java/dev/lukebemish/taskgraphrunner/model/{ListContentsHashStrategy.java => ListOrdering.java} (66%) diff --git a/src/main/java/dev/lukebemish/taskgraphrunner/runtime/TaskInput.java b/src/main/java/dev/lukebemish/taskgraphrunner/runtime/TaskInput.java index 6aea392..37daa34 100644 --- a/src/main/java/dev/lukebemish/taskgraphrunner/runtime/TaskInput.java +++ b/src/main/java/dev/lukebemish/taskgraphrunner/runtime/TaskInput.java @@ -3,9 +3,9 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import dev.lukebemish.taskgraphrunner.model.ListContentsHashStrategy; import dev.lukebemish.taskgraphrunner.model.Input; import dev.lukebemish.taskgraphrunner.model.InputValue; +import dev.lukebemish.taskgraphrunner.model.ListOrdering; import dev.lukebemish.taskgraphrunner.model.PathSensitivity; import dev.lukebemish.taskgraphrunner.model.Value; import dev.lukebemish.taskgraphrunner.model.WorkItem; @@ -252,7 +252,7 @@ public JsonElement recordedValue(Context context) { } } - record SimpleFileListInput(String name, List inputs, ListContentsHashStrategy listContentsHashStrategy) implements FileListInput { + record SimpleFileListInput(String name, List inputs, ListOrdering listOrdering) implements FileListInput { @Override public void hashReference(ByteConsumer digest, Context context) { ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); @@ -268,7 +268,7 @@ public void hashContents(ByteConsumer digest, Context context) { ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); buffer.putInt(inputs.size()); digest.update(buffer); - switch (listContentsHashStrategy) { + switch (listOrdering) { case ORIGINAL -> { for (HasFileInput input : inputs) { input.hashContents(digest, context); @@ -300,7 +300,17 @@ public JsonElement recordedValue(Context context) { @Override public List paths(Context context) { - return inputs.stream().map(input -> input.path(context)).toList(); + var stream = inputs.stream().map(input -> input.path(context)); + if (listOrdering == ListOrdering.CONTENTS) { + stream = stream.sorted((a, b) -> { + var aOutput = new ByteArrayOutputStream(); + HashUtils.hash(a, ByteConsumer.of(aOutput)); + var bOutput = new ByteArrayOutputStream(); + HashUtils.hash(b, ByteConsumer.of(bOutput)); + return Arrays.compare(aOutput.toByteArray(), bOutput.toByteArray()); + }); + } + return stream.toList(); } @Override @@ -379,7 +389,7 @@ static FileListInput files(String name, Input modelInput, WorkItem workItem, Con for (int i = 0; i < listInput.inputs().size(); i++) { fileInputs.add(file(name + "_" + i, listInput.inputs().get(i), workItem, context, pathSensitivity)); } - yield new SimpleFileListInput(name, fileInputs, listInput.listContentsHashStrategy()); + yield new SimpleFileListInput(name, fileInputs, listInput.listOrdering()); } }; } @@ -400,7 +410,7 @@ private static FileListInput fileListFromValue(String name, Context context, Pat throw new IllegalArgumentException("Array parameter `"+ parameterInput.parameter()+"` contains non-string value at index "+i); } } - return new SimpleFileListInput(name, inputs, listValue.listContentsHashStrategy()); + return new SimpleFileListInput(name, inputs, listValue.listOrdering()); } else if (value != null) { if (parameterInput == null) { throw new IllegalArgumentException("Value is not a string or list of strings"); diff --git a/src/model/java/dev/lukebemish/taskgraphrunner/model/Input.java b/src/model/java/dev/lukebemish/taskgraphrunner/model/Input.java index eebfadc..bed73be 100644 --- a/src/model/java/dev/lukebemish/taskgraphrunner/model/Input.java +++ b/src/model/java/dev/lukebemish/taskgraphrunner/model/Input.java @@ -16,9 +16,9 @@ record ParameterInput(String parameter) implements Input {} record DirectInput(Value value) implements Input {} @JsonAdapter(InputAdapter.class) - record ListInput(List inputs, ListContentsHashStrategy listContentsHashStrategy) implements Input { + record ListInput(List inputs, ListOrdering listOrdering) implements Input { public ListInput(List inputs) { - this(inputs, ListContentsHashStrategy.ORIGINAL); + this(inputs, ListOrdering.ORIGINAL); } } } diff --git a/src/model/java/dev/lukebemish/taskgraphrunner/model/InputAdapter.java b/src/model/java/dev/lukebemish/taskgraphrunner/model/InputAdapter.java index 4e7326f..9bc514e 100644 --- a/src/model/java/dev/lukebemish/taskgraphrunner/model/InputAdapter.java +++ b/src/model/java/dev/lukebemish/taskgraphrunner/model/InputAdapter.java @@ -31,7 +31,7 @@ public void write(JsonWriter out, Input value) throws IOException { case Input.TaskInput taskInput -> out.value("task." + taskInput.output().taskName() + "." + taskInput.output().name()); case Input.ListInput listInput -> { - if (listInput.listContentsHashStrategy() == ListContentsHashStrategy.ORIGINAL) { + if (listInput.listOrdering() == ListOrdering.ORIGINAL) { out.beginArray(); for (var i : listInput.inputs()) { write(out, i); @@ -47,8 +47,8 @@ public void write(JsonWriter out, Input value) throws IOException { write(out, i); } out.endArray(); - out.name("listContentsHashStrategy"); - GSON.getAdapter(ListContentsHashStrategy.class).write(out, listInput.listContentsHashStrategy()); + out.name("listOrdering"); + GSON.getAdapter(ListOrdering.class).write(out, listInput.listOrdering()); out.endObject(); } } @@ -101,11 +101,11 @@ public Input read(JsonReader in) throws IOException { for (var element : value.getAsJsonArray()) { inputs.add(GSON.fromJson(element, Input.class)); } - if (!object.has("listContentsHashStrategy")) { + if (!object.has("listOrdering")) { yield new Input.ListInput(inputs); } - var listContentsHashStrategy = GSON.fromJson(object.get("listContentsHashStrategy"), ListContentsHashStrategy.class); - yield new Input.ListInput(inputs, listContentsHashStrategy); + var listOrdering = GSON.fromJson(object.get("listOrdering"), ListOrdering.class); + yield new Input.ListInput(inputs, listOrdering); } default -> throw new IllegalArgumentException("Invalid input type: " + type); }; diff --git a/src/model/java/dev/lukebemish/taskgraphrunner/model/ListContentsHashStrategy.java b/src/model/java/dev/lukebemish/taskgraphrunner/model/ListOrdering.java similarity index 66% rename from src/model/java/dev/lukebemish/taskgraphrunner/model/ListContentsHashStrategy.java rename to src/model/java/dev/lukebemish/taskgraphrunner/model/ListOrdering.java index ff6c18d..20743ea 100644 --- a/src/model/java/dev/lukebemish/taskgraphrunner/model/ListContentsHashStrategy.java +++ b/src/model/java/dev/lukebemish/taskgraphrunner/model/ListOrdering.java @@ -1,6 +1,6 @@ package dev.lukebemish.taskgraphrunner.model; -public enum ListContentsHashStrategy { +public enum ListOrdering { ORIGINAL, CONTENTS } diff --git a/src/model/java/dev/lukebemish/taskgraphrunner/model/Value.java b/src/model/java/dev/lukebemish/taskgraphrunner/model/Value.java index 1952efe..ab050b5 100644 --- a/src/model/java/dev/lukebemish/taskgraphrunner/model/Value.java +++ b/src/model/java/dev/lukebemish/taskgraphrunner/model/Value.java @@ -20,9 +20,9 @@ record StringValue(String value) implements Value {} record BooleanValue(Boolean value) implements Value {} @JsonAdapter(ValueAdapter.class) - record ListValue(List value, ListContentsHashStrategy listContentsHashStrategy) implements Value { + record ListValue(List value, ListOrdering listOrdering) implements Value { public ListValue(List value) { - this(value, ListContentsHashStrategy.ORIGINAL); + this(value, ListOrdering.ORIGINAL); } } diff --git a/src/model/java/dev/lukebemish/taskgraphrunner/model/ValueAdapter.java b/src/model/java/dev/lukebemish/taskgraphrunner/model/ValueAdapter.java index 43d7aa1..2378953 100644 --- a/src/model/java/dev/lukebemish/taskgraphrunner/model/ValueAdapter.java +++ b/src/model/java/dev/lukebemish/taskgraphrunner/model/ValueAdapter.java @@ -11,7 +11,7 @@ import java.util.Map; final class ValueAdapter extends GsonAdapter { - private static final String ORDER_BY_CONTENTS = "__taskgraphrunner.listContentsHashStrategy."+ ListContentsHashStrategy.CONTENTS.name()+"__"; + private static final String ORDER_BY_CONTENTS = "__taskgraphrunner.listOrdering."+ ListOrdering.CONTENTS.name()+"__"; @Override public void write(JsonWriter out, Value value) throws IOException { @@ -21,8 +21,8 @@ public void write(JsonWriter out, Value value) throws IOException { case Value.StringValue stringValue -> out.value(stringValue.value()); case Value.ListValue listValue -> { out.beginArray(); - if (listValue.listContentsHashStrategy() == ListContentsHashStrategy.CONTENTS) { - out.value(ORDER_BY_CONTENTS +listValue.listContentsHashStrategy().name()); + if (listValue.listOrdering() == ListOrdering.CONTENTS) { + out.value(ORDER_BY_CONTENTS +listValue.listOrdering().name()); } for (var v : listValue.value()) { write(out, v); @@ -49,11 +49,11 @@ public Value read(JsonReader in) throws IOException { case BEGIN_ARRAY -> { in.beginArray(); List list = new ArrayList<>(); - ListContentsHashStrategy listContentsHashStrategy = ListContentsHashStrategy.ORIGINAL; + ListOrdering listOrdering = ListOrdering.ORIGINAL; if (in.hasNext()) { var value = read(in); if (value instanceof Value.StringValue stringValue && stringValue.value().equals(ORDER_BY_CONTENTS)) { - listContentsHashStrategy = ListContentsHashStrategy.CONTENTS; + listOrdering = ListOrdering.CONTENTS; } else { list.add(value); } @@ -62,7 +62,7 @@ public Value read(JsonReader in) throws IOException { list.add(read(in)); } in.endArray(); - yield new Value.ListValue(list, listContentsHashStrategy); + yield new Value.ListValue(list, listOrdering); } case BEGIN_OBJECT -> { in.beginObject();