Skip to content

Commit

Permalink
Python: Misc codegen/ changes (#498)
Browse files Browse the repository at this point in the history
  • Loading branch information
lucasmcdonald3 authored Aug 2, 2024
1 parent 0609ad1 commit 4c32e93
Show file tree
Hide file tree
Showing 24 changed files with 401 additions and 220 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,6 @@ static void generateSetup(
PythonSettings settings,
GenerationContext context
) {
var dependencies = SymbolDependency.gatherDependencies(context.writerDelegator().getDependencies().stream());
writePyproject(settings, context.writerDelegator(), dependencies);
writeReadme(settings, context);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ protected void writeInitMethodParameterForOptionalMember(boolean isError, Member
}


private boolean isOptionalDefault(MemberShape member) {
protected boolean isOptionalDefault(MemberShape member) {
// If a member with a default value isn't required, it's optional.
// see: https://smithy.io/2.0/spec/type-refinement-traits.html#smithy-api-default-trait
var target = model.expectShape(member.getTarget());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
/**
* Renders unions.
*/
public final class UnionGenerator implements Runnable {
public class UnionGenerator implements Runnable {

private final Model model;
protected final Model model;
private final SymbolProvider symbolProvider;
private final PythonWriter writer;
protected final PythonWriter writer;
private final UnionShape shape;
private final Set<Shape> recursiveShapes;

Expand All @@ -51,6 +51,10 @@ public UnionGenerator(
this.recursiveShapes = recursiveShapes;
}

protected void writeInitMethodConstraintsChecksForMember(MemberShape member, String memberName) {
// Stub method that can be overridden by other codegens.
}

@Override
public void run() {
var parentName = symbolProvider.toSymbol(shape).getName();
Expand All @@ -70,6 +74,7 @@ public void run() {
writer.writeDocs(trait.getValue());
});
writer.openBlock("def __init__(self, value: $T):", "", targetSymbol, () -> {
writeInitMethodConstraintsChecksForMember(member, memberSymbol.getName());
writer.write("self.value = value");
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ classifiers = [
]
dependencies = [
"awscrt>=0.15,<1.0",
"aiohttp>=3.8.3,<3.9.0"
"aiohttp>=3.9.0"
]

[project.urls]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
awscrt>=0.15,<1.0
aiohttp>=3.8.3,<3.9.0
aiohttp>=3.9.0
3 changes: 3 additions & 0 deletions codegen/smithy-dafny-codegen/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ dependencies {
implementation("software.amazon.awssdk:codegen:2.20.26")
implementation("com.squareup:javapoet:1.13.0")

// Smithy-Python
implementation(project(":smithy-python-codegen"))

// Used for parsing-based tests
testImplementation("org.antlr:antlr4:4.9.2")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,7 @@ public Map<Path, TokenTree> generate() {
namespace
);
final TokenTree typesModuleHeader = Token.of(
"module {:extern \"%s\" } %s".formatted(
DafnyNameResolverHelpers.dafnyExternNamespaceForShapeId(
serviceShape.getId()
),
"module %s".formatted(
typesModuleName
)
);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
TODO-Python: Add more content here

Top-level file overview:

```
├── awssdk - Generates a boto3 wrapper to call from Dafny-generated Python code
├── common - Common code across generation targets
├── localservice - Generates a Smithy client that wraps a Dafny-generated Python localService implementation
└── wrappedlocalservice - Generates a wrapper for the `localservice` code to call the Smithy client from Dafny-generated Python code
```

Each subfolder follows a similar structure:

```
├── customize - Classes referenced from a plugin's `PythonIntegration.customize` function.
│ Generates new files or adds new code to Smithy-Python generated files.
├── extensions - Classes that extend or replace Smithy-Python codegen components.
├── nameresolver - Utility classes to map Smithy model shapes to strings used in generated code.
└── shapevisitor - Classes that generate code to convert to/from Smithy client Python shapes
│ (or AWS SDK shapes) and Dafny implementation shapes.
└── conversionwriter - Classes that generate functions that convert to/from Smithy client Python shapes
(or AWS SDK shapes) and Dafny implementation shapes for StructureShapes and UnionShapes.
```
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
* from the PythonClientCodegenPlugin by not calling `runner.performDefaultCodegenTransforms()` and
* `runner.createDedicatedInputsAndOutputs()`. These methods transform the model in ways such that the
* model does not align with the generated Dafny code. This Plugin also attaches a
* DafnyAwsSdkProtocolTrait to the ServiceShape provided in settings. AWS SDKs do not consistently
* {@link DafnyAwsSdkProtocolTrait} to the ServiceShape provided in settings. AWS SDKs do not consistently
* label a protocol, and Smithy-Python requires that a protocol is assigned. Rather than declare
* that we are using some protocol (e.g. `restJson1`) then not use that in practice, it is more
* proper to define some custom protocol and use that.
Expand Down Expand Up @@ -67,10 +67,10 @@ public String getName() {

@Override
public void execute(PluginContext context) {
CodegenDirector<PythonWriter, PythonIntegration, GenerationContext, PythonSettings> runner =
final CodegenDirector<PythonWriter, PythonIntegration, GenerationContext, PythonSettings> runner =
new CodegenDirector<>();

PythonSettings settings = PythonSettings.from(context.getSettings());
final PythonSettings settings = PythonSettings.from(context.getSettings());
settings.setProtocol(DafnyAwsSdkProtocolTrait.ID);
runner.settings(settings);
runner.directedCodegen(new DirectedDafnyPythonAwsSdkCodegen());
Expand All @@ -81,7 +81,7 @@ public void execute(PluginContext context) {

// Add a DafnyAwsSdkProtocolTrait to the service as a contextual indicator highlighting
// that the DafnyPythonAwsSdk protocol should be used.
ServiceShape serviceShape =
final ServiceShape serviceShape =
context.getModel().expectShape(settings.getService()).asServiceShape().get();
runner.model(addAwsSdkProtocolTrait(context.getModel(), serviceShape));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,17 @@
import software.amazon.smithy.model.shapes.ShapeId;
import software.amazon.smithy.model.shapes.StringShape;
import software.amazon.smithy.model.shapes.UnionShape;
import software.amazon.polymorph.traits.PositionalTrait;
import software.amazon.smithy.model.traits.EnumTrait;
import software.amazon.smithy.model.traits.ErrorTrait;
import software.amazon.smithy.python.codegen.GenerationContext;
import software.amazon.smithy.python.codegen.PythonWriter;
import software.amazon.smithy.utils.CaseUtils;

import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;

import static software.amazon.polymorph.smithydafny.DafnyNameResolver.dafnyTypesModuleName;

/**
Expand All @@ -27,6 +32,31 @@
*/
public class DafnyNameResolver {

// The source of truth here is Dafny's PythonCodeGenerator:
// https://github.com/dafny-lang/dafny/blob/709edd1b938afad604dfad62b86c884d6ec5a44b/Source/DafnyCore/Backends/Python/PythonCodeGenerator.cs#L63-L66
// If a word is in this set, Dafny will apply "mangling" to it by appending an `_`.
// Smithy-Dafny needs to know this set so it can also apply mangling and append an `_` to these words.
private static Set<String> dafnyReservedWords = new HashSet<>(Arrays.asList("False", "None", "True", "and", "as"
, "assert", "async", "await", "break", "class", "continue", "def", "del", "enum", "elif", "else", "except"
, "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass"
, "raise", "return", "try", "while", "with", "yield"));

/**
* "Mangle" any Dafny reserved words by adding an "_".
* This should be applied to all imports for types generated by Dafny.
* A lot of Smithy-Python code aliases imports for types generated by Dafny.
* ex. `import XXX_XXX as DafnyXXX`,
* so this only needs to apply to types imported directly from Dafny.
* @param name
* @return
*/
public static String mangleDafnyType(String name) {
if (dafnyReservedWords.contains(name)) {
return name + "_";
}
return name;
}

/**
* Returns the name of the Python module containing Dafny-generated Python code from the `types`
* module from the same Dafny project for the provided Shape. ex. example.namespace.ExampleShape
Expand Down Expand Up @@ -184,7 +214,9 @@ public static String getDafnyTypeForStringShapeWithEnumTrait(
"Argument is not a StringShape with EnumTrait: " + stringShape.getId());
}

return stringShape.getId().getName() + "_" + enumValue.replace("_", "__");
return mangleDafnyType(stringShape.getId().getName())
+ "_"
+ mangleDafnyType(enumValue.replace("_", "__"));
}

public static void importDafnyTypeForStringShapeWithEnumTrait(
Expand Down Expand Up @@ -234,14 +266,20 @@ else if (context.model().expectShape(shapeId).hasTrait(ErrorTrait.class)) {
importDafnyTypeForError(writer, shapeId, context);
}

else if (context.model().expectShape(shapeId).hasTrait(PositionalTrait.class)) {
// Don't import positional shapes; their underlying types are discovered and imported
return;
}

else {
// When generating a Dafny import, must ALWAYS first import module_ to avoid circular
// dependencies
writer.addStdlibImport(getDafnyGeneratedPathForSmithyNamespace(shapeId.getNamespace()) + ".module_");
String name = shapeId.getName();
writer.addStdlibImport(
getDafnyPythonTypesModuleNameForShape(shapeId, context),
name.replace("_", "__") + "_" + name.replace("_", "__"),
// Mangling for "normal" repeated types only appears to apply to the first name
mangleDafnyType(name.replace("_", "__")) + "_" + name.replace("_", "__"),
getDafnyTypeForShape(shapeId));
}
}
Expand Down Expand Up @@ -415,4 +453,5 @@ public static void importGenericDafnyErrorTypeForNamespace(
writer.addStdlibImport(getDafnyGeneratedPathForSmithyNamespace(namespace) + ".module_");
writer.addStdlibImport(getDafnyTypesModuleNameForSmithyNamespace(namespace, context), "Error");
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ public static String getPythonModuleSmithygeneratedPathForSmithyNamespace(
// In the case of a wrappedLocalService shim in a different namespace,
// the default modulename should not be used,
// and we need a mechanism to override the default modulename.
// If the smithy.api namespace has a dependency-module-name mapping, use that.
// If the smithy.api namespace has a dependency-library-name mapping, use that.
try {
pythonModuleName = getPythonModuleNameForSmithyNamespace(smithyNamespace);
} catch (IllegalArgumentException e) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
package software.amazon.polymorph.smithypython.localservice;

import software.amazon.polymorph.utils.ConstrainTraitUtils;
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.shapes.MemberShape;
import software.amazon.smithy.model.shapes.Shape;
import software.amazon.smithy.model.traits.LengthTrait;
import software.amazon.smithy.model.traits.RangeTrait;
import software.amazon.smithy.python.codegen.PythonWriter;

public class ConstraintUtils {
public static void writeInitMethodConstraintsChecksForMember(
PythonWriter writer, Model model, MemberShape member, String memberName) {
// RangeTrait
Shape targetShape = model.expectShape(member.getTarget());
if (targetShape.hasTrait(RangeTrait.class)) {
RangeTrait rangeTrait = targetShape.getTrait(RangeTrait.class).get();
if (rangeTrait.getMin().isPresent()) {
writeRangeTraitMinCheckForMember(writer, model, member, memberName, rangeTrait);
}
if (rangeTrait.getMax().isPresent()) {
writeRangeTraitMaxCheckForMember(writer, model, member, memberName, rangeTrait);
}
}

// LengthTrait
if (targetShape.hasTrait(LengthTrait.class)) {
LengthTrait lengthTrait = targetShape.getTrait(LengthTrait.class).get();
if (lengthTrait.getMin().isPresent()) {
writeLengthTraitMinCheckForMember(writer, memberName, lengthTrait);
}
if (lengthTrait.getMax().isPresent()) {
writeLengthTraitMaxCheckForMember(writer, memberName, lengthTrait);
}
}
}

/**
* Write validation for {@link LengthTrait} min value. Called from __init__.
*
* @param memberName
* @param lengthTrait
*/
protected static void writeLengthTraitMinCheckForMember(
PythonWriter writer, String memberName, LengthTrait lengthTrait) {
String min = ConstrainTraitUtils.LengthTraitUtils.min(lengthTrait);
writer.openBlock(
"if ($1L is not None) and (len($1L) < $2L):",
"",
memberName,
min,
() -> {
writer.write(
"""
raise ValueError("The size of $1L must be greater than or equal to $2L")
""",
memberName,
min);
});
}

/**
* Write validation for {@link LengthTrait} max value. Called from __init__.
*
* @param memberName
* @param lengthTrait
*/
protected static void writeLengthTraitMaxCheckForMember(
PythonWriter writer, String memberName, LengthTrait lengthTrait) {
String max = ConstrainTraitUtils.LengthTraitUtils.max(lengthTrait);
writer.openBlock(
"if ($1L is not None) and (len($1L) > $2L):",
"",
memberName,
max,
() -> {
writer.write(
"""
raise ValueError("The size of $1L must be less than or equal to $2L")
""",
memberName,
max);
});
}

/**
* Write validation for {@link RangeTrait} min value. Called from __init__.
*
* @param member
* @param memberName
* @param rangeTrait
*/
protected static void writeRangeTraitMinCheckForMember(
PythonWriter writer,
Model model,
MemberShape member,
String memberName,
RangeTrait rangeTrait) {
String min =
ConstrainTraitUtils.RangeTraitUtils.minAsShapeType(
model.expectShape(member.getTarget()), rangeTrait);
writer.openBlock(
"if ($1L is not None) and ($1L < $2L):",
"",
memberName,
min,
() -> {
writer.write(
"""
raise ValueError("$1L must be greater than or equal to $2L")
""",
memberName,
min);
});
}

/**
* Write validation for {@link RangeTrait} max value. Called from __init__.
*
* @param member
* @param memberName
* @param rangeTrait
*/
protected static void writeRangeTraitMaxCheckForMember(
PythonWriter writer,
Model model,
MemberShape member,
String memberName,
RangeTrait rangeTrait) {
String max =
ConstrainTraitUtils.RangeTraitUtils.maxAsShapeType(
model.expectShape(member.getTarget()), rangeTrait);
writer.openBlock(
"if ($1L is not None) and ($1L > $2L):",
"",
memberName,
max,
() -> {
writer.write(
"""
raise ValueError("$1L must be less than or equal to $2L")
""",
memberName,
max);
});
}
}
Loading

0 comments on commit 4c32e93

Please sign in to comment.