Skip to content

Commit

Permalink
Bytecode DSL interpreter migration
Browse files Browse the repository at this point in the history
  • Loading branch information
DSouzaM committed Sep 18, 2024
1 parent b907353 commit 09e02b1
Show file tree
Hide file tree
Showing 93 changed files with 12,262 additions and 1,119 deletions.
2 changes: 1 addition & 1 deletion ci.jsonnet
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "overlay": "2c97f9a352b2773941722f15612bd5c531f63489" }
{ "overlay": "410373502193976c276eb0c68eafd8d9d67fe54b" }
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,9 @@ def run_benchmark(args):
else:
print("### no extra module search paths specified")

if GRAALPYTHON:
print(f"### using bytecode DSL interpreter: {__graalpython__.is_bytecode_dsl_interpreter}")

BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup, warmup_runs=warmup_runs, startup=startup, live_results=live_results).run()


Expand Down
24 changes: 15 additions & 9 deletions graalpython/com.oracle.graal.python.frozen/freeze_modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ def relpath_for_posix_display(path, base):
#######################################
# specs

def parse_frozen_specs():
def parse_frozen_specs(suffix):
seen = {}
for section, specs in FROZEN:
parsed = _parse_specs(specs, section, seen)
Expand All @@ -163,7 +163,7 @@ def parse_frozen_specs():
try:
source = seen[frozenid]
except KeyError:
source = FrozenSource.from_id(frozenid, pyfile)
source = FrozenSource.from_id(frozenid, suffix, pyfile)
seen[frozenid] = source
else:
assert not pyfile or pyfile == source.pyfile, item
Expand Down Expand Up @@ -271,11 +271,11 @@ def iter_subs():
class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')):

@classmethod
def from_id(cls, frozenid, pyfile=None):
def from_id(cls, frozenid, suffix, pyfile=None):
if not pyfile:
pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
#assert os.path.exists(pyfile), (frozenid, pyfile)
frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)
frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR, suffix)
return cls(frozenid, pyfile, frozenfile, STDLIB_DIR)

@classmethod
Expand Down Expand Up @@ -311,7 +311,7 @@ def isbootstrap(self):
return self.id in BOOTSTRAP


def resolve_frozen_file(frozenid, destdir):
def resolve_frozen_file(frozenid, destdir, suffix):
"""Return the filename corresponding to the given frozen ID.
For stdlib modules the ID will always be the full name
Expand All @@ -324,7 +324,7 @@ def resolve_frozen_file(frozenid, destdir):
raise ValueError(f'unsupported frozenid {frozenid!r}')
# We use a consistent naming convention for all frozen modules.
frozen_symbol = FrozenSource.resolve_symbol(frozenid)
frozenfile = f"Frozen{frozen_symbol}.bin"
frozenfile = f"Frozen{frozen_symbol}.{suffix}"

if not destdir:
return frozenfile
Expand Down Expand Up @@ -634,11 +634,17 @@ def main():
STDLIB_DIR = os.path.abspath(parsed_args.python_lib)
FROZEN_MODULES_DIR = os.path.abspath(parsed_args.binary_dir)

if __graalpython__.is_bytecode_dsl_interpreter:
suffix = "bin_dsl"
assert os.path.isdir(parsed_args.binary_dir), "Frozen modules for the DSL should be built after the manual bytecode interpreter."
else:
suffix = "bin"
shutil.rmtree(parsed_args.binary_dir, ignore_errors=True)
os.makedirs(parsed_args.binary_dir)

# create module specs
modules = list(parse_frozen_specs())
modules = list(parse_frozen_specs(suffix))

shutil.rmtree(parsed_args.binary_dir, ignore_errors=True)
os.makedirs(parsed_args.binary_dir)
# write frozen module binary files containing the byte code and class files
# used for importing the binary files
for src in _iter_sources(modules):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -229,11 +229,15 @@ public boolean isNested() {
}

public HashMap<String, Integer> getSymbolsByType(EnumSet<DefUse> expectedFlags, int start) {
return getSymbolsByType(expectedFlags, EnumSet.noneOf(DefUse.class), start);
}

public HashMap<String, Integer> getSymbolsByType(EnumSet<DefUse> expectedFlags, EnumSet<DefUse> unexpectedFlags, int start) {
int i = start;
HashMap<String, Integer> mapping = new HashMap<>();
for (String key : getSortedSymbols()) {
EnumSet<DefUse> keyFlags = getUseOfName(key);
if (!Collections.disjoint(expectedFlags, keyFlags)) {
if (!Collections.disjoint(expectedFlags, keyFlags) && Collections.disjoint(unexpectedFlags, keyFlags)) {
mapping.put(key, i++);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ public class ScopeEnvironment {
final HashMap<SSTNode, Scope> blocks = new HashMap<>();
final ErrorCallback errorCallback;
final EnumSet<FutureFeature> futureFeatures;
final HashMap<Scope, Scope> parents = new HashMap<>();

public static ScopeEnvironment analyze(ModTy moduleNode, ErrorCallback errorCallback, EnumSet<FutureFeature> futureFeatures) {
return new ScopeEnvironment(moduleNode, errorCallback, futureFeatures);
Expand Down Expand Up @@ -128,6 +129,14 @@ public Scope lookupScope(SSTNode node) {
return blocks.get(node);
}

public Scope lookupParent(Scope scope) {
return parents.get(scope);
}

public Scope getTopScope() {
return topScope;
}

private void analyzeBlock(Scope scope, HashSet<String> bound, HashSet<String> free, HashSet<String> global) {
HashSet<String> local = new HashSet<>();
HashMap<String, DefUse> scopes = new HashMap<>();
Expand Down Expand Up @@ -328,6 +337,7 @@ private void enterBlock(String name, Scope.ScopeType type, SSTNode ast) {
if (type == Scope.ScopeType.Annotation) {
return;
}
env.parents.put(scope, prev);
if (prev != null) {
prev.children.add(scope);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public void desugared() {
}

@Test
public void testYieldFrom() {
public void testYieldFromSimple() {
String source = "def gen1():\n" +
" yield 1\n" +
" yield 2\n" +
Expand All @@ -83,4 +83,119 @@ public void testYieldFrom() {
"print(list(gen2()))\n";
assertPrints("[1, 2]\n", source);
}

@Test
public void testYieldFromIterable() {
// yield from should extract an iterator from a non-generator argument
String source = "class Foo:\n" +
" def __init__(self, wrapped):\n" +
" self.wrapped = wrapped\n" +
" def __iter__(self):\n" +
" return iter(self.wrapped)\n" +
"def gen():\n" +
" foo = Foo([1,2,3])\n" +
" yield from foo\n" +
"\n" +
"print(list(gen()))\n";
assertPrints("[1, 2, 3]\n", source);
}

@Test
public void testYieldFromReturn() {
String source = "def gen1():\n" +
" yield 1\n" +
" yield 2\n" +
" return 3\n" +
"\n" +
"def gen2():\n" +
" final = yield from gen1()\n" +
" yield final\n" +
"\n" +
"print(list(gen2()))\n";
assertPrints("[1, 2, 3]\n", source);
}

@Test
public void testYieldFromSend() {
String source = "def gen1(x):\n" +
" yield (yield (yield x))\n" +
"\n" +
"def gen2():\n" +
" yield from gen1(2)\n" +
" yield 8\n" +
"\n" +
"gen = gen2()\n" +
"print(gen.send(None))\n" +
"print(gen.send(4))\n" +
"print(gen.send(6))\n" +
"print(gen.send(42))\n";
assertPrints("2\n4\n6\n8\n", source);
}

@Test
public void testYieldFromThrowCaught() {
String source = "def gen1():\n" +
" try:\n" +
" x = 1\n" +
" while True:\n" +
" x = yield x\n" +
" except ValueError:\n" +
" yield 42\n" +
"\n" +
"def gen2():\n" +
" yield from gen1()\n" +
"\n" +
"gen = gen2()\n" +
"print(gen.send(None))\n" +
"print(gen.send(2))\n" +
"print(gen.send(3))\n" +
"print(gen.throw(ValueError))\n";
assertPrints("1\n2\n3\n42\n", source);
}

@Test
public void testYieldFromThrowUncaught() {
String source = "def gen1():\n" +
" x = 1\n" +
" while True:\n" +
" x = yield x\n" +
"\n" +
"def gen2():\n" +
" yield from gen1()\n" +
"\n" +
"gen = gen2()\n" +
"print(gen.send(None))\n" +
"print(gen.send(2))\n" +
"print(gen.send(3))\n" +
"try:\n" +
" gen.throw(ValueError)\n" +
" print('error')\n" +
"except ValueError:\n" +
" print('success')\n";
assertPrints("1\n2\n3\nsuccess\n", source);
}

@Test
public void testYieldFromClose() {
String source = "def gen1():\n" +
" x = 1\n" +
" try:\n" +
" while True:\n" +
" x = yield x\n" +
" except GeneratorExit:\n" +
" print('gen1 exit')\n" +
"\n" +
"def gen2():\n" +
" try:\n" +
" yield from gen1()\n" +
" except GeneratorExit:\n" +
" print('gen2 exit')\n" +
"\n" +
"gen = gen2()\n" +
"print(gen.send(None))\n" +
"print(gen.send(2))\n" +
"print(gen.send(3))\n" +
"gen.close()\n";
assertPrints("1\n2\n3\ngen1 exit\ngen2 exit\n", source);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package com.oracle.graal.python.test.integration.grammar;

import static com.oracle.graal.python.test.integration.PythonTests.assertPrints;

import org.junit.Test;

public class AsyncTests {
@Test
public void nativeCoroutine() {
String source = "import asyncio\n" +
"async def foo():\n" +
" return 42\n" +
"async def main():\n" +
" print(await foo())\n" +
"asyncio.run(main())";
assertPrints("42\n", source);
}

@Test
public void asyncWith() {
String source = "import asyncio\n" +
"class AsyncContextManager:\n" +
" async def __aenter__(self):\n" +
" await asyncio.sleep(0.01)\n" +
" print(\"entered\")\n" +
" async def __aexit__(self, exc_type, exc_value, traceback):\n" +
" await asyncio.sleep(0.01)\n" +
" if exc_type:\n" +
" print(\"exited exceptionally\")\n" +
" else:\n" +
" print(\"exited normally\")\n" +
" return True\n" +
"async def main(shouldRaise):\n" +
" async with AsyncContextManager():\n" +
" print(\"inside\")\n" +
" if shouldRaise:\n" +
" raise ValueError\n" +
"asyncio.run(main(%s))";
assertPrints("entered\ninside\nexited normally\n", String.format(source, "False"));
assertPrints("entered\ninside\nexited exceptionally\n", String.format(source, "True"));
}

@Test
public void asyncWithExceptional() {
String source = "import asyncio\n" +
"class AsyncContextManager:\n" +
" async def __aenter__(self):\n" +
" await asyncio.sleep(0.01)\n" +
" print(\"entered\")\n" +
" async def __aexit__(self, exc_type, exc_value, traceback):\n" +
" await asyncio.sleep(0.01)\n" +
" print(\"exited\")\n" +
" return False\n" + // don't handle exception
"async def main(shouldRaise):\n" +
" async with AsyncContextManager():\n" +
" print(\"inside\")\n" +
" if shouldRaise:\n" +
" raise ValueError\n" +
"try:\n" +
" asyncio.run(main(%s))\n" +
"except ValueError:\n" +
" print(\"rethrew\")\n";
assertPrints("entered\ninside\nexited\n", String.format(source, "False"));
assertPrints("entered\ninside\nexited\nrethrew\n", String.format(source, "True"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -205,4 +205,21 @@ public void multipleInheritance() {
assertPrints("common\n", source);
}

@Test
public void classDecorator() {
String source = "def wrapper(cls):\n" + //
" orig_init = cls.__init__\n" + //
" def new_init(self):\n" + //
" print('wrapper')\n" + //
" orig_init(self)\n" + //
" cls.__init__ = new_init\n" + //
" return cls\n" + //
"@wrapper\n" + //
"class Foo:\n" + //
" def __init__(self):\n" + //
" print('Foo')\n" + //
"Foo()\n";
assertPrints("wrapper\nFoo\n", source);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -290,4 +290,49 @@ public void testExceptionState6() {
"print(repr(sys.exc_info()[1]))\n";
assertPrints("None\nNone\n", source);
}

@Test
public void testNamedExceptionDeleted() {
String source = "ex = 42\n" +
"try:\n" +
" raise NameError\n" +
"except BaseException as ex:\n" +
" pass\n" +
"try:\n" +
" print(ex)\n" +
" print(\"expected NameError\")\n" +
"except NameError:\n" +
" print(\"hit NameError\")\n";
assertPrints("hit NameError\n", source);
}

@Test
public void testNamedExceptionNotDeleted() {
String source = "ex = 42\n" +
"try:\n" +
" print(\"nothing thrown\")\n" +
"except BaseException as ex:\n" +
" pass\n" +
"try:\n" +
" print(ex)\n" +
"except NameError:\n" +
" print(\"hit unexpected NameError\")\n";
assertPrints("nothing thrown\n42\n", source);
}

@Test
public void testNamedExceptionDeletedByHandler() {
String source = "ex = 42\n" +
"try:\n" +
" raise NameError\n" +
"except BaseException as ex:\n" +
" print(\"deleting exception\")\n" +
" del ex\n" +
"try:\n" +
" print(ex)\n" +
" print(\"expected NameError\")\n" +
"except NameError:\n" +
" print(\"hit NameError\")\n";
assertPrints("deleting exception\nhit NameError\n", source);
}
}
Loading

0 comments on commit 09e02b1

Please sign in to comment.