From 0a483b38c302ebc9fe424e542ec68412e559af16 Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Thu, 30 May 2019 14:44:34 -0600 Subject: [PATCH] Start replacing bash tests with D --- test/Makefile | 19 +- test/dshell/depsprot.d | 18 ++ .../extra-files/depsprot.d | 0 .../extra-files/mul9377a.d | 0 .../extra-files/mul9377b.d | 0 .../extra-files/multi9377.d | 0 .../extra-files/test_shared.d | 0 .../imports/depsprot_default.d | 0 .../imports/depsprot_private.d | 0 .../imports/depsprot_public.d | 0 test/dshell/test9377.d | 8 + test/dshell/test_shared.d | 14 + test/run.d | 77 ++++- test/runnable/depsprot.sh | 35 --- test/runnable/test9377.sh | 7 - test/runnable/test_shared.sh | 12 - test/tools/d_do_test.d | 132 +++++++- test/tools/dshell_prebuilt/dshell_prebuilt.d | 291 ++++++++++++++++++ 18 files changed, 537 insertions(+), 76 deletions(-) create mode 100644 test/dshell/depsprot.d rename test/{runnable => dshell}/extra-files/depsprot.d (100%) rename test/{runnable => dshell}/extra-files/mul9377a.d (100%) rename test/{runnable => dshell}/extra-files/mul9377b.d (100%) rename test/{runnable => dshell}/extra-files/multi9377.d (100%) rename test/{runnable => dshell}/extra-files/test_shared.d (100%) rename test/{runnable => dshell}/imports/depsprot_default.d (100%) rename test/{runnable => dshell}/imports/depsprot_private.d (100%) rename test/{runnable => dshell}/imports/depsprot_public.d (100%) create mode 100644 test/dshell/test9377.d create mode 100644 test/dshell/test_shared.d delete mode 100755 test/runnable/depsprot.sh delete mode 100755 test/runnable/test9377.sh delete mode 100755 test/runnable/test_shared.sh create mode 100644 test/tools/dshell_prebuilt/dshell_prebuilt.d diff --git a/test/Makefile b/test/Makefile index d2356ec31c0e..1d411a81096c 100644 --- a/test/Makefile +++ b/test/Makefile @@ -145,9 +145,12 @@ fail_compilation_tests=$(fail_compilation_tests_long) \ $(wildcard fail_compilation/*.html) fail_compilation_test_results=$(addsuffix .out,$(addprefix $(RESULTS_DIR)/,$(fail_compilation_tests))) +dshell_tests=$(wildcard dshell/*.d) +dshell_test_results=$(addsuffix .out,$(addprefix $(RESULTS_DIR)/,$(dshell_tests))) + all: run_tests -test_tools=$(RESULTS_DIR)/d_do_test$(EXE) $(RESULTS_DIR)/sanitize_json$(EXE) +test_tools=$(RESULTS_DIR)/d_do_test$(EXE) $(RESULTS_DIR)/dshell_prebuilt$(OBJ) $(RESULTS_DIR)/sanitize_json$(EXE) $(RESULTS_DIR)/%.out: % $(RESULTS_DIR)/.created $(test_tools) $(DMD) $(QUIET) $(RESULTS_DIR)/d_do_test $< @@ -165,9 +168,10 @@ $(RESULTS_DIR)/.created: $(QUIET)if [ ! -d $(RESULTS_DIR)/runnable ]; then mkdir $(RESULTS_DIR)/runnable; fi $(QUIET)if [ ! -d $(RESULTS_DIR)/compilable ]; then mkdir $(RESULTS_DIR)/compilable; fi $(QUIET)if [ ! -d $(RESULTS_DIR)/fail_compilation ]; then mkdir $(RESULTS_DIR)/fail_compilation; fi + $(QUIET)if [ ! -d $(RESULTS_DIR)/dshell ]; then mkdir $(RESULTS_DIR)/dshell; fi $(QUIET)touch $(RESULTS_DIR)/.created -run_tests: unit_tests start_runnable_tests start_compilable_tests start_fail_compilation_tests +run_tests: unit_tests start_runnable_tests start_compilable_tests start_fail_compilation_tests start_dshell_tests unit_tests: $(RESULTS_DIR)/unit_test_runner$(EXE) @echo "Running unit tests" @@ -191,7 +195,13 @@ start_fail_compilation_tests: $(RESULTS_DIR)/.created $(test_tools) @echo "Running fail compilation tests" $(QUIET)$(MAKE) $(DMD_TESTSUITE_MAKE_ARGS) --no-print-directory run_fail_compilation_tests -run_all_tests: unit_tests run_runnable_tests run_compilable_tests run_fail_compilation_tests +run_dshell_tests: $(dshell_test_results) + +start_dshell_tests: $(RESULTS_DIR)/.created $(test_tools) + @echo "Running dshell tests" + $(QUIET)$(MAKE) $(DMD_TESTSUITE_MAKE_ARGS) --no-print-directory run_dshell_tests + +run_all_tests: unit_tests run_runnable_tests run_compilable_tests run_fail_compilation_tests run_dshell_tests start_all_tests: $(RESULTS_DIR)/.created $(QUIET)$(MAKE) $(DMD_TESTSUITE_MAKE_ARGS) --no-print-directory $(test_tools) @@ -208,6 +218,9 @@ $(RESULTS_DIR)/d_do_test$(EXE): tools/d_do_test.d $(RESULTS_DIR)/.created $(DMD) -conf= $(MODEL_FLAG) $(DEBUG_FLAGS) -lowmem -od$(RESULTS_DIR) -of$(RESULTS_DIR)$(DSEP)d_do_test$(EXE) $< @wait $(pid) +$(RESULTS_DIR)/dshell_prebuilt$(OBJ): tools/dshell_prebuilt/dshell_prebuilt.d + $(DMD) -conf= $(MODEL_FLAG) -of$(RESULTS_DIR)/dshell_prebuilt$(OBJ) -c $< $(PIC_FLAG) + $(RESULTS_DIR)/sanitize_json$(EXE): tools/sanitize_json.d $(RESULTS_DIR)/.created @echo "Building sanitize_json tool" @echo "OS: '$(OS)'" diff --git a/test/dshell/depsprot.d b/test/dshell/depsprot.d new file mode 100644 index 000000000000..f32aef5c29f9 --- /dev/null +++ b/test/dshell/depsprot.d @@ -0,0 +1,18 @@ +import dshell; +void main() +{ + Vars.set("deps_file", "$OUTPUT_BASE/compile.deps"); + run("$DMD -m$MODEL -deps=$deps_file -Idshell/imports -o- $EXTRA_FILES/$TEST_NAME.d"); + Vars.deps_file + .grep("^$TEST_NAME.*${TEST_NAME}_default") + .grep("private") + .enforceMatches("Default import protection in dependency file should be 'private'"); + Vars.deps_file + .grep("^$TEST_NAME.*${TEST_NAME}_public") + .grep("public") + .enforceMatches("Public import protection in dependency file should be 'public'"); + Vars.deps_file + .grep("^$TEST_NAME.*${TEST_NAME}_private") + .grep("private") + .enforceMatches("Private import protection in dependency file should be 'private'"); +} diff --git a/test/runnable/extra-files/depsprot.d b/test/dshell/extra-files/depsprot.d similarity index 100% rename from test/runnable/extra-files/depsprot.d rename to test/dshell/extra-files/depsprot.d diff --git a/test/runnable/extra-files/mul9377a.d b/test/dshell/extra-files/mul9377a.d similarity index 100% rename from test/runnable/extra-files/mul9377a.d rename to test/dshell/extra-files/mul9377a.d diff --git a/test/runnable/extra-files/mul9377b.d b/test/dshell/extra-files/mul9377b.d similarity index 100% rename from test/runnable/extra-files/mul9377b.d rename to test/dshell/extra-files/mul9377b.d diff --git a/test/runnable/extra-files/multi9377.d b/test/dshell/extra-files/multi9377.d similarity index 100% rename from test/runnable/extra-files/multi9377.d rename to test/dshell/extra-files/multi9377.d diff --git a/test/runnable/extra-files/test_shared.d b/test/dshell/extra-files/test_shared.d similarity index 100% rename from test/runnable/extra-files/test_shared.d rename to test/dshell/extra-files/test_shared.d diff --git a/test/runnable/imports/depsprot_default.d b/test/dshell/imports/depsprot_default.d similarity index 100% rename from test/runnable/imports/depsprot_default.d rename to test/dshell/imports/depsprot_default.d diff --git a/test/runnable/imports/depsprot_private.d b/test/dshell/imports/depsprot_private.d similarity index 100% rename from test/runnable/imports/depsprot_private.d rename to test/dshell/imports/depsprot_private.d diff --git a/test/runnable/imports/depsprot_public.d b/test/dshell/imports/depsprot_public.d similarity index 100% rename from test/runnable/imports/depsprot_public.d rename to test/dshell/imports/depsprot_public.d diff --git a/test/dshell/test9377.d b/test/dshell/test9377.d new file mode 100644 index 000000000000..6ad26fd7d105 --- /dev/null +++ b/test/dshell/test9377.d @@ -0,0 +1,8 @@ +import dshell; +void main() +{ + Vars.set("libname", "$OUTPUT_BASE/a$LIBEXT"); + + run("$DMD -m$MODEL -I$EXTRA_FILES -of$libname -c $EXTRA_FILES/mul9377a.d $EXTRA_FILES/mul9377b.d -lib"); + run("$DMD -m$MODEL -I$EXTRA_FILES -of$OUTPUT_BASE/a$EXE $EXTRA_FILES/multi9377.d $libname"); +} diff --git a/test/dshell/test_shared.d b/test/dshell/test_shared.d new file mode 100644 index 000000000000..465de313f9a1 --- /dev/null +++ b/test/dshell/test_shared.d @@ -0,0 +1,14 @@ +import dshell; +void main() +{ + if (OS != "linux") + { + writefln("Skipping shared library test on %s.", OS); + return; + } + + run("$DMD -m$MODEL -of$OUTPUT_BASE/a$EXE -defaultlib=libphobos2.so $EXTRA_FILES/test_shared.d"); + run("$OUTPUT_BASE/a$EXE", stdout, [ + "LD_LIBRARY_PATH" : "../../phobos/generated/"~OS~"/release/"~MODEL + ]); +} diff --git a/test/run.d b/test/run.d index c1ff7352575d..892062e51738 100755 --- a/test/run.d +++ b/test/run.d @@ -21,7 +21,7 @@ import tools.paths; const scriptDir = __FILE_FULL_PATH__.dirName.buildNormalizedPath; auto testPath(R)(R path) { return buildNormalizedPath(scriptDir, path); } string resultsDir = testPath("test_results"); -immutable testDirs = ["runnable", "compilable", "fail_compilation"]; +immutable testDirs = ["runnable", "compilable", "fail_compilation", "dshell"]; shared bool verbose; // output verbose logging shared bool force; // always run all tests (ignores timestamp checking) shared string hostDMD; // path to host DMD binary (used for building the tools) @@ -33,7 +33,8 @@ enum TestTools { unitTestRunner = TestTool("unit_test_runner", [toolsDir.buildPath("paths")]), testRunner = TestTool("d_do_test"), - jsonSanitizer = TestTool("sanitize_json") + jsonSanitizer = TestTool("sanitize_json"), + dshellPrebuilt = TestTool("dshell_prebuilt", null, Yes.linksWithTests), } immutable struct TestTool @@ -44,6 +45,9 @@ immutable struct TestTool /// Extra arguments that should be supplied to the compiler when compiling the tool. string[] extraArgs; + /// Indicates the tool is a binary that links with tests + Flag!"linksWithTests" linksWithTests; + alias name this; } @@ -92,7 +96,7 @@ Options: if (runUnitTests) { verifyCompilerExists(env); - ensureToolsExists(TestTools.unitTestRunner); + ensureToolsExists(env, TestTools.unitTestRunner); return spawnProcess(unitTestRunnerCommand ~ args).wait(); } @@ -120,7 +124,7 @@ Options: int ret; auto taskPool = new TaskPool(jobs); scope(exit) taskPool.finish(); - ensureToolsExists(EnumMembers!TestTools); + ensureToolsExists(env, EnumMembers!TestTools); foreach (target; taskPool.parallel(targets, 1)) { log("run: %-(%s %)", target.args); @@ -147,27 +151,56 @@ void verifyCompilerExists(string[string] env) Builds the binary of the tools required by the testsuite. Does nothing if the tools already exist and are newer than their source. */ -void ensureToolsExists(const TestTool[] tools ...) +void ensureToolsExists(string[string] env, const TestTool[] tools ...) { resultsDir.mkdirRecurse; shared uint failCount = 0; foreach (tool; tools.parallel(1)) { - const targetBin = resultsDir.buildPath(tool).exeName; - const sourceFile = toolsDir.buildPath(tool ~ ".d"); + string targetBin; + string sourceFile; + if (tool.linksWithTests) + { + targetBin = resultsDir.buildPath(tool).objName; + sourceFile = toolsDir.buildPath(tool, tool ~ ".d"); + } + else + { + targetBin = resultsDir.buildPath(tool).exeName; + sourceFile = toolsDir.buildPath(tool ~ ".d"); + } if (targetBin.timeLastModified.ifThrown(SysTime.init) >= sourceFile.timeLastModified) writefln("%s is already up-to-date", tool); else { - const command = [ - hostDMD, - "-of"~targetBin, - sourceFile - ] ~ tool.extraArgs; + string[] command; + string[string] commandEnv = null; + if (tool.linksWithTests) + { + // This will compile the dshell library thus needs the actual + // DMD compiler under test + command = [ + env["DMD"], + "-conf=", + "-m"~env["MODEL"], + "-of" ~ targetBin, + "-c", + sourceFile + ] ~ getPicFlags(env); + commandEnv = env; + } + else + { + command = [ + hostDMD, + "-of"~targetBin, + sourceFile + ] ~ tool.extraArgs; + } writefln("Executing: %-(%s %)", command); - if (spawnProcess(command).wait) + if (spawnProcess(command, commandEnv).wait) { stderr.writefln("failed to build '%s'", targetBin); atomicOp!"+="(failCount, 1); @@ -431,3 +464,21 @@ auto exeName(T)(T name) name ~= ".exe"; return name; } + +// Add the object filename extension to the given `name` for the current OS. +auto objName(T)(T name) +{ + version(Windows) + return name ~ ".obj"; + else + return name ~ ".o"; +} + +/// Return the correct pic flags as an array of strings +string[] getPicFlags(string[string] env) +{ + const picFlags = env["PIC_FLAGS"]; + if (picFlags.length) + return picFlags.split(); + return cast(string[])[]; +} diff --git a/test/runnable/depsprot.sh b/test/runnable/depsprot.sh deleted file mode 100755 index 8a43563a76ed..000000000000 --- a/test/runnable/depsprot.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -deps_file="${OUTPUT_BASE}.deps" - -# custom error handling -set +eo pipefail - -die() -{ - echo "---- deps file ----" - cat ${deps_file} - echo - echo "$@" - rm -f ${deps_file} - exit 1 -} - -$DMD -m${MODEL} -deps=${deps_file} -Irunnable/imports -o- ${EXTRA_FILES}/${TEST_NAME}.d -test $? -ne 0 && - die "Error compiling" - -grep "^${TEST_NAME}.*${TEST_NAME}_default" ${deps_file} | grep -q private || - die "Default import protection in dependency file should be 'private'" - -grep "^${TEST_NAME}.*${TEST_NAME}_public" ${deps_file} | grep -q public || - die "Public import protection in dependency file should be 'public'" - -grep "^${TEST_NAME}.*${TEST_NAME}_private" ${deps_file} | grep -q private|| - die "Private import protection in dependency file should be 'private'" - -echo "Dependencies file:" -cat ${deps_file} -echo - -rm ${deps_file} diff --git a/test/runnable/test9377.sh b/test/runnable/test9377.sh deleted file mode 100755 index b9fd3f2a24e4..000000000000 --- a/test/runnable/test9377.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -libname=${OUTPUT_BASE}${LIBEXT} - -$DMD -m${MODEL} -I${EXTRA_FILES} -of${libname} -c ${EXTRA_FILES}${SEP}mul9377a.d ${EXTRA_FILES}${SEP}mul9377b.d -lib || exit 1 -$DMD -m${MODEL} -I${EXTRA_FILES} -of${OUTPUT_BASE}${EXE} ${EXTRA_FILES}${SEP}multi9377.d ${libname} || exit 1 -rm_retry ${OUTPUT_BASE}{${LIBEXT},${OBJ},${EXE}} diff --git a/test/runnable/test_shared.sh b/test/runnable/test_shared.sh deleted file mode 100755 index 0804c5d205ef..000000000000 --- a/test/runnable/test_shared.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - - - -if [ ${OS} != "linux" ]; then - echo "Skipping shared library test on ${OS}." - exit 0 -fi - -$DMD -m${MODEL} -of${OUTPUT_BASE}${EXE} -defaultlib=libphobos2.so ${EXTRA_FILES}${SEP}test_shared.d - -LD_LIBRARY_PATH=../../phobos/generated/${OS}/release/${MODEL} ${OUTPUT_BASE}${EXE} diff --git a/test/tools/d_do_test.d b/test/tools/d_do_test.d index 75242943f27d..ed16a93cf811 100755 --- a/test/tools/d_do_test.d +++ b/test/tools/d_do_test.d @@ -5,6 +5,7 @@ import std.algorithm; import std.array; import std.conv; import std.datetime.stopwatch; +import std.datetime.systime; import std.exception; import std.file; import std.format; @@ -17,7 +18,7 @@ import std.stdio; import std.string; import core.sys.posix.sys.wait; -const scriptDir = __FILE_FULL_PATH__.dirName.dirName; +const dmdTestDir = __FILE_FULL_PATH__.dirName.dirName; version(Win32) { @@ -55,7 +56,8 @@ enum TestMode { COMPILE, FAIL_COMPILE, - RUN + RUN, + DSHELL, } struct TestArgs @@ -567,7 +569,7 @@ int tryMain(string[] args) return 1; } - auto test_file = args[1]; + const test_file = args[1]; string input_dir = test_file.dirName(); TestArgs testArgs; @@ -576,8 +578,9 @@ int tryMain(string[] args) case "compilable": testArgs.mode = TestMode.COMPILE; break; case "fail_compilation": testArgs.mode = TestMode.FAIL_COMPILE; break; case "runnable": testArgs.mode = TestMode.RUN; break; + case "dshell": testArgs.mode = TestMode.DSHELL; break; default: - writefln("Error: invalid test directory '%s', expected 'compilable', 'fail_compilation', or 'runnable'", input_dir); + writefln("Error: invalid test directory '%s', expected 'compilable', 'fail_compilation', 'runnable' or 'dshell'", input_dir); return 1; } @@ -610,6 +613,9 @@ int tryMain(string[] args) string output_file = result_path ~ input_file ~ ".out"; string test_app_dmd_base = output_dir ~ envData.sep ~ test_name ~ "_"; + if (testArgs.mode == TestMode.DSHELL) + return runDShellTest(input_dir, test_name, envData, output_dir, output_file); + // envData.sep is required as the results_dir path can be `generated` const absoluteResultDirPath = envData.results_dir.absolutePath ~ envData.sep; const resultsDirReplacement = "{{RESULTS_DIR}}" ~ envData.sep; @@ -965,11 +971,125 @@ int runBashTest(string input_dir, string test_name) version(Windows) { auto process = spawnShell(format("bash %s %s %s", - buildPath(scriptDir, "tools", "sh_do_test.sh"), input_dir, test_name)); + buildPath(dmdTestDir, "tools", "sh_do_test.sh"), input_dir, test_name)); } else { - auto process = spawnProcess([scriptDir.buildPath("tools", "sh_do_test.sh"), input_dir, test_name]); + auto process = spawnProcess([dmdTestDir.buildPath("tools", "sh_do_test.sh"), input_dir, test_name]); } return process.wait(); } + +/// Return the correct pic flags +string[] getPicFlags() +{ + version (Windows) { } else + { + version(X86_64) + return ["-fPIC"]; + if (environment.get("PIC", null) == "1") + return ["-fPIC"]; + } + return cast(string[])[]; +} + +/// Run a dshell test +int runDShellTest(string input_dir, string test_name, const ref EnvData envData, + string output_dir, string output_file) +{ + const testScriptDir = buildPath(dmdTestDir, input_dir); + const testScriptPath = buildPath(testScriptDir, test_name ~ ".d"); + const testOutDir = buildPath(output_dir, test_name); + const testLogName = format("%s/%s.d", input_dir, test_name); + + writefln(" ... %s", testLogName); + + removeIfExists(output_file); + if (exists(testOutDir)) + rmdirRecurse(testOutDir); + mkdirRecurse(testOutDir); + + // create the "dshell" module for the tests + { + auto dshellFile = File(buildPath(testOutDir, "dshell.d"), "w"); + dshellFile.writeln(`module dshell; +public import dshell_prebuilt; +static this() +{ + dshellPrebuiltInit("` ~ input_dir ~ `", "`, test_name , `"); +} +`); + } + + const testScriptExe = buildPath(testOutDir, "run" ~ envData.exe); + const output_file_temp = output_file ~ ".tmp"; + + // + // compile the test + // + { + auto outfile = File(output_file_temp, "w"); + const compile = [envData.dmd, "-conf=", "-m"~envData.model] ~ + getPicFlags ~ [ + "-od" ~ testOutDir, + "-of" ~ testScriptExe, + "-I=" ~ testScriptDir, + "-I=" ~ testOutDir, + "-I=" ~ buildPath(dmdTestDir, "tools", "dshell_prebuilt"), + "-i", + // Causing linker errors for some reason? + "-i=-dshell_prebuilt", buildPath(envData.results_dir, "dshell_prebuilt" ~ envData.obj), + testScriptPath, + ]; + outfile.writeln("[COMPILE_TEST] ", escapeShellCommand(compile)); + // Note that spawnprocess closes the file, so it will need to be re-opened + // below when we run the test + auto compileProc = std.process.spawnProcess(compile, stdin, outfile, outfile); + const exitCode = wait(compileProc); + if (exitCode != 0) + { + printTestFailure(testLogName, output_file_temp); + return exitCode; + } + } + + // + // run the test + // + { + auto outfile = File(output_file_temp, "a"); + const runTest = [testScriptExe]; + outfile.writeln("[RUN_TEST] ", escapeShellCommand(runTest)); + auto runTestProc = std.process.spawnProcess(runTest, stdin, outfile, outfile); + const exitCode = wait(runTestProc); + if (exitCode != 0) + { + printTestFailure(testLogName, output_file_temp); + return exitCode; + } + } + + rename(output_file_temp, output_file); + // TODO: should we remove all the test artifacts if the test passes? rmdirRecurse(testOutDir)? + return 0; +} + +void printTestFailure(string testLogName, string output_file_temp) +{ + writeln("=============================="); + writefln("Test '%s' failed. The logged output:", testLogName); + const output = readText(output_file_temp); + write(output); + if (!output.endsWith("\n")) + writeln(); + writeln("=============================="); + remove(output_file_temp); +} + +/// Make any parent diretories needed for the given `filename` +void mkdirsFor(string filename) +{ + auto dir = dirName(filename); + if (!exists(dir)) + mkdirRecurse(dir); +} diff --git a/test/tools/dshell_prebuilt/dshell_prebuilt.d b/test/tools/dshell_prebuilt/dshell_prebuilt.d new file mode 100644 index 000000000000..62a46b0a6f52 --- /dev/null +++ b/test/tools/dshell_prebuilt/dshell_prebuilt.d @@ -0,0 +1,291 @@ +/** +A small library to help write D shell-like test scripts. +*/ +module dshell_prebuilt; + +public import core.stdc.stdlib : exit; + +public import core.time; +public import core.thread; +public import std.meta; +public import std.exception; +public import std.array; +public import std.string; +public import std.format; +public import std.path; +public import std.file; +public import std.stdio; +public import std.process; + +/** +Emulates bash environment variables. Variables set here will be availble for BASH-like expansion. +*/ +struct Vars +{ + private static __gshared string[string] map; + static void set(string name, string value) + in { assert(value !is null); } do + { + const expanded = shellExpand(value); + assert(expanded !is null, "codebug"); + map[name] = expanded; + } + static string get(const(char)[] name) + { + auto result = map.get(cast(string)name, null); + if (result is null) + assert(0, "Unknown variable '" ~ name ~ "'"); + return result; + } + static string opDispatch(string name)() { return get(name); } +} + +private alias requiredEnvVars = AliasSeq!( + "MODEL", "RESULTS_DIR", + "EXE", "OBJ", + "DMD", "DFLAGS", + "OS", "SEP", "DSEP", +); +private alias allVars = AliasSeq!( + requiredEnvVars, + "TEST_DIR", "TEST_NAME", + "RESULTS_TEST_DIR", + "OUTPUT_BASE", "EXTRA_FILES", + "LIBEXT", +); + +static foreach (var; allVars) +{ + mixin(`string ` ~ var ~ `() { return Vars.` ~ var ~ `; }`); +} + +/// called from the dshell module to initialize environment +void dshellPrebuiltInit(string testDir, string testName) +{ + static foreach (var; requiredEnvVars) + { + mixin(`Vars.set("` ~ var ~ `", requireEnv("` ~ var ~ `"));`); + } + + Vars.set("TEST_DIR", testDir); + Vars.set("TEST_NAME", testName); + // reference to the resulting test_dir folder, e.g .test_results/runnable + Vars.set("RESULTS_TEST_DIR", buildPath(RESULTS_DIR, TEST_DIR)); + // reference to the resulting files without a suffix, e.g. test_results/runnable/test123import test); + Vars.set("OUTPUT_BASE", buildPath(RESULTS_TEST_DIR, TEST_NAME)); + // reference to the extra files directory + Vars.set("EXTRA_FILES", buildPath(TEST_DIR, "extra-files")); + version (Windows) + { + Vars.set("LIBEXT", ".lib"); + } + else + { + Vars.set("LIBEXT", ".a"); + } +} + +private string requireEnv(string name) +{ + const result = environment.get(name, null); + if (result is null) + { + writefln("Error: missing required environment variable '%s'", name); + exit(1); + } + return result; +} + +/// Remove one or more files +void rm(scope const(char[])[] args...) +{ + foreach (arg; args) + { + auto expanded = shellExpand(arg); + if (exists(expanded)) + { + writeln("rm '", expanded, "'"); + // Use loop to workaround issue in windows with removing + // executables after running then + for (int sleepMsecs = 10; ; sleepMsecs *= 2) + { + try { + std.file.remove(expanded); + break; + } catch (Exception e) { + if (sleepMsecs >= 3000) + throw e; + Thread.sleep(dur!"msecs"(sleepMsecs)); + } + } + } + } +} + +/// Make all parent directories needed to create the given `filename` +void mkdirFor(string filename) +{ + auto dir = dirName(filename); + if (!exists(dir)) + { + writefln("[INFO] mkdir -p '%s'", dir); + mkdirRecurse(dir); + } +} + +/** +Run the given command. The `tryRun` variants return the exit code, whereas the `run` variants +will assert on a non-zero exit code. +*/ +auto tryRun(scope const(char[])[] args, File stdout = std.stdio.stdout, string[string] env = null) +{ + std.stdio.stdout.write("[RUN]"); + if (env) + { + foreach (pair; env.byKeyValue) + { + std.stdio.stdout.write(" ", pair.key, "=", pair.value); + } + } + std.stdio.write(" ", escapeShellCommand(args)); + if (stdout != std.stdio.stdout) + { + std.stdio.stdout.write(" > ", stdout.name); + } + std.stdio.stdout.writeln(); + std.stdio.stdout.flush(); + auto proc = spawnProcess(args, stdin, stdout, std.stdio.stderr, env); + return wait(proc); +} +/// ditto +void run(scope const(char[])[] args, File stdout = std.stdio.stdout, string[string] env = null) +{ + const exitCode = tryRun(args, stdout, env); + if (exitCode != 0) + { + writefln("Error: last command exited with code %s", exitCode); + assert(0, "last command failed"); + } +} +/// ditto +void run(string cmd, File stdout = std.stdio.stdout, string[string] env = null) +{ + // TODO: option to disable this? + if (SEP != "/") + cmd = cmd.replace("/", SEP); + run(parseCommand(cmd), stdout, env); +} + +/** +Parse the given string `s` as a command. Performs BASH-like variable expansion. +*/ +string[] parseCommand(string s) +{ + auto rawArgs = s.split(); + auto args = appender!(string[])(); + foreach (rawArg; rawArgs) + { + args.put(shellExpand(rawArg)); + } + return args.data; +} + +/// Expand the given string using BASH-like variable expansion. +string shellExpand(const(char)[] s) +{ + auto expanded = appender!(char[])(); + for (size_t i = 0; i < s.length;) + { + if (s[i] != '$') + { + expanded.put(s[i]); + i++; + } + else + { + i++; + assert(i < s.length, "lone '$' at end of string"); + auto start = i; + if (s[i] == '{') + { + start++; + for (;;) + { + i++; + assert(i < s.length, "unterminated ${..."); + if (s[i] == '}') break; + } + expanded.put(Vars.get(s[start .. i])); + i++; + } + else + { + assert(validVarChar(s[i]), "invalid sequence $'" ~ s[i]); + for (;;) + { + i++; + if (i >= s.length || !validVarChar(s[i])) + break; + } + expanded.put(Vars.get(s[start .. i])); + } + } + } + auto result = expanded.data; + return (result is null) ? "" : result.assumeUnique; +} + +// [a-zA-Z0-9_] +private bool validVarChar(const char c) +{ + import std.ascii : isAlphaNum; + return c.isAlphaNum || c == '_'; +} + +struct GrepResult +{ + string[] matches; + void enforceMatches(string message) + { + if (matches.length == 0) + { + assert(0, message); + } + } +} + +/** +grep the given `file` for the given `pattern`. +*/ +GrepResult grep(string file, string pattern) +{ + const patternExpanded = shellExpand(pattern); + const fileExpanded = shellExpand(file); + writefln("[GREP] file='%s' pattern='%s'", fileExpanded, patternExpanded); + return grepLines(File(fileExpanded, "r").byLine, patternExpanded); +} +/// ditto +GrepResult grep(GrepResult lastResult, string pattern) +{ + auto patternExpanded = shellExpand(pattern); + writefln("[GREP] (%s lines from last grep) pattern='%s'", lastResult.matches.length, patternExpanded); + return grepLines(lastResult.matches, patternExpanded); +} + +private GrepResult grepLines(T)(T lineRange, string finalPattern) +{ + import std.regex; + auto matches = appender!(string[])(); + foreach(line; lineRange) + { + if (matchFirst(line, finalPattern)) + { + static if (is(typeof(lineRange.front()) == string)) + matches.put(line); + else + matches.put(line.idup); + } + } + writefln("[GREP] matched %s lines", matches.data.length); + return GrepResult(matches.data); +}