build: do not run pip install outside the bazel build framework

.bazelrc

@@ -18,14 +18,15 @@ build --incompatible_default_to_explicit_init_py
 # Enable resolution of cc toolchain by go toolchain
 build --incompatible_enable_cc_toolchain_resolution
 build --flag_alias=file_name_version=//:file_name_version
+build  --build_python_zip
 
 # include one of "--define gotags=sqlite_mattn" or "--define gotags=sqlite_modernc"
 # cannot be in common, because query chokes on it.
 build --define gotags=sqlite_modernc,netgo
 build:osx --define gotags=sqlite_modernc
 
 ### options for test
-test --build_tests_only --print_relative_test_log_paths --test_output=errors
+test --build_tests_only --print_relative_test_log_paths --test_output=errors --nobuild_python_zip
 
 ### predefined configurations (use with --config, eg. `bazel test --config=unit_all`)
 test:race --@io_bazel_rules_go//go/config:race

.buildkite/pipeline.yml

@@ -96,7 +96,7 @@ steps:
       - ./tools/licenses.sh /tmp/test-artifacts/licenses
       - diff -rNu3 /tmp/test-artifacts/licenses ./licenses/data
       - echo "--- gomocks"
-      - ./tools/gomocks.py diff
+      - make mocksdiff
       - echo "--- antlr"
       - rm -rf /tmp/test-artifacts/antlr
       - cp -R antlr/ /tmp/test-artifacts/antlr

BUILD.bazel

@@ -209,14 +209,18 @@ pkg_tar(
     package_dir = "",
 )
 
-# This contains all of the binaries needed to run the topology generator.
+# This contains all of the binaries needed to run the topology generator
+# and start a local stack.
 pkg_tar(
     name = "scion-topo",
     srcs = [
-        "//scion-pki/cmd/scion-pki",
+        "//tools:set_ipv6_addr",
+        "//tools:supervisorctl",
+        "//tools:supervisord",
+        "//tools:topodot",
+        "//tools:topogen",
     ],
     mode = "0755",
-    package_dir = "",
 )
 
 # Nogo - Go code analysis tool

Makefile

@@ -1,10 +1,11 @@
-.PHONY: all build build-dev dist-deb antlr clean docker-images gazelle go.mod licenses mocks protobuf scion-topo test test-integration write_all_source_files git-version
+.PHONY: all build build-dev dist-deb antlr clean docker-images gazelle go.mod licenses mocks mocksdiff protobuf scion-topo test test-integration write_all_source_files git-version
 
 build-dev:
 	rm -f bin/*
-	bazel build //:scion //:scion-ci
+	bazel build //:scion //:scion-ci //:scion-topo
 	tar -kxf bazel-bin/scion.tar -C bin
 	tar -kxf bazel-bin/scion-ci.tar -C bin
+	tar -kxf bazel-bin/scion-topo.tar -C bin
 
 build:
 	rm -f bin/*
@@ -75,10 +76,6 @@ docker-images:
 	@echo "Load images"
 	@bazel cquery '//docker:prod union //docker:test' --output=files 2>/dev/null | xargs -I{} docker load --input {}
 
-scion-topo:
-	bazel build //:scion-topo
-	tar --overwrite -xf bazel-bin/scion-topo.tar -C bin
-
 protobuf:
 	rm -rf bazel-bin/pkg/proto/*/go_default_library_/github.com/scionproto/scion/pkg/proto/*
 	bazel build --output_groups=go_generated_srcs //pkg/proto/...
@@ -88,7 +85,10 @@ protobuf:
 	chmod 0644 pkg/proto/*/*.pb.go pkg/proto/*/*/*.pb.go
 
 mocks:
-	tools/gomocks.py
+	bazel run //tools:gomocks
+
+mocksdiff:
+	bazel run //tools:gomocks -- diff
 
 gazelle: go_deps.bzl
 	bazel run //:gazelle --verbose_failures --config=quiet

acceptance/router_benchmark/BUILD.bazel

@@ -53,6 +53,13 @@ py_library(
 py_binary(
     name = "benchmark",
     srcs = ["benchmark.py"],
+    args = [
+        "--brload",
+        "$(location //acceptance/router_benchmark/brload:brload)",
+    ],
+    data = [
+        "//acceptance/router_benchmark/brload",
+    ],
     visibility = ["//visibility:public"],
     deps = [
         "benchmarklib",

acceptance/router_benchmark/benchmark.py

@@ -86,19 +86,22 @@ class RouterBMTool(cli.Application, RouterBM):
     log_level = cli.SwitchAttr(["l", "loglevel"], str, default='warning', help="Logging level")
 
     doit = cli.Flag(["r", "run"],
-                    help="Run the benchmark, as opposed to seeing the instructions.")
+                    help="Run the benchmark, as opposed to seeing the instructions")
     json = cli.Flag(["j", "json"],
-                    help="Output the report in json format.")
+                    help="Output the report in json format")
 
     # Used by the RouterBM mixin:
     coremark = cli.SwitchAttr(["c", "coremark"], int, default=0,
-                              help="The coremark score of the subject machine.")
+                              help="The coremark score of the subject machine")
     mmbm = cli.SwitchAttr(["m", "mmbm"], int, default=0,
-                          help="The mmbm score of the subject machine.")
+                          help="The mmbm score of the subject machine")
     packet_size = cli.SwitchAttr(["s", "size"], int, default=172,
-                                 help="Test packet size (includes all headers - floored at 154).")
+                                 help="Test packet size (includes all headers - floored at 154)")
+    brload_path = cli.SwitchAttr(["b", "brload"], str, default="bin/brload",
+                                 help="Relative path to the brload tool")
+
     intf_map: dict[str, Intf] = {}
-    brload: LocalCommand = local["./bin/brload"]
+    brload: LocalCommand = None
     brload_cpus: list[int] = []
     artifacts = f"{os.getcwd()}/acceptance/router_benchmark"
     prom_address: str = "localhost:9090"
@@ -332,6 +335,9 @@ def instructions(self):
 """)
 
     def main(self, *interfaces: str):
+        # brload cannot be set statically. It need the cli arguments to be
+        # processed.
+        self.brload = local[self.brload_path]
         status = 1
         try:
             logging.basicConfig(level=self.log_level.upper())

scion.sh

@@ -23,11 +23,11 @@ cmd_topology() {
     cmd_topo-clean
 
     echo "Create topology, configuration, and execution files."
-    tools/topogen.py "$@"
+    ./bin/topogen "$@"
 }
 
 cmd_topodot() {
-    ./tools/topodot.py "$@"
+    ./bin/topodot "$@"
 }
 
 start_scion() {

tools/BUILD.bazel

@@ -1,5 +1,6 @@
 load("@rules_python//python:defs.bzl", "py_binary")
 load("@com_github_scionproto_scion_python_deps//:requirements.bzl", "requirement")
+load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
 
 exports_files([
     "gzip_exec_interp",
@@ -26,6 +27,24 @@ py_binary(
     ],
 )
 
+py_console_script_binary(
+    name = "supervisord",
+    pkg = requirement("supervisor"),
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("supervisor-wildcards"),
+    ],
+)
+
+py_console_script_binary(
+    name = "supervisorctl",
+    pkg = requirement("supervisor"),
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("supervisor-wildcards"),
+    ],
+)
+
 py_binary(
     name = "topogen",
     srcs = ["topogen.py"],
@@ -45,3 +64,34 @@ py_binary(
         requirement("pyyaml"),
     ],
 )
+
+py_binary(
+    name = "topodot",
+    srcs = ["topodot.py"],
+    data = [
+    ],
+    main = "topodot.py",
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tools/topology:py_default_library",
+        "@bazel_tools//tools/python/runfiles",
+        requirement("plumbum"),
+    ],
+)
+
+py_binary(
+    name = "set_ipv6_addr",
+    srcs = ["set_ipv6_addr.py"],
+    data = [
+    ],
+    main = "set_ipv6_addr.py",
+    python_version = "PY3",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tools/topology:py_default_library",
+        "@bazel_tools//tools/python/runfiles",
+    ],
+)

tools/gomocks.py

@@ -27,6 +27,7 @@ def rule_to_file(rule: str) -> Tuple[str, str]:
 
 def mock_rules() -> List[str]:
     bazel = plumbum.local['bazel']
+    os.chdir(os.environ.get("BUILD_WORKING_DIRECTORY","/nonexistium"))
     raw_rules = bazel("query", "filter(\"go_default_mock$\", kind(gomock, //...))")
     return raw_rules.splitlines()
 

tools/install_deps

@@ -8,4 +8,4 @@ if which apt-get >& /dev/null; then
 elif which yum >& /dev/null; then
   "$BASE/env/rhel/deps"
 fi
-"$BASE/env/pip3/deps"
+

tools/supervisor.sh

@@ -6,7 +6,7 @@ mkdir -p logs
 OPTIONS="$@"
 CONF_FILE="tools/supervisord.conf"
 if [ ! -e /tmp/supervisor.sock ]; then
-    supervisord -c $CONF_FILE
+    bin/supervisord -c $CONF_FILE
 fi
-supervisorctl -c $CONF_FILE $OPTIONS
+bin/supervisorctl -c $CONF_FILE $OPTIONS