Merge branch 'facebookincubator:main' into main

.circleci/config.yml

@@ -19,6 +19,7 @@ workflows:
   dist-compile:
     jobs:
       - linux-build
+      - linux-pr-fuzzer-run
       - linux-build-options
       - linux-adapters
       - macos-build:
@@ -161,7 +162,7 @@ commands:
       - run:
           name: "Run << parameters.fuzzer_name >> Fuzzer"
           command: |
-            eval " << parameters.fuzzer_exe >> << parameters.fuzzer_args >> " \
+            eval ' << parameters.fuzzer_exe >> << parameters.fuzzer_args >> ' \
                   2>&1 | tee "<< parameters.fuzzer_output >>" || ( \
                     tail -n 1000 "<< parameters.fuzzer_output >>" ; \
                     echo "FAIL: << parameters.fuzzer_name >> run failed"; \
@@ -357,34 +358,6 @@ jobs:
          name: "Run Example Binaries"
          command: |
            find _build/debug/velox/examples/ -maxdepth 1 -type f -executable -exec "{}" \;
-      - run:
-          name: "Build and Test PyVelox"
-          command: |
-            conda init bash
-            source ~/.bashrc
-            conda create -y --name pyveloxenv python=3.7
-            conda activate pyveloxenv
-            LD_LIBRARY_PATH=/usr/local/lib make python-test
-      - run:
-          name: "Check function signatures"
-          command: |
-            source ~/.bashrc
-            conda activate pyveloxenv
-            pip install deepdiff
-            python ./scripts/signature.py export --spark --presto /tmp/pr_signatures.json
-            cp ./scripts/signature.py /tmp/signature.py
-            git remote add upstream https://github.com/facebookincubator/velox
-            git fetch upstream
-            merge_base=$(git merge-base  'upstream/main' `git rev-parse HEAD`) || \
-            { echo "::error::Failed to find merge_base"; exit 1; }
-            echo "Merge Base: $merge_base"
-            git checkout $merge_base
-            git submodule update --init --recursive
-            LD_LIBRARY_PATH=/usr/local/lib make python-clean
-            LD_LIBRARY_PATH=/usr/local/lib make python-build
-            cp /tmp/signature.py ./scripts/signature.py
-            python ./scripts/signature.py export --spark --presto /tmp/main_signatures.json
-            python ./scripts/signature.py diff /tmp/main_signatures.json /tmp/pr_signatures.json
       - post-steps
 
   linux-build-release:
@@ -671,3 +644,78 @@ jobs:
             git config --global user.name "velox"
             cd presto/presto-native-execution
             make runtime-container
+
+  linux-pr-fuzzer-run:
+    executor: build
+    steps:
+      - pre-steps
+      - run:
+          name: "Get merge base function signatures"
+          command: |
+            source ~/.bashrc
+            conda create -y --name pyveloxenv python=3.7
+            conda activate pyveloxenv
+            cp ./scripts/signature.py /tmp/signature.py
+            pip install deepdiff
+            git remote add upstream https://github.com/facebookincubator/velox
+            git fetch upstream
+            merge_base=$(git merge-base  'upstream/main' `git rev-parse HEAD`) || \
+            { echo "::error::Failed to find merge_base"; exit 1; }
+            echo "Merge Base: $merge_base"
+            git checkout $merge_base
+            git submodule update --init --recursive
+            LD_LIBRARY_PATH=/usr/local/lib make python-clean
+            LD_LIBRARY_PATH=/usr/local/lib make python-build
+            python /tmp/signature.py export --spark spark_merge_base_signatures.json
+            python /tmp/signature.py export --presto presto_merge_base_signatures.json
+      - checkout
+      - run:
+          name: "Build"
+          command: |
+              make debug NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=6 EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_ARROW=ON"
+              ccache -s
+          no_output_timeout: 1h
+      - run:
+          name: "Build and test PyVelox"
+          command: |
+            conda init bash
+            source ~/.bashrc
+            conda activate pyveloxenv
+            LD_LIBRARY_PATH=/usr/local/lib make python-test
+      - run:
+          name: "Check and create bias function signatures"
+          command: |
+            source ~/.bashrc
+            conda activate pyveloxenv
+            pip install deepdiff
+            python ./scripts/signature.py export --presto presto_pr_signatures.json
+            python ./scripts/signature.py export --spark spark_pr_signatures.json
+            python ./scripts/signature.py bias presto_merge_base_signatures.json presto_pr_signatures.json /tmp/presto_bias_functions 
+            python ./scripts/signature.py bias spark_merge_base_signatures.json spark_pr_signatures.json /tmp/spark_bias_functions 
+
+      - fuzzer-run:
+          fuzzer_output: "/tmp/fuzzer.log"
+          fuzzer_repro: "/tmp/fuzzer_repro"
+          fuzzer_name: "Expression Bias Run"
+          fuzzer_exe: "if [ -f /tmp/presto_bias_functions ]; then _build/debug/velox/expression/tests/velox_expression_fuzzer_test"
+          fuzzer_args: " --seed ${RANDOM} --lazy_vector_generation_ratio 0.2 \
+          --assign_function_tickets  $(cat /tmp/presto_bias_functions) \
+          --duration_sec 3600 --enable_variadic_signatures \
+          --velox_fuzzer_enable_complex_types \
+          --velox_fuzzer_enable_column_reuse \
+          --velox_fuzzer_enable_expression_reuse \
+          --max_expression_trees_per_step 2 \
+          --retry_with_try \
+          --enable_dereference \
+          --logtostderr=1 --minloglevel=0 \
+          --repro_persist_path=/tmp/fuzzer_repro ; fi"
+
+      - fuzzer-run:
+          fuzzer_output: "/tmp/spark_fuzzer.log"
+          fuzzer_repro: "/tmp/spark_fuzzer_repro"
+          fuzzer_name: "Spark Bias Run"
+          fuzzer_exe: "if [ -f /tmp/spark_bias_functions ];  then _build/debug/velox/expression/tests/spark_expression_fuzzer_test"
+          fuzzer_args: " --seed ${RANDOM} --duration_sec 3600 --logtostderr=1 --minloglevel=0 \
+          --assign_function_tickets  $(cat /tmp/spark_bias_functions) \
+          --repro_persist_path=/tmp/spark_fuzzer_repro ; fi"
+

.github/workflows/benchmark.yml

@@ -193,8 +193,7 @@ jobs:
           ./scripts/benchmark-runner.py compare \
                 --baseline_path ${BASELINE_OUTPUT_PATH} \
                 --contender_path ${CONTENDER_OUTPUT_PATH} \
-                --recursive \
-                --do_not_fail
+                --recursive 
           echo "::endgroup::"
 
       - name: "Save PR number"

build/deps/github_hashes/facebook/folly-rev.txt

@@ -1 +1 @@
-Subproject commit dfeb9e3b20b41ba776d2789e035c1b36c96faa75
+Subproject commit d0254f0af28be32985a43159c3dd8156892f140c

build/fbcode_builder/CMake/FBPythonBinary.cmake

@@ -32,7 +32,7 @@ if(NOT TARGET Python3::Interpreter)
   # We find with QUIET here, since otherwise this generates some noisy warnings
   # on versions of CMake before 3.12
   if (WIN32)
-    # On Windows we need both the Intepreter as well as the Development
+    # On Windows we need both the Interpreter as well as the Development
     # libraries.
     find_package(Python3 COMPONENTS Interpreter Development QUIET)
   else()
@@ -487,7 +487,7 @@ function(add_fb_python_library LIB_NAME)
     #   won't complain if one of the dependencies doesn't exist (since it is
     #   intended to allow passing in file names for plain library files rather
     #   than just targets).
-    # - It ensures that sources for our depencencies are built before any
+    # - It ensures that sources for our dependencies are built before any
     #   executable that depends on us.  Note that we depend on "${dep}.py_lib"
     #   rather than "${dep}.py_sources_built" for this purpose because the
     #   ".py_sources_built" target won't be available for imported targets.

build/fbcode_builder/CMake/fb_py_test_main.py

@@ -262,7 +262,7 @@ def stopTest(self, test):
 
         super(BuckTestResult, self).stopTest(test)
 
-        # If a failure occured during module/class setup, then this "test" may
+        # If a failure occurred during module/class setup, then this "test" may
         # actually be a `_ErrorHolder`, which doesn't contain explicit info
         # about the upcoming test.  Since we really only care about the test
         # name field (i.e. `_testMethodName`), we use that to detect an actual

build/fbcode_builder/getdeps.py

@@ -626,7 +626,7 @@ def run_project_cmd(self, args, loader, manifest):
                     )
                     builder.build(install_dirs, reconfigure=reconfigure)
 
-                    # If we are building the project (not depdendency) and a specific
+                    # If we are building the project (not dependency) and a specific
                     # cmake_target (not 'install') has been requested, then we don't
                     # set the built_marker. This allows subsequent runs of getdeps.py
                     # for the project to run with different cmake_targets to trigger

build/fbcode_builder/getdeps/builder.py

@@ -346,7 +346,7 @@ def _build(self, install_dirs, reconfigure) -> None:
 
 class Iproute2Builder(BuilderBase):
     # ./configure --prefix does not work for iproute2.
-    # Thus, explicitly copy sources from src_dir to build_dir, bulid,
+    # Thus, explicitly copy sources from src_dir to build_dir, build,
     # and then install to inst_dir using DESTDIR
     # lastly, also copy include from build_dir to inst_dir
     def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir) -> None:

build/fbcode_builder/getdeps/cargo.py

@@ -194,7 +194,7 @@ def _patchup_workspace(self, dep_to_git) -> None:
             my-rename-of-crate = { package = "crate", git = "..." }
 
         they can count themselves lucky because the code will raise an
-        Exception. There migh be more cases where the code will silently pass
+        Exception. There might be more cases where the code will silently pass
         producing bad results.
         """
         workspace_dir = self.workspace_dir()
@@ -362,7 +362,7 @@ def _resolve_dep_to_crates(self, build_source_dir, dep_to_git):
 
         dep_to_crates = {}
 
-        # First populate explicit crate paths from depedencies
+        # First populate explicit crate paths from dependencies
         for name, git_conf in dep_to_git.items():
             crates = git_conf["crate_source_map"].keys()
             if crates:

build/fbcode_builder/getdeps/envfuncs.py

@@ -32,7 +32,7 @@ def _key(self, key):
         # project uses `unicode_literals`.  `subprocess` will raise an error
         # if the environment that it is passed has a mixture of byte and
         # unicode strings.
-        # It is simplest to force everthing to be `str` for the sake of
+        # It is simplest to force everything to be `str` for the sake of
         # consistency.
         key = str(key)
         if sys.platform.startswith("win"):

build/fbcode_builder/manifests/lz4

@@ -6,8 +6,8 @@ lz4
 
 [rpms]
 lz4-devel
-# centos (not centos_stream that is Meta internal) 8 is missing this
-[rpms.not(all(distro=centos,distro_vers=8))]
+# centos 8 and centos_stream 9 are missing this rpm
+[rpms.not(any(all(distro=centos,distro_vers=8),all(distro=centos_stream,distro_vers=9)))]
 lz4-static
 
 [debs]

scripts/setup-macos.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script documents setting up a macOS host for presto_cpp
+# This script documents setting up a macOS host for Velox
 # development.  Running it should make you ready to compile.
 #
 # Environment variables:

scripts/signature.py

@@ -34,10 +34,10 @@ class bcolors:
 def export(args):
     """Exports Velox function signatures."""
     if args.spark:
-        pv.register_spark_signatures("spark_")
+        pv.register_spark_signatures()
 
     if args.presto:
-        pv.register_presto_signatures("presto_")
+        pv.register_presto_signatures()
 
     signatures = pv.get_function_signatures()
 
@@ -51,12 +51,15 @@ def export(args):
     return 0
 
 
-def diff(args):
-    """Diffs Velox function signatures."""
-    first_signatures = json.load(args.first)
-    second_signatures = json.load(args.second)
+def diff_signatures(base_signatures, contender_signatures):
+    """Diffs Velox function signatures. Returns a tuple of the delta diff and exit status"""
+
     delta = DeepDiff(
-        first_signatures, second_signatures, ignore_order=True, report_repetition=True
+        base_signatures,
+        contender_signatures,
+        ignore_order=True,
+        report_repetition=True,
+        view="tree",
     )
     exit_status = 0
     if delta:
@@ -93,10 +96,69 @@ def diff(args):
         """
         )
 
-    return exit_status
+    return delta, exit_status
+
+
+def diff(args):
+    """Diffs Velox function signatures."""
+    base_signatures = json.load(args.base)
+    contender_signatures = json.load(args.contender)
+    return diff_signatures(base_signatures, contender_signatures)[1]
+
+
+def bias(args):
+    base_signatures = json.load(args.base)
+    contender_signatures = json.load(args.contender)
+    tickets = args.ticket_value
+    bias_output, status = bias_signatures(
+        base_signatures, contender_signatures, tickets
+    )
+    if status:
+        return status
+
+    if bias_output:
+        with open(args.output_path, "w") as f:
+            print(f"{bias_output}", file=f, end="")
+
+    return 0
 
 
-def parse_args():
+def bias_signatures(base_signatures, contender_signatures, tickets):
+    """Returns newly added functions as string and a status flag.
+    Newly added functions are biased like so `fn_name1=<ticket_count>,fn_name2=<ticket_count>`.
+    If it detects incompatible changes returns 1 in the status and empty string.
+    """
+    delta, status = diff_signatures(base_signatures, contender_signatures)
+
+    # Return if the signature check call flags incompatible changes.
+    if status:
+        return "", status
+
+    if not delta:
+        print(f"{bcolors.BOLD} No changes detected: Nothing to do!")
+        return "", 0
+
+    function_set = set()
+    for items in delta.values():
+        for item in items:
+            function_set.add(item.get_root_key())
+
+    print(f"{bcolors.BOLD}Functions to be biased: {function_set}")
+
+    if function_set:
+        return f"{f'={tickets},'.join(sorted(function_set)) + f'={tickets}'}", 0
+
+    return "", 0
+
+
+def get_tickets(val):
+    tickets = int(val)
+    if tickets < 0:
+        raise argparse.ArgumentTypeError("Cant have negative values!")
+    return tickets
+
+
+def parse_args(args):
     global parser
 
     parser = argparse.ArgumentParser(
@@ -111,16 +173,23 @@ def parse_args():
     export_command_parser.add_argument("output_file", type=argparse.FileType("w"))
 
     diff_command_parser = command.add_parser("diff")
-    diff_command_parser.add_argument("first", type=argparse.FileType("r"))
-    diff_command_parser.add_argument("second", type=argparse.FileType("r"))
-
+    diff_command_parser.add_argument("base", type=argparse.FileType("r"))
+    diff_command_parser.add_argument("contender", type=argparse.FileType("r"))
+
+    bias_command_parser = command.add_parser("bias")
+    bias_command_parser.add_argument("base", type=argparse.FileType("r"))
+    bias_command_parser.add_argument("contender", type=argparse.FileType("r"))
+    bias_command_parser.add_argument("output_path")
+    bias_command_parser.add_argument(
+        "ticket_value", type=get_tickets, default=10, nargs="?"
+    )
     parser.set_defaults(command="help")
 
-    return parser.parse_args()
+    return parser.parse_args(args)
 
 
 def main():
-    args = parse_args()
+    args = parse_args(sys.argv[1:])
     return globals()[args.command](args)