forked from apache/arrow
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
apacheGH-39301: [Archery][CI][Integration] Add nanoarrow to archery +…
… integration setup (apache#39302) ### Rationale for this change The ability to add integration testing was added in nanoarrow however, the infrastructure for running these tests currently lives in the arrow monorepo. ### What changes are included in this PR? - Added the relevant code to Archery such that these tests can be run - Added the relevant scripts/environment variables to CI such that these tests run in the integration CI job ### Are these changes tested? Yes, via the "Integration" CI job. ### Are there any user-facing changes? No. This PR still needs apache#41264 for the integration tests to pass. * Closes: apache#39301 * GitHub Issue: apache#39301 Lead-authored-by: Dewey Dunnington <[email protected]> Co-authored-by: Dewey Dunnington <[email protected]> Signed-off-by: Dewey Dunnington <[email protected]>
- Loading branch information
Showing
8 changed files
with
223 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| #!/usr/bin/env bash | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
| set -e | ||
|
|
||
| arrow_dir=${1} | ||
| source_dir=${1}/nanoarrow | ||
| build_dir=${2}/nanoarrow | ||
|
|
||
| # This file is used to build the nanoarrow binaries needed for the archery | ||
| # integration tests. Testing of the nanoarrow implementation in normal CI is handled | ||
| # by github workflows in the arrow-nanoarrow repository. | ||
|
|
||
| if [ "${ARCHERY_INTEGRATION_WITH_NANOARROW}" -eq "0" ]; then | ||
| echo "=====================================================================" | ||
| echo "Not building nanoarrow" | ||
| echo "=====================================================================" | ||
| exit 0; | ||
| elif [ ! -d "${source_dir}" ]; then | ||
| echo "=====================================================================" | ||
| echo "The nanoarrow source is missing. Please clone the arrow-nanoarrow repository" | ||
| echo "to arrow/nanoarrow before running the integration tests:" | ||
| echo " git clone https://github.com/apache/arrow-nanoarrow.git path/to/arrow/nanoarrow" | ||
| echo "=====================================================================" | ||
| exit 1; | ||
| fi | ||
|
|
||
| set -x | ||
|
|
||
| mkdir -p ${build_dir} | ||
| pushd ${build_dir} | ||
|
|
||
| cmake ${source_dir} -DNANOARROW_BUILD_INTEGRATION_TESTS=ON | ||
| cmake --build . | ||
|
|
||
| popd |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,148 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
| import functools | ||
| import os | ||
|
|
||
| from . import cdata | ||
| from .tester import Tester, CDataExporter, CDataImporter | ||
| from ..utils.source import ARROW_ROOT_DEFAULT | ||
|
|
||
|
|
||
| _NANOARROW_PATH = os.environ.get( | ||
| "ARROW_NANOARROW_PATH", | ||
| os.path.join(ARROW_ROOT_DEFAULT, "nanoarrow/cdata"), | ||
| ) | ||
|
|
||
| _INTEGRATION_DLL = os.path.join( | ||
| _NANOARROW_PATH, "libnanoarrow_c_data_integration" + cdata.dll_suffix | ||
| ) | ||
|
|
||
|
|
||
| class NanoarrowTester(Tester): | ||
| PRODUCER = False | ||
| CONSUMER = False | ||
| FLIGHT_SERVER = False | ||
| FLIGHT_CLIENT = False | ||
| C_DATA_SCHEMA_EXPORTER = True | ||
| C_DATA_ARRAY_EXPORTER = True | ||
| C_DATA_SCHEMA_IMPORTER = True | ||
| C_DATA_ARRAY_IMPORTER = True | ||
|
|
||
| name = "nanoarrow" | ||
|
|
||
| def validate(self, json_path, arrow_path, quirks=None): | ||
| raise NotImplementedError() | ||
|
|
||
| def json_to_file(self, json_path, arrow_path): | ||
| raise NotImplementedError() | ||
|
|
||
| def stream_to_file(self, stream_path, file_path): | ||
| raise NotImplementedError() | ||
|
|
||
| def file_to_stream(self, file_path, stream_path): | ||
| raise NotImplementedError() | ||
|
|
||
| def make_c_data_exporter(self): | ||
| return NanoarrowCDataExporter(self.debug, self.args) | ||
|
|
||
| def make_c_data_importer(self): | ||
| return NanoarrowCDataImporter(self.debug, self.args) | ||
|
|
||
|
|
||
| _nanoarrow_c_data_entrypoints = """ | ||
| const char* nanoarrow_CDataIntegration_ExportSchemaFromJson( | ||
| const char* json_path, struct ArrowSchema* out); | ||
| const char* nanoarrow_CDataIntegration_ImportSchemaAndCompareToJson( | ||
| const char* json_path, struct ArrowSchema* schema); | ||
| const char* nanoarrow_CDataIntegration_ExportBatchFromJson( | ||
| const char* json_path, int num_batch, struct ArrowArray* out); | ||
| const char* nanoarrow_CDataIntegration_ImportBatchAndCompareToJson( | ||
| const char* json_path, int num_batch, struct ArrowArray* batch); | ||
| int64_t nanoarrow_BytesAllocated(void); | ||
| """ | ||
|
|
||
|
|
||
| @functools.lru_cache | ||
| def _load_ffi(ffi, lib_path=_INTEGRATION_DLL): | ||
| ffi.cdef(_nanoarrow_c_data_entrypoints) | ||
| dll = ffi.dlopen(lib_path) | ||
| return dll | ||
|
|
||
|
|
||
| class _CDataBase: | ||
| def __init__(self, debug, args): | ||
| self.debug = debug | ||
| self.args = args | ||
| self.ffi = cdata.ffi() | ||
| self.dll = _load_ffi(self.ffi) | ||
|
|
||
| def _check_nanoarrow_error(self, na_error): | ||
| """ | ||
| Check a `const char*` error return from an integration entrypoint. | ||
| A null means success, a non-empty string is an error message. | ||
| The string is statically allocated on the nanoarrow side and does not | ||
| need to be released. | ||
| """ | ||
| assert self.ffi.typeof(na_error) is self.ffi.typeof("const char*") | ||
| if na_error != self.ffi.NULL: | ||
| error = self.ffi.string(na_error).decode("utf8", errors="replace") | ||
| raise RuntimeError(f"nanoarrow C Data Integration call failed: {error}") | ||
|
|
||
|
|
||
| class NanoarrowCDataExporter(CDataExporter, _CDataBase): | ||
| def export_schema_from_json(self, json_path, c_schema_ptr): | ||
| na_error = self.dll.nanoarrow_CDataIntegration_ExportSchemaFromJson( | ||
| str(json_path).encode(), c_schema_ptr | ||
| ) | ||
| self._check_nanoarrow_error(na_error) | ||
|
|
||
| def export_batch_from_json(self, json_path, num_batch, c_array_ptr): | ||
| na_error = self.dll.nanoarrow_CDataIntegration_ExportBatchFromJson( | ||
| str(json_path).encode(), num_batch, c_array_ptr | ||
| ) | ||
| self._check_nanoarrow_error(na_error) | ||
|
|
||
| @property | ||
| def supports_releasing_memory(self): | ||
| return True | ||
|
|
||
| def record_allocation_state(self): | ||
| return self.dll.nanoarrow_BytesAllocated() | ||
|
|
||
|
|
||
| class NanoarrowCDataImporter(CDataImporter, _CDataBase): | ||
| def import_schema_and_compare_to_json(self, json_path, c_schema_ptr): | ||
| na_error = self.dll.nanoarrow_CDataIntegration_ImportSchemaAndCompareToJson( | ||
| str(json_path).encode(), c_schema_ptr | ||
| ) | ||
| self._check_nanoarrow_error(na_error) | ||
|
|
||
| def import_batch_and_compare_to_json(self, json_path, num_batch, c_array_ptr): | ||
| na_error = self.dll.nanoarrow_CDataIntegration_ImportBatchAndCompareToJson( | ||
| str(json_path).encode(), num_batch, c_array_ptr | ||
| ) | ||
| self._check_nanoarrow_error(na_error) | ||
|
|
||
| @property | ||
| def supports_releasing_memory(self): | ||
| return True |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters