From 91edbcd32b7275451f0541110fdb072a6b962f06 Mon Sep 17 00:00:00 2001 From: Michel Tricot Date: Fri, 8 Sep 2023 12:18:50 -0700 Subject: [PATCH] play --- .../connectors/source-pokeapi/.dockerignore | 4 +- .../connectors/source-pokeapi/Dockerfile | 6 +- .../connectors/source-pokeapi/Makefile | 17 + .../source-pokeapi/acceptance-test-config.yml | 10 +- .../connectors/source-pokeapi/pyproject.toml | 12 +- .../main/groovy/airbyte-python-poetry.gradle | 307 ++++++++++++++++++ 6 files changed, 344 insertions(+), 12 deletions(-) create mode 100644 buildSrc/src/main/groovy/airbyte-python-poetry.gradle diff --git a/airbyte-integrations/connectors/source-pokeapi/.dockerignore b/airbyte-integrations/connectors/source-pokeapi/.dockerignore index 682a873759d3..a2f872013b1b 100644 --- a/airbyte-integrations/connectors/source-pokeapi/.dockerignore +++ b/airbyte-integrations/connectors/source-pokeapi/.dockerignore @@ -1,6 +1,6 @@ * !Dockerfile !main.py -!source_pokeapi -!setup.py +!src/source_pokeapi +!pyproject.toml !secrets diff --git a/airbyte-integrations/connectors/source-pokeapi/Dockerfile b/airbyte-integrations/connectors/source-pokeapi/Dockerfile index 31f309c2e920..fc815038bd51 100644 --- a/airbyte-integrations/connectors/source-pokeapi/Dockerfile +++ b/airbyte-integrations/connectors/source-pokeapi/Dockerfile @@ -1,13 +1,13 @@ FROM python:3.9-slim # Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +#RUN apt-get clean && apt-get update && apt-get install -y bash --allow-insecure-repositories && rm -rf /var/lib/apt/lists/* WORKDIR /airbyte/integration_code -COPY source_pokeapi ./source_pokeapi +COPY src/source_pokeapi ./src/source_pokeapi COPY main.py ./ -COPY setup.py ./ +COPY pyproject.toml ./ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" diff --git a/airbyte-integrations/connectors/source-pokeapi/Makefile b/airbyte-integrations/connectors/source-pokeapi/Makefile index c479a0c50326..951b589e72b3 100644 --- a/airbyte-integrations/connectors/source-pokeapi/Makefile +++ b/airbyte-integrations/connectors/source-pokeapi/Makefile @@ -1,3 +1,6 @@ +DOCKER_REPOSITORY = $(shell cat metadata.yaml | grep 'dockerRepository' | cut -d : -f 2 | tr -d '[[:blank:]]') +PROJECT_TOP_LEVEL = $(shell git rev-parse --show-toplevel) + all: check format-ruff: @@ -15,6 +18,15 @@ lint-ruff: lint-mypy: poetry run mypy src tests || true +build-python: + rm -rf dist build/dist + mkdir -p build + poetry build + mv dist build/ + +build-docker: + docker build . -t $(DOCKER_REPOSITORY):dev + format: format-ruff format-black lint: lint-mypy lint-ruff lint-black @@ -22,4 +34,9 @@ lint: lint-mypy lint-ruff lint-black test: poetry run pytest +integration_test: + source "$(PROJECT_TOP_LEVEL)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" + +build: build-python build-docker + check: lint test diff --git a/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml index 2f23c6db6c7f..12a3fc029e07 100644 --- a/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml @@ -5,19 +5,19 @@ acceptance_tests: bypass_reason: "The spec is currently invalid: it has additionalProperties set to false" connection: tests: - - config_path: "integration_tests/config.json" + - config_path: "tests/integration_tests/config.json" status: "succeed" discovery: tests: - - config_path: "integration_tests/config.json" + - config_path: "tests/integration_tests/config.json" basic_read: tests: - - config_path: "integration_tests/config.json" + - config_path: "tests/integration_tests/config.json" expect_records: bypass_reason: "We should create an expected_records file" full_refresh: tests: - - config_path: "integration_tests/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "tests/integration_tests/config.json" + configured_catalog_path: "tests/integration_tests/configured_catalog.json" incremental: bypass_reason: "This connector does not support incremental syncs." diff --git a/airbyte-integrations/connectors/source-pokeapi/pyproject.toml b/airbyte-integrations/connectors/source-pokeapi/pyproject.toml index 263d6976bfe8..f0f7fa121830 100644 --- a/airbyte-integrations/connectors/source-pokeapi/pyproject.toml +++ b/airbyte-integrations/connectors/source-pokeapi/pyproject.toml @@ -1,12 +1,18 @@ [tool.poetry] -name = "source-pokeapi" +name = "airbyte-source-pokeapi" version = "0.1.0" description = "Source implementation for Pokeapi." authors = ["Airbyte "] +classifiers = [ + "Framework :: Airbyte :: Connectors :: Sources" +] +packages = [ + { include = "source_pokeapi", from = "src" } +] include = [ { path = "src/*/*.json"}, { path = "src/*/schemas/*.json"}, - { path = "schemas/shared/*.json"} + { path = "src/*/schemas/shared/*.json"} ] [tool.poetry.dependencies] @@ -20,6 +26,7 @@ mypy = "^1.4.1" [tool.poetry.group.dev.dependencies] pytest = "~6" +# Could we publish it? connector-acceptance-test = {path = "../../bases/connector-acceptance-test", develop = true} [build-system] @@ -50,3 +57,4 @@ disallow_untyped_defs = "False" testpaths = [ "tests/unit_tests" ] +addopts ="-r a --capture=no -vv --color=yes" diff --git a/buildSrc/src/main/groovy/airbyte-python-poetry.gradle b/buildSrc/src/main/groovy/airbyte-python-poetry.gradle new file mode 100644 index 000000000000..b7884f661ee2 --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-python-poetry.gradle @@ -0,0 +1,307 @@ +import groovy.io.FileType +import groovy.io.FileVisitResult +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.Exec +import ru.vyarus.gradle.plugin.python.task.PythonTask + +class AirbytePythonConfiguration { + String moduleDirectory +} + +class Helpers { + static addTestTaskIfTestFilesFound(Project project, String testFilesDirectory, String taskName, taskDependencies) { + """ + This method verifies if there are test files in a directory before adding the pytest task to run tests on that directory. This is needed + because if there are no tests in that dir and we run pytest on it, it exits with exit code 5 which gradle takes to mean that the process + failed, since it's non-zero. This means that if a module doesn't need a unit or integration test, it still needs to add a dummy test file + like: + + ``` + def make_ci_pass_test(): + assert True + ``` + + So we use this method to leverage pytest's test discovery rules (https://docs.pytest.org/en/6.2.x/goodpractices.html#conventions-for-python-test-discovery) + to selectively run pytest based on whether there seem to be test files in that directory. + Namely, if the directory contains a file whose name is test_*.py or *_test.py then it's a test. + + See https://github.com/airbytehq/airbyte/issues/4979 for original context + """ + + if (project.file(testFilesDirectory).exists()) { + def outputArg = project.hasProperty('reports_folder') ?"-otemp_coverage.xml" : "--skip-empty" + def coverageFormat = project.hasProperty('reports_folder') ? 'xml' : 'report' + def testConfig = project.file('pytest.ini').exists() ? 'pytest.ini' : project.rootProject.file('pyproject.toml').absolutePath + project.projectDir.toPath().resolve(testFilesDirectory).traverse(type: FileType.FILES, nameFilter: ~/(^test_.*|.*_test)\.py$/) { file -> + project.task("_${taskName}Coverage", type: PythonTask, dependsOn: taskDependencies) { + module = "coverage" + command = "run --data-file=${testFilesDirectory}/.coverage.${taskName} --rcfile=${project.rootProject.file('pyproject.toml').absolutePath} -m pytest -s ${testFilesDirectory} -c ${testConfig}" + } + // generation of coverage report is optional and we should skip it if tests are empty + + project.task(taskName, type: Exec){ + commandLine = ".venv/bin/python" + args "-m", "coverage", coverageFormat, "--data-file=${testFilesDirectory}/.coverage.${taskName}", "--rcfile=${project.rootProject.file('pyproject.toml').absolutePath}", outputArg + dependsOn project.tasks.findByName("_${taskName}Coverage") + setIgnoreExitValue true + doLast { + // try to move a generated report to custom report folder if needed + if (project.file('temp_coverage.xml').exists() && project.hasProperty('reports_folder')) { + project.file('temp_coverage.xml').renameTo(project.file("${project.reports_folder}/coverage.xml")) + } + } + + } + // If a file is found, terminate the traversal, thus causing this task to be declared at most once + return FileVisitResult.TERMINATE + } + } + + // If the task doesn't exist then we didn't find a matching file. So add an empty task since other tasks will + // probably rely on this one existing. + if (!project.hasProperty(taskName)) { + project.task(taskName) { + logger.info "Skipping task ${taskName} because ${testFilesDirectory} doesn't exist." + } + } + } +} + +class AirbytePythonPlugin implements Plugin { + + void apply(Project project) { + def extension = project.extensions.create('airbytePython', AirbytePythonConfiguration) + + def venvDirectoryName = '.venv' + project.plugins.apply 'ru.vyarus.use-python' + + project.python { + envPath = venvDirectoryName + minPythonVersion = '3.9' + scope = 'VIRTUALENV' + installVirtualenv = true + pip 'pip:21.3.1' + pip 'mccabe:0.6.1' + // https://github.com/csachs/pyproject-flake8/issues/13 + pip 'flake8:4.0.1' + // flake8 doesn't support pyproject.toml files + // and thus there is the wrapper "pyproject-flake8" for this + pip 'pyproject-flake8:0.0.1a2' + pip 'black:22.3.0' + pip 'mypy:1.4.1' + pip 'isort:5.6.4' + pip 'pytest:6.2.5' + pip 'coverage[toml]:6.3.1' + } + + + project.task('isortFormat', type: PythonTask) { + module = "isort" + command = "--settings-file=${project.rootProject.file('pyproject.toml').absolutePath} ./" + } + + project.task('isortReport', type: PythonTask) { + module = "isort" + command = "--settings-file=${project.rootProject.file('pyproject.toml').absolutePath} --diff --quiet ./" + outputPrefix = '' + } + + project.task('blackFormat', type: PythonTask) { + module = "black" + // the line length should match .isort.cfg + command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} ./" + dependsOn project.rootProject.licenseFormat + dependsOn project.isortFormat + } + + project.task('blackReport', type: PythonTask) { + module = "black" + command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} --diff --quiet ./" + outputPrefix = '' + } + + project.task('flakeCheck', type: PythonTask, dependsOn: project.blackFormat) { + module = "pflake8" + command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} ./" + } + + project.task('flakeReport', type: PythonTask) { + module = "pflake8" + command = "--exit-zero --config ${project.rootProject.file('pyproject.toml').absolutePath} ./" + outputPrefix = '' + } + + project.task("mypyReport", type: Exec){ + commandLine = ".venv/bin/python" + args "-m", "mypy", "--config-file", "${project.rootProject.file('pyproject.toml').absolutePath}", "./" + setIgnoreExitValue true + } + + + + // attempt to install anything in requirements.txt. by convention this should only be dependencies whose source is located in the project. + + if (project.file('requirements.txt').exists()) { + project.task('installLocalReqs', type: PythonTask) { + module = "pip" + command = "install -r requirements.txt" + inputs.file('requirements.txt') + outputs.file('build/installedlocalreqs.txt') + } + } else if (project.file('setup.py').exists()) { + // If requirements.txt does not exists, install from setup.py instead, assume a dev or "tests" profile exists. + // In this case, there is no need to depend on the base python modules since everything should be contained in the setup.py. + project.task('installLocalReqs', type: PythonTask) { + module = "pip" + command = "install .[dev,tests]" + } + } else { + throw new GradleException('Error: Python module lacks requirement.txt and setup.py') + } + + project.task('installReqs', type: PythonTask, dependsOn: project.installLocalReqs) { + module = "pip" + command = "install .[main]" + inputs.file('setup.py') + outputs.file('build/installedreqs.txt') + } + + project.task('installTestReqs', type: PythonTask, dependsOn: project.installReqs) { + module = "pip" + command = "install .[tests]" + inputs.file('setup.py') + outputs.file('build/installedtestreqs.txt') + } + + Helpers.addTestTaskIfTestFilesFound(project, 'unit_tests', 'unitTest', project.installTestReqs) + + Helpers.addTestTaskIfTestFilesFound(project, 'integration_tests', 'customIntegrationTests', project.installTestReqs) + if (!project.tasks.findByName('integrationTest')) { + project.task('integrationTest') + } + project.integrationTest.dependsOn(project.customIntegrationTests) + + if (extension.moduleDirectory) { + project.task('mypyCheck', type: PythonTask) { + module = "mypy" + command = "-m ${extension.moduleDirectory} --config-file ${project.rootProject.file('pyproject.toml').absolutePath}" + } + + project.check.dependsOn mypyCheck + } + + project.task('airbytePythonFormat', type: DefaultTask) { + dependsOn project.blackFormat + dependsOn project.isortFormat + dependsOn project.flakeCheck + } + + project.task('airbytePythonReport', type: DefaultTask) { + dependsOn project.blackReport + dependsOn project.isortReport + dependsOn project.flakeReport + dependsOn project.mypyReport + doLast { + if (project.hasProperty('reports_folder')) { + // Gradles adds some log messages to files and we must remote them + // examples of these lines: + // :airbyte-integrations:connectors: ... + // [python] .venv/bin/python -m black ... + project.fileTree(project.reports_folder).visit { FileVisitDetails details -> + project.println "Found the report file: " + details.file.path + def tempFile = project.file(details.file.path + ".1") + details.file.eachLine { line -> + if ( !line.startsWith(":airbyte") && !line.startsWith("[python]") ) { + tempFile << line + "\n" + } + } + if (!tempFile.exists()) { + // generate empty file + tempFile << "\n" + } + tempFile.renameTo(details.file) + + } + } + } + } + + project.task('airbytePythonApply', type: DefaultTask) { + dependsOn project.installReqs + dependsOn project.airbytePythonFormat + } + + + project.task('airbytePythonTest', type: DefaultTask) { + dependsOn project.airbytePythonApply + dependsOn project.installTestReqs + dependsOn project.unitTest + } + + // Add a task that allows cleaning up venvs to every python project + project.task('cleanPythonVenv', type: Exec) { + commandLine 'rm' + args '-rf', "$project.projectDir.absolutePath/$venvDirectoryName" + } + + // Add a task which can be run at the root project level to delete all python venvs + if (!project.rootProject.hasProperty('cleanPythonVenvs')) { + project.rootProject.task('cleanPythonVenvs') + } + project.rootProject.cleanPythonVenvs.dependsOn(project.cleanPythonVenv) + + project.assemble.dependsOn project.airbytePythonApply + project.assemble.dependsOn project.airbytePythonTest + project.test.dependsOn project.airbytePythonTest + + // saves tools reports to a custom folder + def reportsFolder = project.hasProperty('reports_folder') ? project.reports_folder : '' + if ( reportsFolder != '' ) { + + // clean reports folders + project.file(reportsFolder).deleteDir() + project.file(reportsFolder).mkdirs() + + + + project.tasks.blackReport.configure { + it.logging.addStandardOutputListener(new StandardOutputListener() { + @Override + void onOutput(CharSequence charSequence) { + project.file("$reportsFolder/black.diff") << charSequence + } + }) + } + project.tasks.isortReport.configure { + it.logging.addStandardOutputListener(new StandardOutputListener() { + @Override + void onOutput(CharSequence charSequence) { + project.file("$reportsFolder/isort.diff") << charSequence + } + }) + } + + project.tasks.flakeReport.configure { + it.logging.addStandardOutputListener(new StandardOutputListener() { + @Override + void onOutput(CharSequence charSequence) { + project.file("$reportsFolder/flake.txt") << charSequence + } + }) + } + + project.tasks.mypyReport.configure { + it.logging.addStandardOutputListener(new StandardOutputListener() { + @Override + void onOutput(CharSequence charSequence) { + project.file("$reportsFolder/mypy.log") << charSequence + } + }) + } + + } + } +} +