Skip to content

Commit

Permalink
Revert "Distroless python sdk (apache#32960)"
Browse files Browse the repository at this point in the history
This reverts commit 81f35ab.
  • Loading branch information
shunping committed Nov 12, 2024
1 parent 682eaef commit c414331
Show file tree
Hide file tree
Showing 5 changed files with 2 additions and 87 deletions.
26 changes: 1 addition & 25 deletions sdks/python/container/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -103,33 +103,9 @@ RUN if [ "$pull_licenses" = "true" ] ; then \
python /tmp/license_scripts/pull_licenses_py.py ; \
fi

FROM beam as base
FROM beam
ARG pull_licenses
COPY --from=third_party_licenses /opt/apache/beam/third_party_licenses /opt/apache/beam/third_party_licenses
RUN if [ "$pull_licenses" != "true" ] ; then \
rm -rf /opt/apache/beam/third_party_licenses ; \
fi

ARG TARGETARCH
FROM gcr.io/distroless/python3-debian12:latest-${TARGETARCH} as distroless
ARG py_version

# Contains header files needed by the Python interpreter.
COPY --from=base /usr/local/include /usr/local/include

# Contains the Python interpreter executables.
COPY --from=base /usr/local/bin /usr/local/bin

# Contains the Python library dependencies.
COPY --from=base /usr/local/lib /usr/local/lib

# Python standard library modules.
COPY --from=base /usr/lib/python${py_version} /usr/lib/python${py_version}

# Contains the boot entrypoint and related files such as licenses.
COPY --from=base /opt /opt

ENV PATH "$PATH:/usr/local/bin"

# Despite the ENTRYPOINT set above, need to reset since deriving the layer derives from a different image.
ENTRYPOINT ["/opt/apache/beam/boot"]
9 changes: 1 addition & 8 deletions sdks/python/container/common.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -71,16 +71,10 @@ def copyLauncherDependencies = tasks.register("copyLauncherDependencies", Copy)
}

def pushContainers = project.rootProject.hasProperty(["isRelease"]) || project.rootProject.hasProperty("push-containers")
def baseBuildTarget = 'base'
def buildTarget = project.findProperty('container-build-target') ?: 'base'
var imageName = project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk"
if (buildTarget != baseBuildTarget) {
imageName += "_${buildTarget}"
}

docker {
name containerImageName(
name: imageName,
name: project.docker_image_default_repo_prefix + "python${project.ext.pythonVersion}_sdk",
root: project.rootProject.hasProperty(["docker-repository-root"]) ?
project.rootProject["docker-repository-root"] :
project.docker_image_default_repo_root,
Expand All @@ -96,7 +90,6 @@ docker {
platform(*project.containerPlatforms())
load project.useBuildx() && !pushContainers
push pushContainers
target buildTarget
}

dockerPrepare.dependsOn copyLauncherDependencies
Expand Down
6 changes: 0 additions & 6 deletions sdks/python/test-suites/dataflow/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,6 @@ task validatesContainerTests {
}
}

task validatesDistrolessContainerTests {
getVersionsAsList('distroless_python_versions').each {
dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:validatesDistrolessContainer")
}
}

task examplesPostCommit {
getVersionsAsList('dataflow_examples_postcommit_py_versions').each {
dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:examples")
Expand Down
45 changes: 0 additions & 45 deletions sdks/python/test-suites/dataflow/common.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -380,51 +380,6 @@ task validatesContainer() {
}
}

/**
* Validates the distroless (https://github.com/GoogleContainerTools/distroless) variant of the Python SDK container
* image (sdks/python/container/Dockerfile).
* To test a single version of Python:
* ./gradlew :sdks:python:test-suites:dataflow:py311:validatesDistrolessContainer
* See https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-VirtualEnvironmentSetup
* for more information on setting up different Python versions.
*/
task validatesDistrolessContainer() {
def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
def buildTarget = 'distroless'
def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
def tag = java.time.Instant.now().getEpochSecond()
def imageURL = "${repository}/beam_python${project.ext.pythonVersion}_sdk_${buildTarget}:${tag}"
project.rootProject.ext['docker-repository-root'] = repository
project.rootProject.ext['container-build-target'] = buildTarget
project.rootProject.ext['docker-tag'] = tag
if (project.rootProject.hasProperty('dry-run')) {
println "Running in dry run mode: imageURL: ${imageURL}, pyversion: ${pyversion}, buildTarget: ${buildTarget}, repository: ${repository}, tag: ${tag}, envdir: ${envdir}"
return
}
dependsOn 'initializeForDataflowJob'
dependsOn ":sdks:python:container:py${pyversion}:docker"
dependsOn ":sdks:python:container:py${pyversion}:dockerPush"
def testTarget = "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it"
def argMap = [
"output": "gs://temp-storage-for-end-to-end-tests/py-it-cloud/output",
"project": "apache-beam-testing",
"region": "us-central1",
"runner": "TestDataflowRunner",
"sdk_container_image": "${imageURL}",
"sdk_location": "container",
"staging_location": "gs://temp-storage-for-end-to-end-tests/staging-it",
"temp_location": "gs://temp-storage-for-end-to-end-tests/temp-it",
]
def cmdArgs = mapToArgString(argMap)
doLast {
exec {
workingDir = "${rootDir}/sdks/python"
executable 'sh'
args '-c', ". ${envdir}/bin/activate && pytest ${testTarget} --test-pipeline-options=\"${cmdArgs}\""
}
}
}

task validatesContainerARM() {
def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
dependsOn 'initializeForDataflowJob'
Expand Down
3 changes: 0 additions & 3 deletions sdks/python/test-suites/gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,3 @@ prism_examples_postcommit_py_versions=3.9,3.12

# cross language postcommit python test suites
cross_language_validates_py_versions=3.9,3.12

# Python versions to support distroless variants
distroless_python_versions=3.9,3.10,3.11,3.12

0 comments on commit c414331

Please sign in to comment.