Use certifi to tell urllib where to find CA file (youtube#3974)

Also clean up some unused Docker images.

b/359451816
b/355184050
b/354270960

.pre-commit-config.yaml

@@ -174,6 +174,7 @@ repos:
         pass_filenames: false
         always_run: true
         stages: [push]
+        additional_dependencies: [certifi]
         args: [-m, unittest, cobalt/tools/download_from_gcs_test.py]
     -   id: test-python3-compatibility
         name: Test Python 3 Compatibility

cobalt/tools/download_from_gcs.py

@@ -15,6 +15,7 @@
 """Downloads files from Google Cloud Storage given corresponding sha1s."""
 
 import argparse
+import certifi
 import hashlib
 import logging
 import os
@@ -50,7 +51,8 @@ def ExtractSha1(filename):
 
 def _DownloadFromGcsAndCheckSha1(bucket, sha1):
   url = f'{_BASE_GCS_URL}/{bucket}/{sha1}'
-  with urllib.request.urlopen(url, context=ssl.create_default_context()) as res:
+  ssl_context = ssl.create_default_context(cafile=certifi.where())
+  with urllib.request.urlopen(url, context=ssl_context) as res:
     with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
       shutil.copyfileobj(res, tmp_file)
 

docker/linux/base/build/Dockerfile

@@ -25,6 +25,7 @@ RUN apt update -qqy \
         nasm \
         ninja-build \
         pkgconf \
+        python3-certifi \
         unzip \
     && /opt/clean-after-apt.sh
 

docker/linux/clang-3-9/Dockerfile

@@ -34,5 +34,8 @@ RUN python3.8 -m venv /python38_env
 ENV VIRTUAL_ENV /python38_env
 ENV PATH /python38_env/bin:$PATH
 
+COPY clang-3-9/requirements.txt /opt/requirements.txt
+RUN python3.8 -m pip install --require-hashes --no-deps -r /opt/requirements.txt
+
 CMD gn gen ${OUTDIR}/${PLATFORM}_${CONFIG} --args="target_platform=\"${PLATFORM}\" build_type=\"${CONFIG}\" using_old_compiler=true build_with_separate_cobalt_toolchain=true" && \
     ninja -v -j ${NINJA_PARALLEL} -C ${OUTDIR}/${PLATFORM}_${CONFIG} ${TARGET:-cobalt_install}

docker/linux/linux-x64x11/requirements.in → docker/linux/clang-3-9/requirements.in

@@ -4,5 +4,4 @@
 # $ pip3 install pip-tools
 # $ pip-compile --allow-unsafe --generate-hashes path/to/requirements.in
 
-selenium==3.141.0
-Brotli==1.0.9
+certifi

docker/linux/clang-3-9/requirements.txt

@@ -0,0 +1,10 @@
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+#    pip-compile --allow-unsafe --generate-hashes docker/linux/clang-3-9/requirements.in
+#
+certifi==2024.7.4 \
+    --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \
+    --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90
+    # via -r docker/linux/clang-3-9/requirements.in

docker/linux/gcc-6-3/Dockerfile

@@ -34,5 +34,8 @@ RUN python3.8 -m venv /python38_env
 ENV VIRTUAL_ENV /python38_env
 ENV PATH /python38_env/bin:$PATH
 
+COPY gcc-6-3/requirements.txt /opt/requirements.txt
+RUN python3.8 -m pip install --require-hashes --no-deps -r /opt/requirements.txt
+
 CMD gn gen ${OUTDIR}/${PLATFORM}${SB_API_VERSION:+-sbversion-$SB_API_VERSION}_${CONFIG:-debug} --args="target_platform=\"${PLATFORM}\" build_type=\"${CONFIG:-debug}\" build_with_separate_cobalt_toolchain=true using_old_compiler=true ${SB_API_VERSION:+sb_api_version=$SB_API_VERSION}" && \
     ninja -v -j ${NINJA_PARALLEL} -C ${OUTDIR}/${PLATFORM}${SB_API_VERSION:+-sbversion-$SB_API_VERSION}_${CONFIG:-debug} ${TARGET:-cobalt_install}

docker/linux/gcc-6-3/requirements.in

@@ -0,0 +1,7 @@
+# IMPORTANT: Remember to also update requirements.txt when updating this file:
+# $ docker run -it --mount type=bind,source=${COBALT_SRC},target=/code -w /code python:3.8 /bin/bash
+# In the container:
+# $ pip3 install pip-tools
+# $ pip-compile --allow-unsafe --generate-hashes path/to/requirements.in
+
+certifi

docker/linux/gcc-6-3/requirements.txt

@@ -0,0 +1,10 @@
+#
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
+#
+#    pip-compile --allow-unsafe --generate-hashes docker/linux/gcc-6-3/requirements.in
+#
+certifi==2024.7.4 \
+    --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \
+    --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90
+    # via -r docker/linux/gcc-6-3/requirements.in

docker/linux/linux-x64x11/Dockerfile

@@ -27,8 +27,5 @@ RUN apt update -qqy \
         libxi-dev \
     && /opt/clean-after-apt.sh
 
-COPY ./linux-x64x11/requirements.txt /opt/requirements.txt
-RUN python3 -m pip install --require-hashes --no-deps -r /opt/requirements.txt
-
 CMD gn gen ${OUTDIR}/${PLATFORM}${SB_API_VERSION:+-sbversion-$SB_API_VERSION}_${CONFIG:-debug} --args="target_platform=\"${PLATFORM}\" build_type=\"${CONFIG:-debug}\" build_with_separate_cobalt_toolchain=true ${SB_API_VERSION:+sb_api_version=$SB_API_VERSION}" && \
     ninja -j ${NINJA_PARALLEL} -C ${OUTDIR}/${PLATFORM}${SB_API_VERSION:+-sbversion-$SB_API_VERSION}_${CONFIG:-debug} ${TARGET:-cobalt_install}

docker/linux/unittest/requirements.in

@@ -5,5 +5,7 @@
 # $ pip-compile --allow-unsafe --generate-hashes path/to/requirements.in
 
 selenium==3.141.0
-Brotli==1.0.9
 junitparser==2.8.0
+
+# Integration tests.
+brotli==1.0.9

docker/linux/unittest/requirements.txt

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile --generate-hashes requirements.in
+#    pip-compile --allow-unsafe --generate-hashes docker/linux/unittest/requirements.in
 #
 brotli==1.0.9 \
     --hash=sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d \
@@ -67,17 +67,17 @@ brotli==1.0.9 \
     --hash=sha256:e4c4e92c14a57c9bd4cb4be678c25369bf7a092d55fd0866f759e425b9660806 \
     --hash=sha256:ec1947eabbaf8e0531e8e899fc1d9876c179fc518989461f5d24e2223395a9e3 \
     --hash=sha256:f909bbbc433048b499cb9db9e713b5d8d949e8c109a2a548502fb9aa8630f0b1
-    # via -r requirements.in
+    # via -r docker/linux/unittest/requirements.in
 future==0.18.3 \
     --hash=sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307
     # via junitparser
 junitparser==2.8.0 \
     --hash=sha256:e1a7d41e0c92ca032c46eed07a6268c478f90ec1f411ea8a76b69f245e57cfb6
-    # via -r requirements.in
+    # via -r docker/linux/unittest/requirements.in
 selenium==3.141.0 \
     --hash=sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c \
     --hash=sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d
-    # via -r requirements.in
+    # via -r docker/linux/unittest/requirements.in
 urllib3==1.26.18 \
     --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \
     --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0

docker/precommit_hooks/requirements.txt

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with Python 3.7
+# This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
-#    pip-compile --allow-unsafe --generate-hashes requirements.in
+#    pip-compile --allow-unsafe --generate-hashes docker/precommit_hooks/requirements.in
 #
 astroid==2.15.6 \
     --hash=sha256:389656ca57b6108f939cf5d2f9a2a825a3be50ba9d589670f393236e0a03b91c \
@@ -15,7 +15,7 @@ cfgv==3.3.1 \
 cpplint==1.6.1 \
     --hash=sha256:00ddc86d6e4de2a9dcfa272402dcbe21593363a93b7c475bc391e335062f34b1 \
     --hash=sha256:d430ce8f67afc1839340e60daa89e90de08b874bc27149833077bba726dfc13a
-    # via -r requirements.in
+    # via -r docker/precommit_hooks/requirements.in
 dill==0.3.7 \
     --hash=sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e \
     --hash=sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03
@@ -35,10 +35,7 @@ identify==2.5.24 \
 importlib-metadata==6.7.0 \
     --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \
     --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5
-    # via
-    #   pre-commit
-    #   virtualenv
-    #   yapf
+    # via yapf
 isort==5.11.5 \
     --hash=sha256:6be1f76a507cb2ecf16c7cf14a37e41609ca082330be4e3436a18ef74add55db \
     --hash=sha256:ba1d72fb2595a01c7895a5128f9585a5cc4b6d395f1c8d514989b9a7eb2a8746
@@ -99,11 +96,11 @@ platformdirs==3.10.0 \
 pre-commit==2.21.0 \
     --hash=sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658 \
     --hash=sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad
-    # via -r requirements.in
+    # via -r docker/precommit_hooks/requirements.in
 pylint==2.17.5 \
     --hash=sha256:73995fb8216d3bed149c8d51bba25b2c52a8251a2c8ac846ec668ce38fab5413 \
     --hash=sha256:f7b601cbc06fef7e62a754e2b41294c2aa31f1cb659624b9a85bcba29eaf8252
-    # via -r requirements.in
+    # via -r docker/precommit_hooks/requirements.in
 pyyaml==6.0.1 \
     --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \
     --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \
@@ -156,56 +153,11 @@ tomlkit==0.12.1 \
     --hash=sha256:38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86 \
     --hash=sha256:712cbd236609acc6a3e2e97253dfc52d4c2082982a88f61b640ecf0817eab899
     # via pylint
-typed-ast==1.5.5 \
-    --hash=sha256:042eb665ff6bf020dd2243307d11ed626306b82812aba21836096d229fdc6a10 \
-    --hash=sha256:045f9930a1550d9352464e5149710d56a2aed23a2ffe78946478f7b5416f1ede \
-    --hash=sha256:0635900d16ae133cab3b26c607586131269f88266954eb04ec31535c9a12ef1e \
-    --hash=sha256:118c1ce46ce58fda78503eae14b7664163aa735b620b64b5b725453696f2a35c \
-    --hash=sha256:16f7313e0a08c7de57f2998c85e2a69a642e97cb32f87eb65fbfe88381a5e44d \
-    --hash=sha256:1efebbbf4604ad1283e963e8915daa240cb4bf5067053cf2f0baadc4d4fb51b8 \
-    --hash=sha256:2188bc33d85951ea4ddad55d2b35598b2709d122c11c75cffd529fbc9965508e \
-    --hash=sha256:2b946ef8c04f77230489f75b4b5a4a6f24c078be4aed241cfabe9cbf4156e7e5 \
-    --hash=sha256:335f22ccb244da2b5c296e6f96b06ee9bed46526db0de38d2f0e5a6597b81155 \
-    --hash=sha256:381eed9c95484ceef5ced626355fdc0765ab51d8553fec08661dce654a935db4 \
-    --hash=sha256:429ae404f69dc94b9361bb62291885894b7c6fb4640d561179548c849f8492ba \
-    --hash=sha256:44f214394fc1af23ca6d4e9e744804d890045d1643dd7e8229951e0ef39429b5 \
-    --hash=sha256:48074261a842acf825af1968cd912f6f21357316080ebaca5f19abbb11690c8a \
-    --hash=sha256:4bc1efe0ce3ffb74784e06460f01a223ac1f6ab31c6bc0376a21184bf5aabe3b \
-    --hash=sha256:57bfc3cf35a0f2fdf0a88a3044aafaec1d2f24d8ae8cd87c4f58d615fb5b6311 \
-    --hash=sha256:597fc66b4162f959ee6a96b978c0435bd63791e31e4f410622d19f1686d5e769 \
-    --hash=sha256:5f7a8c46a8b333f71abd61d7ab9255440d4a588f34a21f126bbfc95f6049e686 \
-    --hash=sha256:5fe83a9a44c4ce67c796a1b466c270c1272e176603d5e06f6afbc101a572859d \
-    --hash=sha256:61443214d9b4c660dcf4b5307f15c12cb30bdfe9588ce6158f4a005baeb167b2 \
-    --hash=sha256:622e4a006472b05cf6ef7f9f2636edc51bda670b7bbffa18d26b255269d3d814 \
-    --hash=sha256:6eb936d107e4d474940469e8ec5b380c9b329b5f08b78282d46baeebd3692dc9 \
-    --hash=sha256:7f58fabdde8dcbe764cef5e1a7fcb440f2463c1bbbec1cf2a86ca7bc1f95184b \
-    --hash=sha256:83509f9324011c9a39faaef0922c6f720f9623afe3fe220b6d0b15638247206b \
-    --hash=sha256:8c524eb3024edcc04e288db9541fe1f438f82d281e591c548903d5b77ad1ddd4 \
-    --hash=sha256:94282f7a354f36ef5dbce0ef3467ebf6a258e370ab33d5b40c249fa996e590dd \
-    --hash=sha256:b445c2abfecab89a932b20bd8261488d574591173d07827c1eda32c457358b18 \
-    --hash=sha256:be4919b808efa61101456e87f2d4c75b228f4e52618621c77f1ddcaae15904fa \
-    --hash=sha256:bfd39a41c0ef6f31684daff53befddae608f9daf6957140228a08e51f312d7e6 \
-    --hash=sha256:c631da9710271cb67b08bd3f3813b7af7f4c69c319b75475436fcab8c3d21bee \
-    --hash=sha256:cc95ffaaab2be3b25eb938779e43f513e0e538a84dd14a5d844b8f2932593d88 \
-    --hash=sha256:d09d930c2d1d621f717bb217bf1fe2584616febb5138d9b3e8cdd26506c3f6d4 \
-    --hash=sha256:d40c10326893ecab8a80a53039164a224984339b2c32a6baf55ecbd5b1df6431 \
-    --hash=sha256:d41b7a686ce653e06c2609075d397ebd5b969d821b9797d029fccd71fdec8e04 \
-    --hash=sha256:d5c0c112a74c0e5db2c75882a0adf3133adedcdbfd8cf7c9d6ed77365ab90a1d \
-    --hash=sha256:e1a976ed4cc2d71bb073e1b2a250892a6e968ff02aa14c1f40eba4f365ffec02 \
-    --hash=sha256:e48bf27022897577d8479eaed64701ecaf0467182448bd95759883300ca818c8 \
-    --hash=sha256:ed4a1a42df8a3dfb6b40c3d2de109e935949f2f66b19703eafade03173f8f437 \
-    --hash=sha256:f0aefdd66f1784c58f65b502b6cf8b121544680456d1cebbd300c2c813899274 \
-    --hash=sha256:fc2b8c4e1bc5cd96c1a823a885e6b158f8451cf6f5530e1829390b4d27d0807f \
-    --hash=sha256:fd946abf3c31fb50eee07451a6aedbfff912fcd13cf357363f5b4e834cc5e71a \
-    --hash=sha256:fe58ef6a764de7b4b36edfc8592641f56e69b7163bba9f9c8089838ee596bfb2
-    # via astroid
 typing-extensions==4.7.1 \
     --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \
     --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2
     # via
     #   astroid
-    #   importlib-metadata
-    #   platformdirs
     #   pylint
 virtualenv==20.24.2 \
     --hash=sha256:43a3052be36080548bdee0b42919c88072037d50d56c28bd3f853cbe92b953ff \
@@ -291,7 +243,7 @@ wrapt==1.15.0 \
 yapf==0.40.1 \
     --hash=sha256:958587eb5c8ec6c860119a9c25d02addf30a44f75aa152a4220d30e56a98037c \
     --hash=sha256:b8bfc1f280949153e795181768ca14ef43d7312629a06c43e7abd279323af313
-    # via -r requirements.in
+    # via -r docker/precommit_hooks/requirements.in
 zipp==3.15.0 \
     --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \
     --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556

docker/pytest/requirements.in

@@ -14,3 +14,4 @@ six==1.12.0
 atomicwrites==1.4.1
 parameterized>=0.9.0
 protobuf>4
+certifi