platform(general): expose retry and timeout configuration for interac…

…tion with the platform (#5585) * expose retry and timeout configuration for interaction with the platform * fix lint * update docs Co-authored-by: Taylor <[email protected]> --------- Co-authored-by: Taylor <[email protected]>
bridgecrewio · Sep 21, 2023 · 780d848 · 780d848
1 parent 4950b14
commit 780d848
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 38 deletions.
diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py
@@ -41,9 +41,20 @@
 from checkov.common.util.consts import PRISMA_PLATFORM, BRIDGECREW_PLATFORM, CHECKOV_RUN_SCA_PACKAGE_SCAN_V2
 from checkov.common.util.data_structures_utils import merge_dicts
 from checkov.common.util.dockerfile import is_dockerfile
-from checkov.common.util.http_utils import normalize_prisma_url, get_auth_header, get_default_get_headers, \
-    get_user_agent_header, get_default_post_headers, get_prisma_get_headers, get_prisma_auth_header, \
-    get_auth_error_message, normalize_bc_url
+from checkov.common.util.http_utils import (
+    normalize_prisma_url,
+    get_auth_header,
+    get_default_get_headers,
+    get_user_agent_header,
+    get_default_post_headers,
+    get_prisma_get_headers,
+    get_prisma_auth_header,
+    get_auth_error_message,
+    normalize_bc_url,
+    REQUEST_CONNECT_TIMEOUT,
+    REQUEST_READ_TIMEOUT,
+    REQUEST_RETRIES,
+)
 from checkov.common.util.type_forcers import convert_prisma_policy_filter_to_dict, convert_str_to_bool
 from checkov.version import version as checkov_version
 
@@ -116,6 +127,8 @@ def __init__(self) -> None:
         self.use_s3_integration = False
         self.platform_integration_configured = False
         self.http: urllib3.PoolManager | urllib3.ProxyManager | None = None
+        self.http_timeout = urllib3.Timeout(connect=REQUEST_CONNECT_TIMEOUT, read=REQUEST_READ_TIMEOUT)
+        self.http_retry = urllib3.Retry(REQUEST_RETRIES, redirect=3)
         self.bc_skip_mapping = False
         self.cicd_details: _CicdDetails = {}
         self.support_flag_enabled = False
@@ -222,22 +235,39 @@ def setup_http_manager(self, ca_certificate: str | None = None, no_cert_verify:
             logging.debug(f'Using CA cert {ca_certificate} and cert_reqs {cert_reqs}')
             try:
                 parsed_url = urllib3.util.parse_url(os.environ['https_proxy'])
-                self.http = urllib3.ProxyManager(os.environ['https_proxy'],
-                                                 cert_reqs=cert_reqs,
-                                                 ca_certs=ca_certificate,
-                                                 proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth))  # type:ignore[no-untyped-call]
+                self.http = urllib3.ProxyManager(
+                    os.environ['https_proxy'],
+                    cert_reqs=cert_reqs,
+                    ca_certs=ca_certificate,
+                    proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth),  # type:ignore[no-untyped-call]
+                    timeout=self.http_timeout,
+                    retries=self.http_retry,
+                )
             except KeyError:
-                self.http = urllib3.PoolManager(cert_reqs=cert_reqs, ca_certs=ca_certificate)
+                self.http = urllib3.PoolManager(
+                    cert_reqs=cert_reqs,
+                    ca_certs=ca_certificate,
+                    timeout=self.http_timeout,
+                    retries=self.http_retry,
+                )
         else:
             cert_reqs = 'CERT_NONE' if no_cert_verify else None
             logging.debug(f'Using cert_reqs {cert_reqs}')
             try:
                 parsed_url = urllib3.util.parse_url(os.environ['https_proxy'])
-                self.http = urllib3.ProxyManager(os.environ['https_proxy'],
-                                                 cert_reqs=cert_reqs,
-                                                 proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth))  # type:ignore[no-untyped-call]
+                self.http = urllib3.ProxyManager(
+                    os.environ['https_proxy'],
+                    cert_reqs=cert_reqs,
+                    proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth),  # type:ignore[no-untyped-call]
+                    timeout=self.http_timeout,
+                    retries=self.http_retry,
+                )
             except KeyError:
-                self.http = urllib3.PoolManager(cert_reqs=cert_reqs)
+                self.http = urllib3.PoolManager(
+                    cert_reqs=cert_reqs,
+                    timeout=self.http_timeout,
+                    retries=self.http_retry,
+                )
         logging.debug('Successfully set up HTTP manager')
 
     def setup_bridgecrew_credentials(

diff --git a/checkov/common/util/http_utils.py b/checkov/common/util/http_utils.py
@@ -18,14 +18,20 @@
 from checkov.common.util.consts import DEV_API_GET_HEADERS, DEV_API_POST_HEADERS, PRISMA_API_GET_HEADERS, \
     PRISMA_PLATFORM, BRIDGECREW_PLATFORM
 from checkov.common.util.data_structures_utils import merge_dicts
+from checkov.common.util.type_forcers import force_int, force_float
 from checkov.version import version as checkov_version
 
 if TYPE_CHECKING:
     from checkov.common.bridgecrew.bc_source import SourceType
     from requests import Response
 
 # https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
-DEFAULT_TIMEOUT = (3.1, 30)
+REQUEST_CONNECT_TIMEOUT = force_float(os.getenv("CHECKOV_REQUEST_CONNECT_TIMEOUT")) or 3.1
+REQUEST_READ_TIMEOUT = force_int(os.getenv("CHECKOV_REQUEST_READ_TIMEOUT")) or 30
+DEFAULT_TIMEOUT = (REQUEST_CONNECT_TIMEOUT, REQUEST_READ_TIMEOUT)
+
+# https://urllib3.readthedocs.io/en/stable/user-guide.html#retrying-requests
+REQUEST_RETRIES = force_int(os.getenv("CHECKOV_REQUEST_RETRIES")) or 3
 
 logger = logging.getLogger(__name__)
 add_resource_code_filter_to_logger(logger)

diff --git a/checkov/common/vcs/base_vcs_dal.py b/checkov/common/vcs/base_vcs_dal.py
@@ -10,7 +10,7 @@
 import urllib3
 
 from checkov.common.util.data_structures_utils import merge_dicts
-from checkov.common.util.http_utils import get_user_agent_header
+from checkov.common.util.http_utils import get_user_agent_header, REQUEST_CONNECT_TIMEOUT, REQUEST_READ_TIMEOUT, REQUEST_RETRIES
 
 
 class BaseVCSDAL:
@@ -29,6 +29,8 @@ def __init__(self) -> None:
         self.org_complementary_metadata: dict[str, Any] = {}
         self.repo_complementary_metadata: dict[str, Any] = {}
         self.http: urllib3.PoolManager | None = None
+        self.http_timeout = urllib3.Timeout(connect=REQUEST_CONNECT_TIMEOUT, read=REQUEST_READ_TIMEOUT)
+        self.http_retry = urllib3.Retry(REQUEST_RETRIES, redirect=3)
         self.setup_http_manager(ca_certificate=os.getenv('BC_CA_BUNDLE', None))
         self.discover()
         self.setup_conf_dir()
@@ -51,18 +53,32 @@ def setup_http_manager(self, ca_certificate: str | None = None) -> None:
             os.environ['REQUESTS_CA_BUNDLE'] = ca_certificate
             try:
                 parsed_url = urllib3.util.parse_url(os.environ['https_proxy'])
-                self.http = urllib3.ProxyManager(os.environ['https_proxy'], cert_reqs='REQUIRED',
-                                                 ca_certs=ca_certificate,
-                                                 proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth))  # type:ignore[no-untyped-call]
+                self.http = urllib3.ProxyManager(
+                    os.environ['https_proxy'],
+                    cert_reqs='REQUIRED',
+                    ca_certs=ca_certificate,
+                    proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth),  # type:ignore[no-untyped-call]
+                    timeout=self.http_timeout,
+                    retries=self.http_retry,
+                )
             except KeyError:
-                self.http = urllib3.PoolManager(cert_reqs='REQUIRED', ca_certs=ca_certificate)
+                self.http = urllib3.PoolManager(
+                    cert_reqs='REQUIRED',
+                    ca_certs=ca_certificate,
+                    timeout=self.http_timeout,
+                    retries=self.http_retry,
+                )
         else:
             try:
                 parsed_url = urllib3.util.parse_url(os.environ['https_proxy'])
-                self.http = urllib3.ProxyManager(os.environ['https_proxy'],
-                                                 proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth))  # type:ignore[no-untyped-call]
+                self.http = urllib3.ProxyManager(
+                    os.environ['https_proxy'],
+                    proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth),  # type:ignore[no-untyped-call]
+                    timeout=self.http_timeout,
+                    retries=self.http_retry,
+                )
             except KeyError:
-                self.http = urllib3.PoolManager()
+                self.http = urllib3.PoolManager(timeout=self.http_timeout, retries=self.http_retry)
 
     def _request(self, endpoint: str, allowed_status_codes: list[int]) -> dict[str, Any] | None:
         if allowed_status_codes is None:

diff --git a/docs/2.Basics/Visualizing Checkov Output.md b/docs/2.Basics/Visualizing Checkov Output.md
@@ -48,23 +48,26 @@ The table below details the arguments used when executing the API token:
 
 To enrich Bridgecrew's context with CI/CD systems data, we strongly recommend that Checkov uses environment variables.
 
-| Environment Variable | Description | Example |
-| -------- | ----------- | ----------- |
-| BC_FROM_BRANCH | Source branch | feature/foo |
-| BC_TO_BRANCH | Target branch | main |
-| BC_PR_ID | Pull request identifier | 825 |
-| BC_PR_URL | Link to pull request/merge request | https://github.com/bridgecrewio/checkov/pull/825 |
-| BC_COMMIT_HASH | Commit identifier | 5df50ab857e7a255e4e731877748b539915ad489 |
-| BC_COMMIT_URL | Link to commit in CI/VCS system | https://github.com/bridgecrewio/checkov/commit/5df50ab857e7a255e4e731877748b539915ad489 |
-| BC_AUTHOR_NAME | User associated with the CI trigger | schosterbarak |
-| BC_AUTHOR_URL | Link to the user profile page | https://github.com/schosterbarak |
-| BC_RUN_ID | CI run identifier | 525220526 |
-| BC_RUN_URL | Link to the run in the CI system | https://github.com/bridgecrewio/checkov/actions/runs/525220526 |
-| BC_REPOSITORY_URL | Link to the GitHub repository | https://github.com/bridgecrewio/checkov/ |
-| BC_SOURCE | Name of CI system being integrated | githubActions |
-| BC_API_URL | URL of BC app for platform integration | https://www.bridgecrew.cloud |
-| PRISMA_API_URL | URL of Prisma app for platform integration | https://app3.prismacloud.io |
-| SLS_FILE_MASK | File names mask for all serverless files | serverless.yaml,serverless.yml |
+| Environment Variable            | Description                                                                    | Example                                                                                 |
+|---------------------------------|--------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------|
+| BC_FROM_BRANCH                  | Source branch                                                                  | feature/foo                                                                             |
+| BC_TO_BRANCH                    | Target branch                                                                  | main                                                                                    |
+| BC_PR_ID                        | Pull request identifier                                                        | 825                                                                                     |
+| BC_PR_URL                       | Link to pull request/merge request                                             | https://github.com/bridgecrewio/checkov/pull/825                                        |
+| BC_COMMIT_HASH                  | Commit identifier                                                              | 5df50ab857e7a255e4e731877748b539915ad489                                                |
+| BC_COMMIT_URL                   | Link to commit in CI/VCS system                                                | https://github.com/bridgecrewio/checkov/commit/5df50ab857e7a255e4e731877748b539915ad489 |
+| BC_AUTHOR_NAME                  | User associated with the CI trigger                                            | schosterbarak                                                                           |
+| BC_AUTHOR_URL                   | Link to the user profile page                                                  | https://github.com/schosterbarak                                                        |
+| BC_RUN_ID                       | CI run identifier                                                              | 525220526                                                                               |
+| BC_RUN_URL                      | Link to the run in the CI system                                               | https://github.com/bridgecrewio/checkov/actions/runs/525220526                          |
+| BC_REPOSITORY_URL               | Link to the GitHub repository                                                  | https://github.com/bridgecrewio/checkov/                                                |
+| BC_SOURCE                       | Name of CI system being integrated                                             | githubActions                                                                           |
+| BC_API_URL                      | URL of BC app for platform integration                                         | https://www.bridgecrew.cloud                                                            |
+| PRISMA_API_URL                  | URL of Prisma app for platform integration                                     | https://app3.prismacloud.io                                                             |
+| SLS_FILE_MASK                   | File names mask for all serverless files                                       | serverless.yaml,serverless.yml                                                          |
+| CHECKOV_REQUEST_CONNECT_TIMEOUT | Number of seconds requests will wait to establish a connection to the platform | 3.1                                                                                     |
+| CHECKOV_REQUEST_READ_TIMEOUT    | Number of seconds requests will wait for the platform to send a response. This duration matches our timeout settings, so changes are likely unnecessary.      | 30                                                                                      |
+| CHECKOV_REQUEST_RETRIES         | Number of retries requests will do towards the platform                        | 3                                                                                       |
 
 ## Bridgecrew platform view