Avoid jenkinsapi trying to fetch all jobs. (#1045)

* Use lazy argument to prevent an initial "poll" of Jenkins jobs.

By default the jenkinsapi Jenkins class will trigger an eager loading of
all Jenkins jobs via the API. Our Jenkins server buckles under the
weight of this and so we do not want to do this until we need to and
ideally future changes will replace unfiltered polling completely.

* Avoid triggering spurious HTTP request of all jenkins jobs.

There's a malinteraction with the jenkinsapi library and Python
convention which has been causing extremely large HTTP requests whenever
the truthiness of an instance of the Jenkins class is tested.

Jenkins defines a `__len__` method which is used to determine
truthiness. But this method triggers a fetch of every job on our Jenkins
host which can take 2 - 5 minutes to return due to the sheer volume of
jobs.

By avoiding this in favor of an explicit check whether jenkins is an
instance of the JenkinsProxy class we can save taking a really big
runtime hit and cut down on spurious failures due to timeouts.

* Whitespace.

ros_buildfarm/ci_job.py

@@ -29,6 +29,7 @@
 from ros_buildfarm.config import get_ci_build_files
 from ros_buildfarm.config import get_index as get_config_index
 from ros_buildfarm.git import get_repository
+from ros_buildfarm.jenkins import JenkinsProxy
 from ros_buildfarm.templates import expand_template
 from rosdistro import get_index
 
@@ -80,7 +81,7 @@ def _configure_ci_jobs(
         ci_view_name: configure_ci_view(
             jenkins, ci_view_name, dry_run=dry_run)
     }
-    if not jenkins:
+    if not isinstance(jenkins, JenkinsProxy):
         view_configs.update(views)
     groovy_data = {
         'dry_run': dry_run,

ros_buildfarm/devel_job.py

@@ -33,6 +33,7 @@
 from ros_buildfarm.config import get_index as get_config_index
 from ros_buildfarm.config import get_source_build_files
 from ros_buildfarm.git import get_repository
+from ros_buildfarm.jenkins import JenkinsProxy
 from ros_buildfarm.templates import expand_template
 from rosdistro import get_distribution_cache
 from rosdistro import get_index
@@ -90,7 +91,7 @@ def configure_devel_jobs(
     if build_file.test_pull_requests_force is not False:
         views[pull_request_view_name] = configure_devel_view(
             jenkins, pull_request_view_name, dry_run=dry_run)
-    if not jenkins:
+    if not isinstance(jenkins, JenkinsProxy):
         view_configs.update(views)
     groovy_data = {
         'dry_run': dry_run,

ros_buildfarm/doc_job.py

@@ -33,6 +33,7 @@
 from ros_buildfarm.config import get_global_doc_build_files
 from ros_buildfarm.config import get_index as get_config_index
 from ros_buildfarm.git import get_repository
+from ros_buildfarm.jenkins import JenkinsProxy
 from ros_buildfarm.templates import expand_template
 from rosdistro import get_distribution_cache
 from rosdistro import get_index
@@ -83,7 +84,7 @@ def configure_doc_jobs(
     views = {}
     views[doc_view_name] = configure_doc_view(
         jenkins, doc_view_name, dry_run=dry_run)
-    if not jenkins:
+    if not isinstance(jenkins, JenkinsProxy):
         view_configs.update(views)
     groovy_data = {
         'dry_run': dry_run,

ros_buildfarm/jenkins.py

@@ -38,6 +38,8 @@ class JenkinsProxy(Jenkins):
     def __init__(self, *args, **kwargs):  # noqa: D107
         requester_kwargs = copy.copy(kwargs)
         requester_kwargs['baseurl'] = args[0]
+        # Don't trigger jenkinsapi poll on initialization
+        kwargs['lazy'] = True
         kwargs['requester'] = CrumbRequester(**requester_kwargs)
         kwargs.setdefault('timeout', 120)
         super(JenkinsProxy, self).__init__(*args, **kwargs)
@@ -88,7 +90,7 @@ def configure_view(
     view_config = get_view_config(
         template_name, view_name, include_regex=include_regex,
         filter_queue=filter_queue)
-    if not jenkins:
+    if not isinstance(jenkins, JenkinsProxy):
         _cached_views[key] = view_config
         return view_config
     view_type = _get_view_type(view_config)

ros_buildfarm/release_job.py

@@ -43,6 +43,7 @@
 from ros_buildfarm.config import get_index as get_config_index
 from ros_buildfarm.config import get_release_build_files
 from ros_buildfarm.git import get_repository
+from ros_buildfarm.jenkins import JenkinsProxy
 from ros_buildfarm.package_repo import get_package_repo_data
 from ros_buildfarm.templates import expand_template
 from rosdistro import get_cached_distribution
@@ -144,14 +145,14 @@ def configure_release_jobs(
             job_name, job_config = configure_import_package_job(
                 config_url, rosdistro_name, release_build_name,
                 config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)
-            if not jenkins:
+            if not isinstance(jenkins, JenkinsProxy):
                 all_job_configs[job_name] = job_config
             break
 
     job_name, job_config = configure_sync_packages_to_main_job(
         config_url, rosdistro_name, release_build_name,
         config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run)
-    if not jenkins:
+    if not isinstance(jenkins, JenkinsProxy):
         all_job_configs[job_name] = job_config
 
     for os_name, os_code_name in platforms:
@@ -161,7 +162,7 @@ def configure_release_jobs(
                 os_name, os_code_name, arch,
                 config=config, build_file=build_file, jenkins=jenkins,
                 dry_run=dry_run)
-            if not jenkins:
+            if not isinstance(jenkins, JenkinsProxy):
                 all_job_configs[job_name] = job_config
 
     targets = []
@@ -172,7 +173,7 @@ def configure_release_jobs(
     views = configure_release_views(
         jenkins, rosdistro_name, release_build_name, targets,
         dry_run=dry_run)
-    if not jenkins:
+    if not isinstance(jenkins, JenkinsProxy):
         all_view_configs.update(views)
     groovy_data = {
         'dry_run': dry_run,