Upgrade to python3 (#504)

* upgrade to python3 and remove python2 support

.gitignore

@@ -4,7 +4,7 @@
 *egg
 *egg-info
 .coverage
-.venv
+.venv3
 .build
 build/
 docs/.ropeproject

.travis.yml

@@ -1,14 +1,18 @@
 language: python
 python:
-  - "2.7"
+  - "3.7"
+env:
+  global:
+    # pypi api token split in 2 because of travisci issue https://travis-ci.community/t/travis-encrypt-data-too-large-for-pypi-tokens-with-older-repos/5792/5
+    - secure: "gZ9TDdXbvpw+84CC1Hx4nS44XpDqlao6E1fw44g4DIAKtFwdBd/eDntexWjPCP7X1qVuY3ssswKTVV2gAKUzDPROhZUVTv6L6nrNruKn75MXKNyb4XTf6fatfUptWRA9kVu/mFi8M6Ptp1ySULvEmI0bUSuGZ9AuQISwjfhTqPA="
+    - secure: "dpoI6IGPp78nrvuZpgvqkt+PbVIPKb5QrnR+XGOXKRHYpSwNb6DOeZ55tzNZ16e3T/3LVTFZ+VZsn8iv26uUYGBQctP5HBh4zzO8qOuZKgqkN2tmxCifcUxgcWh+Prjj71Toj/+gUdy8ph7zEcLfj3Q2bRTdCfuM56/MoI6Y/5E="
 # https://github.com/travis-ci/travis-ci/issues/7940#issuecomment-310759657
 before_install:
   - sudo rm -f /etc/boto.cfg
 install:
+  - pip install pip --upgrade
   - make build
-  - python setup.py install
 script: make test
-sudo: false
 before_deploy:
   - make kingpin.zip
 deploy:
@@ -17,20 +21,19 @@ deploy:
       secure: "TMRRd3PeZIRf4wvD2Bh+ykvvBVIztE6M6JE89WBb/CbaIbdUqoDoldYjybYbbXDPPPs+ybVOYZTwMylx6TDy40WsYlzlaabnbTZedvUWDC3GcqD3E4I5XMBVBColP1cKZqHYaa/p23V9QfJFoXCJzGJ5VWOXclj0A0NEvb+oUpU="
     file: kingpin.zip
     overwrite: true
-    skip_cleanup: true
+    cleanup: false
     on:
       tags: true
-      python: '2.7'
+      python: '3.7'
       all_branches: true
       condition: $TRAVIS_TAG =~ ^(v[0-9]+.[0-9]+.[0-9]+[a-z]?)|pre_release$
       repo: Nextdoor/kingpin
   - provider: pypi
-    user: nextdoor
-    password:
-      secure: "Xi9h1g1AECHV8dzV+Pz0JnD46lFCYkIaX3IYnWjrknbX8KAbcUxvcRmx1NfTxNTI7/jm4Sl0UakEmvG9GFleldLp0vXrxEIqyuZtT3bNlD6xD6SVt4XXZ6lIGFuzVfMhE2FJSkPlqSjKOzEzGSIUwmO3D7ZSqLl7LYBYQKwNslM="
+    user: __token__
+    password: $TOKEN_1$TOKEN_2
     on:
       tags: true
-      python: '2.7'
+      python: '3.7'
       all_branches: true
       condition: $TRAVIS_TAG =~ ^v[0-9]+.[0-9]+.[0-9]+[a-z]?$
       repo: Nextdoor/kingpin

Makefile

@@ -12,7 +12,6 @@ INTEGRATION_TESTS ?= aws,rightscale,http,rollbar,slack,spotinst
 export URLLIB_DEBUG=true
 
 all: build
-
 build: .build
 
 .build: requirements.test.txt
@@ -27,23 +26,23 @@ clean:
 	$(MAKE) -C docs clean
 
 test: build docs
-	python setup.py test pep8 pyflakes
+	python3 setup.py test pep8 pyflakes
 	# A few simple dry-tests of yaml and json scripts to make sure that the
 	# full commandline actually works.
 	PYTHONPATH=$(HERE) python kingpin/bin/deploy.py --dry --script examples/test/sleep.json
 	PYTHONPATH=$(HERE) python kingpin/bin/deploy.py --dry --script examples/test/sleep.yaml
 
 integration: build
 	INTEGRATION_TESTS=$(INTEGRATION_TESTS) PYFLAKES_NODOCTEST=True \
-		python setup.py integration pep8 pyflakes
+		python3 setup.py integration pep8 pyflakes
 
 pack: kingpin.zip
-	@python kingpin.zip --help 2>&1 >/dev/null && echo Success || echo Fail
+	python kingpin.zip --help 2>&1 >/dev/null && echo Success || echo Fail
 
 kingpin.zip:
 	rm -rf zip
 	mkdir -p zip
-	pip install --target ./zip ./
+	pip install -r requirements.txt --target ./zip ./
 	find ./zip -name '*.pyc' -delete
 	find ./zip -name '*.egg-info' | xargs rm -rf
 	cd zip; ln -sf kingpin/bin/deploy.py ./__main__.py

docs/modules.rst

@@ -43,8 +43,6 @@ Full Module Docs
    :members:
 .. automodule:: kingpin.actors.spotinst
    :members:
-.. automodule:: kingpin.actors.support.api
-   :members:
 .. automodule:: kingpin.actors.utils
    :members:
 .. automodule:: kingpin.constants

kingpin/actors/aws/base.py

@@ -34,7 +34,9 @@
 
 import json
 import logging
-import urllib
+import urllib.request
+import urllib.parse
+import urllib.error
 import re
 
 from boto import exception as boto_exception
@@ -248,7 +250,7 @@ def _wrap_boto_exception(self, e):
                 msg = 'Access credentials have expired'
                 return exceptions.InvalidCredentials(msg)
 
-            msg = '%s: %s' % (e.error_code, e.message)
+            msg = '%s: %s' % (e.error_code, str(e))
             if e.status == 403:
                 return exceptions.InvalidCredentials(msg)
         elif isinstance(e, boto3_exceptions.Boto3Error):
@@ -277,7 +279,7 @@ def _find_elb(self, name):
             elbs = yield self.api_call(self.elb_conn.get_all_load_balancers,
                                        load_balancer_names=name)
         except boto_exception.BotoServerError as e:
-            msg = '%s: %s' % (e.error_code, e.message)
+            msg = '%s: %s' % (e.error_code, str(e))
             log.error('Received exception: %s' % msg)
 
             if e.status == 400:
@@ -312,7 +314,7 @@ def _find_target_group(self, arn):
             trgts = yield self.api_call(self.elbv2_conn.describe_target_groups,
                                         Names=[arn])
         except botocore_exceptions.ClientError as e:
-            raise exceptions.UnrecoverableActorFailure(e.message)
+            raise exceptions.UnrecoverableActorFailure(str(e))
 
         arns = [t['TargetGroupArn'] for t in trgts['TargetGroups']]
 
@@ -348,7 +350,7 @@ def _policy_doc_to_dict(self, policy):
         args:
             policy: The policy string returned by Boto
         """
-        return json.loads(urllib.unquote(policy))
+        return json.loads(urllib.parse.unquote(policy))
 
     def _parse_policy_json(self, policy):
         """Parse a single JSON file into an Amazon policy.

kingpin/actors/aws/cloudformation.py

@@ -262,15 +262,15 @@ def _validate_template(self, body=None, url=None):
             try:
                 yield self.api_call(self.cf3_conn.validate_template, **cfg)
             except ClientError as e:
-                raise InvalidTemplate(e.message)
+                raise InvalidTemplate(e)
 
         if url is not None:
             cfg = {'TemplateURL': url}
             self.log.info('Validating template (%s) with AWS...' % url)
             try:
                 yield self.api_call(self.cf3_conn.validate_template, **cfg)
             except ClientError as e:
-                raise InvalidTemplate(e.message)
+                raise InvalidTemplate(e)
 
     def _create_parameters(self, parameters):
         """Converts a simple Key/Value dict into Amazon CF Parameters.
@@ -297,7 +297,7 @@ def _create_parameters(self, parameters):
         new_params = [
             {'ParameterKey': k,
              'ParameterValue': v}
-            for k, v in parameters.items()]
+            for k, v in list(parameters.items())]
         sorted_params = sorted(new_params, key=lambda k: k['ParameterKey'])
         return sorted_params
 
@@ -321,7 +321,7 @@ def _get_stack(self, stack):
                 queue_name='describe_stacks',
                 StackName=stack)
         except ClientError as e:
-            if 'does not exist' in e.message:
+            if 'does not exist' in str(e):
                 raise gen.Return(None)
 
             raise CloudFormationError(e)
@@ -442,7 +442,7 @@ def _delete_stack(self, stack):
             ret = yield self.api_call(
                 self.cf3_conn.delete_stack, StackName=stack)
         except ClientError as e:
-            raise CloudFormationError(e.message)
+            raise CloudFormationError(str(e))
 
         req_id = ret['ResponseMetadata']['RequestId']
         self.log.info('Stack delete requested: %s' % req_id)
@@ -490,7 +490,7 @@ def _create_stack(self, stack):
                 EnableTerminationProtection=enable_termination_protection,
                 **cfg)
         except ClientError as e:
-            raise CloudFormationError(e.message)
+            raise CloudFormationError(str(e))
 
         # Now wait until the stack creation has finished. If the creation
         # fails, get the logs from Amazon for the user.

kingpin/actors/aws/ecs.py

@@ -216,7 +216,7 @@ def __init__(self, *args, **kwargs):
         super(ECSBaseActor, self).__init__(*args, **kwargs)
 
         count = self.option('count')
-        if type(count) is str or type(count) is unicode:
+        if type(count) is str or type(count) is str:
             try:
                 self._options['count'] = int(count)
             except ValueError:
@@ -916,7 +916,7 @@ def _wait_for_deployment_update(self, service_name, task_definition_name):
             try:
                 service = yield self._describe_service(service_name)
             except ServiceNotFound as e:
-                self.log.info('Service Not Found: %s' % e.message)
+                self.log.info('Service Not Found: %s' % str(e))
                 yield gen.sleep(2)
                 continue
 

kingpin/actors/aws/elb.py

@@ -596,8 +596,8 @@ def _find_instance_elbs(self, instances):
         elbs_with_members = []
 
         for instance in instances:
-            elbs = filter(lambda lb: instance in [i.id for i in lb.instances],
-                          all_elbs)
+            elbs = [lb for lb in all_elbs
+                    if instance in [i.id for i in lb.instances]]
             self.log.debug('%s is a member of %s' % (instance, elbs))
             elbs_with_members.extend(elbs)
 

kingpin/actors/aws/elbv2.py

@@ -106,7 +106,7 @@ def _add(self, arn, targets):
                 TargetGroupArn=arn,
                 Targets=targets)
         except botocore.exceptions.ClientError as e:
-            raise exceptions.UnrecoverableActorFailure(e.message)
+            raise exceptions.UnrecoverableActorFailure(str(e))
 
     @gen.coroutine
     def _execute(self):
@@ -190,7 +190,7 @@ def _remove(self, arn, targets):
                 TargetGroupArn=arn,
                 Targets=targets)
         except botocore.exceptions.ClientError as e:
-            raise exceptions.UnrecoverableActorFailure(e.message)
+            raise exceptions.UnrecoverableActorFailure(str(e))
 
     @gen.coroutine
     def _execute(self):

kingpin/actors/aws/iam/entities.py

@@ -150,7 +150,7 @@ def _parse_inline_policies(self, policies):
 
         # If a single policy was supplied (ie, maybe on a command line) then
         # turn it into a list.
-        if isinstance(policies, basestring):
+        if isinstance(policies, str):
             policies = [policies]
 
         # Run through any supplied Inline IAM Policies and verify that they're
@@ -594,7 +594,7 @@ def _ensure_groups(self, name, groups):
             name: The user we're managing
             groups: The list (or single) of groups to join be members of
         """
-        if isinstance(groups, basestring):
+        if isinstance(groups, str):
             groups = [groups]
 
         current_groups = set()
@@ -614,14 +614,19 @@ def _ensure_groups(self, name, groups):
 
         # Find any groups that we're not already a member of, and add us
         tasks = []
-        for new_group in set(groups) - current_groups:
-            tasks.append(self._add_user_to_group(name, new_group))
+        try:
+            for new_group in set(groups) - current_groups:
+                tasks.append(self._add_user_to_group(name, new_group))
+        except StopIteration:
+            pass
+
         yield tasks
 
         # Find any group memberships we didn't know about, and purge them
         tasks = []
         for bad_group in current_groups - set(groups):
             tasks.append(self._remove_user_from_group(name, bad_group))
+
         yield tasks
 
     @gen.coroutine
@@ -1075,7 +1080,10 @@ def _ensure_role(self, name, role):
         if not existing and not role:
             raise gen.Return()
         elif existing and not role:
-            yield self._remove_role(name, existing)
+            try:
+                yield self._remove_role(name, existing)
+            except StopIteration:
+                return
         elif not existing and role:
             yield self._add_role(name, role)
         elif existing != role:

kingpin/actors/aws/iam/test/test_certs.py

@@ -7,6 +7,7 @@
 from kingpin.actors import exceptions
 from kingpin.actors.aws import settings
 from kingpin.actors.aws.iam import certs
+import importlib
 
 log = logging.getLogger(__name__)
 
@@ -18,7 +19,7 @@ def setUp(self):
         settings.AWS_ACCESS_KEY_ID = 'unit-test'
         settings.AWS_SECRET_ACCESS_KEY = 'unit-test'
         settings.RETRYING_SETTINGS = {'stop_max_attempt_number': 1}
-        reload(certs)
+        importlib.reload(certs)
 
     @testing.gen_test
     def test_execute(self):
@@ -38,7 +39,7 @@ def test_execute(self):
         yield actor._execute()
 
         # call count is 1 -- one extra retry due to BotoServerError above.
-        self.assertEquals(actor.iam_conn.upload_server_cert.call_count, 1)
+        self.assertEqual(actor.iam_conn.upload_server_cert.call_count, 1)
         actor.iam_conn.upload_server_cert.assert_called_with(
             path=None,
             private_key=actor.readfile(),
@@ -64,7 +65,7 @@ def test_execute_dry(self):
         with open_patcher:
             yield actor._execute()
 
-        self.assertEquals(actor.iam_conn.upload_server_cert.call_count, 0)
+        self.assertEqual(actor.iam_conn.upload_server_cert.call_count, 0)
 
 
 class TestDeleteCert(testing.AsyncTestCase):
@@ -74,7 +75,7 @@ def setUp(self):
         settings.AWS_ACCESS_KEY_ID = 'unit-test'
         settings.AWS_SECRET_ACCESS_KEY = 'unit-test'
         settings.RETRYING_SETTINGS = {'stop_max_attempt_number': 1}
-        reload(certs)
+        importlib.reload(certs)
 
     @testing.gen_test(timeout=60)
     def test_delete_cert_dry(self):
@@ -84,7 +85,7 @@ def test_delete_cert_dry(self):
         yield actor.execute()
 
         actor.iam_conn.get_server_certificate.assert_called_with('test')
-        self.assertEquals(actor.iam_conn.get_server_certificate.call_count, 1)
+        self.assertEqual(actor.iam_conn.get_server_certificate.call_count, 1)
 
         err = BotoServerError('400', 'Broken!')
         actor.iam_conn.get_server_certificate.side_effect = err