TDL-15774 : Update refresh tokens to chain (dev-mode)

.circleci/config.yml

@@ -47,7 +47,7 @@ jobs:
             source /usr/local/share/virtualenvs/dev_env.sh
             # BUG https://jira.talendforge.org/browse/TDL-15395
             echo "pylint is skipping the following: $PYLINT_DISABLE_LIST"
-            pylint tap_jira -d "$PYLINT_DISABLE_LIST,unsupported-assignment-operation,unsupported-membership-test,unsubscriptable-object,dangerous-default-value,too-many-instance-attributes"
+            pylint tap_jira -d "$PYLINT_DISABLE_LIST,unsupported-assignment-operation,unsupported-membership-test,unsubscriptable-object,dangerous-default-value,too-many-instance-attributes,unspecified-encoding"
       - slack/notify-on-failure:
           only_for_branches: master
 

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Changelog
 
+## [v2.2.0]
+  * Add support for dev mode [#104](https://github.com/singer-io/tap-jira/pull/104)
 ## [v2.1.5]
   * Skipped the record for out of range date values [#87](https://github.com/singer-io/tap-jira/pull/87)
 ## [v2.1.4]

setup.py

@@ -2,14 +2,14 @@
 from setuptools import setup, find_packages
 
 setup(name="tap-jira",
-      version="2.1.5",
+      version="2.2.0",
       description="Singer.io tap for extracting data from the Jira API",
       author="Stitch",
       url="http://singer.io",
       classifiers=["Programming Language :: Python :: 3 :: Only"],
       py_modules=["tap_jira"],
       install_requires=[
-          "singer-python==5.12.1",
+          "singer-python==5.13.0",
           "requests==2.20.0",
           "dateparser"
       ],

tap_jira/__init__.py

@@ -13,7 +13,6 @@
 REQUIRED_CONFIG_KEYS_CLOUD = ["start_date",
                               "user_agent",
                               "cloud_id",
-                              "access_token",
                               "refresh_token",
                               "oauth_client_id",
                               "oauth_client_secret"]
@@ -113,10 +112,12 @@ def sync():
 @singer.utils.handle_top_exception(LOGGER)
 def main():
     args = get_args()
+    if args.dev:
+        LOGGER.warning("Executing Tap in Dev mode")
 
     jira_config = args.config
     # jira client instance
-    jira_client = Client(jira_config)
+    jira_client = Client(args.config_path, jira_config, args.dev)
 
     # Setup Context
     Context.client = jira_client

tap_jira/http.py

@@ -2,6 +2,7 @@
 import time
 import threading
 import re
+import json
 from requests.exceptions import (HTTPError, Timeout)
 from requests.auth import HTTPBasicAuth
 import requests
@@ -157,13 +158,14 @@ def get_request_timeout(config):
     return request_timeout
 
 class Client():
-    def __init__(self, config):
+    def __init__(self, config_path, config, dev_mode = False):
         self.is_cloud = 'oauth_client_id' in config.keys()
         self.session = requests.Session()
         self.next_request_at = datetime.now()
         self.user_agent = config.get("user_agent")
         self.login_timer = None
         self.timeout = get_request_timeout(config)
+        self.config_path = config_path
 
         # Assign False for cloud Jira instance
         self.is_on_prem_instance = False
@@ -178,10 +180,14 @@ def __init__(self, config):
             self.oauth_client_id = config.get('oauth_client_id')
             self.oauth_client_secret = config.get('oauth_client_secret')
 
+            if dev_mode and not self.access_token:
+                raise Exception("Access token config property is missing")
+
             # Only appears to be needed once for any 6 hour period. If
             # running the tap for more than 6 hours is needed this will
             # likely need to be more complicated.
-            self.refresh_credentials()
+            if not dev_mode:
+                self.refresh_credentials()
             self.test_credentials_are_authorized()
         else:
             LOGGER.info("Using Basic Auth API authentication")
@@ -262,6 +268,8 @@ def refresh_credentials(self):
                 timeout=self.timeout)
             resp.raise_for_status()
             self.access_token = resp.json()['access_token']
+            self.refresh_token = resp.json()['refresh_token']
+            self._write_config()
         except Exception as ex:
             error_message = str(ex)
             if resp:
@@ -284,6 +292,19 @@ def test_basic_credentials_are_authorized(self):
         # Assign True value to is_on_prem_instance property for on-prem Jira instance
         self.is_on_prem_instance = self.request("users","GET","/rest/api/2/serverInfo").get('deploymentType') == "Server"
 
+    def _write_config(self):
+        LOGGER.info("Credentials Refreshed")
+
+        # Update config at config_path
+        with open(self.config_path) as file:
+            config = json.load(file)
+
+        config['refresh_token'] = self.refresh_token
+        config['access_token'] = self.access_token
+
+        with open(self.config_path, 'w') as file:
+            json.dump(config, file, indent=2)
+
 class Paginator():
     def __init__(self, client, page_num=0, order_by=None, items_key="values"):
         self.client = client

tests/unittests/test_basic_auth_in_discover.py

@@ -13,6 +13,8 @@ def __init__(self):
         self.properties = False
         self.config = {}
         self.state = False
+        self.dev = False
+        self.config_path = "mock_config.json"
 
 # Mock response
 def get_mock_http_response(status_code, content={}):

tests/unittests/test_dev_mode.py

@@ -0,0 +1,74 @@
+import json
+import os
+import unittest
+from unittest.mock import patch, MagicMock
+
+import singer
+
+from tap_jira.http import Client
+
+
+LOGGER = singer.get_logger()
+
+
+class TestClientDevMode(unittest.TestCase):
+    """Test the dev mode functionality."""
+
+    def setUp(self):
+        """Creates a sample config for test execution"""
+        # Data to be written
+        self.mock_config = {
+            "oauth_client_secret": "sample_client_secret",
+            "user_agent": "test_user_agent",
+            "oauth_client_id": "sample_client_id",
+            "access_token": "sample_access_token",
+            "cloud_id": "1234567890",
+            "refresh_token": "sample_refresh_token",
+            "start_date": "2017-12-04T19:19:32Z",
+            "request_timeout": 300,
+            "groups": "jira-administrators, site-admins, jira-software-users",
+        }
+
+        self.tmp_config_filename = "sample_jira_config.json"
+
+        # Serializing json
+        json_object = json.dumps(self.mock_config, indent=4)
+        # Writing to sample_quickbooks_config.json
+        with open(self.tmp_config_filename, "w") as outfile:
+            outfile.write(json_object)
+
+    def tearDown(self):
+        """Deletes the sample config"""
+        if os.path.isfile(self.tmp_config_filename):
+            os.remove(self.tmp_config_filename)
+
+    @patch("tap_jira.http.Client.request", return_value=MagicMock(status_code=200))
+    @patch("requests.Session.post", return_value=MagicMock(status_code=200))
+    @patch("tap_jira.http.Client._write_config")
+    def test_client_with_dev_mode(
+        self, mock_write_config, mock_post_request, mock_request
+    ):
+        """Checks the dev mode implementation and verifies write config functionality is
+        not called"""
+        Client(
+            config_path=self.tmp_config_filename, config=self.mock_config, dev_mode=True
+        )
+
+        # _write_config function should never be called as it will update the config
+        self.assertEqual(mock_write_config.call_count, 0)
+
+    @patch("tap_jira.http.Client.request", return_value=MagicMock(status_code=200))
+    @patch("requests.Session.post", side_effect=Exception())
+    def test_client_dev_mode_missing_access_token(
+        self, mock_post_request, mock_request
+    ):
+        """Exception should be raised if missing access token"""
+
+        del self.mock_config["access_token"]
+
+        with self.assertRaises(Exception):
+            Client(
+                config_path=self.tmp_config_filename,
+                config=self.mock_config,
+                dev_mode=True,
+            )

tests/unittests/test_error_handling.py

@@ -72,7 +72,9 @@ def test_request_with_handling_for_400_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraBadRequestError as e:
             expected_error_message = "HTTP-error-code: 400, Error: A validation exception has occurred."
@@ -85,7 +87,9 @@ def test_request_with_handling_for_401_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraUnauthorizedError as e:
             expected_error_message = "HTTP-error-code: 401, Error: Invalid authorization credentials."
@@ -98,7 +102,9 @@ def test_request_with_handling_for_403_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraForbiddenError as e:
             expected_error_message = "HTTP-error-code: 403, Error: User does not have permission to access the resource."
@@ -112,7 +118,9 @@ def test_request_with_handling_for_404_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraNotFoundError as e:
             expected_error_message = "HTTP-error-code: 404, Error: The resource you have specified cannot be found."
@@ -124,7 +132,9 @@ def test_request_with_handling_for_409_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraConflictError as e:
             expected_error_message = "HTTP-error-code: 409, Error: The request does not match our state in some way."
@@ -136,7 +146,9 @@ def test_request_with_handling_for_429_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraRateLimitError as e:
             expected_error_message = "HTTP-error-code: 429, Error: The API rate limit for your organisation/application pairing has been exceeded."
@@ -149,7 +161,9 @@ def test_request_with_handling_for_449_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraSubRequestFailedError as e:
             expected_error_message = "HTTP-error-code: 449, Error: The API was unable to process every part of the request."
@@ -162,7 +176,9 @@ def test_request_with_handling_for_500_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraInternalServerError as e:
             expected_error_message = "HTTP-error-code: 500, Error: The server encountered an unexpected condition which prevented it from fulfilling the request."
@@ -175,7 +191,9 @@ def test_request_with_handling_for_501_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraNotImplementedError as e:
             expected_error_message = "HTTP-error-code: 501, Error: The server does not support the functionality required to fulfill the request."
@@ -188,7 +206,9 @@ def test_request_with_handling_for_502_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraBadGatewayError as e:
             expected_error_message = "HTTP-error-code: 502, Error: Server received an invalid response."
@@ -201,7 +221,9 @@ def test_request_with_handling_for_503_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraServiceUnavailableError as e:
             expected_error_message = "HTTP-error-code: 503, Error: API service is currently unavailable."
@@ -214,7 +236,9 @@ def test_request_with_handling_for_504_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraGatewayTimeoutError as e:
             expected_error_message = "HTTP-error-code: 504, Error: API service time out, please check Jira server."
@@ -227,7 +251,9 @@ def test_request_with_handling_for_505_exceptin_handling(self,mock_send):
         try:
             tap_stream_id = "tap_jira"
             mock_config = {"username":"mock_username","password":"mock_password","base_url": "mock_base_url"}
-            mock_client = http.Client(mock_config)
+            mock_config_path = "mock_config.json"
+            mock_dev_mode = False
+            mock_client = http.Client(mock_config_path, mock_config, mock_dev_mode)
             mock_client.request(tap_stream_id)
         except http.JiraError as e:
             expected_error_message = "HTTP-error-code: 505, Error: Unknown Error"