Merge pull request #2 from storebrand/tests

Added unittest

.github/workflows/ci.yaml

@@ -0,0 +1,62 @@
+### A CI workflow template that runs linting and python testing
+### TODO: Modify as needed or as desired.
+
+name: Test tap-sharepointsites
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  linting:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # Only lint using the primary version used for dev
+        python-version: ["3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      run: |
+        python -m pip install --upgrade pip
+        pip install poetry==1.2.*
+        pip install tox
+    - name: Run lint command from tox.ini
+      run: |
+        poetry run tox -e lint
+    - name: Commit changes
+      uses: stefanzweifel/git-auto-commit-action@v4
+      with:
+        commit_message: Apply code formatting
+  pytest:
+    runs-on: ubuntu-latest
+    needs: linting
+    strategy:
+      max-parallel: 5
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      run: |
+        python -m pip install --upgrade pip
+        pip install poetry==1.2.*
+    - name: Install dependencies
+      run: |
+        poetry install
+    - name: Test with pytest
+      run: |
+        poetry run pytest --capture=no

.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+poetry.lock
+.coverage
+.tox

pyproject.toml

@@ -19,6 +19,12 @@ requests = "~=2.31.0"
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.0"
 singer-sdk = { version="~=0.32.0", extras = ["testing"] }
+responses = ">0.1.0"
+pytest-cov = ">=3.0.0"
+tox = "^3.14.3"
+isort = "^5.10.1"
+flake8 = "^3.9.2"
+black = "^22.3.0"
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

tap_pxwebapi/streams.py

@@ -2,19 +2,17 @@
 
 from __future__ import annotations
 
-import typing as t
-from pathlib import Path
 import hashlib
+from functools import cached_property
+from pathlib import Path
+from typing import Iterable
+
+import requests
 from singer_sdk import typing as th  # JSON Schema typing helpers
-from typing import Any, Callable, Iterable
+
 from tap_pxwebapi.client import pxwebapiStream
-import requests
-from functools import cached_property
 
-# TODO: Delete this is if not using json files for schema definition
 SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
-# TODO: - Override `UsersStream` and `GroupsStream` with your own stream definition.
-#       - Copy-paste as many times as needed to create multiple stream types.
 
 
 class TablesStream(pxwebapiStream):
@@ -33,7 +31,6 @@ def __init__(self, *args, **kwargs):
 
         super().__init__(*args, **kwargs)
 
-
     @property
     def url_base(self) -> str:
         return self.config["base_url"]
@@ -42,19 +39,19 @@ def url_base(self) -> str:
     def path(self) -> str:
         """Return API endpoint path string."""
         return f"en/table/{self.table_config['table_id']}"
-    
+
     @property
     def name(self) -> str:
         """Return a human-readable name for this stream."""
         return self.table_config["table_name"]
-    
+
     @staticmethod
     def json_stat2_to_rows(json_stat2):
         rows = []
         dimensions = json_stat2["dimension"]
         values = json_stat2["value"]
         dimension_keys = list(dimensions.keys())
-        
+
         def recursive_build_row(dim_index, current_row):
             if dim_index == len(dimension_keys):
                 current_row["value"] = values[len(rows)]
@@ -87,31 +84,20 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]:
         rows = self.json_stat2_to_rows(json_stat2)
 
         for i, row in enumerate(rows):
-            hash_object = hashlib.sha256()
-
             row["_sdc_row_hash"] = self.create_hash_from_dict(row)
             yield row
 
-
     def prepare_request_payload(
-        self, context: dict | None, next_page_token: _TToken | None
+        self, context: dict | None, next_page_token
     ) -> dict | None:
         """Prepare the data payload for the REST API request."""
 
-        base_payload = {
-            "query": [],
-              "response": {
-                "format": "json-stat2"
-            }
-        }
+        base_payload = {"query": [], "response": {"format": "json-stat2"}}
 
         for select in self.table_config.get("select", []):
             column_payload = {
                 "code": select["code"],
-                "selection": {
-                    "filter": "item",
-                    "values": select["values"]
-                }
+                "selection": {"filter": "item", "values": select["values"]},
             }
             base_payload["query"].append(column_payload)
 
@@ -120,52 +106,46 @@ def prepare_request_payload(
 
         if not last_time:
             return base_payload
-        
+
         self.logger.info("time_items: " + str(self.time_items))
-        
+
         if len(self.time_items) == 1:
-            new_times = [item for item in self.time_items[0]["values"] if item > last_time]
+            new_times = [
+                item for item in self.time_items[0]["values"] if item > last_time
+            ]
             self.logger.info("new_times: " + str(new_times))
 
             if not new_times:
                 # Difficult to abort the stream here, so we just fetch the latest period again
                 self.logger.info("No new times, fetching latest period")
                 time_payload = {
                     "code": self.time_items[0]["code"],
-                    "selection": {
-                        "filter": "item",
-                        "values": [last_time]
-                    }
+                    "selection": {"filter": "item", "values": [last_time]},
                 }
             else:
                 self.logger.info(f"New times found, fetching new times {new_times}")
                 time_payload = {
                     "code": self.time_items[0]["code"],
-                    "selection": {
-                        "filter": "item",
-                        "values": new_times
-                    }
+                    "selection": {"filter": "item", "values": new_times},
                 }
 
             base_payload["query"].append(time_payload)
 
             self.logger.info("payload: " + str(base_payload))
 
             return base_payload
-
-
 
     @cached_property
     def schema(self) -> th.PropertiesList:
-        
+
         r = requests.get(self.url_base + self.path)
         r.raise_for_status()
 
         time_variable = [item for item in r.json()["variables"] if item.get("time")]
         self.time_items = time_variable
 
         properties = th.PropertiesList()
-        
+
         for item in r.json()["variables"]:
 
             properties.append(
@@ -188,18 +168,8 @@ def schema(self) -> th.PropertiesList:
 
         properties.append(
             th.Property(
-                "_sdc_row_hash",
-                th.StringType,
-                description="Row number",
-                required=True
+                "_sdc_row_hash", th.StringType, description="Row number", required=True
             )
         )
 
         return properties.to_dict()
-
-
-
-
-
-
-

tap_pxwebapi/tap.py

@@ -45,15 +45,15 @@ class Tappxwebapi(Tap):
                                 th.Property(
                                     "values",
                                     th.ArrayType(th.StringType),
-                                )
+                                ),
                             )
-                        )
-                    )
+                        ),
+                    ),
                 )
             ),
             required=True,
             description="Tables to read",
-        )
+        ),
     ).to_dict()
 
     def discover_streams(self) -> list[streams.TablesStream]:
@@ -64,10 +64,7 @@ def discover_streams(self) -> list[streams.TablesStream]:
         """
 
         for table in self.config["tables"]:
-            yield streams.TablesStream(
-                tap=self,
-                table_config=table
-            )
+            yield streams.TablesStream(tap=self, table_config=table)
 
 
 if __name__ == "__main__":