Merge pull request #36 from h4ndzdatm0ld/develop

## [0.1.4] - 2022-11-06 ### Added - CodeQL Action - `SUSPENDED` state as a default to `Offline` ### Changed - Bumped version of Poetry Action - Added a Nautobot version to the testing matrix in CI - Bumped `oauthlib` to 3.2.1 - Unpinned `responses` testing library ### Fixed - Exception raised when a vm doesn't have any disks. - Filtering logic was missing to aggregate VM's from nautobot, if defined in job args - Documentation pointed to invalid configuration values ### Removed
h4ndzdatm0ld · Nov 6, 2022 · b7e6062 · b7e6062
2 parents 8743f84 + 4f579d0
commit b7e6062
Show file tree

Hide file tree

Showing 13 changed files with 790 additions and 346 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v3"
       - name: "Linting: black"
         run: "poetry run invoke black"
   bandit:
@@ -27,7 +27,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v3"
       - name: "Linting: bandit"
         run: "poetry run invoke bandit"
   pydocstyle:
@@ -38,7 +38,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v3"
       - name: "Linting: pydocstyle"
         run: "poetry run invoke pydocstyle"
   flake8:
@@ -49,7 +49,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v3"
       - name: "Linting: flake8"
         run: "poetry run invoke flake8"
   yamllint:
@@ -60,7 +60,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v3"
       - name: "Linting: yamllint"
         run: "poetry run invoke yamllint"
   pylint:
@@ -83,7 +83,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v3"
       - name: "Set up Docker Buildx"
         id: "buildx"
         uses: "docker/setup-buildx-action@v1"
@@ -112,7 +112,7 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ["3.7", "3.8", "3.9"]
-        nautobot-version: ["1.3.3"]
+        nautobot-version: ["1.3.3", "1.4.1"]
     runs-on: "ubuntu-20.04"
     env:
       INVOKE_NAUTOBOT_SSOT_VSPHERE_PYTHON_VER: "${{ matrix.python-version }}"
@@ -121,7 +121,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v3"
       - name: "Set up Docker Buildx"
         id: "buildx"
         uses: "docker/setup-buildx-action@v1"

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,74 @@
+---
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: ["develop"]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: ["develop"]
+  schedule:
+    - cron: "43 5 * * 2"
+
+jobs:
+  analyze:
+    name: "Analyze"
+    runs-on: "ubuntu-latest"
+    permissions:
+      actions: "read"
+      contents: "read"
+      security-events: "write"
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["python"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+      - name: "Checkout repository"
+        uses: "actions/checkout@v3"
+
+      # Initializes the CodeQL tools for scanning.
+      - name: "Initialize CodeQL"
+        uses: "github/codeql-action/init@v2"
+        with:
+          languages: "${{ matrix.language }}"
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+
+          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: "Autobuild"
+        uses: "github/codeql-action/autobuild@v2"
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+      # - run: |
+      #   echo "Run, Build Application using script"
+      #   ./location_of_script_within_repo/buildscript.sh
+
+      - name: "Perform CodeQL Analysis"
+        uses: "github/codeql-action/analyze@v2"
+        with:
+          category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -14,7 +14,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v2"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v2"
+        uses: "networktocode/gh-action-setup-poetry-environment@v3"
       - name: "Sphinx Build"
         run: "poetry run sphinx-build -vvv -b html ./docs ./docs/public"
       - name: "Deploy auto generated documentation to GH-Pages"

diff --git a/README.md b/README.md
@@ -47,7 +47,7 @@ The plugin behavior can be controlled with additional configuration settings
 ```bash
 - `VSPHERE_TYPE` Defaults to `VMWare vSphere`
 - `ENFORCE_CLUSTER_GROUP_TOP_LEVEL` Defaults to True
-- `VSPHERE_VM_STATUS_MAP` Defaults to {"POWERED_OFF": "Offline", "POWERED_ON": "Active"}
+- `VSPHERE_VM_STATUS_MAP` Defaults to {"POWERED_OFF": "Offline", "POWERED_ON": "Active", "SUSPENDED": "Suspended"}
 - `VSPHERE_IP_STATUS_MAP` Defaults to {"PREFERRED": "Active", "UNKNOWN": "Reserved"}
 - `VSPHERE_VM_INTERFACE_MAP` Defaults to {"NOT_CONNECTED": False, "CONNECTED": True}
 - `PRIMARY_IP_SORT_BY` Defaults to "Lowest"

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -4,6 +4,28 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.4] - 2022-11-06
+
+### Added
+
+- CodeQL Action
+- `SUSPENDED` state as a default to `Offline`
+
+### Changed
+
+- Bumped version of Poetry Action
+- Added a Nautobot version to the testing matrix in CI
+- Bumped `oauthlib` to 3.2.1
+- Unpinned `responses` testing library
+
+### Fixed
+
+- Exception raised when a vm doesn't have any disks.
+- Filtering logic was missing to aggregate VM's from nautobot, if defined in job args
+- Documentation pointed to invalid configuration values
+
+### Removed
+
 ## [0.1.3] - 2022-08-13
 
 ### Added

diff --git a/docs/overview.md b/docs/overview.md
@@ -1,7 +1,11 @@
 # Overview of Nautobot SSoT vSphere
 
+A plugin for [Nautobot](https://github.com/nautobot/nautobot) that leverages the SSoT plugin to create Virtual Machines, VMInterfaces, IPAddresses, Clusters, and Cluster Groups from VMWare vSphere.
+
 ## Configuration Settings
 
+The following options are user configurable to _attempt_ to provide the user with a better synchronization experience, depending on what the environment's deployment looks like. All of the options below have default values set and you most likely for the most common deployment of vSphere / DataCenters / Clusters.
+
 `VSPHERE_TYPE`
 
 - This is the default Cluster Type that will be created in Nautobot and associated to every Cluster discovered.
@@ -10,14 +14,26 @@
 
 - `Diffsync` has a concept of `Top Level`. This application defaults to the `ClusterGroup` as Top Level, which translates to a vSphere DataCenter. This can be changed to have a Cluster in vSphere be the Top Level and ignore DataCenters.
 
-`VSPHERE_VM_STATUS_MAP, VSPHERE_IP_STATUS_MAP, VSPHERE_VM_INTERFACE_MAP`
+`DEFAULT_VM_STATUS_MAP, DEFAULT_IP_STATUS_MAP, VSPHERE_VM_INTERFACE_MAP`
 
 - These are dictionary (maps) that translate the vSphere value to something Nautobot can understand. You must be sure that the values provided are valid before manipulating these maps.
 
-```bash
-- `VSPHERE_VM_STATUS_MAP` Defaults to {"POWERED_OFF": "Offline", "POWERED_ON": "Active"}
-- `VSPHERE_IP_STATUS_MAP` Defaults to {"PREFERRED": "Active", "UNKNOWN": "Reserved"}
-- `VSPHERE_VM_INTERFACE_MAP` Defaults to {"NOT_CONNECTED": False, "CONNECTED": True}
+- `DEFAULT_VM_STATUS_MAP` Defaults to
+
+```json
+{ "POWERED_OFF": "Offline", "POWERED_ON": "Active", "SUSPENDED": "Suspended" }
+```
+
+- `DEFAULT_IP_STATUS_MAP` Defaults to
+
+```json
+{ "PREFERRED": "Active", "UNKNOWN": "Reserved" }
+```
+
+- `VSPHERE_VM_INTERFACE_MAP` Defaults to
+
+```json
+{"NOT_CONNECTED": False, "CONNECTED": True}
 ```
 
 `PRIMARY_IP_SORT_BY`
@@ -31,3 +47,11 @@
 `DEFAULT_IGNORE_LINK_LOCAL`
 
 - Defaults to `True` and drops any link-local address found on a vSphere vm that has an IPv6 address assigned.
+
+`DEFAULT_IP_STATUS_MAP`
+
+- This allows you to change what the `Status` of an interface maps to from vSphere -> Nautobot. Defaults to the following:
+
+```json
+{ "PREFERRED": "Active", "UNKNOWN": "Reserved" }
+```
diff --git a/nautobot_ssot_vsphere/__init__.py b/nautobot_ssot_vsphere/__init__.py
@@ -32,7 +32,7 @@ class NautobotSsotVsphereConfig(PluginConfig):
     min_version = "1.2.0"
     max_version = "1.9999"
     default_settings = {
-        "VSPHERE_VM_STATUS_MAP": {"POWERED_OFF": "Offline", "POWERED_ON": "Active"},
+        "VSPHERE_VM_STATUS_MAP": {"POWERED_OFF": "Offline", "POWERED_ON": "Active", "SUSPENDED": "Suspended"},
         "VSPHERE_VM_INTERFACE_MAP": {"NOT_CONNECTED": False, "CONNECTED": True},
         "ENFORCE_CLUSTER_GROUP_TOP_LEVEL": True,
         "VSPHERE_IP_STATUS_MAP": {"PREFERRED": "Active", "UNKNOWN": "Reserved"},

diff --git a/nautobot_ssot_vsphere/diffsync/adapters/adapter_nautobot.py b/nautobot_ssot_vsphere/diffsync/adapters/adapter_nautobot.py
@@ -108,9 +108,40 @@ def load_clustergroups(self):
         for diffsync_clustergroup in cluster_groups:
             self.get_or_instantiate(self.diffsync_clustergroup, {"name": diffsync_clustergroup.name})
 
-    def load_virtual_machines(self):
+    # TODO: (HUGO) Create a generic filtering func to repeat this logic for other objects
+    def get_initial_vms(self, ssot_tag: Tag):
+        """Identify the VirtualMachine objects based on user defined job inputs.
+
+        Args:
+            ssot_tag (Tag): Tag used for filtering
+        """
+        # Simple check / validate Tag is present.
+        if self.sync_vsphere_tagged_only:
+            self.job.log_debug(f"Filtering VirtualMachines by Tag `{ssot_tag.slug}`")
+            virtual_machines = VirtualMachine.objects.filter(tags__slug=ssot_tag.slug)
+            self.job.log_debug(f"Found VirtualMachines by Tag {virtual_machines.count()}")
+            if self.cluster_filter:
+                self.job.log_debug(
+                    f"Filtering VirtualMachines by Tag `{ssot_tag.slug}` and Cluster `{self.cluster_filter.name}`"
+                )
+                virtual_machines = VirtualMachine.objects.filter(
+                    Q(cluster=self.cluster_filter) & Q(tags__slug=ssot_tag.slug)
+                )
+                if not virtual_machines:
+                    self.job.log_warning(
+                        message=f"{self.cluster_filter.name} was used to filter, alongside SSoT Tag. {self.cluster_filter.name} is potentially not tagged. No objects found."  # NOQA
+                    )
+        elif not self.sync_vsphere_tagged_only:
+            if self.cluster_filter:
+                virtual_machines = VirtualMachine.objects.filter(name=self.cluster_filter.name)
+            else:
+                virtual_machines = VirtualMachine.objects.all()
+        return virtual_machines
+
+    def load_virtual_machines(self, ssot_tag: Tag):
         """Load Nautobot Virtual Machines."""
-        virtual_machines = VirtualMachine.objects.all()
+        # Capture virtual machines with conditional logic based on user input | filters
+        virtual_machines = self.get_initial_vms(ssot_tag)
         self.job.log_debug(f"Found {virtual_machines.count()} Virtual Machine objects")
         for virtual_machine in virtual_machines:
             diffsync_virtualmachine, _ = self.get_or_instantiate(
@@ -192,9 +223,9 @@ def load_data(self):
         self.load_clustergroups()
         # Load Clusters. This can used to filter by TAG or Cluster to minimize
         # The number of Virtual Machines from vSphere coming into Nautobot.
-        self.load_clusters(ssot_tag)
+        self.load_clusters(ssot_tag=ssot_tag)
         # Load Virtual Machine
-        self.load_virtual_machines()
+        self.load_virtual_machines(ssot_tag=ssot_tag)
 
     def load(self):
         """Load data from Nautobot."""

diff --git a/nautobot_ssot_vsphere/diffsync/adapters/adapter_vsphere.py b/nautobot_ssot_vsphere/diffsync/adapters/adapter_vsphere.py
@@ -58,7 +58,9 @@ def load_virtualmachines(self, cluster, diffsync_cluster):
                 {
                     "vcpus": virtual_machine["cpu_count"],
                     "memory": virtual_machine["memory_size_MiB"],
-                    "disk": get_disk_total(virtual_machine_details["disks"]),
+                    "disk": get_disk_total(virtual_machine_details["disks"])
+                    if virtual_machine_details.get("disks")
+                    else None,
                     "status": defaults.DEFAULT_VM_STATUS_MAP[virtual_machine_details["power_state"]],
                     "cluster": cluster["name"],
                 },
@@ -206,7 +208,9 @@ def load_standalone_vms(self):
                 {
                     "vcpus": virtual_machine["cpu_count"],
                     "memory": virtual_machine["memory_size_MiB"],
-                    "disk": get_disk_total(virtual_machine_details["disks"]),
+                    "disk": get_disk_total(virtual_machine_details["disks"])
+                    if virtual_machine_details.get("disks")
+                    else None,
                     "status": defaults.DEFAULT_VM_STATUS_MAP[virtual_machine_details["power_state"]],
                     "cluster": defaults.DEFAULT_CLUSTER_NAME,
                 },

diff --git a/nautobot_ssot_vsphere/diffsync/defaults.py b/nautobot_ssot_vsphere/diffsync/defaults.py
@@ -8,7 +8,9 @@
 VSPHERE_PASSWORD = CONFIG["VSPHERE_PASSWORD"]
 VSPHERE_VERIFY_SSL = CONFIG.get("VSPHERE_VERIFY_SSL", False)
 VSPHERE_URI = CONFIG["VSPHERE_URI"]
-DEFAULT_VM_STATUS_MAP = CONFIG.get("VSPHERE_VM_STATUS_MAP", {"POWERED_OFF": "Offline", "POWERED_ON": "Active"})
+DEFAULT_VM_STATUS_MAP = CONFIG.get(
+    "VSPHERE_VM_STATUS_MAP", {"POWERED_OFF": "Offline", "POWERED_ON": "Active", "SUSPENDED": "Suspended"}
+)
 DEFAULT_IP_STATUS_MAP = CONFIG.get("VSPHERE_IP_STATUS_MAP", {"PREFERRED": "Active", "UNKNOWN": "Reserved"})
 VSPHERE_VM_INTERFACE_MAP = CONFIG.get("VSPHERE_VM_INTERFACE_MAP", {"NOT_CONNECTED": False, "CONNECTED": True})
 PRIMARY_IP_SORT_BY = CONFIG.get("PRIMARY_IP_SORT_BY", "Lowest")

diff --git a/nautobot_ssot_vsphere/signals.py b/nautobot_ssot_vsphere/signals.py
@@ -1,3 +1,4 @@
+# pylint: disable=too-many-locals
 """Signal handlers for nautobot_ssot_vsphere."""
 
 from typing import List, Optional
@@ -56,6 +57,14 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs):  # pylint: disa
     ClusterType = apps.get_model("virtualization", "ClusterType")
     VMInterface = apps.get_model("virtualization", "VMInterface")
     IPAddress = apps.get_model("ipam", "IPAddress")
+    Status = apps.get_model("extras", "Status")
+    ContentType = apps.get_model("contenttypes", "ContentType")  # pylint:disable=invalid-name
+
+    status, _ = Status.objects.get_or_create(
+        name="Suspended", slug="suspended", description="Machine is in a suspended state"
+    )
+    status.content_types.add(ContentType.objects.get_for_model(VirtualMachine))
+    status.save()
 
     tag, _ = Tag.objects.get_or_create(
         slug="ssot-synced-from-vsphere",