Merge pull request #318 from SUNET/develop

Release version 1.5.0

.github/workflows/run-unit-tests.yml

@@ -32,7 +32,7 @@ jobs:
 #      - name: Create code coverage dir
 #        run: mkdir -p docker/coverage/
 #
-      - uses: satackey/action-docker-layer-caching@v0.0.11
+      - uses: jpribyl/action-docker-layer-caching@v0.1.1
         # Ignore the failure of a step and avoid terminating the job.
         continue-on-error: true
 
@@ -44,7 +44,7 @@ jobs:
       - name: Get branch name (pull request)
         if: github.event_name == 'pull_request'
         shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
+        run: echo "BRANCH_NAME=pr/$(echo ${GITHUB_REF#refs/pull/} | cut -d '/' -f1)" >> $GITHUB_ENV
 
       - name: Get repo URL
         shell: bash

alembic/versions/2f9faee221a7_add_ipv6_gw_field_to_mgmtdomain.py

@@ -0,0 +1,24 @@
+"""Add ipv6_gw field to Mgmtdomain
+
+Revision ID: 2f9faee221a7
+Revises: b7629362583c
+Create Date: 2022-10-26 13:52:12.466111
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "2f9faee221a7"
+down_revision = "b7629362583c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("mgmtdomain", sa.Column("ipv6_gw", sa.Unicode(43)))
+
+
+def downgrade():
+    op.drop_column("mgmtdomain", "ipv6_gw")

alembic/versions/adcce7d9baaa_add_device_secondary_management_ip_column.py

@@ -0,0 +1,28 @@
+""""Add secondary_management_ip to Device
+
+Revision ID: adcce7d9baaa
+Revises: 2f9faee221a7
+Create Date: 2023-01-11 15:18:12.188994
+
+"""
+import sqlalchemy as sa
+import sqlalchemy_utils
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "adcce7d9baaa"
+down_revision = "2f9faee221a7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "device",
+        sa.Column("secondary_management_ip", sqlalchemy_utils.types.ip_address.IPAddressType(length=50), nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_column("device", "secondary_management_ip")

docker/api/Dockerfile

@@ -29,6 +29,7 @@ RUN apt-get update \
       psmisc \
       supervisor \
       uwsgi-plugin-python3 \
+      ssh-client \
     && pip3 install --no-cache-dir uwsgi
 
 # Prepare for supervisord, ngninx

docker/api/config/api.yml

@@ -9,3 +9,7 @@ certpath: /tmp/devicecerts/
 global_unique_vlans: True
 init_mgmt_timeout: 30
 mgmtdomain_reserved_count: 5
+mgmtdomain_primary_ip_version: 4
+commit_confirmed_mode: 1
+commit_confirmed_timeout: 300
+commit_confirmed_wait: 1

docker/api/pytest.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-PYTESTARGS=()
+PYTESTARGS=("-vv" "--showlocals")
 
 if [ ! -z "$NO_EQUIPMENTTEST" ] ; then
 	PYTESTARGS+=("-m" "not equipment")

docker/dhcpd/dhcpd.sh

@@ -6,7 +6,12 @@ if [ ! -z "$GITREPO_ETC" ]
 then
 	cd /opt/cnaas
 	rm -rf /opt/cnaas/etc
-	git clone $GITREPO_ETC etc
+        base_url=$(echo $GITREPO_ETC | cut -d\# -f1)
+        branch=$(echo $GITREPO_ETC | cut -d\# -s -f2)
+        if [ -n "$branch" ]; then
+           branch="-b $branch"
+        fi
+	git clone $branch $base_url etc
 	if [ -f "/opt/cnaas/etc/dhcpd/dhcpd.conf" ]
 	then
 		cp /opt/cnaas/etc/dhcpd/dhcpd.conf /opt/cnaas/dhcpd.conf

docs/apiref/devices.rst

@@ -190,6 +190,32 @@ This will return both the generated configuration based on the template for
 this device type, and also a list of available vaiables that could be used
 in the template.
 
+Running config
+--------------
+
+To get the latest running config from the device you can use running_config:
+
+::
+
+  curl https://hostname/api/v1.0/device/<device_hostname>/running_config
+
+You can also specify to only get the running config for a specific interface:
+
+::
+
+  curl https://hostname/api/v1.0/device/<device_hostname>/running_config?interface=Ethernet1
+
+Example output:
+
+::
+
+   {
+       "status": "success",
+       "data": {
+           "config": "no switchport\nvrf MGMT\nip address 10.100.2.101/24\nno lldp transmit\nno lldp receive"
+       }
+   }
+
 View previous config
 --------------------
 

docs/apiref/jobs.rst

@@ -107,6 +107,18 @@ It's also possible to query a single job by job ID:
    curl http://hostname/api/v1.0/job/5
 
 
+Abort scheduled job
+-------------------
+
+To abort a scheduled job send a request like this:
+
+::
+
+   curl http://hostname/api/v1.0/job/5 -X PUT -d '{"action": "ABORT"}' -H "Content-Type: application/json"
+
+You can also send the request to a running job, but currently only firmware
+upgrade will listen for aborts while running and devices that already passed
+the check will continue with their upgrades.
 
 Locks
 -----

docs/apiref/settings.rst

@@ -92,3 +92,40 @@ Output:
      "status": "error",
      "message": "Validation error for setting radius_servers->0->host, bad value: 10.0.0.500 (value origin: API POST data)\nMessage: string does not match regex \"^((?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|([a-z0-9-]{1,63}\\.)([a-z-][a-z0-9-]{1,62}\\.?)+)$\", field should be: Hostname, FQDN or IP address\n"
    }
+
+
+Server settings:
+----------------
+
+To get configuration settings for the API server, send a GET request to:
+
+::
+
+   curl https://hostname/api/v1.0/settings/server
+
+Output will very depending on version of NMS running, example:
+
+::
+
+   {
+     "api": {
+       "HOST": "172.30.0.1",
+       "HTTPD_URL": "https://cnaas_httpd:1443/api/v1.0/firmware",
+       "VERIFY_TLS": true,
+       "VERIFY_TLS_DEVICE": false,
+       "JWT_CERT": "/etc/cnaas-nms/public.pem",
+       "CAFILE": "/etc/cnaas-nms/certs/rootCA.crt",
+       "CAKEYFILE": "/etc/cnaas-nms/certs/rootCA.key",
+       "CERTPATH": "/etc/cnaas-nms/certs",
+       "ALLOW_APPLY_CONFIG_LIVERUN": false,
+       "FIRMWARE_URL": "https://cnaas_httpd:1443/api/v1.0/firmware",
+       "JWT_ENABLED": true,
+       "PLUGIN_FILE": "/etc/cnaas-nms/plugins.yml",
+       "GLOBAL_UNIQUE_VLANS": true,
+       "INIT_MGMT_TIMEOUT": 30,
+       "MGMTDOMAIN_RESERVED_COUNT": 5,
+       "COMMIT_CONFIRMED_MODE": 2,
+       "COMMIT_CONFIRMED_TIMEOUT": 300,
+       "SETTINGS_OVERRIDE": null
+     }
+   }

docs/apiref/syncto.rst

@@ -36,8 +36,27 @@ The status success in this case only means that the job was scheduled successful
 you have to poll the job API to see that result of what was done, the job itself might still
 fail.
 
-Arguments:
-----------
+Configuration changes can be made in a way that requires a separate confirm call since version 1.5.
+If the change can not be confirmed because the device is not unreachable for example, the device
+will roll back the configuration. Before version 1.5 this concept was not supported, but from this
+version it's supported and enabled by default using mode 1.
+
+.. _commit_confirm_modes:
+
+**Commit confirm modes:**
+
+ - 0 = No confirm commit (default up to version 1.4)
+ - 1 = Commit is immediately confirmed for each device when that device is configured
+   (default from version 1.5)
+ - 2 = Commit is confirmed after all devices in the job has been configured, but only if all were
+   successful. This mode is only supported for EOS and JunOS so far, and only supported for small
+   number of devices per commit (max 50). If mode 2 is specified and an unsupported device is
+   selected that device will use mode 1 instead.
+
+Commit confirm mode can be specified in the configuration file, but it's also possible to override
+that setting for a specific job using the API argument confirm_mode (see below).
+
+**Arguments:**
 
  - hostname: Optional, the hostname of a device
  - device_type: Optional, a device type (access, dist or core)
@@ -59,6 +78,59 @@ Arguments:
    This should be a string with max 255 characters.
  - ticket_ref: Optionally reference a service ticket associated with this job.
    This should be a string with max 32 characters.
+ - confirm_mode: Optionally override the default commit confirm mode (see above) for this job.
+   Must be an integer 0, 1 or 2 if specified.
 
 If neither hostname or device_type is specified all devices that needs to be sycnhronized
 will be selected.
+
+Sync history
+------------
+
+When an API call causes a device to become unsynchronized a synchronization event is created
+in the synchistory log. You can query or manually add events from this history using the API.
+
+Get synchistory events:
+
+::
+
+   curl https://hostname/api/v1.0/device_synchistory?hostname=eosaccess
+
+Example output:
+
+::
+
+   {
+     "status": "success",
+     "data": {
+       "hostnames": {
+         "eosaccess": [
+           {
+             "cause": "refresh_settings",
+             "timestamp": 1688458956.684019,
+             "by": "indy",
+             "job_id": 123
+           }
+         ]
+       }
+     }
+   }
+
+If the query parameter "hostname" is left out the API will return events for
+all devices.
+
+"cause" is a text string reference to the thing that caused the device to become
+unsynchronized. For more details on events see :ref:`sync_status_tutorial`.
+"timestamp" is a floating point number representing the seconds since Unix epoch (UTC).
+"by" is string referring to what user triggered the event. "job_id" is an integer
+referring to a job if this event was triggered by a job, or otherwise it's null.
+
+Manually adding a synchistory event:
+
+::
+
+   curl https://hostname/api/v1.0/device_synchistory -d '{"hostname": "eosaccess", "cause": "oob", "by": "indy"}'
+   -X POST -H "Content-Type: application/json"
+
+The "time" paramater can optionally be specified as a floating point number of seconds
+since Unix epoch (UTC). If not provided the current time will be used.

docs/changelog/index.rst

@@ -1,6 +1,28 @@
 Changelog
 =========
 
+Version 1.5.0
+-------------
+
+New features:
+
+ - Add commit confirm modes: mode 0 - no commit confirm (previous behavior), mode 1 - confirm each device individually
+   after it has commited, mode 2 - confirm devices only when all devices in the job has comitted successfully
+ - Add API to query configured API settings
+ - New settings for:
+  * users: username, ssh_key, password_hash etc to handle device user accounts
+  * dot1x_multi_host, poe_reboot_maintain
+  * prefix_sets and routing_policies to describe policies for router peerings etc
+ - Sync history events, save what event caused devices to become unsynchronized
+ - API to query running config
+
+Bug fixes:
+
+ - Don't return duplicates of neighbor entries
+ - Fix error "Error in repository: HEAD is a detached" when refreshing settings repo
+ - Mark init neighbors as unsync if they have local changes
+ - Log events was not always sent from redis to websocket
+
 Version 1.4.2
 -------------
 

docs/configuration/index.rst

@@ -38,6 +38,15 @@ Defines parameters for the API:
   each defined management domain when assigning new management IP addresses to devices.
   Defaults to 5 (e.g. meaning 10.0.0.1 through 10.0.0.5 would remain unassigned on
   a domain for 10.0.0.0/24).
+- mgmtdomain_primary_ip_version: For dual stack management domains, this setting
+  defines whether IP version 4 or 6 is preferred when an access device's primary
+  management address is assigned. The only valid values are therefore 4 and 6.
+- commit_confirmed_mode: Integer specifying default commit confirm mode
+  (see :ref:`commit_confirm_modes`). Defaults to 1.
+- commit_confirmed_timeout: Time to wait before rolling back an unconfirmed commit,
+  specified in seconds. Defaults to 300.
+- commit_confirmed_wait: Time to wait between comitting configuration and checking
+  that the device is still reachable, specified in seconds. Defaults to 1.
 
 /etc/cnaas-nms/repository.yml
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^