Merge branch 'v3.26' into fix/js-sdk-delete-query

.github/actions/install-node-and-deps/action.yml

@@ -4,7 +4,7 @@ inputs:
   node-version:
     description: Node Version
     required: true
-    default: '~16'
+    default: 18
 runs:
   using: composite
   steps:

.github/dependabot.yml

@@ -13,3 +13,31 @@ updates:
       interval: "monthly"
     commit-message:
       prefix: "dev"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "dev"
+
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "dev"
+
+  - package-ecosystem: "gomod"
+    directory: "/tools"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "dev"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "dev"

.github/workflows/go.yml

@@ -85,7 +85,7 @@ jobs:
           args: -h postgres -U root ttn_lorawan_is_store_test
       - name: Configure AWS Credentials
         if: "${{ env.AWS_REGION != '' }}"
-        uses: aws-actions/configure-aws-credentials@v1-node16
+        uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-region: "${{ secrets.AWS_REGION }}"
           role-to-assume: "arn:aws:iam::${{ secrets.AWS_USER_ID }}:role/${{ secrets.AWS_ROLE_NAME }}"

.github/workflows/release-rc.yml

@@ -64,9 +64,9 @@ jobs:
           export RC_VERSION=${RC_VERSION#release/v}
           echo "rc_version=$RC_VERSION" >> $GITHUB_OUTPUT
       - name: Run Goreleaser
-        uses: goreleaser/goreleaser-action@v3
+        uses: goreleaser/goreleaser-action@v4
         with:
-          version: "v1.9.2"
+          version: "v1.18.2"
           args: release --config .goreleaser.snapshot.yml --snapshot --timeout 60m
         env:
           SIGN_KEY_NAME: ${{ secrets.SIGN_KEY_NAME }}

.github/workflows/release-snapshot.yml

@@ -55,9 +55,9 @@ jobs:
           printf '%s' '${{ secrets.SIGNATURE_PASSPHRASE }}' >/tmp/gpg_passphrase
           cat /tmp/gpg_passphrase | gpg --passphrase-fd 0 --no-tty --batch --import gpg_signing_key
       - name: Run Goreleaser
-        uses: goreleaser/goreleaser-action@v3
+        uses: goreleaser/goreleaser-action@v4
         with:
-          version: "v1.9.2"
+          version: "v1.18.2"
           args: release --config .goreleaser.snapshot.yml --snapshot --timeout 60m
         env:
           SIGN_KEY_NAME: ${{ secrets.SIGN_KEY_NAME }}

.github/workflows/release-tag.yml

@@ -45,7 +45,7 @@ jobs:
             git push origin v${{ steps.version.outputs.result }}
           fi
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1-node16
+        uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-region: "${{ secrets.AWS_REGION }}"
           role-to-assume: "arn:aws:iam::${{ secrets.AWS_USER_ID }}:role/${{ secrets.AWS_ROLE_NAME }}"
@@ -105,9 +105,9 @@ jobs:
           printf '%s' '${{ secrets.SIGNATURE_PASSPHRASE }}' >/tmp/gpg_passphrase
           cat /tmp/gpg_passphrase | gpg --passphrase-fd 0 --no-tty --batch --import gpg_signing_key
       - name: Run Goreleaser
-        uses: goreleaser/goreleaser-action@v3
+        uses: goreleaser/goreleaser-action@v4
         with:
-          version: "v1.9.2"
+          version: "v1.18.2"
           args: release --config .goreleaser.release.yml --release-notes /tmp/release-notes.md --timeout 120m
         env:
           SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_LOGIN }}

CHANGELOG.md

@@ -11,6 +11,8 @@ For details about compatibility between different releases, see the **Commitment
 
 ### Added
 
+- New Admin Panel in the Console.
+
 ### Changed
 
 ### Deprecated
@@ -21,10 +23,34 @@ For details about compatibility between different releases, see the **Commitment
 
 - Console not applying webhook field masks when creating a webhook from a template that has field masks set.
 - Removing user invitations not working in the user management panel for administrators.
+- Fix payload formatter page launching malformed requests in the Console.
+- HTTP API routes for parsing QR codes for the QR Generator service. We exercise our right to break compatibility with third party HTTP clients since this is a bug.
+  - `/qr-code/end-devices/parse` is changed to `/qr-codes/end-devices/parse`.
+  - `/qr-code/end-devices/{format_id}/parse` is changed to `/qr-codes/end-devices/{format_id}/parse`.
 
 ### Security
 
-## [3.25.2] - unreleased
+## [3.26.1] - 2023-06-20
+
+### Added
+
+- Support claim in device import in the Console.
+
+## [3.26.0] - 2023-06-06
+
+### Added
+
+- Support for scanning a QR code that only contains the hexadecimal encoded DevEUI.
+- Experimental flag `ns.adr.auto_narrow_steer`. When enabled, end devices which do not have an explicit channel steering mode will be steered towards the LoRa narrow channels.
+
+### Fixed
+
+- Console not applying webhook field masks when creating a webhook from a template that has field masks set.
+- LoRa Basics Station `PONG` messages will now contain the application payload of the associated `PING`, as required by the WebSockets specification.
+  - This fix enables `PING`/`PONG` behavior for non reference implementations of the LNS protocol.
+- Fix crash of "Edit webhook" view due to invalid Authorization header encoding in the Console.
+
+## [3.25.2] - 2023-05-16
 
 ### Added
 
@@ -98,7 +124,7 @@ For details about compatibility between different releases, see the **Commitment
 - Device claiming that transfer devices between applications is now deprecated and will be removed in a future version of The Things Stack. Device claiming on Join Servers, including The Things Join Server, remains functional. This deprecates the following components:
   - API for managing application claim authorization (`EndDeviceClaimingServer.AuthorizeApplication` and `EndDeviceClaimingServer.UnauthorizeApplication`)
   - CLI commands to manage application claim settings (`ttn-lw-cli application claim [authorize|unauthorize]`)
-  - CLI command to claim end devices  (`ttn-lw-cli devices claim`)
+  - CLI command to claim end devices (`ttn-lw-cli devices claim`)
 
 ### Fixed
 
@@ -232,10 +258,10 @@ For details about compatibility between different releases, see the **Commitment
 ### Changed
 
 - Option to ignore logs from selected gRPC methods now supports ignoring logs for selected errors on method.
-    Examples:
-    - `--grpc.log-ignore-methods="/ttn.lorawan.v3.GsNs/HandleUplink"`: log is skipped when no error occurs.
-    - `--grpc.log-ignore-methods="/ttn.lorawan.v3.GsNs/HandleUplink:pkg/networkserver:duplicate_uplink;pkg/networkserver:device_not_found"`: log is skipped when either `pkg/networkserver:duplicate_uplink` or `pkg/networkserver:device_not_found` error occurs (but not on success).
-    - `--grpc.log-ignore-methods="/ttn.lorawan.v3.GsNs/HandleUplink:;pkg/networkserver:duplicate_uplink"`: log is skipped on success or when `pkg/networkserver:duplicate_uplink` error occurs.
+  Examples:
+  - `--grpc.log-ignore-methods="/ttn.lorawan.v3.GsNs/HandleUplink"`: log is skipped when no error occurs.
+  - `--grpc.log-ignore-methods="/ttn.lorawan.v3.GsNs/HandleUplink:pkg/networkserver:duplicate_uplink;pkg/networkserver:device_not_found"`: log is skipped when either `pkg/networkserver:duplicate_uplink` or `pkg/networkserver:device_not_found` error occurs (but not on success).
+  - `--grpc.log-ignore-methods="/ttn.lorawan.v3.GsNs/HandleUplink:;pkg/networkserver:duplicate_uplink"`: log is skipped on success or when `pkg/networkserver:duplicate_uplink` error occurs.
 - The Gateway Server now takes into consideration the extra duty cycle checks present in the LoRa Basics Station forwarder. Previously the Gateway Server may accept the scheduling of downlinks which the packet forwarder would silently drop.
   - Note that in some rare cases in which the LoRa Basics Station duty cycle is stricter than the windowed approach used by The Things Stack, the scheduling will fail and this will be visible via `ns.down.data.schedule.fail` events. Note that this is actually a positive outcome - it allows the Network Server to schedule the downlink via another gateway, while previously the downlink would be scheduled but get silently dropped on the gateway.
 
@@ -2582,7 +2608,9 @@ For details about compatibility between different releases, see the **Commitment
 NOTE: These links should respect backports. See https://github.com/TheThingsNetwork/lorawan-stack/pull/1444/files#r333379706.
 -->
 
-[unreleased]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.25.2...v3.25
+[unreleased]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.26.1...v3.26
+[3.26.1]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.26.0...v3.26.1
+[3.26.0]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.25.2...v3.26.0
 [3.25.2]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.25.1...v3.25.2
 [3.25.1]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.25.0...v3.25.1
 [3.25.0]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.24.2...v3.25.0

CODEOWNERS

@@ -11,17 +11,17 @@
 /go.mod @johanstokking @adriansmares
 /go.sum @johanstokking @adriansmares
 /package.json @kschiffer @ryaplots
-/yarn.lock @kschiffer @ryaplots
-/sdk/js/package.json @kschiffer @ryaplots
-/sdk/js/yarn.lock @kschiffer @ryaplots
+/yarn.lock @kschiffer @ryaplots @mjamescompton
+/sdk/js/package.json @kschiffer @ryaplots @mjamescompton
+/sdk/js/yarn.lock @kschiffer @ryaplots @mjamescompton
 
 # Build pipeline and tooling
 /Makefile @adriansmares
 /tools @adriansmares
 /.github/workflows @adriansmares
-/config @kschiffer @ryaplots
+/config @kschiffer @ryaplots @mjamescompton
 /config/stack @benolayinka @KrishnaIyer
-/cypress @kschiffer @ryaplots
+/cypress @kschiffer @ryaplots @mjamescompton
 
 # API, configuration, CLI and storage compatibility
 /api/*.proto @johanstokking @adriansmares
@@ -49,7 +49,7 @@
 /pkg/rpcmiddleware @johanstokking @adriansmares
 /pkg/rpcserver @johanstokking @adriansmares
 /pkg/types @johanstokking
-/pkg/web @kschiffer @ryaplots
+/pkg/web @kschiffer @ryaplots @mjamescompton
 /pkg/webmiddleware @KrishnaIyer @kschiffer
 /pkg/webui @kschiffer @ryaplots @mjamescompton
 /pkg/ratelimit @johanstokking @KrishnaIyer
@@ -77,7 +77,7 @@
 
 /pkg/joinserver @johanstokking @adriansmares
 
-/pkg/console @kschiffer @ryaplots
+/pkg/console @kschiffer @ryaplots @mjamescompton
 
 /pkg/packetbrokeragent @johanstokking @adriansmares
 

DEVELOPMENT.md

@@ -55,7 +55,7 @@ The Things Stack components are primarily built in Go, we use React for web fron
 
 ## Development Environment
 
-The Things Network's development tooling uses [Mage](https://magefile.org/). Under the hood, `mage` calls other tools such as `git`, `go`, `yarn`, `docker` etc. Recent versions are supported; Node v16.x and Go v1.18.x.
+The Things Network's development tooling uses [Mage](https://magefile.org/). Under the hood, `mage` calls other tools such as `git`, `go`, `yarn`, `docker` etc. Recent versions are supported; Node v18.x and Go v1.18.x.
 
 - Follow [Go's installation guide](https://golang.org/doc/install) to install Go.
 - Download Node.js [from their website](https://nodejs.org) and install it.

Dockerfile

@@ -12,7 +12,7 @@ RUN rm -rf /srv/ttn-lorawan/lorawan-frequency-plans/.git
 COPY data/lorawan-webhook-templates /srv/ttn-lorawan/lorawan-webhook-templates
 RUN rm -rf /srv/ttn-lorawan/lorawan-webhook-templates/.git
 
-FROM alpine:3.17
+FROM alpine:3.18
 
 RUN addgroup -g 886 thethings && adduser -u 886 -S -G thethings thethings
 
@@ -29,7 +29,7 @@ COPY public /srv/ttn-lorawan/public
 COPY --from=builder /srv/ttn-lorawan/lorawan-frequency-plans /srv/ttn-lorawan/lorawan-frequency-plans
 COPY --from=builder /srv/ttn-lorawan/lorawan-webhook-templates /srv/ttn-lorawan/lorawan-webhook-templates
 COPY data/lorawan-devices-index /srv/ttn-lorawan/lorawan-devices-index
-RUN chown thethings:thethings -R /srv/ttn-lorawan/lorawan-devices-index
+RUN chmod 755 -R /srv/ttn-lorawan/lorawan-devices-index
 
 EXPOSE 1700/udp 1881 8881 1882 8882 1883 8883 1884 8884 1885 8885 1887 8887
 

api/api.md

@@ -53,6 +53,7 @@
   - [Message `ALCSyncCommand.AppTimeReq`](#ttn.lorawan.v3.ALCSyncCommand.AppTimeReq)
   - [Enum `ALCSyncCommandIdentifier`](#ttn.lorawan.v3.ALCSyncCommandIdentifier)
 - [File `lorawan-stack/api/applicationserver_integrations_storage.proto`](#lorawan-stack/api/applicationserver_integrations_storage.proto)
+  - [Message `ContinuationTokenPayload`](#ttn.lorawan.v3.ContinuationTokenPayload)
   - [Message `GetStoredApplicationUpCountRequest`](#ttn.lorawan.v3.GetStoredApplicationUpCountRequest)
   - [Message `GetStoredApplicationUpCountResponse`](#ttn.lorawan.v3.GetStoredApplicationUpCountResponse)
   - [Message `GetStoredApplicationUpCountResponse.CountEntry`](#ttn.lorawan.v3.GetStoredApplicationUpCountResponse.CountEntry)
@@ -1294,6 +1295,19 @@ The NsAs service connects a Network Server to an Application Server.
 
 ## <a name="lorawan-stack/api/applicationserver_integrations_storage.proto">File `lorawan-stack/api/applicationserver_integrations_storage.proto`</a>
 
+### <a name="ttn.lorawan.v3.ContinuationTokenPayload">Message `ContinuationTokenPayload`</a>
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| `limit` | [`google.protobuf.UInt32Value`](#google.protobuf.UInt32Value) |  |  |
+| `after` | [`google.protobuf.Timestamp`](#google.protobuf.Timestamp) |  |  |
+| `before` | [`google.protobuf.Timestamp`](#google.protobuf.Timestamp) |  |  |
+| `f_port` | [`google.protobuf.UInt32Value`](#google.protobuf.UInt32Value) |  |  |
+| `order` | [`string`](#string) |  |  |
+| `field_mask` | [`google.protobuf.FieldMask`](#google.protobuf.FieldMask) |  |  |
+| `last` | [`google.protobuf.Duration`](#google.protobuf.Duration) |  |  |
+| `last_received_id` | [`int64`](#int64) |  |  |
+
 ### <a name="ttn.lorawan.v3.GetStoredApplicationUpCountRequest">Message `GetStoredApplicationUpCountRequest`</a>
 
 | Field | Type | Label | Description |
@@ -1339,13 +1353,15 @@ The NsAs service connects a Network Server to an Application Server.
 | `order` | [`string`](#string) |  | Order results. |
 | `field_mask` | [`google.protobuf.FieldMask`](#google.protobuf.FieldMask) |  | The names of the upstream message fields that should be returned. See the API reference for allowed field names for each type of upstream message. |
 | `last` | [`google.protobuf.Duration`](#google.protobuf.Duration) |  | Query upstream messages that have arrived in the last minutes or hours. Cannot be used in conjunction with after and before. |
+| `continuation_token` | [`string`](#string) |  | The continuation token, which is used to retrieve the next page. If provided, other fields are ignored. |
 
 #### Field Rules
 
 | Field | Validations |
 | ----- | ----------- |
 | `type` | <p>`string.in`: `[ uplink_message uplink_normalized join_accept downlink_ack downlink_nack downlink_sent downlink_failed downlink_queued downlink_queue_invalidated location_solved service_data]`</p> |
 | `order` | <p>`string.in`: `[ -received_at received_at]`</p> |
+| `continuation_token` | <p>`string.max_len`: `16000`</p> |
 
 ### <a name="ttn.lorawan.v3.ApplicationUpStorage">Service `ApplicationUpStorage`</a>
 
@@ -2717,13 +2733,6 @@ in a future version of The Things Stack.
 | `qr_code` | [`bytes`](#bytes) |  | Raw QR code contents. |
 | `target_application_ids` | [`ApplicationIdentifiers`](#ttn.lorawan.v3.ApplicationIdentifiers) |  | Application identifiers of the target end device. |
 | `target_device_id` | [`string`](#string) |  | End device ID of the target end device. If empty, use the source device ID. |
-| `target_network_server_address` | [`string`](#string) |  | The address of the Network Server where the device will be registered. If set and if the source device is currently registered on a Network Server, settings will be transferred. If not set, the device shall not be registered on a Network Server. |
-| `target_network_server_kek_label` | [`string`](#string) |  | The KEK label of the Network Server to use for wrapping network session keys. |
-| `target_application_server_address` | [`string`](#string) |  | The address of the Application Server where the device will be registered. If set and if the source device is currently registered on an Application Server, settings will be transferred. If not set, the device shall not be registered on an Application Server. |
-| `target_application_server_kek_label` | [`string`](#string) |  | The KEK label of the Application Server to use for wrapping the application session key. |
-| `target_application_server_id` | [`string`](#string) |  | The AS-ID of the Application Server to use. |
-| `target_net_id` | [`bytes`](#bytes) |  | Home NetID. |
-| `invalidate_authentication_code` | [`bool`](#bool) |  | If set, invalidate the authentication code with which the device gets claimed. This prohibits subsequent claiming requests. |
 
 #### Field Rules
 
@@ -2732,12 +2741,6 @@ in a future version of The Things Stack.
 | `qr_code` | <p>`bytes.min_len`: `0`</p><p>`bytes.max_len`: `1024`</p> |
 | `target_application_ids` | <p>`message.required`: `true`</p> |
 | `target_device_id` | <p>`string.max_len`: `36`</p><p>`string.pattern`: `^[a-z0-9](?:[-]?[a-z0-9]){2,}$|^$`</p> |
-| `target_network_server_address` | <p>`string.pattern`: `^(?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])(?::[0-9]{1,5})?$|^$`</p> |
-| `target_network_server_kek_label` | <p>`string.max_len`: `2048`</p> |
-| `target_application_server_address` | <p>`string.pattern`: `^(?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])(?::[0-9]{1,5})?$|^$`</p> |
-| `target_application_server_kek_label` | <p>`string.max_len`: `2048`</p> |
-| `target_application_server_id` | <p>`string.max_len`: `100`</p> |
-| `target_net_id` | <p>`bytes.len`: `3`</p> |
 
 ### <a name="ttn.lorawan.v3.ClaimEndDeviceRequest.AuthenticatedIdentifiers">Message `ClaimEndDeviceRequest.AuthenticatedIdentifiers`</a>
 
@@ -8672,8 +8675,8 @@ The Pba service allows clients to manage peering through Packet Broker.
 | `GetFormat` | `GET` | `/api/v3/qr-codes/end-devices/formats/{format_id}` |  |
 | `ListFormats` | `GET` | `/api/v3/qr-codes/end-devices/formats` |  |
 | `Generate` | `POST` | `/api/v3/qr-codes/end-devices` | `*` |
-| `Parse` | `POST` | `/api/v3/qr-code/end-devices/parse` | `*` |
-| `Parse` | `POST` | `/api/v3/qr-code/end-devices/{format_id}/parse` | `*` |
+| `Parse` | `POST` | `/api/v3/qr-codes/end-devices/parse` | `*` |
+| `Parse` | `POST` | `/api/v3/qr-codes/end-devices/{format_id}/parse` | `*` |
 
 ## <a name="lorawan-stack/api/regional.proto">File `lorawan-stack/api/regional.proto`</a>