diff --git a/subsys/net/l2/ieee802154/ieee802154.c b/subsys/net/l2/ieee802154/ieee802154.c
index 09fdf8d28cfc9c..b5b372ebe5c10a 100644
--- a/subsys/net/l2/ieee802154/ieee802154.c
+++ b/subsys/net/l2/ieee802154/ieee802154.c
@@ -247,9 +247,9 @@ int ieee802154_radio_send(struct net_if *iface, struct net_pkt *pkt, struct net_
 	return -EIO;
 }
 
-static inline void swap_and_set_pkt_ll_addr(struct net_linkaddr *addr, bool has_pan_id,
-					    enum ieee802154_addressing_mode mode,
-					    struct ieee802154_address_field *ll)
+static inline void set_pkt_ll_addr(struct net_linkaddr *addr, bool has_pan_id,
+				   enum ieee802154_addressing_mode mode,
+				   struct ieee802154_address_field *ll)
 {
 	addr->type = NET_LINK_IEEE802154;
 
@@ -270,16 +270,6 @@ static inline void swap_and_set_pkt_ll_addr(struct net_linkaddr *addr, bool has_
 		addr->len = 0U;
 		addr->addr = NULL;
 	}
-
-	/* The net stack expects link layer addresses to be in
-	 * big endian format for posix compliance so we must swap it.
-	 * This is ok as the L2 address field comes from the header
-	 * part of the packet buffer which will not be directly accessible
-	 * once the packet reaches the upper layers.
-	 */
-	if (addr->len > 0) {
-		sys_mem_swap(addr->addr, addr->len);
-	}
 }
 
 /**
@@ -431,19 +421,20 @@ static enum net_verdict ieee802154_recv(struct net_if *iface, struct net_pkt *pk
 		return NET_OK;
 	}
 
+	set_pkt_ll_addr(net_pkt_lladdr_src(pkt), !fs->fc.pan_id_comp, fs->fc.src_addr_mode,
+			mpdu.mhr.src_addr);
+
+	set_pkt_ll_addr(net_pkt_lladdr_dst(pkt), true, fs->fc.dst_addr_mode, mpdu.mhr.dst_addr);
+
 	if (!ieee802154_decipher_data_frame(iface, pkt, &mpdu)) {
 		return NET_DROP;
 	}
 
-	/* Setting L2 addresses must be done after packet authentication and internal
-	 * packet handling as it will mangle the package header to comply with upper
-	 * network layers' (POSIX) requirement to represent network addresses in big endian.
+	/* The net stack expects link layer addresses to be in
+	 * big endian format for posix compliance so we must swap it.
 	 */
-	swap_and_set_pkt_ll_addr(net_pkt_lladdr_src(pkt), !fs->fc.pan_id_comp,
-				 fs->fc.src_addr_mode, mpdu.mhr.src_addr);
-
-	swap_and_set_pkt_ll_addr(net_pkt_lladdr_dst(pkt), true, fs->fc.dst_addr_mode,
-				 mpdu.mhr.dst_addr);
+	sys_mem_swap(net_pkt_lladdr_src(pkt)->addr, net_pkt_lladdr_src(pkt)->len);
+	sys_mem_swap(net_pkt_lladdr_dst(pkt)->addr, net_pkt_lladdr_dst(pkt)->len);
 
 	net_pkt_set_ll_proto_type(pkt, ETH_P_IEEE802154);
 
diff --git a/subsys/net/l2/ieee802154/ieee802154_frame.c b/subsys/net/l2/ieee802154/ieee802154_frame.c
index 002e3c0be22546..11f4d5c73fef1f 100644
--- a/subsys/net/l2/ieee802154/ieee802154_frame.c
+++ b/subsys/net/l2/ieee802154/ieee802154_frame.c
@@ -959,7 +959,6 @@ bool ieee802154_decipher_data_frame(struct net_if *iface, struct net_pkt *pkt,
 	uint8_t authtag_len = level_2_authtag_len[level];
 	uint8_t ll_hdr_len = (uint8_t *)mpdu->payload - net_pkt_data(pkt);
 	uint8_t payload_len = net_pkt_get_len(pkt) - ll_hdr_len - authtag_len;
-	uint8_t ext_addr_le[IEEE802154_EXT_ADDR_LENGTH];
 
 	/* TODO: Handle src short address.
 	 * This will require to look up in nbr cache with short addr
@@ -970,9 +969,8 @@ bool ieee802154_decipher_data_frame(struct net_if *iface, struct net_pkt *pkt,
 		goto out;
 	}
 
-	sys_memcpy_swap(ext_addr_le, net_pkt_lladdr_src(pkt)->addr, net_pkt_lladdr_src(pkt)->len);
 	if (!ieee802154_decrypt_auth(&ctx->sec_ctx, net_pkt_data(pkt), ll_hdr_len, payload_len,
-				     authtag_len, ext_addr_le,
+				     authtag_len, net_pkt_lladdr_src(pkt)->addr,
 				     sys_le32_to_cpu(mpdu->mhr.aux_sec->frame_counter))) {
 		NET_ERR("Could not decipher the frame");
 		goto out;