librz/arch: migrate Xtensa to Capstone (#4654)

* Add xtensa arch capstone plugin
* Remove xtensa-gnu
* fix: `asm.cpu` is not set correctly, when there is no “,” in asm plugin -> cpus
* fix core_disassembly when n_bytes==0
* fix xtensa_analyze_op_esil
* fix xtensa esil pc mod
* fix xtensa esil l32r
* fix xtensa esil call0
* fix xtensa esil extui
* fix xtensa esil store
* fix xtensa stack inc and ret
* keep `asm.cpu` when valid in cb_asmarch
* fix cb_asmarch
* fix max op size
* xtensa: add cpu select
* xtensa: mark PUSH POP
* xtensa: set src dst
* xtensa: set dir
* xtensa: fix stackframe

librz/arch/isa/xtensa/xtensa.c

@@ -0,0 +1,68 @@
+// SPDX-FileCopyrightText: 2024 billow <[email protected]>
+// SPDX-License-Identifier: LGPL-3.0-only
+
+#include "xtensa.h"
+
+bool xtensa_init(void **user) {
+	if (*user) {
+		return true;
+	}
+	XtensaContext *ctx = RZ_NEW0(XtensaContext);
+	if (!ctx) {
+		return false;
+	}
+	*user = ctx;
+	return true;
+}
+
+bool xtensa_fini(void *user) {
+	XtensaContext *ctx = user;
+	cs_close(&ctx->handle);
+	free(ctx);
+	return true;
+}
+
+bool xtensa_open(XtensaContext *ctx, const char *cpu, bool big_endian) {
+	if (!ctx) {
+		return false;
+	}
+	cs_mode mode = big_endian ? CS_MODE_BIG_ENDIAN : CS_MODE_LITTLE_ENDIAN;
+	if (RZ_STR_ISEMPTY(cpu)) {
+		mode |= xtensa_cpu_modes[0].mode;
+	} else {
+		for (int i = 0; i < RZ_ARRAY_SIZE(xtensa_cpu_modes); ++i) {
+			if (RZ_STR_EQ(cpu, xtensa_cpu_modes[i].cpu)) {
+				mode |= xtensa_cpu_modes[i].mode;
+				break;
+			}
+		}
+	}
+	if (mode == ctx->mode) {
+		return true;
+	}
+	if (cs_open(CS_ARCH_XTENSA, mode, &ctx->handle) != CS_ERR_OK) {
+		return false;
+	}
+	ctx->mode = mode;
+	if (cs_option(ctx->handle, CS_OPT_DETAIL, CS_OPT_ON) != CS_ERR_OK) {
+		return false;
+	}
+	return true;
+}
+
+bool xtensa_disassemble(XtensaContext *self, const ut8 *buf, int len, ut64 addr) {
+	self->count = cs_disasm(self->handle, buf, len, addr, 1, &self->insn);
+	if (self->count == 0) {
+		return false;
+	}
+	return true;
+}
+
+void xtensa_disassemble_fini(XtensaContext *self) {
+	if (!self->insn) {
+		return;
+	}
+	cs_free(self->insn, self->count);
+	self->insn = NULL;
+	self->count = 0;
+}

librz/arch/isa/xtensa/xtensa.h

@@ -0,0 +1,67 @@
+// SPDX-FileCopyrightText: 2024 billow <[email protected]>
+// SPDX-License-Identifier: LGPL-3.0-only
+
+#ifndef RZ_XTENSA_H
+#define RZ_XTENSA_H
+
+#include <capstone/capstone.h>
+#include <rz_asm.h>
+
+typedef struct {
+	const char *cpu;
+	cs_mode mode;
+} XtensaCPUMode;
+
+static const XtensaCPUMode xtensa_cpu_modes[] = {
+	{ .cpu = "esp32", .mode = CS_MODE_XTENSA_ESP32 },
+	{ .cpu = "esp32s2", .mode = CS_MODE_XTENSA_ESP32S2 },
+	{ .cpu = "esp8266", .mode = CS_MODE_XTENSA_ESP8266 },
+};
+
+typedef struct xtensa_context_t {
+	cs_mode mode;
+	csh handle;
+	cs_insn *insn;
+	size_t count;
+} XtensaContext;
+
+bool xtensa_init(void **user);
+bool xtensa_fini(void *user);
+bool xtensa_open(XtensaContext *ctx, const char *cpu, bool big_endian);
+bool xtensa_disassemble(XtensaContext *self, const ut8 *buf, int len, ut64 addr);
+void xtensa_disassemble_fini(XtensaContext *self);
+void xtensa_analyze_op_esil(XtensaContext *ctx, RzAnalysisOp *op);
+
+static inline cs_xtensa_op_mem *xtensa_op_mem(cs_insn *insn, unsigned int index) {
+	cs_xtensa_op *op = &insn->detail->xtensa.operands[index];
+	rz_warn_if_fail(op->type == XTENSA_OP_MEM);
+	return &op->mem;
+}
+
+static inline xtensa_reg xtensa_op_reg(cs_insn *insn, unsigned int index) {
+	cs_xtensa_op *op = &insn->detail->xtensa.operands[index];
+	rz_warn_if_fail(op->type == XTENSA_OP_REG);
+	return op->reg;
+}
+
+static inline int32_t xtensa_op_imm(cs_insn *insn, unsigned int index) {
+	cs_xtensa_op *op = &insn->detail->xtensa.operands[index];
+	rz_warn_if_fail(op->type == XTENSA_OP_IMM);
+	return op->imm;
+}
+
+static inline int32_t xtensa_op_l32r(cs_insn *insn, unsigned int index) {
+	cs_xtensa_op *op = &insn->detail->xtensa.operands[index];
+	rz_warn_if_fail(op->type == XTENSA_OP_L32R);
+	return op->imm;
+}
+
+#define XOP(I)    (ctx->insn->detail->xtensa.operands + I)
+#define MEM(I)    xtensa_op_mem(ctx->insn, I)
+#define REGI(I)   xtensa_op_reg(ctx->insn, I)
+#define REGN(I)   cs_reg_name(ctx->handle, (xtensa_op_reg(ctx->insn, I)))
+#define IMM(I)    xtensa_op_imm(ctx->insn, I)
+#define L32R(I)   xtensa_op_l32r(ctx->insn, I)
+#define INSN_SIZE (ctx->insn->size)
+
+#endif // RZ_XTENSA_H