HW VDF (#148)

Co-authored-by: Rostislav <[email protected]>
Co-authored-by: sean-sn <[email protected]>
Co-authored-by: Mike Schaffstein <[email protected]>

.github/workflows/hw-build.yml

@@ -0,0 +1,103 @@
+name: HW Build
+
+on:
+  push:
+    branches:
+      - main
+  release:
+    types: [published]
+  pull_request:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+concurrency:
+  # SHA is added to the end if on `main` to let all main workflows run
+  group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/long_lived/')) && github.sha || '' }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+
+jobs:
+  build-hw:
+    name: Build HW VDF Client
+    runs-on: [ubuntu-latest]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set Env
+        uses: Chia-Network/actions/setjobenv@main
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake libgmp-dev libboost-system-dev
+
+      - name: Download USB drivers
+        run: |
+          mkdir libft4222
+          cd libft4222
+          wget https://download.chia.net/vdf/libft4222-linux-1.4.4.170.tgz
+          tar -xvzf libft4222-linux-1.4.4.170.tgz
+          ln -s ${{ github.workspace }}/libft4222 ${{ github.workspace }}/src/hw/libft4222
+          ln -s ${{ github.workspace }}/libft4222/build-x86_64/libft4222.so.1.4.4.170 ${{ github.workspace }}/libft4222/build-x86_64/libft4222.so
+
+      - name: Compile
+        working-directory: "${{ github.workspace }}/src"
+        run: make -f Makefile.vdf-client emu_hw_test hw_test emu_hw_vdf_client hw_vdf_client
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: hw-vdf
+          path: |
+            ${{ github.workspace }}/src/emu_hw_test
+            ${{ github.workspace }}/src/hw_test
+            ${{ github.workspace }}/src/hw_vdf_client
+            ${{ github.workspace }}/src/emu_hw_vdf_client
+            ${{ github.workspace }}/src/hw/libft4222/build-x86_64/libft4222.so
+
+      - name: Assemble .deb
+        env:
+          INSTALLER_VERSION: "${{ env.RELEASE_TAG || format('0.0.1-{0}', github.run_id) }}"
+          PLATFORM: "amd64"
+        run: |
+          pip install j2cli
+          CLI_DEB_BASE="chiavdf-hw_$INSTALLER_VERSION-1_$PLATFORM"
+          mkdir -p "dist/$CLI_DEB_BASE/usr/bin"
+          mkdir -p "dist/$CLI_DEB_BASE/usr/lib"
+          mkdir -p "dist/$CLI_DEB_BASE/DEBIAN"
+          mkdir -p "dist/$CLI_DEB_BASE/etc/udev/rules.d"
+          j2 -o "dist/$CLI_DEB_BASE/DEBIAN/control" assets/deb/control.j2
+
+          cp ${{ github.workspace }}/src/emu_hw_test dist/$CLI_DEB_BASE/usr/bin/
+          cp ${{ github.workspace }}/src/hw_test dist/$CLI_DEB_BASE/usr/bin/
+          cp ${{ github.workspace }}/src/hw_vdf_client dist/$CLI_DEB_BASE/usr/bin/
+          cp ${{ github.workspace }}/src/emu_hw_vdf_client dist/$CLI_DEB_BASE/usr/bin/
+          cp ${{ github.workspace }}/src/hw/libft4222/build-x86_64/libft4222.so dist/$CLI_DEB_BASE/usr/lib/
+          echo 'SUBSYSTEM=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="601c", MODE="0666"' > dist/$CLI_DEB_BASE/etc/udev/rules.d/99-chiavdf.rules
+          dpkg-deb --build --root-owner-group "dist/$CLI_DEB_BASE"
+
+          echo "DEB_NAME=$CLI_DEB_BASE.deb" >> $GITHUB_ENV
+
+      - name: Upload Installer
+        uses: actions/upload-artifact@v3
+        with:
+          name: installer
+          path: |
+            ${{ github.workspace }}/dist/${{ env.DEB_NAME }}
+
+      - name: Upload release artifacts
+        if: env.RELEASE == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload \
+            $RELEASE_TAG \
+            dist/${{ env.DEB_NAME }}

.gitignore

@@ -25,6 +25,10 @@ src/vdf_client
 src/1weso_test
 src/2weso_test
 src/prover_test
+src/emu_hw_test
+src/hw_test
+src/emu_hw_vdf_client
+src/hw_vdf_client
 /verifier
 /verifier_test
 /build/*
@@ -33,6 +37,9 @@ src/prover_test
 *.o
 *.s
 
+# external library
+/src/hw/libft4222
+
 # pyenv
 .python-version
 .eggs

README_ASIC.md

@@ -0,0 +1,76 @@
+# ASIC timelord user guide
+
+## Initial setup
+
+Download and unpack LibFT4222 library:
+```bash
+# in chiavdf directory
+wget https://ftdichip.com/wp-content/uploads/2022/06/libft4222-linux-1.4.4.170.tgz
+mkdir src/hw/libft4222
+tar -C src/hw/libft4222 -xf libft4222-linux-1.4.4.170.tgz
+ln -s libft4222.so.1.4.4.170 src/hw/libft4222/build-x86_64/libft4222.so
+```
+
+Build binaries:
+```bash
+cd src
+make -f Makefile.vdf-client emu_hw_test hw_test emu_hw_vdf_client hw_vdf_client
+```
+
+Connect the Chia VDF ASIC device and verify that it is detected:
+```bash
+# in chiavdf/src/ directory
+LD_LIBRARY_PATH=hw/libft4222/build-x86_64 ./hw_vdf_client --list
+```
+
+If the device is shown in the list, check if it's working:
+```bash
+LD_LIBRARY_PATH=hw/libft4222/build-x86_64 ./hw_test
+```
+
+Output should contain lines similar to the following:
+```
+VDF 0: 1000000 HW iters done in 2s, HW speed: 714720 ips
+```
+
+## Running
+
+You should have a Chia full node running and synced.
+
+Start timelord (but not timelord-launcher) in chia-blockchain:
+```bash
+chia start timelord-only
+```
+
+Start hardware VDF client (`8000` specifies timelord's port number):
+```bash
+# in chiavdf/src/ directory
+LD_LIBRARY_PATH=hw/libft4222/build-x86_64 ./hw_vdf_client 8000
+```
+
+The VDF client accepts a number of options:
+```
+Usage: ./hw_vdf_client [OPTIONS] PORT [N_VDFS]
+List of options [default, min - max]:
+  --freq N - set ASIC frequency [1100, 200 - 2200]
+  --voltage N - set board voltage [0.88, 0.7 - 1.0]
+  --ip A.B.C.D - timelord IP address [localhost]
+        Allows connecting to a timelord running on a remote host. Useful when running multiple machines with VDF hardware connecting to a single timelord.
+  --vdfs-mask N - mask for enabling VDF engines [7, 1 - 7]
+        The ASIC has 3 VDF engines numbered 0, 1, 2. If not running all 3 engines, the mask can be specified to enable specific engines. It must be the result of bitwise OR of the engine bits (1, 2, 4 for engines 0, 1, 2).
+  --vdf-threads N - number of software threads per VDF engine [4, 2 - 64]
+        Number of software threads computing intermediate values and proofs per VDF engine.
+  --proof-threads N - number of proof threads per VDF engine
+        Number of software threads only computing proofs per VDF engine. Must be less than --vdf-threads.
+  --auto-freq-period N - auto-adjust frequency every N seconds [0, 10 - inf]
+  --list - list available devices and exit
+```
+
+## Shutting down
+
+Stop timelord:
+```bash
+chia stop timelord-only
+```
+
+Stop the VDF client by pressing Control-C or killing the process with `SIGTERM`.

assets/deb/control.j2

@@ -0,0 +1,6 @@
+Package: chiavdf-hw
+Version: {{ INSTALLER_VERSION }}
+Architecture: {{ PLATFORM }}
+Maintainer: Chia Network Inc <hello@chia.net>
+Description: Hardware VDF Client
+Depends: libgmp-dev, libboost-system-dev

src/Makefile.vdf-client

@@ -6,14 +6,14 @@ else
 NOPIE = -no-pie
 endif
 
-LDFLAGS += -flto $(NOPIE)
+LDFLAGS += -flto $(NOPIE) -g
 LDLIBS += -lgmpxx -lgmp -lboost_system -pthread
 CXXFLAGS += -flto -std=c++1z -D VDF_MODE=0 -D FAST_MACHINE=1 -pthread $(NOPIE) -fvisibility=hidden
 ifeq ($(UNAME),Darwin)
 CXXFLAGS += -D CHIAOSX=1
 endif
 
-OPT_CFLAGS = -O3
+OPT_CFLAGS = -O3 -g
 
 ifneq ($(ASAN),)
 LDFLAGS += -fsanitize=address -fsanitize=undefined
@@ -27,15 +27,16 @@ endif
 
 .PHONY: all clean
 
-all: vdf_client prover_test 1weso_test 2weso_test vdf_bench
+BINS = vdf_client prover_test 1weso_test 2weso_test vdf_bench
+all: $(BINS)
 
 clean:
-	rm -f *.o vdf_client prover_test 1weso_test 2weso_test compile_asm vdf_bench
+	rm -f *.o hw/*.o $(BINS) compile_asm emu_hw_test hw_test hw_vdf_client emu_hw_vdf_client
 
-vdf_client vdf_bench prover_test 1weso_test 2weso_test avx512_test: %: %.o lzcnt.o asm_compiled.o avx2_asm_compiled.o avx512_asm_compiled.o
+$(BINS) avx512_test: %: %.o lzcnt.o asm_compiled.o avx2_asm_compiled.o avx512_asm_compiled.o
 	$(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS)
 
-vdf_client.o vdf_bench.o prover_test.o 1weso_test.o 2weso_test.o avx512_test.o: CXXFLAGS += $(OPT_CFLAGS)
+$(addsuffix .o,$(BINS)) avx512_test.o: CXXFLAGS += $(OPT_CFLAGS)
 
 lzcnt.o: refcode/lzcnt.c
 	$(CC) -c refcode/lzcnt.c
@@ -51,3 +52,21 @@ avx512_asm_compiled.s: compile_asm
 
 compile_asm: compile_asm.o
 	$(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS)
+
+HW_OBJS = $(addprefix hw/,hw_util.o hw_proof.o hw_interface.o chia_driver.o ftdi_driver.o vdf_driver.o pll_freqs.o) vdf_base.o lzcnt.o
+EMU_OBJS = hw/emu_funcs.o hw/emu_runner.o
+HW_LIB = hw/libft4222/build-x86_64/libft4222.so
+
+hw_test: hw/hw_test.o $(HW_OBJS) $(HW_LIB) hw/real_hw.o
+	$(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS)
+
+emu_hw_test: hw/hw_test.o $(HW_OBJS) $(EMU_OBJS)
+	$(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS)
+
+hw_vdf_client: hw/hw_vdf_client.o $(HW_OBJS) $(HW_LIB) hw/real_hw.o
+	$(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS)
+
+emu_hw_vdf_client: hw/hw_vdf_client.o $(HW_OBJS) $(EMU_OBJS)
+	$(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS)
+
+hw/hw_test.o hw/hw_vdf_client.o $(HW_OBJS) $(EMU_OBJS): CXXFLAGS += -I. -Ihw -Ihw/libft4222 $(OPT_CFLAGS) -Wall

src/Reducer.h

@@ -64,7 +64,7 @@ class alignas(64) Reducer {
   /**
    * @brief run - runs reduction algorithm for cg context params
    */
-  inline void run() {
+  void run() {
     while (!isReduced()) {
       int_fast64_t a, b, c;
       {
@@ -229,7 +229,7 @@ bool bLZCHasHW=false;
 
       // The condition (abs(v_) | abs(x_)) <= THRESH protects against
       // overflow
-      below_threshold = (abs(v_) | abs(x_)) <= THRESH ? 1 : 0;
+      below_threshold = (labs(v_) | labs(x_)) <= THRESH ? 1 : 0;
     } while (below_threshold && a > c && c > 0);
 
     if (below_threshold) {

src/bqfc.c

@@ -161,7 +161,7 @@ enum BQFC_FLAGS {
  */
 int bqfc_serialize_only(uint8_t *out_str, const struct qfb_c *c, size_t d_bits)
 {
-    size_t offset, bytes, size, g_size;
+    size_t offset, g_size;
 
     d_bits = (d_bits + 31) & ~(size_t)31;
 

src/bqfc.h

@@ -27,6 +27,6 @@ int bqfc_serialize_only(uint8_t *out_str, const struct qfb_c *c, size_t d_bits);
 int bqfc_deserialize_only(struct qfb_c *out_c, const uint8_t *str, size_t d_bits);
 
 int bqfc_serialize(uint8_t *out_str, mpz_t a, mpz_t b, size_t d_bits);
-int bqfc_deserialize(mpz_t out_a, mpz_t out_b, const mpz_t D, const uint8_t *str, size_t d_bits);
+int bqfc_deserialize(mpz_t out_a, mpz_t out_b, const mpz_t D, const uint8_t *str, size_t size, size_t d_bits);
 
 #endif // BQFC_H

src/hw/chia_driver.cpp

@@ -0,0 +1,39 @@
+#include "chia_driver.hpp"
+
+void ChiaDriver::SerializeJob(uint8_t *buf,
+                              uint32_t job_id,
+                              uint64_t iteration_count,
+                              mpz_t a,
+                              mpz_t f,
+                              mpz_t d,
+                              mpz_t l) {
+  size_t offset = 0;
+  const size_t reg_size = REG_BYTES;
+  offset += write_bytes(reg_size, offset, buf, job_id);
+  offset += write_bytes(2 * reg_size, offset, buf, iteration_count);
+  offset += write_bytes(reg_size * CHIA_VDF_CMD_A_MULTIREG_COUNT,
+                        offset, buf, a, NUM_2X_COEFFS);
+  offset += write_bytes(reg_size * CHIA_VDF_CMD_F_MULTIREG_COUNT,
+                        offset, buf, f, NUM_2X_COEFFS);
+  offset += write_bytes(reg_size * CHIA_VDF_CMD_D_MULTIREG_COUNT,
+                        offset, buf, d, NUM_4X_COEFFS);
+  offset += write_bytes(reg_size * CHIA_VDF_CMD_L_MULTIREG_COUNT,
+                        offset, buf, l, NUM_1X_COEFFS);
+  // The last word is 0x1 to start the job
+  offset += write_bytes(reg_size, offset, buf, (uint32_t)0x1);
+}
+
+void ChiaDriver::DeserializeJob(uint8_t *buf,
+                                uint32_t &job_id,
+                                uint64_t &iteration_count,
+                                mpz_t a,
+                                mpz_t f) {
+  size_t offset = 0;
+  const size_t reg_size = REG_BYTES;
+  offset += read_bytes(reg_size, offset, buf, job_id);
+  offset += read_bytes(2 * reg_size, offset, buf, iteration_count);
+  offset += read_bytes(reg_size * CHIA_VDF_STATUS_A_MULTIREG_COUNT,
+                       offset, buf, a, NUM_2X_COEFFS);
+  offset += read_bytes(reg_size * CHIA_VDF_STATUS_F_MULTIREG_COUNT,
+                       offset, buf, f, NUM_2X_COEFFS);
+}

src/hw/chia_driver.hpp

@@ -0,0 +1,44 @@
+#ifndef CHIA_DRIVER_HPP
+#define CHIA_DRIVER_HPP
+
+#include <gmp.h>
+
+#include "vdf_driver.hpp"
+#include "chia_registers.hpp"
+
+class ChiaDriver : public VdfDriver {
+public:
+  const static unsigned NUM_1X_COEFFS = 18;
+  const static unsigned NUM_2X_COEFFS = 34;
+  const static unsigned NUM_3X_COEFFS = 52;
+  const static unsigned NUM_4X_COEFFS = 68;
+
+  ChiaDriver() :
+    VdfDriver(16 /*WORD_BITS*/, 19 /*REDUNDANT_BITS*/, true) {
+  }
+
+  virtual size_t CmdSize() {
+    return (CHIA_VDF_CMD_START_REG_OFFSET -
+            CHIA_VDF_CMD_JOB_ID_REG_OFFSET + REG_BYTES);
+  }
+
+  virtual size_t StatusSize() {
+    return (CHIA_VDF_STATUS_END_REG_OFFSET -
+            CHIA_VDF_STATUS_ITER_0_REG_OFFSET);
+  }
+
+  void SerializeJob(uint8_t *buf,
+                    uint32_t job_id,
+                    uint64_t iteration_count,
+                    mpz_t a,
+                    mpz_t f,
+                    mpz_t d,
+                    mpz_t l);
+
+  void DeserializeJob(uint8_t *buf,
+                      uint32_t &job_id,
+                      uint64_t &iteration_count,
+                      mpz_t a,
+                      mpz_t f);
+};
+#endif