Add tokenizers_srcdir_relative build tag to allow static library path

Makefile

@@ -1,7 +1,7 @@
 build:
 	@cargo build --release
 	@cp target/release/libtokenizers.a .
-	@go build .
+	@go build -tags tokenizers_srcdir_relative .
 
 build-example:
 	@docker build -f ./example/Dockerfile . -t tokenizers-example

README.md

@@ -6,6 +6,12 @@ Go bindings for the [HuggingFace Tokenizers](https://github.com/huggingface/toke
 
 `make build` to build `libtokenizers.a` that you need to run your application that uses bindings.
 
+To use `libtokenizers.a` in your go application, either:
+* Place `libtokenizers.a` in /usr/lib/, and compile your app as usual with `go build`.
+* Place `libtokenizers.a` in the go source directory of the tokenizer module 
+(e.g. /home/user/go/pkg/mod/github.com/daulet/[email protected]/), and compile with
+`go build -tags tokenizers_srcdir_relative`.
+
 ### Using pre-built binaries
 
 Build your Go application using pre-built native binaries: `docker build --platform=linux/amd64 -f example/Dockerfile .`

release/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.3
 
-FROM rust:1.71 as builder-rust
+FROM rust:1.74 as builder-rust
 ARG TARGETPLATFORM
 WORKDIR /workspace
 COPY ./src ./src
@@ -16,5 +16,5 @@ COPY ./test/data ./test/data
 RUN go mod download
 COPY --from=builder-rust \
     /workspace/target/release/libtokenizers.a \
-    /go/pkg/mod/github.com/daulet/[email protected]/libtokenizers.a
+    /usr/lib/libtokenizers.a
 RUN go run main.go

tokenizer.go

@@ -3,7 +3,8 @@ package tokenizers
 // TODO packaging: how do we build the rust lib for distribution?
 
 /*
-#cgo LDFLAGS: ${SRCDIR}/libtokenizers.a -ldl -lm -lstdc++
+#cgo tokenizers_srcdir_relative LDFLAGS: ${SRCDIR}/libtokenizers.a -ldl -lm -lstdc++
+#cgo !tokenizers_srcdir_relative LDFLAGS: /usr/lib/libtokenizers.a -ldl -lm -lstdc++
 #include <stdlib.h>
 #include "tokenizers.h"
 */