Update to allow for platform dependent libs in CGO

Makefile

@@ -16,9 +16,10 @@ release-darwin-%:
 	cp artifacts/darwin-$*/libtokenizers.darwin-$*.tar.gz artifacts/all/libtokenizers.darwin-$*.tar.gz
 
 release-linux-%:
-	docker buildx build --platform linux/$* -f release/Dockerfile . -t tokenizers.linux-$*
+	docker buildx build --platform linux/$* --build-arg="DOCKER_TARGETPLATFORM=linux/$*" -f release/Dockerfile . -t tokenizers.linux-$*
 	mkdir -p artifacts/linux-$*
-	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint cp tokenizers.linux-$* /workspace/tokenizers/libtokenizers.a /mnt/libtokenizers.a
+	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint ls tokenizers.linux-$* /workspace/tokenizers/lib/linux
+	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint cp tokenizers.linux-$* /workspace/tokenizers/lib/linux/$*/libtokenizers.a /mnt/libtokenizers.a
 	cd artifacts/linux-$* && \
 		tar -czf libtokenizers.linux-$*.tar.gz libtokenizers.a
 	mkdir -p artifacts/all

README.md

@@ -11,6 +11,7 @@ Go bindings for the [HuggingFace Tokenizers](https://github.com/huggingface/toke
 Build your Go application using pre-built native binaries: `docker build --platform=linux/amd64 -f example/Dockerfile .`
 
 Available binaries:
+
 * [darwin-arm64](https://github.com/daulet/tokenizers/releases/latest/download/libtokenizers.darwin-arm64.tar.gz)
 * [linux-arm64](https://github.com/daulet/tokenizers/releases/latest/download/libtokenizers.linux-arm64.tar.gz)
 * [linux-amd64](https://github.com/daulet/tokenizers/releases/latest/download/libtokenizers.linux-amd64.tar.gz)
@@ -20,6 +21,7 @@ Available binaries:
 TLDR: [working example](example/main.go).
 
 Load a tokenizer from a JSON config:
+
 ```go
 import "github.com/daulet/tokenizers"
 
@@ -32,6 +34,7 @@ defer tk.Close()
 ```
 
 Encode text and decode tokens:
+
 ```go
 fmt.Println("Vocab size:", tk.VocabSize())
 // Vocab size: 30522
@@ -44,18 +47,19 @@ fmt.Println(tk.Decode([]uint32{2829, 4419, 14523, 2058, 1996, 13971, 3899}, true
 ```
 
 ## Benchmarks
+
 ```bash
 go test . -bench=. -benchmem -benchtime=10s
 
 goos: darwin
 goarch: arm64
 pkg: github.com/daulet/tokenizers
-BenchmarkEncodeNTimes-10     	  996556	     11851 ns/op	     116 B/op	       6 allocs/op
-BenchmarkEncodeNChars-10      1000000000	     2.446 ns/op	       0 B/op	       0 allocs/op
-BenchmarkDecodeNTimes-10     	 7286056	      1657 ns/op	     112 B/op	       4 allocs/op
-BenchmarkDecodeNTokens-10    	65191378	     211.0 ns/op	       7 B/op	       0 allocs/op
+BenchmarkEncodeNTimes-10        996556      11851 ns/op      116 B/op        6 allocs/op
+BenchmarkEncodeNChars-10      1000000000      2.446 ns/op        0 B/op        0 allocs/op
+BenchmarkDecodeNTimes-10       7286056       1657 ns/op      112 B/op        4 allocs/op
+BenchmarkDecodeNTokens-10     65191378      211.0 ns/op        7 B/op        0 allocs/op
 PASS
-ok  	github.com/daulet/tokenizers	126.681s
+ok   github.com/daulet/tokenizers 126.681s
 ```
 
 ## Contributing

example/Dockerfile

@@ -8,6 +8,6 @@ RUN curl -fsSL https://github.com/daulet/tokenizers/releases/download/${VERSION}
 COPY ./example .
 COPY ./test/data ./test/data
 RUN go mod download
-RUN mv ./libtokenizers.a /go/pkg/mod/github.com/daulet/tokenizers@${VERSION}/libtokenizers.a
+RUN mv ./libtokenizers.a /go/pkg/mod/github.com/daulet/tokenizers@${VERSION}/lib/$(echo ${TARGETPLATFORM} | tr / -)/libtokenizers.a
 # mounting Go cache won't work since we mutate it above
 RUN go run main.go

release/Dockerfile

@@ -9,7 +9,7 @@ COPY ./Cargo.lock ./Cargo.lock
 RUN cargo build --release
 
 FROM golang:1.21 as builder-go
-ARG TARGETPLATFORM
+ARG DOCKER_TARGETPLATFORM
 WORKDIR /workspace
 COPY ./release/go.mod   .
 COPY ./release/main.go  .
@@ -18,6 +18,6 @@ COPY tokenizer.go       ./tokenizers/
 COPY tokenizers.h       ./tokenizers/
 COPY --from=builder-rust \
     /workspace/target/release/libtokenizers.a \
-    ./tokenizers/
+    ./tokenizers/lib/${DOCKER_TARGETPLATFORM}/
 COPY ./test/data        ./test/data
-RUN go run .
+RUN go run -ldflags="-extldflags '-L./tokenizers/lib/${DOCKER_TARGETPLATFORM}'" .

tokenizer.go

@@ -3,7 +3,7 @@ package tokenizers
 // TODO packaging: how do we build the rust lib for distribution?
 
 /*
-#cgo LDFLAGS: ${SRCDIR}/libtokenizers.a -ldl -lm -lstdc++
+#cgo LDFLAGS: -ltokenizers -ldl -lm -lstdc++
 #include <stdlib.h>
 #include "tokenizers.h"
 */