testing daulet/tokenizers

livepeer · Aug 1, 2024 · aa7c9d1 · aa7c9d1
1 parent f6e5909
commit aa7c9d1
Show file tree

Hide file tree

Showing 4 changed files with 342 additions and 4 deletions.
diff --git a/.gitignore b/.gitignore
@@ -43,3 +43,5 @@ run_*.sh
 
 # mac autogenerated file
 .DS_Store
+
+cmd/livepeer/__debug*
diff --git a/go.mod b/go.mod
@@ -66,6 +66,7 @@ require (
 	github.com/containerd/containerd v1.7.0-beta.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
+	github.com/daulet/tokenizers v0.8.0
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
@@ -241,6 +242,6 @@ require (
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 
-replace github.com/livepeer/lpms => /home/ricks/development/livepeer/ai/lpms
+replace github.com/livepeer/lpms => /home/administrator/AI_SPE/lpms
 
-replace github.com/livepeer/ai-worker => /home/ricks/development/livepeer/ai/ai-worker
+replace github.com/livepeer/ai-worker => /home/administrator/AI_SPE/ai-worker