fix build:

- added tokenizer lib linking
- go package pulling from neuralmagic internal repo

Author: vMaroon

.gitignore

@@ -31,4 +31,8 @@ go.work.sum
 # generated docs
 site
 
+# tokenizer lib
+lib
+
+# local configuration files
 .envrc

.tekton/buildah-build.yaml

@@ -44,6 +44,15 @@ spec:
         USERNAME=$(jq -r '.auths["quay.io"].username' /root/.docker/config.json)
         PASSWORD=$(jq -r '.auths["quay.io"].password' /root/.docker/config.json)
 
+        echo "🔐 Extracting Git credentials from workspace..."
+        GIT_USER=$(cat /workspace/git-auth/username)
+        GIT_TOKEN=$(cat /workspace/git-auth/token)
+
+        if [ -z "$GIT_USER" ] || [ -z "$GIT_TOKEN" ]; then
+          echo "❌ Error: Missing git-auth credentials"
+          exit 1
+        fi
+        
         if [ "$USERNAME" = "null" ] || [ "$PASSWORD" = "null" ]; then
           echo "❌ Error: Missing registry credentials"
           exit 1
@@ -56,8 +65,10 @@ spec:
         export DOCKER_CONFIG=/root/.docker
         export BUILDER=buildah
         export IMG=$(params.image_tag_base):$(params.dev-version)
-
+        export GIT_NM_USER=$GIT_USER 
+        export NM_TOKEN=$GIT_TOKEN
+        
         echo "🚀 Calling make buildah-build with IMG=$IMG..."
-        make buildah-build IMG=$IMG
+        make buildah-build IMG=$IMG 
 
         echo "$IMG" > /tekton/results/image-url

.tekton/go-build-task.yaml

@@ -28,5 +28,8 @@ spec:
         git config --global url."https://${GIT_USER}:${GIT_TOKEN}@github.com".insteadOf "https://github.com"
         git config --global --add safe.directory "$(pwd)"
 
+        # required for go build with tokenizer lib linking
+        dnf install -y gcc-c++ libstdc++ libstdc++-devel && dnf clean all
+        
         go env -w GOFLAGS=-buildvcs=false
         make build

.tekton/pipelinerun.yaml

@@ -331,6 +331,8 @@ spec:
             workspace: registry-secret
           - name: container-storage
             workspace: container-storage
+          - name: git-auth
+            workspace: git-auth
 
       - name: vulnerability-scan
         when:

Makefile

@@ -439,11 +439,20 @@ lint: check-golangci-lint ## Run lint
 	golangci-lint run
 
 ##@ Build
+LDFLAGS ?= -extldflags '-L$(shell pwd)/lib'
+CGO_ENABLED=1 # Enable CGO
+
+.PHONY: download-tokenizer
+download-tokenizer: ## Download the HuggingFace tokenizer bindings.
+	@echo "Downloading HuggingFace tokenizer bindings..."
+	mkdir -p lib
+	curl -L https://github.com/daulet/tokenizers/releases/download/v1.20.2/libtokenizers.$(TARGETOS)-$(TARGETARCH).tar.gz | tar -xz -C lib
+	ranlib lib/*.a
 
 .PHONY: build
-build: check-go ##
+build: check-go download-tokenizer ##
 	@printf "\033[33;1m==== Building ====\033[0m\n"
-	go build -o bin/epp cmd/epp/main.go cmd/epp/health.go
+	go build -ldflags="$(LDFLAGS)" -o bin/epp cmd/epp/main.go cmd/epp/health.go
 
 ##@ Container Build/Push
 
@@ -456,7 +465,12 @@ buildah-build: check-builder load-version-json ## Build and push image (multi-ar
 	  for arch in amd64; do \
 	    ARCH_TAG=$$FINAL_TAG-$$arch; \
 	    echo "📦 Building for architecture: $$arch"; \
-		buildah build --arch=$$arch --os=linux --layers -t $(IMG)-$$arch . || exit 1; \
+		buildah build \
+			--arch=$$arch \
+			--build-arg GIT_NM_USER=$(GIT_NM_USER) \
+            --build-arg NM_TOKEN=$(NM_TOKEN) \
+			--os=linux \
+			--layers -t $(IMG)-$$arch . || exit 1; \
 	    echo "🚀 Pushing image: $(IMG)-$$arch"; \
 	    buildah push $(IMG)-$$arch docker://$(IMG)-$$arch || exit 1; \
 	  done; \
@@ -474,7 +488,11 @@ buildah-build: check-builder load-version-json ## Build and push image (multi-ar
 	  sed -e '1 s/\(^FROM\)/FROM --platform=$${BUILDPLATFORM}/' Dockerfile > Dockerfile.cross; \
 	  - docker buildx create --use --name image-builder || true; \
 	  docker buildx use image-builder; \
-	  docker buildx build --push --platform=$(PLATFORMS) --tag $(IMG) -f Dockerfile.cross . || exit 1; \
+	  docker buildx build --push \
+	  			--platform=$(PLATFORMS) \
+	  			--build-arg GIT_NM_USER=$(GIT_NM_USER)\
+                --build-arg NM_TOKEN=$(NM_TOKEN) \
+                --tag $(IMG) -f Dockerfile.cross . || exit 1; \
 	  docker buildx rm image-builder || true; \
 	  rm Dockerfile.cross; \
 	elif [ "$(BUILDER)" = "podman" ]; then \