Merge pull request #22 from StrongMonkey/tweak-ci

Chore: add cross compile script and CI

.github/workflows/main.yml

@@ -1,38 +1,20 @@
-name: Test & Release
+name: Release
 
 on:
-  - push
-  - workflow_dispatch
+  push:
+    tags:
+      - "v*"
 
 env:
   GO_VERSION: "1.22.x"
 
 jobs:
-  test-suite:
-    timeout-minutes: 30
-    name: Full Test Suite
-    runs-on: ubuntu-22.04
-    steps:
-      # Setup
-      - uses: actions/checkout@v4
-      - name: Setup Go environment
-        uses: actions/setup-go@v5
-        with:
-          go-version: "${{ env.GO_VERSION }}"
-      - name: Setup CI Tools
-        run: make ci-setup
-      # Tests
-      - name: Run Go Tests
-        run: make test
-      # Builds
-      - name: Test Platform Builds
-        run: make build-cross
-
-
   release-github:
     name: Build & Release Binaries
-    # Only run on tags
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-22.04, macos-latest ]
     steps:
       # Setup
       - uses: actions/checkout@v4
@@ -42,9 +24,18 @@ jobs:
           go-version: "${{ env.GO_VERSION }}"
       - name: Setup CI Tools
         run: make ci-setup
+      - name: Run Go Tests
+        run: make test
+      - name: Install Homebrew on macOS
+        if: runner.os == 'macOS'
+        run: |
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> ~/.zprofile
+          eval "$(/opt/homebrew/bin/brew shellenv)"
+          brew install mingw-w64
       # Go Build
       - name: Build Binary
-        run: make gen-checksum build-cross
+        run: make build-cross
       # Create Git Release
       - name: Extract Tag from Ref
         if: startsWith(github.ref, 'refs/tags/')
@@ -63,7 +54,6 @@ jobs:
           allowUpdates: true
           artifactErrorsFailBuild: true
           artifacts: dist/*
-          # discussionCategory: releases
           makeLatest: ${{ steps.semver.outputs.prerelease == '' }}
           generateReleaseNotes: true
           prerelease: ${{ steps.semver.outputs.prerelease != '' }}

.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Test
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  GO_VERSION: "1.22.x"
+
+jobs:
+  test-suite:
+    timeout-minutes: 30
+    name: Full Test Suite
+    runs-on: ubuntu-22.04
+    steps:
+      # Setup
+      - uses: actions/checkout@v4
+      - name: Setup Go environment
+        uses: actions/setup-go@v5
+        with:
+          go-version: "${{ env.GO_VERSION }}"
+      - name: Setup CI Tools
+        run: make ci-setup
+      - name: Build
+        run: make build
+      # Tests
+      - name: Run Go Tests
+        run: make test

Makefile

@@ -12,7 +12,7 @@ endif
 GO_TAGS := netgo
 LD_FLAGS := -s -w -X github.com/gptscript-ai/knowledge/version.Version=${GIT_TAG}
 build:
-	go build -o bin/knowledge -tags "${GO_TAGS}" -ldflags '$(LD_FLAGS)' .
+	go build -o bin/knowledge -tags "${GO_TAGS}" -ldflags '$(LD_FLAGS) ' .
 
 run: build
 	bin/knowledge server
@@ -36,19 +36,11 @@ lint:
 test:
 	go test -v ./...
 
-# cross-compilation for all targets
-TARGETS ?= darwin/amd64 darwin/arm64 linux/amd64 linux/386 linux/arm linux/arm64 windows/amd64
-build-cross: LD_FLAGS += -extldflags "-static"
 build-cross:
-	CGO_ENABLED=0 gox -parallel=3 -output="dist/knowledge-{{.OS}}-{{.Arch}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(GO_TAGS),-tags '$(TAGS)',) -ldflags '$(LD_FLAGS)'
-gen-checksum:	build-cross
-	$(eval ARTIFACTS_TO_PUBLISH := $(shell ls dist/*))
-	$$(sha256sum $(ARTIFACTS_TO_PUBLISH) > dist/checksums.txt)
+	GIT_TAG=${GIT_TAG} ./scripts/cross-build.sh
 
 ci-setup:
 	@echo "### Installing Go tools..."
 	@echo "### -> Installing golangci-lint..."
 	curl -sfL $(PKG_GOLANGCI_LINT_SCRIPT) | sh -s -- -b $(GOENVPATH)/bin v$(PKG_GOLANGCI_LINT_VERSION)
 
-	@echo "### -> Installing gox..."
-	./scripts/install-tools.sh gox

install.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -e
+
+REPO="gptscript-ai/knowledge"
+INSTALL_DIR="/usr/local/bin"
+
+# Function to determine the OS and architecture
+get_os_arch() {
+    OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+    ARCH=$(uname -m)
+
+    case $ARCH in
+        x86_64)
+            ARCH="amd64"
+            ;;
+        aarch64)
+            ARCH="arm64"
+            ;;
+        arm64)
+            ARCH="arm64"
+            ;;
+        *)
+            echo "Unsupported architecture: $ARCH"
+            exit 1
+            ;;
+    esac
+
+    echo "${OS}-${ARCH}"
+}
+
+# Function to download the latest release
+download_latest_release() {
+    OS_ARCH=$1
+    LATEST_RELEASE_URL="https://api.github.com/repos/$REPO/releases/latest"
+
+    DOWNLOAD_URL=$(curl -s $LATEST_RELEASE_URL | grep "browser_download_url.*$OS_ARCH" | cut -d '"' -f 4)
+
+    if [[ -z "$DOWNLOAD_URL" ]]; then
+        echo "No binary found for $OS_ARCH"
+        exit 1
+    fi
+
+    TEMP_DIR=$(mktemp -d)
+    TEMP_FILE="$TEMP_DIR/knowledge"
+
+    echo "Downloading $DOWNLOAD_URL..."
+    curl -sL "$DOWNLOAD_URL" -o "$TEMP_FILE"
+
+    chmod +x $TEMP_FILE
+
+    mv $TEMP_FILE "$INSTALL_DIR/knowledge"
+
+    rm -rf "$TEMP_DIR"
+
+    echo "Installed knowledge to $INSTALL_DIR/knowledge"
+}
+
+# Ensure the script is run as root
+if [[ $EUID -ne 0 ]]; then
+    echo "This script must be run as root. Please run with sudo."
+    exit 1
+fi
+
+OS_ARCH=$(get_os_arch)
+download_latest_release "$OS_ARCH"

scripts/cross-build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -ex
+
+GO_TAGS="netgo"
+LD_FLAGS="-s -w -X github.com/gptscript-ai/knowledge/version.Version=${GIT_TAG}"
+
+if [ "$(go env GOOS)" = "linux" ]; then
+  CGO_ENABLED=1 GOARCH=amd64 go build -o dist/knowledge-linux-amd64 -tags "${GO_TAGS}" -ldflags "${LD_FLAGS}\" -extldflags \"-static\" " .
+else
+  CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 go build -o dist/knowledge-windows-amd64 -tags "${GO_TAGS}" -ldflags "${LD_FLAGS}" .
+  CGO_ENABLED=1 GOARCH=amd64 go build -o dist/knowledge-darwin-amd64 -tags "${GO_TAGS}" -ldflags "${LD_FLAGS}" .
+  CGO_ENABLED=1 GOARCH=arm64 go build -o dist/knowledge-darwin-arm64 -tags "${GO_TAGS}" -ldflags "${LD_FLAGS}" .
+fi

tests/test.py

@@ -295,11 +295,11 @@ def test_imagejon7_dataset(setup_imagejon7_dataset, judge_client, question, answ
     ("What was net income?", "$14,845 million or $14.845 billion"),
     ("How many hours were volunteered, and across how many countries to help confronting society’s challenge?", "115,000, 84"),
     ("How much did Citi finance for affordable housing in the U.S.?", "$6, billion"),
-    ("What were total liabilities of Citigroup as of Dec 31 2022?", "$2,214,838, million"),
+    ("What were total liabilities of Citigroup as of Dec 31 2022?", "$2,214,838 million"),
     ("What were total assets of Citigroup as of Dec 31 2022?", "2,416,676, million"),
     ("On what page are Basel III Revisions?", "49"),
     ("How many employees are at Citi?", "240,000"),
-    ("What was the revenue from legacy franchises", "8.5, billion"),
+    ("What was the revenue from legacy franchises", "$8.5 billion or $8.472 billion"),
     ("How large is the new stress capital buffer?", "4.0%"),
     ("What were total revenues of Citigroup in 2022?", "$75,338 million or $75.338 billion or $75.3 billion"),
 ])