diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 76f4cb8e..40e11944 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -24,13 +24,24 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.PUBLISH_PACKAGE_PAT }} - - name: Build Docker Images + - name: Build PacNexus Docker Images run: | - docker build --build-arg version="${SHORT_SHA}-dev" --no-cache -t webserver . - docker tag webserver "ghcr.io/pacstall/webserver:$SHORT_SHA" - docker tag webserver ghcr.io/pacstall/webserver:development + docker build --build-arg version="${SHORT_SHA}-dev" --no-cache -t pacnexus pacnexus.Dockerfile + docker tag pacnexus "ghcr.io/pacstall/pacnexus:$SHORT_SHA" + docker tag pacnexus ghcr.io/pacstall/pacnexus:development - - name: Push Images + - name: Push PacNexus Images run: | - docker push "ghcr.io/pacstall/webserver:$SHORT_SHA" - docker push ghcr.io/pacstall/webserver:development \ No newline at end of file + docker push "ghcr.io/pacstall/pacnexus:$SHORT_SHA" + docker push ghcr.io/pacstall/pacnexus:development + + - name: Build PacSight Docker Images + run: | + docker build --build-arg version="${SHORT_SHA}-dev" --no-cache -t pacsight pacsight.Dockerfile + docker tag pacsight "ghcr.io/pacstall/pacsight:$SHORT_SHA" + docker tag pacsight ghcr.io/pacstall/pacsight:development + + - name: Push PacSight Images + run: | + docker push "ghcr.io/pacstall/pacsight:$SHORT_SHA" + docker push ghcr.io/pacstall/pacsight:development \ No newline at end of file diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index 07f61da2..9c4f54f3 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -22,13 +22,24 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.PUBLISH_PACKAGE_PAT }} - - name: Build Docker image + - name: Build PacNexus Docker image run: | - docker build --build-arg VITE_VERSION="${{ github.event.inputs.version }}" --no-cache -t webserver . - docker tag webserver "ghcr.io/pacstall/webserver:${{ github.event.inputs.version }}" - docker tag webserver ghcr.io/pacstall/webserver:latest + docker build --build-arg VITE_VERSION="${{ github.event.inputs.version }}" --no-cache -t pacnexus pacnexus.Dockerfile + docker tag pacnexus "ghcr.io/pacstall/pacnexus:${{ github.event.inputs.version }}" + docker tag pacnexus ghcr.io/pacstall/pacnexus:latest - - name: Push Image + - name: Build PacSight Docker image run: | - docker push "ghcr.io/pacstall/webserver:${{ github.event.inputs.version }}" - docker push ghcr.io/pacstall/webserver:latest + docker build --build-arg VITE_VERSION="${{ github.event.inputs.version }}" --no-cache -t pacsight pacsight.Dockerfile + docker tag pacsight "ghcr.io/pacstall/pacsight:${{ github.event.inputs.version }}" + docker tag pacsight ghcr.io/pacstall/pacsight:latest + + - name: Push PacNexus Image + run: | + docker push "ghcr.io/pacstall/pacnexus:${{ github.event.inputs.version }}" + docker push ghcr.io/pacstall/pacnexus:latest + + - name: Push PacSight Image + run: | + docker push "ghcr.io/pacstall/pacsight:${{ github.event.inputs.version }}" + docker push ghcr.io/pacstall/pacsight:latest diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5792e289..3743c1a7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,17 +1,27 @@ -version: '3.2' - services: mariadb: image: mariadb # inherited from docker-compose.yml - webserver: + pacnexus: + # overrides the image used from docker-compose.yml + build: + context: . + dockerfile: pacnexus.Dockerfile + args: + VITE_VERSION: "${VITE_VERSION}" + environment: + PACSTALL_LOG_LEVEL: "debug" + # inherited from docker-compose.yml + pacsight: # overrides the image used from docker-compose.yml build: context: . - dockerfile: Dockerfile + dockerfile: pacsight.Dockerfile args: VITE_VERSION: "${VITE_VERSION}" + environment: + PACSTALL_LOG_LEVEL: "debug" # inherited from docker-compose.yml matomo: diff --git a/docker-compose.yml b/docker-compose.yml index 909d2f75..27f4d541 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.2' - services: mariadb: @@ -14,7 +12,7 @@ services: - target: 3306 published: 3306 - webserver: + pacnexus: image: ghcr.io/pacstall/webserver:latest depends_on: - mariadb @@ -28,10 +26,8 @@ services: max-file: "5" max-size: "10m" environment: - PACSTALL_DISCORD_TOKEN: "" - PACSTALL_DISCORD_CHANNEL_ID: "" - PACSTALL_DISCORD_ENABLED: "false" - PACSTALL_DISCORD_TAGS: "" # See https://discord.com/developers/docs/reference#message-formatting + PACSTALL_PACSIGHT_PORT: 3301 + PACSTALL_PACSIGHT_HOST: pacsight PACSTALL_DATABASE_HOST: mariadb PACSTALL_DATABASE_USER: root PACSTALL_DATABASE_PASSWORD: changeme @@ -40,7 +36,25 @@ services: PACSTALL_MATOMO_ENABLED: "true" MATOMO_DOMAIN: "http://matomo" MATOMO_SITE_ID: "1" - PACSTALL_PROGRAMS_GIT_BRANCH: "master" + PACSTALL_LOG_LEVEL: "info" + + pacsight: + image: ghcr.io/pacstall/pacsight:latest + logging: + driver: "json-file" + options: + max-file: "5" + max-size: "10m" + environment: + PACSTALL_PACSIGHT_PORT: 3301 + PACSTALL_REPOLOGY_CACHE_PATH: "/var/cache/pacsight" + PACSTALL_LOG_LEVEL: "info" + deploy: + resources: + limits: + memory: 6g + volumes: + - pacsight_cache:/var/cache/pacsight matomo: image: matomo @@ -60,3 +74,4 @@ services: volumes: db: matomo: + pacsight_cache: \ No newline at end of file diff --git a/Dockerfile b/pacnexus.Dockerfile similarity index 91% rename from Dockerfile rename to pacnexus.Dockerfile index ff1aac44..ca7019fb 100644 --- a/Dockerfile +++ b/pacnexus.Dockerfile @@ -41,5 +41,5 @@ WORKDIR /root/dist/ RUN ls -al /root/dist -CMD echo "Starting webserver in a few seconds..." && sleep 3 && "./webserver" +CMD echo "Starting pacnexus in a few seconds..." && sleep 3 && "./pacnexus" EXPOSE 3300 diff --git a/pacsight.Dockerfile b/pacsight.Dockerfile new file mode 100644 index 00000000..2e8cd080 --- /dev/null +++ b/pacsight.Dockerfile @@ -0,0 +1,31 @@ +FROM golang:1.23-alpine AS server + +ARG VITE_VERSION +ENV VITE_VERSION="${VITE_VERSION}" +ENV NODE_ENV="production" + +WORKDIR /root/ + +COPY ./server ./server + +RUN apk add --no-cache make gcc musl-dev + +WORKDIR /root/server +RUN make dist/pacsight + +FROM ubuntu:22.04 +WORKDIR /root/ + +RUN apt update +RUN apt install wget curl -y + +COPY --from=server /root/server/dist/ /root/server/dist/ + +RUN apt update && apt install make git jq -y + +WORKDIR /root/server/dist + +RUN ls -al /root/server/dist + +CMD "./pacsight" +EXPOSE 8080 diff --git a/server/.gitignore b/server/.gitignore index dc5734a9..7a976121 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -4,3 +4,4 @@ tmp/ dist/ programs/ main +repology_cache.json diff --git a/server/Makefile b/server/Makefile index c80a59b7..e2002640 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,69 +1,45 @@ LDFLAGS= \ -w \ -s \ - -X 'pacstall.dev/webserver/config/build.UpdateInterval=900' \ - -X 'pacstall.dev/webserver/config/build.TempDir=./tmp' \ - -X 'pacstall.dev/webserver/config/build.MaxOpenFiles=100' \ - -X 'pacstall.dev/webserver/config/build.Port=3300' \ - -X 'pacstall.dev/webserver/config/build.PublicDir=./public' \ - -X 'pacstall.dev/webserver/config/build.Production=true' \ - -X 'pacstall.dev/webserver/config/build.GitURL=https://github.com/pacstall/pacstall-programs.git' \ - -X 'pacstall.dev/webserver/config/build.GitClonePath=./programs' \ - -X 'pacstall.dev/webserver/config/build.Version=${VERSION}' + -X 'pacstall.dev/webserver/pkg/common/config/build.Production=true' \ + -X 'pacstall.dev/webserver/pkg/common/config/build.Version=${VERSION}' -all: dist/webserver +all: dist/pacnexus dist/pacsight test: - PACSTALL_DATABASE_HOST=localhost \ - PACSTALL_DATABASE_PORT=3306 \ - PACSTALL_DATABASE_USER=root \ - PACSTALL_DATABASE_PASSWORD=changeme \ - PACSTALL_DATABASE_NAME=pacstall \ - PACSTALL_DISCORD_ENABLED=false \ - PACSTALL_DISCORD_TOKEN="" \ - PACSTALL_DISCORD_CHANNEL_ID="" \ - PACSTALL_DISCORD_TAGS="" \ - PACSTALL_MATOMO_ENABLED="false" \ - PACSTALL_REPOLOGY_ENABLED="false" \ - PACSTALL_PROGRAMS_GIT_BRANCH="master" \ GO_ENV=test go test -v types/pac/parser/parse_test.go test_internal: - PACSTALL_DATABASE_HOST=localhost \ - PACSTALL_DATABASE_PORT=3306 \ - PACSTALL_DATABASE_USER=root \ - PACSTALL_DATABASE_PASSWORD=changeme \ - PACSTALL_DATABASE_NAME=pacstall \ - PACSTALL_DISCORD_ENABLED=false \ - PACSTALL_DISCORD_TOKEN="" \ - PACSTALL_DISCORD_CHANNEL_ID="" \ - PACSTALL_DISCORD_TAGS="" \ - PACSTALL_MATOMO_ENABLED="false" \ - PACSTALL_REPOLOGY_ENABLED="false" \ - PACSTALL_PROGRAMS_GIT_BRANCH="master" \ GO_ENV=test go test -v types/pac/parser/pacsh/internal/git_version_test.go -run: - (cd .. && docker compose up -d mariadb) +run-database: + @(cd .. && docker compose up -d mariadb) - PACSTALL_DATABASE_HOST=localhost \ - PACSTALL_DATABASE_PORT=3306 \ - PACSTALL_DATABASE_USER=root \ - PACSTALL_DATABASE_PASSWORD=changeme \ - PACSTALL_DATABASE_NAME=pacstall \ - PACSTALL_DISCORD_ENABLED=false \ - PACSTALL_DISCORD_TOKEN="" \ - PACSTALL_DISCORD_CHANNEL_ID="" \ - PACSTALL_DISCORD_TAGS="" \ - PACSTALL_MATOMO_ENABLED="false" \ - PACSTALL_REPOLOGY_ENABLED="false" \ - PACSTALL_PROGRAMS_GIT_BRANCH="master" \ - go run bin/webserver/main.go +run-pacnexus: run-database + @go run cmd/pacnexus/main.go + +run-pacsight: + @go run cmd/pacsight/main.go + + +run: + @trap 'kill 0' SIGINT; \ + $(MAKE) run-pacsight | sed 's/^/[pacsight] /' & \ + $(MAKE) run-pacnexus | sed 's/^/[pacnexus] /' & \ + wait + @echo "Both executables completed or interrupted." + +dist/pacnexus: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f) + CGO_ENABLED=0 go build -o dist/pacnexus -ldflags "${LDFLAGS}" cmd/pacnexus/main.go + +dist/pacsight: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f) + CGO_ENABLED=0 go build -o dist/pacsight -ldflags "${LDFLAGS}" cmd/pacsight/main.go -dist/webserver: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f) - CGO_ENABLED=0 go build -o dist/webserver -ldflags "${LDFLAGS}" bin/webserver/main.go clean: - [ -d ./dist ] && rm -r dist || : + @[ -d ./dist ] && rm -rf dist || : + @[ -d ./programs ] && rm -rf programs || : + @[ -d ./tmp ] && rm -rf tmp || : + @echo "Cleaned up." fmt: - go fmt ./... + \ No newline at end of file diff --git a/server/bin/webserver/main.go b/server/bin/webserver/main.go deleted file mode 100644 index 97008335..00000000 --- a/server/bin/webserver/main.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import ( - "fmt" - "time" - - "github.com/fatih/color" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/repology" - "pacstall.dev/webserver/server" - ps_api "pacstall.dev/webserver/server/api/pacscripts" - repology_api "pacstall.dev/webserver/server/api/repology" - urlshortener "pacstall.dev/webserver/server/api/url_shortener" - pac_ssr "pacstall.dev/webserver/server/ssr/pacscript" - "pacstall.dev/webserver/types/pac/parser" -) - -func printLogo() { - logoColor := color.New(color.FgHiMagenta, color.Bold).SprintFunc() - fmt.Println(logoColor(` - ____ __ ____ - / __ \____ ___________/ /_____ _/ / / - / /_/ / __ '/ ___/ ___/ __/ __ '/ / / - / ____/ /_/ / /__(__ ) /_/ /_/ / / / -/_/ _\__,_/\___/____/\__/\__,_/_/_/ -| | / /__ / /_ / ___/___ ______ _____ _____ -| | /| / / _ \/ __ \\__ \/ _ \/ ___/ | / / _ \/ ___/ -| |/ |/ / __/ /_/ /__/ / __/ / | |/ / __/ / -|__/|__/\___/_.___/____/\___/_/ |___/\___/_/ - coded by saenai255, owned by Pacstall Org - `)) -} - -func setupRequests() { - router := server.Router() - - /* Packages */ - pac_ssr.EnableSSR() - router.HandleFunc("/api/repology", repology_api.GetRepologyPackageListHandle).Methods("GET") - router.HandleFunc("/api/packages", ps_api.GetPacscriptListHandle).Methods("GET") - router.HandleFunc("/api/packages/{name}", ps_api.GetPacscriptHandle).Methods("GET") - router.HandleFunc("/api/packages/{name}/requiredBy", ps_api.GetPacscriptRequiredByHandle).Methods("GET") - router.HandleFunc("/api/packages/{name}/dependencies", ps_api.GetPacscriptDependenciesHandle).Methods("GET") - - /* Shortened Links - Must be last as it functions as a catch-all trap */ - router.HandleFunc("/q/{linkId}", urlshortener.GetShortenedLinkRedirectHandle).Methods("GET") -} - -func main() { - if config.Production { - log.SetLogLevel(log.Level.Info) - } else { - log.SetLogLevel(log.Level.Debug) - } - - startedAt := time.Now() - - printLogo() - - setupRequests() - log.Info("registered http requests") - - log.Info("attempting to start TCP listener") - - server.OnServerOnline(func() { - log.NotifyCustom("🚀 Startup 🧑‍🚀", "successfully started up.") - log.Info("server is now online on port %v.\n", config.Port) - - log.Info("booted in %v\n", color.GreenString("%v", time.Since(startedAt))) - - parser.ScheduleRefresh(config.UpdateInterval) - log.Info("scheduled pacscripts to auto-refresh every %v", config.UpdateInterval) - - if config.Repology.Enabled { - repology.ScheduleRefresh(config.RepologyUpdateInterval) - log.Info("scheduled repology to auto-refresh every %v", config.RepologyUpdateInterval) - } else { - log.Warn("repository repology is disabled") - } - }) - - server.Listen(config.Port) -} diff --git a/server/cmd/pacnexus/main.go b/server/cmd/pacnexus/main.go new file mode 100644 index 00000000..b6e69a1d --- /dev/null +++ b/server/cmd/pacnexus/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "time" + + "github.com/fatih/color" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/internal/pacnexus/server" + ps_api "pacstall.dev/webserver/internal/pacnexus/server/api/pacscripts" + repology_api "pacstall.dev/webserver/internal/pacnexus/server/api/repology" + urlshortener "pacstall.dev/webserver/internal/pacnexus/server/api/url_shortener" + pac_ssr "pacstall.dev/webserver/internal/pacnexus/server/ssr/pacscript" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser" + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/pacsight" +) + +func printLogo() { + logoColor := color.New(color.FgHiMagenta, color.Bold).SprintFunc() + fmt.Println(logoColor(` +88888b. 8888b. .d8888b 88888b. .d88b. 888 888 888 888 .d8888b +888 "88b "88b d88P" 888 "88b d8P Y8b Y8bd8P' 888 888 88K +888 888 .d888888 888 888 888 88888888 X88K 888 888 "Y8888b. +888 d88P 888 888 Y88b. 888 888 Y8b. .d8""8b. Y88b 888 X88 +88888P" "Y888888 "Y8888P 888 888 "Y8888 888 888 "Y88888 88888P' +888 +888 +888 + + coded by saenai255, owned by Pacstall Org + `)) +} + +func setupRequests() { + router := server.Router() + + /* Packages */ + pac_ssr.EnableSSR() + router.HandleFunc("/api/repology", repology_api.GetRepologyPackageListHandle).Methods("GET") + router.HandleFunc("/api/packages", ps_api.GetPacscriptListHandle).Methods("GET") + router.HandleFunc("/api/packages/{name}", ps_api.GetPacscriptHandle).Methods("GET") + router.HandleFunc("/api/packages/{name}/requiredBy", ps_api.GetPacscriptRequiredByHandle).Methods("GET") + router.HandleFunc("/api/packages/{name}/dependencies", ps_api.GetPacscriptDependenciesHandle).Methods("GET") + + /* Shortened Links - Must be last as it functions as a catch-all trap */ + router.HandleFunc("/q/{linkId}", urlshortener.GetShortenedLinkRedirectHandle).Methods("GET") +} + +func main() { + startedAt := time.Now() + + config.Init() + printLogo() + + setupRequests() + log.Info("registered http requests") + + pacsightRpc, err := pacsight.NewPacsightRpcService(config.PacSight.Host, config.PacSight.Port) + if err != nil { + log.Error("failed to create pacsight rpc service: %+v", err) + return + } + log.Info("connected to pacsight rpc service") + + log.Info("attempting to start tcp listener") + + server.OnServerOnline(func() { + log.Info("server is now online on port %v.\n", config.PacNexus.Port) + + log.Info("booted in %v\n", color.GreenString("%v", time.Since(startedAt))) + + parser.ScheduleRefresh(config.PacstallPrograms.UpdateInterval, pacsightRpc) + log.Info("scheduled pacscripts to auto-refresh every %v", config.PacstallPrograms.UpdateInterval) + }) + + server.Listen(config.PacNexus.Port) +} diff --git a/server/cmd/pacsight/main.go b/server/cmd/pacsight/main.go new file mode 100644 index 00000000..6500d937 --- /dev/null +++ b/server/cmd/pacsight/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "net" + "net/http" + + "github.com/fatih/color" + "pacstall.dev/webserver/internal/pacsight/config" + "pacstall.dev/webserver/internal/pacsight/repology" + "pacstall.dev/webserver/internal/pacsight/rpcall" + "pacstall.dev/webserver/pkg/common/log" +) + +func printLogo() { + logoColor := color.New(color.FgHiMagenta, color.Bold).SprintFunc() + fmt.Println(logoColor(` + d8b 888 888 + Y8P 888 888 + 888 888 +88888b. 8888b. .d8888b .d8888b 888 .d88b. 88888b. 888888 +888 "88b "88b d88P" 88K 888 d88P"88b 888 "88b 888 +888 888 .d888888 888 "Y8888b. 888 888 888 888 888 888 +888 d88P 888 888 Y88b. X88 888 Y88b 888 888 888 Y88b. +88888P" "Y888888 "Y8888P 88888P' 888 "Y88888 888 888 "Y888 +888 888 +888 Y8b d88P +888 "Y88P" + + coded by saenai255, owned by Pacstall Org + `)) +} + +func main() { + config.Init() + printLogo() + + log.Info("booting up pacsight") + + go repology.ScheduleRefresh(config.Repology.RepologyUpdateInterval) + rpcall.RegisterService() + + port := fmt.Sprintf(":%v", config.PacSight.Port) + listener, err := net.Listen("tcp", port) + if err != nil { + log.Fatal("failed to start server: %+v", err) + } + + log.Info("server started on port " + port) + + if err := http.Serve(listener, nil); err != nil { + log.Fatal("failed to serve: %+v", err) + } + +} diff --git a/server/config/build/vars.go b/server/config/build/vars.go deleted file mode 100644 index 1b1fc378..00000000 --- a/server/config/build/vars.go +++ /dev/null @@ -1,14 +0,0 @@ -package build - -var Production = "false" - -var UpdateInterval = "900" -var RepologyUpdateInterval = "43200" -var TempDir = "./tmp" -var MaxOpenFiles = "100" -var GitURL = "https://github.com/pacstall/pacstall-programs.git" -var GitClonePath = "./programs" - -var Port = "3300" -var PublicDir = "../client/dist" -var Version = "development" diff --git a/server/config/env.go b/server/config/env.go deleted file mode 100644 index a70e0865..00000000 --- a/server/config/env.go +++ /dev/null @@ -1,49 +0,0 @@ -package config - -// Configuration for the discord integration -var Discord = struct { - Token string - ChannelID string - Enabled bool - Tags string -}{ - Token: getEnvString("PACSTALL_DISCORD_TOKEN"), - ChannelID: getEnvString("PACSTALL_DISCORD_CHANNEL_ID"), - Enabled: getEnvBool("PACSTALL_DISCORD_ENABLED"), - Tags: getEnvString("PACSTALL_DISCORD_TAGS"), -} - -var PacstallPrograms = struct { - Branch string -}{ - Branch: getEnvString("PACSTALL_PROGRAMS_GIT_BRANCH"), -} - -// Configuration for the database -var Database = struct { - Host string - Port int - User string - Password string - Name string -}{ - Host: getEnvString("PACSTALL_DATABASE_HOST"), - Port: getEnvInt("PACSTALL_DATABASE_PORT"), - User: getEnvString("PACSTALL_DATABASE_USER"), - Password: getEnvString("PACSTALL_DATABASE_PASSWORD"), - Name: getEnvString("PACSTALL_DATABASE_NAME"), -} - -// Configuration for the Matomo API -var Matomo = struct { - Enabled bool -}{ - Enabled: getEnvBool("PACSTALL_MATOMO_ENABLED"), -} - -// Configuration for the Repology API -var Repology = struct { - Enabled bool -}{ - Enabled: getEnvBoolOrDefault("PACSTALL_REPOLOGY_ENABLED", true), -} diff --git a/server/config/vars.go b/server/config/vars.go deleted file mode 100644 index d41d7f64..00000000 --- a/server/config/vars.go +++ /dev/null @@ -1,21 +0,0 @@ -package config - -import ( - "time" - - "pacstall.dev/webserver/config/build" -) - -var Production = toBool(build.Production) - -var UpdateInterval = time.Duration(toInt(build.UpdateInterval)) * time.Second -var RepologyUpdateInterval = time.Duration(toInt(build.RepologyUpdateInterval)) * time.Second -var TempDir = build.TempDir -var MaxOpenFiles = toInt(build.MaxOpenFiles) -var GitURL = build.GitURL -var GitClonePath = build.GitClonePath - -var Port = toInt(build.Port) -var PublicDir = build.PublicDir - -var Version = build.Version diff --git a/server/go.mod b/server/go.mod index 665a53bf..185a8735 100644 --- a/server/go.mod +++ b/server/go.mod @@ -32,6 +32,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pacstall/go-srcinfo v0.0.0-20240916164747-6f75fe362117 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/server/internal/pacnexus/config/env.go b/server/internal/pacnexus/config/env.go new file mode 100644 index 00000000..a183dfeb --- /dev/null +++ b/server/internal/pacnexus/config/env.go @@ -0,0 +1,110 @@ +package config + +import ( + "time" + + "pacstall.dev/webserver/pkg/common/env" +) + +var PacSight = struct { + Port int + Host string +}{} + +func initPacSightEnv() { + PacSight.Port = env.GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 3301) + PacSight.Host = env.GetEnvStringOrDefault("PACSTALL_PACSIGHT_HOST", "localhost") +} + +var PacNexus = struct { + Port int + PublicDir string +}{} + +func initPacNexusEnv() { + PacNexus.Port = env.GetEnvIntOrDefault("PACSTALL_PACNEXUS_PORT", 3300) + PacNexus.PublicDir = env.GetEnvStringOrDefault("PACSTALL_PACNEXUS_PUBLIC_DIR", "./public") +} + +// Configuration for the discord integration +var Discord = struct { + Token string + ChannelID string + Enabled bool + Tags string +}{} + +func initDiscordEnv() { + Discord.Enabled = env.GetEnvBoolOrDefault("PACSTALL_DISCORD_ENABLED", false) + + if !Discord.Enabled { + return + } + + Discord.Token = env.GetEnvString("PACSTALL_DISCORD_TOKEN") + Discord.ChannelID = env.GetEnvString("PACSTALL_DISCORD_CHANNEL_ID") + Discord.Tags = env.GetEnvString("PACSTALL_DISCORD_TAGS") +} + +var PacstallPrograms = struct { + Branch string + UpdateInterval time.Duration + TempDir string + MaxOpenFiles int + GitURL string + GitClonePath string +}{} + +func initPacstallProgramsEnv() { + PacstallPrograms.Branch = env.GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_BRANCH", "master") + PacstallPrograms.UpdateInterval = time.Duration(env.GetEnvIntOrDefault("PACSTALL_PROGRAMS_UPDATE_INTERVAL", 15*60)) * time.Second + PacstallPrograms.TempDir = env.GetEnvStringOrDefault("PACSTALL_PROGRAMS_TEMP_DIR", "./tmp") + PacstallPrograms.MaxOpenFiles = env.GetEnvIntOrDefault("PACSTALL_PROGRAMS_MAX_OPEN_FILES", 100) + PacstallPrograms.GitURL = env.GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_URL", "https://github.com/pacstall/pacstall-programs.git") + PacstallPrograms.GitClonePath = env.GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_CLONE_PATH", "./programs") +} + +// Configuration for the database +var Database = struct { + Host string + Port int + User string + Password string + Name string +}{} + +func initDatabaseEnv() { + Database.Host = env.GetEnvStringOrDefault("PACSTALL_DATABASE_HOST", "localhost") + Database.Port = env.GetEnvIntOrDefault("PACSTALL_DATABASE_PORT", 3306) + Database.User = env.GetEnvStringOrDefault("PACSTALL_DATABASE_USER", "root") + Database.Password = env.GetEnvStringOrDefault("PACSTALL_DATABASE_PASSWORD", "changeme") + Database.Name = env.GetEnvStringOrDefault("PACSTALL_DATABASE_NAME", "pacstall") +} + +// Configuration for the Matomo API +var Matomo = struct { + Enabled bool +}{} + +func initMatomoEnv() { + Matomo.Enabled = env.GetEnvBoolOrDefault("PACSTALL_MATOMO_ENABLED", false) +} + +// Configuration for the Repology API +var Repology = struct { + Enabled bool +}{} + +func initRepologyEnv() { + Repology.Enabled = env.GetEnvBoolOrDefault("PACSTALL_REPOLOGY_ENABLED", false) +} + +func Init() { + initPacNexusEnv() + initPacSightEnv() + initDatabaseEnv() + initDiscordEnv() + initMatomoEnv() + initPacstallProgramsEnv() + initRepologyEnv() +} diff --git a/server/consts/consts.go b/server/internal/pacnexus/consts/consts.go similarity index 100% rename from server/consts/consts.go rename to server/internal/pacnexus/consts/consts.go diff --git a/server/model/connection.go b/server/internal/pacnexus/model/connection.go similarity index 60% rename from server/model/connection.go rename to server/internal/pacnexus/model/connection.go index 614d7f55..61e60dd7 100644 --- a/server/model/connection.go +++ b/server/internal/pacnexus/model/connection.go @@ -7,18 +7,21 @@ import ( "gorm.io/driver/mysql" "gorm.io/gorm" "gorm.io/gorm/logger" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/pkg/common/log" ) var database *gorm.DB = nil -var connectionString = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", - config.Database.User, - config.Database.Password, - config.Database.Host, - config.Database.Port, - config.Database.Name, -) + +func getConnectionString() string { + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", + config.Database.User, + config.Database.Password, + config.Database.Host, + config.Database.Port, + config.Database.Name, + ) +} func Instance() *gorm.DB { if database != nil { @@ -26,7 +29,7 @@ func Instance() *gorm.DB { } err := retry(5, func() (err error) { - database, err = gorm.Open(mysql.Open(connectionString), &gorm.Config{ + database, err = gorm.Open(mysql.Open(getConnectionString()), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) @@ -34,7 +37,7 @@ func Instance() *gorm.DB { }) if err != nil { - panic(fmt.Sprintf("failed to connect database: %v", err)) + panic(fmt.Sprintf("failed to connect database: %+v", err)) } log.Info("connected to database.") diff --git a/server/model/repology_project.go b/server/internal/pacnexus/model/repology_project.go similarity index 100% rename from server/model/repology_project.go rename to server/internal/pacnexus/model/repology_project.go diff --git a/server/model/repology_project_provider.go b/server/internal/pacnexus/model/repology_project_provider.go similarity index 100% rename from server/model/repology_project_provider.go rename to server/internal/pacnexus/model/repology_project_provider.go diff --git a/server/model/shortened_link.go b/server/internal/pacnexus/model/shortened_link.go similarity index 100% rename from server/model/shortened_link.go rename to server/internal/pacnexus/model/shortened_link.go diff --git a/server/server/api/pacscripts/dependencies.go b/server/internal/pacnexus/server/api/pacscripts/dependencies.go similarity index 85% rename from server/server/api/pacscripts/dependencies.go rename to server/internal/pacnexus/server/api/pacscripts/dependencies.go index 07ddc522..14221f28 100644 --- a/server/server/api/pacscripts/dependencies.go +++ b/server/internal/pacnexus/server/api/pacscripts/dependencies.go @@ -5,10 +5,10 @@ import ( "net/http" "github.com/gorilla/mux" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) type pacscriptDependencies struct { diff --git a/server/server/api/pacscripts/package.go b/server/internal/pacnexus/server/api/pacscripts/package.go similarity index 73% rename from server/server/api/pacscripts/package.go rename to server/internal/pacnexus/server/api/pacscripts/package.go index c3fbfad8..09dee1e6 100644 --- a/server/server/api/pacscripts/package.go +++ b/server/internal/pacnexus/server/api/pacscripts/package.go @@ -5,10 +5,10 @@ import ( "net/http" "github.com/gorilla/mux" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) func GetPacscriptHandle(w http.ResponseWriter, req *http.Request) { diff --git a/server/server/api/pacscripts/package_list.go b/server/internal/pacnexus/server/api/pacscripts/package_list.go similarity index 89% rename from server/server/api/pacscripts/package_list.go rename to server/internal/pacnexus/server/api/pacscripts/package_list.go index c35c1e41..550dcba9 100644 --- a/server/server/api/pacscripts/package_list.go +++ b/server/internal/pacnexus/server/api/pacscripts/package_list.go @@ -5,11 +5,11 @@ import ( "math" "net/http" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/server/query" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" - "pacstall.dev/webserver/types/pac/parser" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/server/query" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser" ) type packageListPage struct { diff --git a/server/server/api/pacscripts/required_by.go b/server/internal/pacnexus/server/api/pacscripts/required_by.go similarity index 79% rename from server/server/api/pacscripts/required_by.go rename to server/internal/pacnexus/server/api/pacscripts/required_by.go index ccf66a52..06c5c7da 100644 --- a/server/server/api/pacscripts/required_by.go +++ b/server/internal/pacnexus/server/api/pacscripts/required_by.go @@ -5,10 +5,10 @@ import ( "net/http" "github.com/gorilla/mux" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) func GetPacscriptRequiredByHandle(w http.ResponseWriter, req *http.Request) { diff --git a/server/server/api/repology/repology.go b/server/internal/pacnexus/server/api/repology/repology.go similarity index 74% rename from server/server/api/repology/repology.go rename to server/internal/pacnexus/server/api/repology/repology.go index 82b8415e..ce72f761 100644 --- a/server/server/api/repology/repology.go +++ b/server/internal/pacnexus/server/api/repology/repology.go @@ -4,10 +4,10 @@ import ( "fmt" "net/http" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) func GetRepologyPackageListHandle(w http.ResponseWriter, req *http.Request) { diff --git a/server/server/api/repology/types.go b/server/internal/pacnexus/server/api/repology/types.go similarity index 97% rename from server/server/api/repology/types.go rename to server/internal/pacnexus/server/api/repology/types.go index de81339c..3c5a4d6f 100644 --- a/server/server/api/repology/types.go +++ b/server/internal/pacnexus/server/api/repology/types.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" ) type maintainerDetails struct { diff --git a/server/server/api/url_shortener/url_shortener.go b/server/internal/pacnexus/server/api/url_shortener/url_shortener.go similarity index 83% rename from server/server/api/url_shortener/url_shortener.go rename to server/internal/pacnexus/server/api/url_shortener/url_shortener.go index a31576b1..06de3bdf 100644 --- a/server/server/api/url_shortener/url_shortener.go +++ b/server/internal/pacnexus/server/api/url_shortener/url_shortener.go @@ -7,12 +7,11 @@ import ( "github.com/gorilla/mux" "github.com/treelightsoftware/go-matomo" "gorm.io/gorm" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/model" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/internal/pacnexus/model" + "pacstall.dev/webserver/pkg/common/log" ) -var db = model.Instance() var incrementVisitsExpression = gorm.Expr(model.ShortenedLinkColumns.Visits+" + ?", 1) var pathParams = struct { @@ -40,7 +39,7 @@ func GetShortenedLinkRedirectHandle(w http.ResponseWriter, req *http.Request) { } var shortenedLink model.ShortenedLink - if result := db.Where(model.ShortenedLink{LinkId: linkId}).First(&shortenedLink); result.Error != nil { + if result := model.Instance().Where(model.ShortenedLink{LinkId: linkId}).First(&shortenedLink); result.Error != nil { w.WriteHeader(404) return } @@ -51,7 +50,7 @@ func GetShortenedLinkRedirectHandle(w http.ResponseWriter, req *http.Request) { return } - db.Model(&shortenedLink).Update(model.ShortenedLinkColumns.Visits, incrementVisitsExpression) + model.Instance().Model(&shortenedLink).Update(model.ShortenedLinkColumns.Visits, incrementVisitsExpression) if config.Matomo.Enabled { pingMatomoTracker(req.RemoteAddr, req.UserAgent(), req.Referer(), linkId) } diff --git a/server/server/headers.go b/server/internal/pacnexus/server/headers.go similarity index 95% rename from server/server/headers.go rename to server/internal/pacnexus/server/headers.go index a1a8b105..60a3d598 100644 --- a/server/server/headers.go +++ b/server/internal/pacnexus/server/headers.go @@ -5,7 +5,7 @@ import ( "net/http" "strings" - "pacstall.dev/webserver/config" + "pacstall.dev/webserver/pkg/common/config" ) type AlreadyResponded = bool diff --git a/server/server/health_check.go b/server/internal/pacnexus/server/health_check.go similarity index 95% rename from server/server/health_check.go rename to server/internal/pacnexus/server/health_check.go index 669e6e73..702b787d 100644 --- a/server/server/health_check.go +++ b/server/internal/pacnexus/server/health_check.go @@ -5,7 +5,7 @@ import ( "net/http" "time" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) func registerHealthCheck() { diff --git a/server/server/query/main.go b/server/internal/pacnexus/server/query/main.go similarity index 100% rename from server/server/query/main.go rename to server/internal/pacnexus/server/query/main.go diff --git a/server/server/sitemap.go b/server/internal/pacnexus/server/sitemap.go similarity index 96% rename from server/server/sitemap.go rename to server/internal/pacnexus/server/sitemap.go index 74321039..21adac94 100644 --- a/server/server/sitemap.go +++ b/server/internal/pacnexus/server/sitemap.go @@ -4,7 +4,7 @@ import ( "fmt" "net/http" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" ) type SitemapEntry struct { diff --git a/server/server/spa.go b/server/internal/pacnexus/server/spa.go similarity index 96% rename from server/server/spa.go rename to server/internal/pacnexus/server/spa.go index 35e6a9d4..75baa352 100644 --- a/server/server/spa.go +++ b/server/internal/pacnexus/server/spa.go @@ -8,7 +8,7 @@ import ( text "text/template" - "pacstall.dev/webserver/server/ssr" + "pacstall.dev/webserver/internal/pacnexus/server/ssr" ) type spaHandler struct { diff --git a/server/server/ssr/pacscript/main.go b/server/internal/pacnexus/server/ssr/pacscript/main.go similarity index 100% rename from server/server/ssr/pacscript/main.go rename to server/internal/pacnexus/server/ssr/pacscript/main.go diff --git a/server/server/ssr/pacscript/package.go b/server/internal/pacnexus/server/ssr/pacscript/package.go similarity index 84% rename from server/server/ssr/pacscript/package.go rename to server/internal/pacnexus/server/ssr/pacscript/package.go index 4edeafb7..02414077 100644 --- a/server/server/ssr/pacscript/package.go +++ b/server/internal/pacnexus/server/ssr/pacscript/package.go @@ -5,11 +5,11 @@ import ( "regexp" "strings" - "pacstall.dev/webserver/consts" - r "pacstall.dev/webserver/server/ssr" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/consts" + r "pacstall.dev/webserver/internal/pacnexus/server/ssr" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) func registerPacscriptSSRData() { diff --git a/server/server/ssr/pacscript/packages.go b/server/internal/pacnexus/server/ssr/pacscript/packages.go similarity index 88% rename from server/server/ssr/pacscript/packages.go rename to server/internal/pacnexus/server/ssr/pacscript/packages.go index 8ba86c15..5e4f9da1 100644 --- a/server/server/ssr/pacscript/packages.go +++ b/server/internal/pacnexus/server/ssr/pacscript/packages.go @@ -3,7 +3,7 @@ package ssr import ( "regexp" - r "pacstall.dev/webserver/server/ssr" + r "pacstall.dev/webserver/internal/pacnexus/server/ssr" ) func registerPacscriptListSSRData() { diff --git a/server/server/ssr/templates.go b/server/internal/pacnexus/server/ssr/templates.go similarity index 100% rename from server/server/ssr/templates.go rename to server/internal/pacnexus/server/ssr/templates.go diff --git a/server/server/webserver.go b/server/internal/pacnexus/server/webserver.go similarity index 80% rename from server/server/webserver.go rename to server/internal/pacnexus/server/webserver.go index 1602491a..a1e6a177 100644 --- a/server/server/webserver.go +++ b/server/internal/pacnexus/server/webserver.go @@ -10,8 +10,9 @@ import ( "time" "github.com/gorilla/mux" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/internal/pacnexus/config" + globalConfig "pacstall.dev/webserver/pkg/common/config" + "pacstall.dev/webserver/pkg/common/log" ) var router mux.Router = *mux.NewRouter() @@ -27,7 +28,7 @@ func Listen(port int) { Router().Use(func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("X-Pacstall-Version", config.Version) + w.Header().Add("X-Pacstall-Version", globalConfig.Version) if strings.Contains(r.URL.Path, "/api") { w.Header().Add("Content-Type", "application/json") @@ -38,10 +39,10 @@ func Listen(port int) { go triggerServerOnline(port) - if config.Production { - path, err := filepath.Abs(config.PublicDir) + if globalConfig.Production { + path, err := filepath.Abs(config.PacNexus.PublicDir) if err != nil { - log.Fatal("failed to find client public dir at path '%s'. err: %+v", config.PublicDir, err) + log.Fatal("failed to find client public dir at path '%s'. err: %+v", config.PacNexus.PublicDir, err) } Router().PathPrefix("/").Handler(spaHandler{staticPath: path}) diff --git a/server/types/pac/pacstore/store.go b/server/internal/pacnexus/types/pac/pacstore/store.go similarity index 88% rename from server/types/pac/pacstore/store.go rename to server/internal/pacnexus/types/pac/pacstore/store.go index e38bf0fe..e331e9e9 100644 --- a/server/types/pac/pacstore/store.go +++ b/server/internal/pacnexus/types/pac/pacstore/store.go @@ -3,8 +3,8 @@ package pacstore import ( "time" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/array" ) var lastModified time.Time diff --git a/server/types/pac/parser/git/lib.go b/server/internal/pacnexus/types/pac/parser/git/lib.go similarity index 98% rename from server/types/pac/parser/git/lib.go rename to server/internal/pacnexus/types/pac/parser/git/lib.go index 0852cac7..92d62ec8 100644 --- a/server/types/pac/parser/git/lib.go +++ b/server/internal/pacnexus/types/pac/parser/git/lib.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/joomcode/errorx" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) func hardResetAndPull(path, branch string) error { diff --git a/server/types/pac/parser/last_updated.go b/server/internal/pacnexus/types/pac/parser/last_updated.go similarity index 87% rename from server/types/pac/parser/last_updated.go rename to server/internal/pacnexus/types/pac/parser/last_updated.go index adf1bb8d..f29314d7 100644 --- a/server/types/pac/parser/last_updated.go +++ b/server/internal/pacnexus/types/pac/parser/last_updated.go @@ -9,11 +9,11 @@ import ( "time" "github.com/joomcode/errorx" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/parser/pacsh" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/log" ) type packageLastUpdatedTuple struct { @@ -27,7 +27,7 @@ func getPackageLastUpdatedTuples() ([]packageLastUpdatedTuple, error) { return nil, errorx.Decorate(err, "failed to get absolute path to wording directory") } - programsPath := path.Join(wordingDirectoryAbsolute, config.GitClonePath) + programsPath := path.Join(wordingDirectoryAbsolute, config.PacstallPrograms.GitClonePath) script := fmt.Sprintf(` cd %v for i in ./packages/*/*.pacscript; do echo $i; git log -1 --pretty=\"%%at\" $i; done diff --git a/server/types/pac/parser/pacscript.go b/server/internal/pacnexus/types/pac/parser/pacscript.go similarity index 84% rename from server/types/pac/parser/pacscript.go rename to server/internal/pacnexus/types/pac/parser/pacscript.go index 82c3032b..2ee7efb5 100644 --- a/server/types/pac/parser/pacscript.go +++ b/server/internal/pacnexus/types/pac/parser/pacscript.go @@ -1,8 +1,8 @@ package parser import ( - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/array" ) func computeRequiredBy(script *pac.Script, scripts []*pac.Script) { diff --git a/server/types/pac/parser/pacscript_list.go b/server/internal/pacnexus/types/pac/parser/pacscript_list.go similarity index 76% rename from server/types/pac/parser/pacscript_list.go rename to server/internal/pacnexus/types/pac/parser/pacscript_list.go index d109ec37..945bf265 100644 --- a/server/types/pac/parser/pacscript_list.go +++ b/server/internal/pacnexus/types/pac/parser/pacscript_list.go @@ -1,6 +1,6 @@ package parser -import "pacstall.dev/webserver/types/pac" +import "pacstall.dev/webserver/internal/pacnexus/types/pac" type PacscriptListWrapper []*pac.Script diff --git a/server/types/pac/parser/pacsh/exec_sh.go b/server/internal/pacnexus/types/pac/parser/pacsh/exec_sh.go similarity index 94% rename from server/types/pac/parser/pacsh/exec_sh.go rename to server/internal/pacnexus/types/pac/parser/pacsh/exec_sh.go index e268df76..3b9482ac 100644 --- a/server/types/pac/parser/pacsh/exec_sh.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/exec_sh.go @@ -4,7 +4,7 @@ import ( "os" "github.com/joomcode/errorx" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) var removeFile = os.Remove diff --git a/server/types/pac/parser/pacsh/git_version.go b/server/internal/pacnexus/types/pac/parser/pacsh/git_version.go similarity index 76% rename from server/types/pac/parser/pacsh/git_version.go rename to server/internal/pacnexus/types/pac/parser/pacsh/git_version.go index c40d00f9..dc99e292 100644 --- a/server/types/pac/parser/pacsh/git_version.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/git_version.go @@ -2,9 +2,9 @@ package pacsh import ( "github.com/joomcode/errorx" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/parser/pacsh/internal" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh/internal" + "pacstall.dev/webserver/pkg/common/array" ) func ApplyGitVersion(p *pac.Script) error { diff --git a/server/types/pac/parser/pacsh/internal/git_version.go b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go similarity index 96% rename from server/types/pac/parser/pacsh/internal/git_version.go rename to server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go index 6c80ee32..daeba4b3 100644 --- a/server/types/pac/parser/pacsh/internal/git_version.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go @@ -6,8 +6,8 @@ import ( "time" "github.com/joomcode/errorx" - "pacstall.dev/webserver/types/pac/parser/git" - "pacstall.dev/webserver/types/pac/parser/parallelism/timeout" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/git" + "pacstall.dev/webserver/pkg/common/parallelism/timeout" ) type GitSourceInfo struct { diff --git a/server/types/pac/parser/pacsh/internal/git_version_test.go b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version_test.go similarity index 97% rename from server/types/pac/parser/pacsh/internal/git_version_test.go rename to server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version_test.go index 6e3d7f2f..ca87ff16 100644 --- a/server/types/pac/parser/pacsh/internal/git_version_test.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version_test.go @@ -3,7 +3,7 @@ package internal_test import ( "testing" - "pacstall.dev/webserver/types/pac/parser/pacsh/internal" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh/internal" ) func assertGitSourceInfoEquals(t *testing.T, expected, actual internal.GitSourceInfo) { diff --git a/server/types/pac/parser/pacsh/parse_pac_output.go b/server/internal/pacnexus/types/pac/parser/pacsh/parse_pac_output.go similarity index 83% rename from server/types/pac/parser/pacsh/parse_pac_output.go rename to server/internal/pacnexus/types/pac/parser/pacsh/parse_pac_output.go index 21fd336f..197bd0cd 100644 --- a/server/types/pac/parser/pacsh/parse_pac_output.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/parse_pac_output.go @@ -2,7 +2,7 @@ package pacsh import ( "github.com/pacstall/go-srcinfo" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" ) func ParsePacOutput(data []byte) (*pac.Script, error) { diff --git a/server/types/pac/parser/pacsh/pretty-name.go b/server/internal/pacnexus/types/pac/parser/pacsh/pretty-name.go similarity index 86% rename from server/types/pac/parser/pacsh/pretty-name.go rename to server/internal/pacnexus/types/pac/parser/pacsh/pretty-name.go index d1e12030..a75414d7 100644 --- a/server/types/pac/parser/pacsh/pretty-name.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/pretty-name.go @@ -3,8 +3,8 @@ package pacsh import ( "strings" - "pacstall.dev/webserver/types" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/types" ) func getPrettyName(p *pac.Script) string { diff --git a/server/types/pac/parser/pacsh/temp_dir.go b/server/internal/pacnexus/types/pac/parser/pacsh/temp_dir.go similarity index 100% rename from server/types/pac/parser/pacsh/temp_dir.go rename to server/internal/pacnexus/types/pac/parser/pacsh/temp_dir.go diff --git a/server/types/pac/parser/pacsh/temp_exec.go b/server/internal/pacnexus/types/pac/parser/pacsh/temp_exec.go similarity index 96% rename from server/types/pac/parser/pacsh/temp_exec.go rename to server/internal/pacnexus/types/pac/parser/pacsh/temp_exec.go index b87028e6..ab404214 100644 --- a/server/types/pac/parser/pacsh/temp_exec.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/temp_exec.go @@ -7,7 +7,7 @@ import ( "path" "github.com/joomcode/errorx" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) var createFile = os.Create diff --git a/server/types/pac/parser/parse.go b/server/internal/pacnexus/types/pac/parser/parse.go similarity index 60% rename from server/types/pac/parser/parse.go rename to server/internal/pacnexus/types/pac/parser/parse.go index 26eca283..e7461cc6 100644 --- a/server/types/pac/parser/parse.go +++ b/server/internal/pacnexus/types/pac/parser/parse.go @@ -6,25 +6,25 @@ import ( "strings" "github.com/joomcode/errorx" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/consts" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/repology" - "pacstall.dev/webserver/types" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" - "pacstall.dev/webserver/types/pac/parser/git" - "pacstall.dev/webserver/types/pac/parser/pacsh" - "pacstall.dev/webserver/types/pac/parser/parallelism/batch" - "pacstall.dev/webserver/types/pac/parser/parallelism/channels" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/internal/pacnexus/consts" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/git" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/pacsight" + "pacstall.dev/webserver/pkg/common/parallelism/batch" + "pacstall.dev/webserver/pkg/common/parallelism/channels" + "pacstall.dev/webserver/pkg/common/types" ) const PACKAGE_LIST_FILE_NAME = "./packagelist" const MAX_GIT_VERSION_CONCURRENCY = 5 -func ParseAll() error { - if err := git.RefreshPrograms(config.GitClonePath, config.GitURL, config.PacstallPrograms.Branch); err != nil { +func ParseAll(pacsightRpc *pacsight.PacsightRpcService) error { + if err := git.RefreshPrograms(config.PacstallPrograms.GitClonePath, config.PacstallPrograms.GitURL, config.PacstallPrograms.Branch); err != nil { return errorx.Decorate(err, "could not update repository 'pacstall-programs'") } @@ -33,7 +33,7 @@ func ParseAll() error { return errorx.Decorate(err, "failed to parse packagelist") } - loadedPacscripts, err := parsePacscriptFiles(pkgList) + loadedPacscripts, err := parsePacscriptFiles(pkgList, pacsightRpc) if err != nil { return errorx.Decorate(err, "failed to parse pacscripts") } @@ -72,7 +72,7 @@ func ParseAll() error { } func readKnownPacscriptNames() ([]string, error) { - pkglistPath := path.Join(config.GitClonePath, PACKAGE_LIST_FILE_NAME) + pkglistPath := path.Join(config.PacstallPrograms.GitClonePath, PACKAGE_LIST_FILE_NAME) bytes, err := os.ReadFile(pkglistPath) if err != nil { return nil, err @@ -86,18 +86,26 @@ func readKnownPacscriptNames() ([]string, error) { return names, nil } -func parsePacscriptFiles(names []string) ([]*pac.Script, error) { - if err := pacsh.CreateTempDirectory(config.TempDir); err != nil { +func parsePacscriptFiles(names []string, pacsightRpc *pacsight.PacsightRpcService) ([]*pac.Script, error) { + if err := pacsh.CreateTempDirectory(config.PacstallPrograms.TempDir); err != nil { return nil, errorx.Decorate(err, "failed to create temporary directory") } log.Info("parsing pacscripts...") - outChan := batch.Run(int(config.MaxOpenFiles), names, func(pacName string) (*pac.Script, error) { - out, err := ParsePacscriptFile(config.GitClonePath, pacName) + outChan := batch.Run(int(config.PacstallPrograms.MaxOpenFiles), names, func(pacName string) (*pac.Script, error) { + out, err := ParsePacscriptFile(config.PacstallPrograms.GitClonePath, pacName) + + if config.Repology.Enabled && len(out.Repology) > 0 { + project, err := pacsightRpc.GetRepologyProject(out.Repology) + if err != nil { + log.Debug("failed to get repology project %v. Error: %+v", out.Repology, err) + } else { + if err := UpdateScriptVersion(project, out); err != nil { + log.Warn("failed to update script version %v with repology. Error: %+v", pacName, err) + } + + out.LatestVersion = &project.Version - if config.Repology.Enabled { - if err := repology.Sync(out); err != nil { - log.Debug("failed to sync %v with repology. Error: %+v", pacName, err) } } diff --git a/server/internal/pacnexus/types/pac/parser/scheduler.go b/server/internal/pacnexus/types/pac/parser/scheduler.go new file mode 100644 index 00000000..7d1af143 --- /dev/null +++ b/server/internal/pacnexus/types/pac/parser/scheduler.go @@ -0,0 +1,28 @@ +package parser + +import ( + "time" + + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/pacsight" +) + +func ScheduleRefresh(every time.Duration, pacsightRpc *pacsight.PacsightRpcService) { + go refresh(every, pacsightRpc) +} + +func refresh(every time.Duration, pacsightRpc *pacsight.PacsightRpcService) { + for { + err := ParseAll(pacsightRpc) + if err != nil { + log.Error("parse error: %+v", err) + + retryIn := time.Second * 30 + log.Info("retrying in %v", retryIn) + time.Sleep(retryIn) + continue + } + + time.Sleep(every) + } +} diff --git a/server/types/pac/parser/search.go b/server/internal/pacnexus/types/pac/parser/search.go similarity index 96% rename from server/types/pac/parser/search.go rename to server/internal/pacnexus/types/pac/parser/search.go index f1b99407..f5f7595e 100644 --- a/server/types/pac/parser/search.go +++ b/server/internal/pacnexus/types/pac/parser/search.go @@ -3,8 +3,8 @@ package parser import ( "strings" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/array" ) const DEFAULT = "default" diff --git a/server/repology/sync.go b/server/internal/pacnexus/types/pac/parser/sync.go similarity index 89% rename from server/repology/sync.go rename to server/internal/pacnexus/types/pac/parser/sync.go index c1bd83c8..c4aceec0 100644 --- a/server/repology/sync.go +++ b/server/internal/pacnexus/types/pac/parser/sync.go @@ -1,11 +1,11 @@ -package repology +package parser import ( "strings" "github.com/hashicorp/go-version" - "pacstall.dev/webserver/model" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/types" ) func compareNonStandardVersion(current, latest string) pac.UpdateStatusValue { @@ -21,7 +21,7 @@ func compareNonStandardVersion(current, latest string) pac.UpdateStatusValue { return pac.UpdateStatus.Major } -func updateScriptVersion(project model.RepologyProjectProvider, script *pac.Script) (err error) { +func UpdateScriptVersion(project types.RepologyApiProject, script *pac.Script) (err error) { script.LatestVersion = &project.Version if *script.LatestVersion == script.Version { @@ -80,7 +80,7 @@ func updateScriptVersion(project model.RepologyProjectProvider, script *pac.Scri return } -func versionCompare(current, latest string) pac.UpdateStatusValue { +func GetUpdateStatus(current, latest string) pac.UpdateStatusValue { min := func(a, b int) int { if a < b { return a diff --git a/server/types/pac/script.go b/server/internal/pacnexus/types/pac/script.go similarity index 98% rename from server/types/pac/script.go rename to server/internal/pacnexus/types/pac/script.go index 96b43298..b7272974 100644 --- a/server/types/pac/script.go +++ b/server/internal/pacnexus/types/pac/script.go @@ -5,8 +5,8 @@ import ( "time" "github.com/pacstall/go-srcinfo" - "pacstall.dev/webserver/types" - "pacstall.dev/webserver/types/array" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/types" ) type updateStatus struct { diff --git a/server/internal/pacsight/config/env.go b/server/internal/pacsight/config/env.go new file mode 100644 index 00000000..9e3f2ddb --- /dev/null +++ b/server/internal/pacsight/config/env.go @@ -0,0 +1,31 @@ +package config + +import ( + "time" + + "pacstall.dev/webserver/pkg/common/env" +) + +var PacSight = struct { + Port int +}{} + +func initPacSightEnv() { + PacSight.Port = env.GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 3301) +} + +var Repology = struct { + RepologyUpdateInterval time.Duration + CachePath string + MaxOpenFiles int +}{} + +func initRepologyEnv() { + Repology.RepologyUpdateInterval = time.Duration(env.GetEnvIntOrDefault("PACSTALL_REPOLOGY_UPDATE_INTERVAL", 60*60*6)) * time.Second + Repology.CachePath = env.GetEnvStringOrDefault("PACSTALL_REPOLOGY_CACHE_PATH", "./repology_cache") +} + +func Init() { + initPacSightEnv() + initRepologyEnv() +} diff --git a/server/internal/pacsight/repology/exporter.go b/server/internal/pacsight/repology/exporter.go new file mode 100644 index 00000000..127f9ba5 --- /dev/null +++ b/server/internal/pacsight/repology/exporter.go @@ -0,0 +1,5 @@ +package repology + +import "pacstall.dev/webserver/internal/pacsight/repology/types" + +var ExportRepologyDatabase = types.ExportRepologyDatabase diff --git a/server/internal/pacsight/repology/fetch.go b/server/internal/pacsight/repology/fetch.go new file mode 100644 index 00000000..27898493 --- /dev/null +++ b/server/internal/pacsight/repology/fetch.go @@ -0,0 +1,97 @@ +package repology + +import ( + "strings" + + "github.com/joomcode/errorx" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/types" +) + +func parseRepologyFilter(filter string) (string, string) { + idx := strings.Index(filter, ":") + return strings.TrimSpace(filter[:idx]), strings.TrimSpace(filter[idx+1:]) +} + +const ( + RF_REPO = "repo" + RF_SUBREPO = "subrepo" + RF_STATUS = "status" + RF_SRCNAME = "srcname" + RF_BINNAME = "binname" + RF_VERSION = "version" + RF_ORIGVERSION = "origversion" + RF_VISIBLENAME = "visiblename" + RF_SUMMARY = "summary" +) + +func FindRepologyProject(projectsMap types.RepologyApiProjectSearchResponse, search []string) (types.RepologyApiProject, error) { + if len(search) == 0 { + return types.RepologyApiProject{}, errorx.IllegalArgument.New("no search terms provided") + } + + _, projectName := parseRepologyFilter(search[0]) + + projects, ok := projectsMap[projectName] + if !ok { + return types.RepologyApiProject{}, errorx.DataUnavailable. + New("project not found"). + WithProperty(errorx.RegisterProperty("project"), projectName) + } + + for _, filter := range search[1:] { + field, value := parseRepologyFilter(filter) + + projects = array.Filter(projects, func(i *array.Iterator[types.RepologyApiProject]) bool { + switch field { + case RF_REPO: + return i.Value.Repository == value + case RF_SUBREPO: + return i.Value.SubRepository != nil && *i.Value.SubRepository == value + case RF_STATUS: + return i.Value.Status == value + case RF_SRCNAME: + return i.Value.SourceName != nil && *i.Value.SourceName == value + case RF_BINNAME: + return i.Value.BinaryName != nil && *i.Value.BinaryName == value + case RF_VERSION: + return i.Value.Version == value + case RF_ORIGVERSION: + return i.Value.OriginalVersion == value + case RF_VISIBLENAME: + return i.Value.VisibleName != nil && *i.Value.VisibleName == value + case RF_SUMMARY: + return i.Value.Summary == value + default: + return false + } + }) + } + + projects = sortByStatus(projects) + + if len(projects) == 0 { + return types.RepologyApiProject{}, errorx.IllegalArgument.New("no projects found") + } + + return projects[0], nil +} + +var repologyStatusPriority = map[string]int{ + "newest": 0, + "rolling": 1, + "devel": 3, + "legacy": 4, + "outdated": 5, + "unique": 6, + "noscheme": 7, + "incorrect": 7, + "untrusted": 7, + "ignored": 7, +} + +func sortByStatus(projects []types.RepologyApiProject) []types.RepologyApiProject { + return array.SortBy(array.Clone(projects), func(p1, p2 types.RepologyApiProject) bool { + return repologyStatusPriority[p1.Status] < repologyStatusPriority[p2.Status] + }) +} diff --git a/server/internal/pacsight/repology/scheduler.go b/server/internal/pacsight/repology/scheduler.go new file mode 100644 index 00000000..1f670305 --- /dev/null +++ b/server/internal/pacsight/repology/scheduler.go @@ -0,0 +1,149 @@ +package repology + +import ( + "bufio" + "encoding/gob" + "io/fs" + "os" + "path" + "time" + + "github.com/joomcode/errorx" + "pacstall.dev/webserver/internal/pacsight/config" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/types" +) + +var Packages types.RepologyApiProjectSearchResponse = make(map[string][]types.RepologyApiProject) + +func ScheduleRefresh(every time.Duration) { + log.Info("attempting to read repology cache...") + cache, err := readCache() + if err != nil { + log.Warn("no cache found. this is normal on clean runs: %+v", err) + } else { + log.Info("repology cache read successfully. %d projects found", len(cache)) + Packages = cache + } + + go func() { + for { + log.Info("refreshing Repology database...") + out := types.RepologyApiProjectSearchResponse{} + resultsChan, errChan := ExportRepologyDatabase() + + chanLoop: + for { + select { + case pair, ok := <-resultsChan: + if !ok { + break chanLoop + } + + out[pair.ProjectName] = pair.Projects + + if err := cachePair(pair.ProjectName, pair.Projects); err != nil { + log.Error("failed to cache Repology projects: %+v", err) + } else { + log.Trace("cached repology project '%s'", pair.ProjectName) + } + case err := <-errChan: + if err != nil { + log.Error("failed to export Repology projects: %+v", err) + break chanLoop + } + } + } + + log.Info("repology database refreshed successfully. %d projects found", len(out)) + Packages = out + time.Sleep(every) + } + }() + + log.Info("scheduled repology refresh every %v", every) +} + +func readCache() (types.RepologyApiProjectSearchResponse, error) { + ensureCacheDirectoryExists() + + var err error + var dirEntries []fs.DirEntry + if dirEntries, err = os.ReadDir(config.Repology.CachePath); err != nil { + return nil, err + } + + dirEntries = array.Filter(dirEntries, func(entry *array.Iterator[fs.DirEntry]) bool { + return !entry.Value.IsDir() + }) + + if len(dirEntries) == 0 { + return nil, errorx.DataUnavailable.New("no cache files found") + } + + files := array.SwitchMap(dirEntries, func(entry *array.Iterator[fs.DirEntry]) string { + return entry.Value.Name() + }) + + cache := types.RepologyApiProjectSearchResponse{} + + for _, fileName := range files { + cacheFilePath := path.Join(config.Repology.CachePath, fileName) + file, err := os.Open(cacheFilePath) + if err != nil { + return nil, err + } + + defer file.Close() + + reader := bufio.NewReader(file) + decoder := gob.NewDecoder(reader) + + projects := []types.RepologyApiProject{} + if err = decoder.Decode(&projects); err != nil { + return nil, err + } + + projectName := fileName[:len(fileName)-4] // remove .gob extension + + cache[projectName] = projects + } + + return cache, nil +} + +func cachePair(projectName string, projects []types.RepologyApiProject) error { + ensureCacheDirectoryExists() + fileName := projectName + ".gob" + cacheFilePath := path.Join(config.Repology.CachePath, fileName) + + // delete existing cache file if it exists + os.RemoveAll(cacheFilePath) + + file, err := os.Create(cacheFilePath) + if err != nil { + return err + } + + defer file.Close() + + writer := bufio.NewWriter(file) + encoder := gob.NewEncoder(writer) + + if err = encoder.Encode(projects); err != nil { + return err + } + + if err = writer.Flush(); err != nil { + return err + } + + return nil +} + +func ensureCacheDirectoryExists() { + if err := os.MkdirAll(config.Repology.CachePath, 0755); err != nil { + log.Fatal("failed to create cache directory: %+v", err) + } +} diff --git a/server/repology/internal/api.go b/server/internal/pacsight/repology/types/api.go similarity index 83% rename from server/repology/internal/api.go rename to server/internal/pacsight/repology/types/api.go index 6201a441..6ba33d0b 100644 --- a/server/repology/internal/api.go +++ b/server/internal/pacsight/repology/types/api.go @@ -1,4 +1,4 @@ -package internal +package types import ( "encoding/json" @@ -7,12 +7,13 @@ import ( "net/url" "github.com/joomcode/errorx" + "pacstall.dev/webserver/pkg/common/types" ) const _USER_AGENT = "Pacstall/WebServer/Exporter" -func getProjectSearch(projectName string) (RepologyApiProjectSearchResponse, error) { - var response RepologyApiProjectSearchResponse +func getProjectSearch(projectName string) (types.RepologyApiProjectSearchResponse, error) { + var response types.RepologyApiProjectSearchResponse request := http.Request{ Method: "GET", diff --git a/server/internal/pacsight/repology/types/exporter.go b/server/internal/pacsight/repology/types/exporter.go new file mode 100644 index 00000000..c53e15c7 --- /dev/null +++ b/server/internal/pacsight/repology/types/exporter.go @@ -0,0 +1,119 @@ +package types + +import ( + "math" + "strings" + "time" + + "github.com/joomcode/errorx" + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/types" +) + +// Ends up waiting `sum(1^2, 2^2, 3^2, ..., RETRY_COUNT^2)` = 385 seconds (6.4 minutes) at most +const RETRY_COUNT = 10 +const REPOLOGY_PROJECT_FETCH_THROTTLE = time.Second + +type RepologyProjectPair struct { + ProjectName string + Projects []types.RepologyApiProject +} + +func ExportRepologyDatabase() (<-chan RepologyProjectPair, <-chan error) { + out := make(chan RepologyProjectPair) + errs := make(chan error) + + go func() { + page := 1 + lastProjectName := "" + + lastRepoFetch := time.Now() + var repologyProjectsMap = make(types.RepologyApiProjectSearchResponse) + + for { + if time.Since(lastRepoFetch) < REPOLOGY_PROJECT_FETCH_THROTTLE { + time.Sleep(REPOLOGY_PROJECT_FETCH_THROTTLE - time.Since(lastRepoFetch)) + } + + log.Trace("page %v | cursor at: %v", page, lastProjectName) + + var projectPage map[string][]types.RepologyApiProject + var err error + + retry: + for i := 1; i <= RETRY_COUNT; i += 1 { + projectPage, err = getProjectSearch(lastProjectName) + if err == nil { + break retry + } + + retryDelay := time.Duration(math.Pow(float64(i), 2)) * REPOLOGY_PROJECT_FETCH_THROTTLE + log.Debug("failed to fetch repology project page '%s'. retrying in %v", lastProjectName, retryDelay) + time.Sleep(retryDelay) + } + + if err != nil { + errs <- errorx.Decorate(err, "failed to fetch repology project page '%s'", lastProjectName) + close(out) + close(errs) + return + } + + lastRepoFetch = time.Now() + + shouldStop := false + for projectName, apiProjectProvider := range projectPage { + if _, ok := repologyProjectsMap[projectName]; !ok { + repologyProjectsMap[projectName] = []types.RepologyApiProject{} + } + + repologyProjectsMap[projectName] = append(repologyProjectsMap[projectName], apiProjectProvider...) + + out <- RepologyProjectPair{ + ProjectName: projectName, + Projects: apiProjectProvider, + } + + shouldStop = projectName == lastProjectName + lastProjectName = identityOrSkipProject(projectName) + } + + if shouldStop { + break + } + + page += 1 + } + + close(errs) + close(out) + }() + + return out, errs +} + +var projectNamesToSkipToNextCussor = map[string]string{ + "emacs:": "emacsa", + "go:": "goa", + "haskell:": "haskella", + "lisp:": "lispa", + "node:": "nodea", + "ocaml:": "ocamla", + "perl:": "perla", + "php:": "phpa", + "python:": "pythona", + "r:": "ra", + "ruby:": "rubya", + "rust:": "rusta", + "texlive:": "texlivea", +} + +func identityOrSkipProject(name string) string { + for prefix, skipTo := range projectNamesToSkipToNextCussor { + if strings.HasPrefix(name, prefix) { + return skipTo + } + } + + return name +} diff --git a/server/internal/pacsight/rpcall/service.go b/server/internal/pacsight/rpcall/service.go new file mode 100644 index 00000000..d46fddf1 --- /dev/null +++ b/server/internal/pacsight/rpcall/service.go @@ -0,0 +1,26 @@ +package rpcall + +import ( + "net/rpc" + + "github.com/joomcode/errorx" + "pacstall.dev/webserver/internal/pacsight/repology" + "pacstall.dev/webserver/pkg/common/pacsight" +) + +func RegisterService() { + rpc.Register(new(Service)) + rpc.HandleHTTP() +} + +type Service int + +func (s *Service) GetRepologyProject(args *pacsight.GetRepologyProjectArgs, reply *pacsight.GetRepologyProjectReply) error { + project, err := repology.FindRepologyProject(repology.Packages, args.Filters) + if err != nil { + return errorx.ExternalError.Wrap(err, "pacsight rpc call failed") + } + + reply.Project = project + return nil +} diff --git a/server/log/discord.go b/server/log/discord.go deleted file mode 100644 index 02b1f33c..00000000 --- a/server/log/discord.go +++ /dev/null @@ -1,16 +0,0 @@ -package log - -import ( - "log" - - "github.com/bwmarrin/discordgo" -) - -func connect(token string) *discordgo.Session { - client, err := discordgo.New("Bot " + token) - if err != nil { - log.Fatalf("failed to connect to discord\n%v\n", err) - } - - return client -} diff --git a/server/log/lib.go b/server/log/lib.go deleted file mode 100644 index 189325be..00000000 --- a/server/log/lib.go +++ /dev/null @@ -1,139 +0,0 @@ -package log - -import ( - "fmt" - glog "log" - "os" - - "github.com/bwmarrin/discordgo" - "github.com/fatih/color" - "pacstall.dev/webserver/config" -) - -var ( - logInfo = color.CyanString("INFO") - logError = color.RedString("ERROR") - logDebug = color.GreenString("DEBUG") - logFatal = color.New(color.BgHiRed, color.FgBlack).Sprintf("FATAL") - logWarn = color.YellowString("WARN") -) - -const ( - logDiscordError = "❌ Error ❌" - logDiscordWarn = "⚠ïļ Warning" - logDiscordFatal = "💀â˜ĒïļðŸ’Ĩ Fatal ðŸŠĶ⚰ïļðŸ§Ÿâ€â™‚ïļ" - logDiscordNotify = "ðŸ“Ē Notification" -) - -type tLogLevel uint8 - -var Level = struct { - Info tLogLevel - Error tLogLevel - Debug tLogLevel - Fatal tLogLevel - Warn tLogLevel -}{ - Debug: 0, - Info: 1, - Warn: 2, - Error: 3, - Fatal: 4, -} - -var logLevel = Level.Debug - -func SetLogLevel(level tLogLevel) { - logLevel = level -} - -var logger = glog.New(os.Stdout, "", glog.Ldate|glog.Ltime) - -func doLog(level, message string, args ...any) { - msg := fmt.Sprintf("%s: %s\n", level, fmt.Sprintf(message, args...)) - - if level == logFatal { - logger.Fatal(msg) - } else { - logger.Print(msg) - } -} - -func Info(message string, args ...any) { - if logLevel > Level.Info { - return - } - - doLog(logInfo, message, args...) -} - -func Error(message string, args ...any) { - if logLevel > Level.Error { - return - } - - doLog(logError, message, args...) - go sendDiscordMessage(true, logDiscordError, message, args...) -} - -func Fatal(message string, args ...any) { - if logLevel > Level.Fatal { - return - } - - sendDiscordMessage(true, logDiscordFatal, message, args...) - doLog(logFatal, message, args...) -} - -func Warn(message string, args ...any) { - if logLevel > Level.Warn { - return - } - - doLog(logWarn, message, args...) - go sendDiscordMessage(true, logDiscordWarn, message, args...) -} - -func Debug(message string, args ...any) { - if logLevel > Level.Debug { - return - } - - doLog(logDebug, message, args...) -} - -func Notify(message string, args ...any) { - go sendDiscordMessage(false, logDiscordNotify, message, args...) -} - -func NotifyCustom(level, message string, args ...any) { - go sendDiscordMessage(false, level, message, args...) -} - -func sendDiscordMessage(tag bool, level, message string, args ...any) { - if !config.Discord.Enabled { - return - } - - msg := fmt.Sprintf("Webserver - %s: %s\n", level, fmt.Sprintf(message, args...)) - if tag { - msg = fmt.Sprintf("%s %s", config.Discord.Tags, msg) - } - - _, err := discordClient.ChannelMessageSend( - config.Discord.ChannelID, - msg, - ) - - if err != nil { - panic(fmt.Sprintf("failed to send discord message\n%v", err)) - } -} - -var discordClient = func() *discordgo.Session { - if config.Discord.Enabled { - return connect(config.Discord.Token) - } - - return nil -}() diff --git a/server/types/array/array.go b/server/pkg/common/array/array.go similarity index 100% rename from server/types/array/array.go rename to server/pkg/common/array/array.go diff --git a/server/types/array/array_test.go b/server/pkg/common/array/array_test.go similarity index 100% rename from server/types/array/array_test.go rename to server/pkg/common/array/array_test.go diff --git a/server/types/array/sort.go b/server/pkg/common/array/sort.go similarity index 100% rename from server/types/array/sort.go rename to server/pkg/common/array/sort.go diff --git a/server/pkg/common/config/build/vars.go b/server/pkg/common/config/build/vars.go new file mode 100644 index 00000000..5a88ce0f --- /dev/null +++ b/server/pkg/common/config/build/vars.go @@ -0,0 +1,4 @@ +package build + +var Production = "false" +var Version = "development" diff --git a/server/pkg/common/config/vars.go b/server/pkg/common/config/vars.go new file mode 100644 index 00000000..d184c784 --- /dev/null +++ b/server/pkg/common/config/vars.go @@ -0,0 +1,11 @@ +package config + +import ( + "pacstall.dev/webserver/pkg/common/config/build" + "pacstall.dev/webserver/pkg/common/env" +) + +var Production = build.Production == "true" || build.Production == "1" +var Version = build.Version + +var LogLevel = env.GetEnvStringOrDefault("PACSTALL_LOG_LEVEL", "debug") diff --git a/server/config/util.go b/server/pkg/common/env/util.go similarity index 60% rename from server/config/util.go rename to server/pkg/common/env/util.go index 5d5ec124..051bca49 100644 --- a/server/config/util.go +++ b/server/pkg/common/env/util.go @@ -1,9 +1,10 @@ -package config +package env import ( "fmt" "os" "strconv" + "strings" ) func toInt(str string) int { @@ -59,34 +60,59 @@ func getEnvVar[T any](key string, format formatter[T]) T { panic(fmt.Sprintf("could not find environment variable '%s'", key)) } - return format.Format(val) + formattedValue := format.Format(val) + printEnvVariable(key, formattedValue) + + return formattedValue +} + +func printEnvVariable(key string, value interface{}) { + key = strings.ToUpper(key) + if strings.Contains(key, "PASSWORD") || strings.Contains(key, "PRIVATE") || strings.Contains(key, "SECRET") || strings.Contains(key, "HIDDEN") { + fmt.Printf("env[%s]=%s\n", key, "********") + return + } + + fmt.Printf("env[%s]=%#v\n", key, value) } func getEnvOrDefault[T any](key string, defaultValue T, format formatter[T]) T { val, ok := os.LookupEnv(key) if !ok { + printEnvVariable(key, defaultValue) return defaultValue } - return format.Format(val) + formattedValue := format.Format(val) + printEnvVariable(key, formattedValue) + + return formattedValue } var _stringFormatter formatter[string] = stringFormatter{} var _intFormatter formatter[int] = intFormatter{} var _boolFormatter formatter[bool] = boolFormatter{} -func getEnvString(key string) string { +func GetEnvString(key string) string { return getEnvVar(key, _stringFormatter) } -func getEnvInt(key string) int { +func GetEnvStringOrDefault(key, defaultValue string) string { + return getEnvOrDefault(key, defaultValue, _stringFormatter) +} + +func GetEnvInt(key string) int { return getEnvVar(key, _intFormatter) } -func getEnvBool(key string) bool { +func GetEnvIntOrDefault(key string, defaultValue int) int { + return getEnvOrDefault(key, defaultValue, _intFormatter) +} + +func GetEnvBool(key string) bool { return getEnvVar(key, _boolFormatter) } -func getEnvBoolOrDefault(key string, defaultValue bool) bool { +func GetEnvBoolOrDefault(key string, defaultValue bool) bool { return getEnvOrDefault(key, defaultValue, _boolFormatter) } diff --git a/server/pkg/common/log/lib.go b/server/pkg/common/log/lib.go new file mode 100644 index 00000000..ba9cf1a0 --- /dev/null +++ b/server/pkg/common/log/lib.go @@ -0,0 +1,110 @@ +package log + +import ( + "fmt" + glog "log" + "os" + "strings" + + "github.com/fatih/color" + "pacstall.dev/webserver/pkg/common/config" +) + +var ( + logInfo = color.CyanString("INFO") + logError = color.RedString("ERROR") + logDebug = color.GreenString("DEBUG") + logTrace = color.HiGreenString("TRACE") + logFatal = color.New(color.BgHiRed, color.FgBlack).Sprintf("FATAL") + logWarn = color.YellowString("WARN") +) + +type tLogLevel uint8 + +const ( + _LEVEL_TRACE tLogLevel = iota + _LEVEL_DEBUG + _LEVEL_INFO + _LEVEL_WARN + _LEVEL_ERROR + _LEVEL_FATAL +) + +var logLevels = map[string]tLogLevel{ + "TRACE": _LEVEL_TRACE, + "DEBUG": _LEVEL_DEBUG, + "INFO": _LEVEL_INFO, + "WARN": _LEVEL_WARN, + "ERROR": _LEVEL_ERROR, + "FATAL": _LEVEL_FATAL, +} + +func getLogLevel() tLogLevel { + l, ok := logLevels[strings.ToUpper(config.LogLevel)] + if !ok { + doLog(logWarn, "unknown log level '%s'. defaulting to INFO", config.LogLevel) + return _LEVEL_INFO + } + + return l +} + +var logger = glog.New(os.Stdout, "", glog.Ldate|glog.Ltime) + +func doLog(level, message string, args ...any) { + msg := fmt.Sprintf("%s: %s\n", level, fmt.Sprintf(message, args...)) + + if level == logFatal { + logger.Fatal(msg) + } else { + logger.Print(msg) + } +} + +func Info(message string, args ...any) { + if getLogLevel() > _LEVEL_INFO { + return + } + + doLog(logInfo, message, args...) +} + +func Error(message string, args ...any) { + if getLogLevel() > _LEVEL_ERROR { + return + } + + doLog(logError, message, args...) +} + +func Fatal(message string, args ...any) { + if getLogLevel() > _LEVEL_FATAL { + return + } + + doLog(logFatal, message, args...) +} + +func Warn(message string, args ...any) { + if getLogLevel() > _LEVEL_WARN { + return + } + + doLog(logWarn, message, args...) +} + +func Debug(message string, args ...any) { + if getLogLevel() > _LEVEL_DEBUG { + return + } + + doLog(logDebug, message, args...) +} + +func Trace(message string, args ...any) { + if getLogLevel() > _LEVEL_TRACE { + return + } + + doLog(logTrace, message, args...) +} diff --git a/server/pkg/common/pacsight/rpc.go b/server/pkg/common/pacsight/rpc.go new file mode 100644 index 00000000..37a820d9 --- /dev/null +++ b/server/pkg/common/pacsight/rpc.go @@ -0,0 +1,42 @@ +package pacsight + +import ( + "net/rpc" + "strconv" + + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/pkg/common/types" +) + +type GetRepologyProjectArgs struct { + Filters []string +} + +type GetRepologyProjectReply struct { + Project types.RepologyApiProject +} + +type PacsightRpcService struct { + client *rpc.Client +} + +func NewPacsightRpcService(addr string, port int) (*PacsightRpcService, error) { + if !config.Repology.Enabled { + return nil, nil + } + + client, err := rpc.DialHTTP("tcp", addr+":"+strconv.FormatInt(int64(port), 10)) + return &PacsightRpcService{client}, err +} + +func (s *PacsightRpcService) GetRepologyProject(filters []string) (types.RepologyApiProject, error) { + args := &GetRepologyProjectArgs{Filters: filters} + reply := &GetRepologyProjectReply{} + + err := s.client.Call("Service.GetRepologyProject", args, reply) + if err != nil { + return types.RepologyApiProject{}, err + } + + return reply.Project, nil +} diff --git a/server/types/pac/parser/parallelism/batch/run.go b/server/pkg/common/parallelism/batch/run.go similarity index 93% rename from server/types/pac/parser/parallelism/batch/run.go rename to server/pkg/common/parallelism/batch/run.go index 95adb852..bf071c8a 100644 --- a/server/types/pac/parser/parallelism/batch/run.go +++ b/server/pkg/common/parallelism/batch/run.go @@ -4,7 +4,7 @@ import ( "sync/atomic" "time" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) func Run[T any, E any](batchSize int, items []T, fn func(T) (E, error)) <-chan E { diff --git a/server/types/pac/parser/parallelism/batch/run_test.go b/server/pkg/common/parallelism/batch/run_test.go similarity index 72% rename from server/types/pac/parser/parallelism/batch/run_test.go rename to server/pkg/common/parallelism/batch/run_test.go index 7268e397..88d069ca 100644 --- a/server/types/pac/parser/parallelism/batch/run_test.go +++ b/server/pkg/common/parallelism/batch/run_test.go @@ -4,9 +4,9 @@ import ( "testing" "time" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac/parser/parallelism/batch" - "pacstall.dev/webserver/types/pac/parser/parallelism/channels" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/batch" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/channels" + "pacstall.dev/webserver/pkg/common/array" ) func Test_Batch_Run(t *testing.T) { diff --git a/server/types/pac/parser/parallelism/channels/exhaust.go b/server/pkg/common/parallelism/channels/exhaust.go similarity index 100% rename from server/types/pac/parser/parallelism/channels/exhaust.go rename to server/pkg/common/parallelism/channels/exhaust.go diff --git a/server/types/pac/parser/parallelism/channels/to_slice.go b/server/pkg/common/parallelism/channels/to_slice.go similarity index 100% rename from server/types/pac/parser/parallelism/channels/to_slice.go rename to server/pkg/common/parallelism/channels/to_slice.go diff --git a/server/types/pac/parser/parallelism/channels/to_slice_test.go b/server/pkg/common/parallelism/channels/to_slice_test.go similarity index 100% rename from server/types/pac/parser/parallelism/channels/to_slice_test.go rename to server/pkg/common/parallelism/channels/to_slice_test.go diff --git a/server/types/pac/parser/parallelism/timeout/timeout.go b/server/pkg/common/parallelism/timeout/timeout.go similarity index 100% rename from server/types/pac/parser/parallelism/timeout/timeout.go rename to server/pkg/common/parallelism/timeout/timeout.go diff --git a/server/types/equals.go b/server/pkg/common/types/equals.go similarity index 100% rename from server/types/equals.go rename to server/pkg/common/types/equals.go diff --git a/server/types/percentage.go b/server/pkg/common/types/percentage.go similarity index 100% rename from server/types/percentage.go rename to server/pkg/common/types/percentage.go diff --git a/server/types/pkgtype.go b/server/pkg/common/types/pkgtype.go similarity index 100% rename from server/types/pkgtype.go rename to server/pkg/common/types/pkgtype.go diff --git a/server/repology/internal/api_types.go b/server/pkg/common/types/repology.go similarity index 97% rename from server/repology/internal/api_types.go rename to server/pkg/common/types/repology.go index 57d48214..89bd5a81 100644 --- a/server/repology/internal/api_types.go +++ b/server/pkg/common/types/repology.go @@ -1,4 +1,4 @@ -package internal +package types type RepologyApiProject struct { Repository string `json:"repo"` diff --git a/server/repology/exporter.go b/server/repology/exporter.go deleted file mode 100644 index 177add9f..00000000 --- a/server/repology/exporter.go +++ /dev/null @@ -1,5 +0,0 @@ -package repology - -import "pacstall.dev/webserver/repology/internal" - -var ExportRepologyDatabase = internal.ExportRepologyDatabase diff --git a/server/repology/fetch.go b/server/repology/fetch.go deleted file mode 100644 index 71b29024..00000000 --- a/server/repology/fetch.go +++ /dev/null @@ -1,78 +0,0 @@ -package repology - -import ( - "fmt" - "strings" - - "github.com/joomcode/errorx" - "pacstall.dev/webserver/model" - "pacstall.dev/webserver/types/array" -) - -func parseRepologyFilter(filter string) (string, string) { - idx := strings.Index(filter, ":") - return strings.TrimSpace(filter[:idx]), strings.TrimSpace(filter[idx+1:]) -} - -var repologyFilterToColumn = map[string]string{ - "repo": model.RepologyProjectProviderColumns.Repository, - "subrepo": model.RepologyProjectProviderColumns.SubRepository, - "status": model.RepologyProjectProviderColumns.Status, - "srcname": model.RepologyProjectProviderColumns.SourceName, - "binname": model.RepologyProjectProviderColumns.BinaryName, - "version": model.RepologyProjectProviderColumns.Version, - "origversion": model.RepologyProjectProviderColumns.OriginalVersion, - "visiblename": model.RepologyProjectProviderColumns.VisibleName, - "summary": model.RepologyProjectProviderColumns.Summary, -} - -func findRepologyProject(search []string) (model.RepologyProjectProvider, error) { - var result model.RepologyProjectProvider - - if len(search) == 0 { - return result, errorx.IllegalArgument.New("no search terms provided") - } - - db := model.Instance() - _, projectName := parseRepologyFilter(search[0]) - - query := db.Where("project_name = ?", projectName).Where(fmt.Sprintf("%v = ?", model.RepologyProjectProviderColumns.Active), true) - for _, filter := range search[1:] { - filterName, filterValue := parseRepologyFilter(filter) - column, ok := repologyFilterToColumn[filterName] - if !ok { - return result, errorx.IllegalArgument.New("invalid filter '%v'", filterName) - } - - query = query.Where(fmt.Sprintf("%v = ?", column), filterValue) - } - - var results []model.RepologyProjectProvider - if err := query.Order("version desc").Find(&results).Error; err != nil || len(results) == 0 { - return result, errorx.Decorate(err, "failed to fetch repology project") - } - - results = sortByStatus(results) - result = results[0] - - return result, nil -} - -var repologyStatusPriority = map[string]int{ - "newest": 0, - "rolling": 1, - "devel": 3, - "legacy": 4, - "outdated": 5, - "unique": 6, - "noscheme": 7, - "incorrect": 7, - "untrusted": 7, - "ignored": 7, -} - -func sortByStatus(projects []model.RepologyProjectProvider) []model.RepologyProjectProvider { - return array.SortBy(array.Clone(projects), func(p1, p2 model.RepologyProjectProvider) bool { - return repologyStatusPriority[p1.Status] < repologyStatusPriority[p2.Status] - }) -} diff --git a/server/repology/internal/exporter.go b/server/repology/internal/exporter.go deleted file mode 100644 index a55ecd02..00000000 --- a/server/repology/internal/exporter.go +++ /dev/null @@ -1,186 +0,0 @@ -package internal - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/joomcode/errorx" - "gorm.io/gorm" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/model" -) - -const RETRY_COUNT = 5 - -func ExportRepologyDatabase(db *gorm.DB) error { - err := migrateTables(db) - if err != nil { - return errors.Join(errors.New("failed to reset repology tables"), err) - } - - it := 1 - lastProjectName := "" - - const REPOLOGY_PROJECT_FETCH_THROTTLE = time.Second - - lastRepoFetch := time.Now() - - for { - if time.Since(lastRepoFetch) < REPOLOGY_PROJECT_FETCH_THROTTLE { - time.Sleep(REPOLOGY_PROJECT_FETCH_THROTTLE - time.Since(lastRepoFetch)) - } - - log.Debug("page %v | cursor at: %v", it, lastProjectName) - - var projectPage map[string][]RepologyApiProject - var err error - - retry: - for i := 0; i < RETRY_COUNT; i += 1 { - projectPage, err = getProjectSearch(lastProjectName) - if err == nil { - break retry - } - - retryDelay := time.Duration(i+1) * REPOLOGY_PROJECT_FETCH_THROTTLE - log.Debug("failed to fetch repology project page '%s'. retrying in %v", lastProjectName, retryDelay) - time.Sleep(retryDelay) - } - - if err != nil { - return errorx.Decorate(err, "failed to fetch repology project page '%s'", lastProjectName) - } - - lastRepoFetch = time.Now() - - var projects []model.RepologyProject - var projectProviders []model.RepologyProjectProvider - for projectName, apiProjectProvider := range projectPage { - - lastProjectName = identityOrSkipProject(projectName) - for _, apiProjectProvider := range apiProjectProvider { - // Save project provider as inactive - projectProvider := mapRepologyApiProjectProviderToModel(projectName, apiProjectProvider) - projectProviders = append(projectProviders, projectProvider) - - project := model.RepologyProject{ - Name: projectName, - } - - projects = append(projects, project) - } - } - - if len(projects) <= 1 { - break - } - - err = db.Save(&projects).Error - if err != nil { - return errors.Join(errors.New("failed to create repology projects"), err) - } - - err = db.CreateInBatches(&projectProviders, 90).Error - if err != nil { - return errors.Join(errors.New("failed to create repology project providers"), err) - } - - it++ - } - - // Delete active (old) repology project providers - if err := db.Debug().Where(fmt.Sprintf("%v = ?", model.RepologyProjectProviderColumns.Active), true).Delete(&model.RepologyProjectProvider{}).Error; err != nil { - return errors.Join(errors.New("failed to delete old repology project providers"), err) - } - - // Mark new repology project providers as active - if err := db.Debug().Exec( - fmt.Sprintf( - "UPDATE %s SET %s = 1", - model.RepologyProjectProviderTableName, - model.RepologyProjectProviderColumns.Active, - ), - ).Error; err != nil { - return errors.Join(errors.New("failed to update new repology project providers"), err) - } - - return nil -} - -var projectNamesToSkipToNextCussor = map[string]string{ - "emacs:": "emacsa", - "go:": "goa", - "haskell:": "haskella", - "lisp:": "lispa", - "node:": "nodea", - "ocaml:": "ocamla", - "perl:": "perla", - "php:": "phpa", - "python:": "pythona", - "r:": "ra", - "ruby:": "rubya", - "rust:": "rusta", - "texlive:": "texlivea", -} - -func identityOrSkipProject(name string) string { - for prefix, skipTo := range projectNamesToSkipToNextCussor { - if strings.HasPrefix(name, prefix) { - return skipTo - } - } - - return name -} - -func migrateTables(db *gorm.DB) error { - err := db.AutoMigrate(&model.RepologyProject{}) - if err != nil { - return err - } - - if err = truncateTable(db, model.RepologyProjectTableName); err != nil { - return err - } - - err = db.AutoMigrate(&model.RepologyProjectProvider{}) - if err != nil { - return err - } - - if err = truncateTable(db, model.RepologyProjectProviderTableName); err != nil { - return err - } - - return nil -} - -func truncateTable(db *gorm.DB, tableName string) error { - log.Debug("attempting to truncate table %v", tableName) - err := db.Exec("TRUNCATE TABLE " + tableName).Error - if err != nil { - return errors.Join(fmt.Errorf("failed to truncate table %v", tableName), err) - } - - log.Info("successfully truncated table %v", tableName) - return nil -} - -func mapRepologyApiProjectProviderToModel(projectName string, apiProjectProvider RepologyApiProject) model.RepologyProjectProvider { - projectProvider := model.RepologyProjectProvider{ - ProjectName: projectName, - Repository: apiProjectProvider.Repository, - SubRepository: apiProjectProvider.SubRepository, - SourceName: apiProjectProvider.SourceName, - VisibleName: apiProjectProvider.VisibleName, - BinaryName: apiProjectProvider.BinaryName, - Version: apiProjectProvider.Version, - OriginalVersion: apiProjectProvider.OriginalVersion, - Status: apiProjectProvider.Status, - Summary: apiProjectProvider.Summary, - Active: false, - } - return projectProvider -} diff --git a/server/repology/lib.go b/server/repology/lib.go deleted file mode 100644 index 9153cdc1..00000000 --- a/server/repology/lib.go +++ /dev/null @@ -1,18 +0,0 @@ -package repology - -import ( - "pacstall.dev/webserver/types/pac" -) - -func Sync(script *pac.Script) error { - if len(script.Repology) == 0 { - return nil - } - - project, err := findRepologyProject(script.Repology) - if err != nil { - return err - } - - return updateScriptVersion(project, script) -} diff --git a/server/repology/scheduler.go b/server/repology/scheduler.go deleted file mode 100644 index b917befb..00000000 --- a/server/repology/scheduler.go +++ /dev/null @@ -1,25 +0,0 @@ -package repology - -import ( - "time" - - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/model" -) - -func ScheduleRefresh(every time.Duration) { - db := model.Instance() - go func() { - - for { - log.Info("refreshing Repology database...") - err := ExportRepologyDatabase(db) - if err != nil { - log.Error("failed to export Repology projects: %+v", err) - } else { - log.Info("repology database refreshed successfully") - } - time.Sleep(every) - } - }() -} diff --git a/server/types/pac/parser/parse_test.go b/server/types/pac/parser/parse_test.go deleted file mode 100644 index 984664ce..00000000 --- a/server/types/pac/parser/parse_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package parser_test - -import ( - "encoding/json" - "fmt" - "os" - "path" - "testing" - - "pacstall.dev/webserver/types" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/parser" - "pacstall.dev/webserver/types/pac/parser/pacsh" -) - -var FIXTURES_DIR = func() string { - dir, err := os.Getwd() - if err != nil { - panic(err) - } - - return path.Join(dir, "../../../fixtures") -}() - -var TEST_PROGRAMS_DIR = path.Join(FIXTURES_DIR, "test-programs") - -func assertEquals(t *testing.T, what string, expected interface{}, actual interface{}) { - if actual != expected { - t.Errorf("pacscript.%v: expected '%#v', got '%#v'", what, expected, actual) - } -} - -func assertArrayEquals[T types.Equaller](t *testing.T, what string, expected []T, actual []T) { - if len(actual) == len(expected) && len(actual) == 0 { - return - } - - if len(actual) != len(expected) { - t.Errorf("pacscript.%v expected len '%v', got len '%v' (expected '%#v', got '%#v')", what, len(expected), len(actual), expected, actual) - return - } - - for idx := range expected { - if !expected[idx].Equals(actual[idx]) { - t.Errorf("pacscript.%v[%v] expected '%#v', got '%#v'", what, idx, expected, actual) - } - } -} - -func assertStringArrayEquals(t *testing.T, what string, expected []string, actual []string) { - if len(actual) == len(expected) && len(actual) == 0 { - return - } - - if len(actual) != len(expected) { - t.Errorf("pacscript.%v expected len '%v', got len '%v' (expected '%#v', got '%#v')", what, len(expected), len(actual), expected, actual) - return - } - - for idx := range expected { - if expected[idx] != actual[idx] { - t.Errorf("pacscript.%v[%v] expected '%#v', got '%#v'", what, idx, expected, actual) - } - } -} - -func assertPacscriptEquals(t *testing.T, expected *pac.Script, actual *pac.Script) { - assertEquals(t, "package name", expected.PackageName, actual.PackageName) - assertStringArrayEquals(t, "maintainers", expected.Maintainers, actual.Maintainers) - assertEquals(t, "description", expected.Description, actual.Description) - assertEquals(t, "gives", expected.Gives, actual.Gives) - assertEquals(t, "version", expected.Version, actual.Version) - assertArrayEquals(t, "breaks", expected.Breaks, actual.Breaks) - assertArrayEquals(t, "conflicts", expected.Conflicts, actual.Conflicts) - assertArrayEquals(t, "replaces", expected.Replaces, actual.Replaces) - assertEquals(t, "pretty name", expected.PrettyName, actual.PrettyName) - assertArrayEquals(t, "sources", expected.Source, actual.Source) - assertArrayEquals(t, "runtime dependencies", expected.RuntimeDependencies, actual.RuntimeDependencies) - assertArrayEquals(t, "build dependencies", expected.BuildDependencies, actual.BuildDependencies) - assertArrayEquals(t, "optional dependencies", expected.OptionalDependencies, actual.OptionalDependencies) - assertArrayEquals(t, "pacstall dependencies", expected.PacstallDependencies, actual.PacstallDependencies) - assertStringArrayEquals(t, "required by", expected.RequiredBy, actual.RequiredBy) - assertStringArrayEquals(t, "repology", expected.Repology, actual.Repology) - assertEquals(t, "update status", expected.UpdateStatus, actual.UpdateStatus) -} - -func loadSnapshot(snapshotPath string) (*pac.Script, error) { - bytes, err := os.ReadFile(snapshotPath) - if err != nil { - return nil, err - } - - var out pac.Script - if err := json.Unmarshal(bytes, &out); err != nil { - return nil, err - } - - return &out, nil -} - -func assertPacscriptMatchesSnapshot(t *testing.T, pkgname string) { - t.Helper() - - if pacsh.CreateTempDirectory("./tmp") != nil { - t.Errorf("failed to create temp directory") - return - } - - actual, err := parser.ParsePacscriptFile(TEST_PROGRAMS_DIR, pkgname) - if err != nil { - t.Errorf("expected no error, got %v", err) - return - } - - snapshotPath := path.Join(TEST_PROGRAMS_DIR, "packages", pkgname, fmt.Sprintf("%v.snapshot.json", pkgname)) - expected, err := loadSnapshot(snapshotPath) - if err != nil { - bytes, err := json.Marshal(actual) - if err != nil { - t.Errorf("failed to serialize snapshot. %v", err) - return - } - - if err := os.WriteFile(snapshotPath, bytes, 0644); err != nil { - t.Errorf("failed to write snapshot. %v", err) - return - } - - t.Errorf("missing snapshot. a new one has been generated. rerun tests") - return - } - - assertPacscriptEquals(t, expected, actual) -} - -func Test_PacscriptSnapshots(t *testing.T) { - dirEntries, err := os.ReadDir(path.Join(TEST_PROGRAMS_DIR, "packages")) - if err != nil { - t.Errorf("failed to read test packages. %v", err) - return - } - - for _, dirEntry := range dirEntries { - if !dirEntry.IsDir() { - continue - } - - t.Logf("==> Running snapshot test for file: %v", dirEntry.Name()) - assertPacscriptMatchesSnapshot(t, dirEntry.Name()) - } -} diff --git a/server/types/pac/parser/scheduler.go b/server/types/pac/parser/scheduler.go deleted file mode 100644 index 9edd7925..00000000 --- a/server/types/pac/parser/scheduler.go +++ /dev/null @@ -1,27 +0,0 @@ -package parser - -import ( - "time" - - "pacstall.dev/webserver/log" -) - -func ScheduleRefresh(every time.Duration) { - go refresh(every) -} - -func refresh(every time.Duration) { - for { - err := ParseAll() - if err != nil { - log.Error("parse error: %+v", err) - - retryIn := time.Second * 30 - log.Info("retrying in %v", retryIn) - time.Sleep(retryIn) - continue - } - - time.Sleep(every) - } -}