From 567187951cc6c3912c8f54b68f9ef7a361fb4d1a Mon Sep 17 00:00:00 2001 From: Krishan Thisera Date: Sun, 12 Nov 2023 20:19:17 +1030 Subject: [PATCH] use S3 as the backend --- backend/filesystem.go | 4 - backend/models.go | 12 ++ backend/s3.go | 43 ++++- go.mod | 34 +++- go.sum | 60 +++++- output.txt | 432 ++++++++++++++++++++++++++++++++++++++++++ pilot/models.go | 19 +- pilot/pilot.go | 13 +- pilot/render-cache.go | 15 +- pilot/utils.go | 105 ++++++++++ render/models.go | 6 +- render/render.go | 5 +- 12 files changed, 701 insertions(+), 47 deletions(-) create mode 100644 output.txt create mode 100644 pilot/utils.go diff --git a/backend/filesystem.go b/backend/filesystem.go index 8f7225b..1c2c588 100644 --- a/backend/filesystem.go +++ b/backend/filesystem.go @@ -5,10 +5,6 @@ import ( "path/filepath" ) -type FileSystem struct { - BaseDir string -} - func (f FileSystem) Put(u string, data []byte) error { p, err := generateRelativePath(u) if err != nil { diff --git a/backend/models.go b/backend/models.go index b6c2576..58cc444 100644 --- a/backend/models.go +++ b/backend/models.go @@ -1,6 +1,18 @@ package backend +import "github.com/aws/aws-sdk-go-v2/service/s3" + type Backend interface { Put(url string, data []byte) error Get(url string) ([]byte, error) } + +type S3 struct { + BucketName string `yaml:"bucketName"` + Region string `yaml:"region"` + S3Client *s3.Client `yaml:"-"` +} + +type FileSystem struct { + BaseDir string `yaml:"baseDir"` +} diff --git a/backend/s3.go b/backend/s3.go index 7fc8d72..771bd53 100644 --- a/backend/s3.go +++ b/backend/s3.go @@ -1,17 +1,42 @@ package backend -import "fmt" +import ( + "bytes" + "context" + "fmt" + "io" -type S3 struct { - BucketName string -} + "github.com/aws/aws-sdk-go-v2/service/s3" +) -func (s S3) Put(url string, data []byte) error { - fmt.Println(string(data)) +func (b S3) Put(url string, data []byte) error { + _, err := b.S3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: &b.BucketName, + Key: &url, + Body: bytes.NewReader(data), + }) + if err != nil { + fmt.Println(err) + return err + } return nil } -func (s S3) Get(url string) ([]byte, error) { - fmt.Printf("Getting %s to S3", s.BucketName) - return nil, nil +func (b S3) Get(url string) ([]byte, error) { + res, err := b.S3Client.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: &b.BucketName, + Key: &url, + }) + + if err != nil { + return nil, err + } + + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + return body, nil } diff --git a/go.mod b/go.mod index cbd7a62..9222beb 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,35 @@ module github.com/krishanthisera/grender go 1.21.3 require ( + github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 + github.com/chromedp/cdproto v0.0.0-20231025043423-5615e204d422 + github.com/chromedp/chromedp v0.9.3 + github.com/gin-gonic/gin v1.9.1 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.22.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/config v1.23.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect + github.com/aws/smithy-go v1.16.0 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/chromedp/cdproto v0.0.0-20231025043423-5615e204d422 // indirect - github.com/chromedp/chromedp v0.9.3 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect @@ -30,10 +51,11 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.13.0 // indirect google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 194bc28..cd92ef7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,39 @@ +github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI= +github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ= +github.com/aws/aws-sdk-go-v2/config v1.23.0 h1:kqzEfGGDIrRJpfJckgwuZfFTbU9NB1jZnRcaO9MpOqE= +github.com/aws/aws-sdk-go-v2/config v1.23.0/go.mod h1:p7wbxKXXjS1GGQOss7VXOazVMFF9bjUGq85/4wR/fSw= +github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI= +github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c= +github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0 h1:hwZB07/beLiCopuRKF0t+dEHmP39iN4YtDh3X5d3hrg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0/go.mod h1:rdAuXeHWhI/zkpYcO5n8WCpaIgY9MUxFyBsuqq3kjyA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4= +github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= +github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -12,6 +48,7 @@ github.com/chromedp/chromedp v0.9.3/go.mod h1:NipeUkUcuzIdFbBP8eNNvl9upcceOfWzoJ github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= @@ -19,6 +56,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -35,6 +74,8 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -43,6 +84,7 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= @@ -55,9 +97,11 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -68,6 +112,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= @@ -76,21 +121,24 @@ github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/output.txt b/output.txt new file mode 100644 index 0000000..fe0916e --- /dev/null +++ b/output.txt @@ -0,0 +1,432 @@ +package pilot + +import ( + "github.com/krishanthisera/grender/backend" +) + +func (c *renderAndCacheConfig) RenderAndCache(url string) ([]byte, error) { + + res, err := backend.Backend.Get(*c.backend, url) + + // If errored te app must render the page on the fly + if err != nil { + page, err := c.render.Render(url) + if err != nil { + return nil, err + } + // If the page is rendered successfully, save it to the backend + if err := backend.Backend.Put(*c.backend, url, []byte(*page)); err != nil { + return []byte(*page), err + } + return []byte(*page), nil + } + + return res, nil +} +package pilot + +import ( + "github.com/krishanthisera/grender/backend" + "github.com/krishanthisera/grender/render" +) + +// Config struct to represent the overall YAML configuration +type Config struct { + Version string `yaml:"version"` + RenderingConfig render.Config `yaml:"renderingConfig"` + Server struct { + Port string `yaml:"port"` + } `yaml:"server"` + Backend struct { + S3 backend.S3 `yaml:"s3"` + FileSystem backend.FileSystem `yaml:"fileSystem"` + } `yaml:"backend"` +} + +type renderAndCacheConfig struct { + backend *backend.Backend + render *render.Config +} +package pilot + +import ( + "context" + "fmt" + "os" + + aws "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/krishanthisera/grender/backend" + "gopkg.in/yaml.v2" +) + +func createBackendFromConfig(backendConfig interface{}) (backend.Backend, error) { + switch b := backendConfig.(type) { + case backend.S3: + cfg, err := aws.LoadDefaultConfig(context.TODO(), aws.WithRegion(fmt.Sprintf(b.Region))) + bucket := backend.S3{BucketName: "grender.io", S3Client: s3.NewFromConfig(cfg)} + return &bucket, err + case backend.FileSystem: + fs := backend.FileSystem{BaseDir: b.BaseDir} + return &fs, nil + default: + + panic(fmt.Sprintf("Unknown backend: %T", b)) + } +} + +// TO DO: Refactor this to use the backend package +// func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { +// var buf map[string]interface{} +// err := unmarshal(&buf) +// if err != nil { +// return err +// } + +// c.Version = buf["version"].(string) + +// renderingConfig, ok := buf["renderingConfig"].(map[interface{}]interface{}) +// if !ok { +// return errors.New("renderingConfig field not found or is not a map") +// } + +// renderingConfigData, err := yaml.Marshal(renderingConfig) +// if err != nil { +// return err +// } +// err = yaml.Unmarshal(renderingConfigData, &c.RenderingConfig) +// if err != nil { +// return err +// } + +// server, ok := buf["server"].(map[interface{}]interface{}) +// if !ok { +// return errors.New("server field not found or is not a map") +// } + +// c.Server.Port, ok = server["port"].(string) +// if !ok { +// return errors.New("port field not found or is not a string") +// } + +// backend, ok := buf["backend"].(map[interface{}]interface{}) +// if !ok { +// return errors.New("backend field not found or is not a map") +// } + +// if s3, ok := backend["s3"].(map[interface{}]interface{}); ok { +// s3Data, err := yaml.Marshal(s3) +// if err != nil { +// return err +// } +// err = yaml.Unmarshal(s3Data, &c.Backend.S3) +// if err != nil { +// return err +// } + +// } else if fs, ok := backend["fileSystem"].(map[interface{}]interface{}); ok { +// fsData, err := yaml.Marshal(fs) +// if err != nil { +// return err +// } +// err = yaml.Unmarshal(fsData, &c.Backend.FileSystem) +// if err != nil { +// return err +// } +// } else { +// return errors.New("unknown backend type") +// } + +// return nil +// } + +func GenerateConfig(path string) (*Config, error) { + yamlData, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var config Config + err = yaml.Unmarshal(yamlData, &config) + if err != nil { + return nil, fmt.Errorf("error unmarshalling YAML: %v", err) + } + return &config, nil +} +package pilot + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" +) + +func (C *Config) Grender() { + + router := gin.Default() + router.GET("/render/*url", func(c *gin.Context) { + url := c.Param("url") + fmt.Println(url) + + be, err := createBackendFromConfig(C.Backend.S3) + if err != nil { + fmt.Println(err) + } + rac := renderAndCacheConfig{backend: &be, render: &C.RenderingConfig} + renderedHTML, err := rac.RenderAndCache(url) + + // Page is rendered successfully + if renderedHTML != nil { + c.Data(http.StatusOK, "text/html", []byte(renderedHTML)) + // Page cannot be cached + if err != nil { + c.String(http.StatusInternalServerError, "Error caching URL: %v", err) + return + } + return + } else { + // Page cannot be rendered + c.String(http.StatusInternalServerError, "Error rendering URL: %v", err) + return + } + + }) + + router.Run(fmt.Sprintf(":%s", C.Server.Port)) +} +package main + +import ( + "fmt" + + "github.com/krishanthisera/grender/pilot" +) + +func main() { + fmt.Println("Grender is starting...") + grenderConfig, err := pilot.GenerateConfig("grender.yaml") + fmt.Print(grenderConfig) + if err != nil { + panic(err) + } + grenderConfig.Grender() +} +package backend + +import "github.com/aws/aws-sdk-go-v2/service/s3" + +type Backend interface { + Put(url string, data []byte) error + Get(url string) ([]byte, error) +} + +type S3 struct { + BucketName string `yaml:"bucketName"` + Region string `yaml:"region"` + S3Client *s3.Client `yaml:"-"` +} + +type FileSystem struct { + BaseDir string `yaml:"baseDir"` +} +package backend + +import ( + "net/url" + "path" + "path/filepath" + "strings" +) + +func generateRelativePath(u string) (string, error) { + url, err := url.Parse(u) + if err != nil { + return "", err + } + path := path.Join(url.Host, url.Path) + + return strings.TrimSuffix(path, filepath.Ext(path)), nil +} +package backend + +type Redis struct { + Chanel string +} + +func (r Redis) Put() error { + // Redis put logic + return nil +} + +func (r Redis) Get() ([]byte, error) { + // Redis get logic + return nil, nil +} +package backend + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +func (b S3) Put(url string, data []byte) error { + _, err := b.S3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: &b.BucketName, + Key: &url, + Body: bytes.NewReader(data), + }) + if err != nil { + fmt.Println(err) + return err + } + return nil +} + +func (b S3) Get(url string) ([]byte, error) { + res, err := b.S3Client.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: &b.BucketName, + Key: &url, + }) + + if err != nil { + return nil, err + } + + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + return body, nil +} +package backend + +import ( + "os" + "path/filepath" +) + +func (f FileSystem) Put(u string, data []byte) error { + p, err := generateRelativePath(u) + if err != nil { + return err + } + + // Getting directory path + dir := filepath.Dir(filepath.Join(f.BaseDir, p)) + os.MkdirAll(dir, os.ModePerm) + + // Writing file to the system + file, err := os.Create(filepath.Join(f.BaseDir, p) + ".html") + if err != nil { + return err + } + + _, err = file.Write(data) + if err != nil { + return err + } + + defer file.Close() + return nil + +} + +func (f FileSystem) Get(url string) ([]byte, error) { + // FileSystem get logic + p, err := generateRelativePath(url) + if err != nil { + return nil, err + } + + b, err := os.ReadFile(filepath.Join(f.BaseDir, p) + ".html") + if err != nil { + return nil, err + } + return b, nil +} +package render + +type Config struct { + PageWaitTime float32 `yaml:"pageWaitTime"` // Seconds + PageWailCondition string `yaml:"pageWaitCondition"` +} +package render + +import ( + "context" + "fmt" + "log" + "net/url" + "time" + + "github.com/chromedp/cdproto/dom" + "github.com/chromedp/chromedp" +) + +func (config Config) Render(webAddr string) (*string, error) { + // Validate the URL + webAddr = func(val string) string { + u, err := url.Parse(val) + if err != nil { + panic(err) + } + if u.Scheme == "" { + u.Scheme = "https" + } + return u.String() + }(webAddr) + + // Create a new Chrome headless instance + ctx, cancel := chromedp.NewContext( + context.Background(), + chromedp.WithLogf(log.Printf), + ) + defer cancel() + + var html string + if err := chromedp.Run(ctx, pageRender(webAddr, config.PageWailCondition, time.Duration(config.PageWaitTime*float32(time.Second)), &html)); err != nil { + return nil, err + } + return &html, nil +} + +// This is the function that does the actual rendering +func pageRender(webAddr string, waitCondition string, pageWaitTime time.Duration, html *string) chromedp.Tasks { + return chromedp.Tasks{ + chromedp.Navigate(webAddr), + chromedp.ActionFunc(func(ctx context.Context) error { + // Wait Condition + + var result bool + startTime := time.Now() + for time.Since(startTime) < pageWaitTime { + if err := chromedp.Evaluate(waitCondition, &result).Do(ctx); err != nil { + return err + } + if result { + break // The condition is met, exit the loop. + } + + // Sleep for a short duration before re-evaluating the condition. + time.Sleep(500 * time.Millisecond) + } + + if !result { + return fmt.Errorf("timeout [%v] waiting for window.prerenderReady to become true", pageWaitTime) + } + + node, err := dom.GetDocument().Do(ctx) + if err != nil { + return err + } + *html, err = dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx) + return err + }), + } +} diff --git a/pilot/models.go b/pilot/models.go index 126125b..a394bba 100644 --- a/pilot/models.go +++ b/pilot/models.go @@ -1,9 +1,24 @@ package pilot import ( + "github.com/krishanthisera/grender/backend" "github.com/krishanthisera/grender/render" ) -type Pilot struct { - RenderConfigs render.RenderingConfigs `yaml:"rendering_configs"` +// Config struct to represent the overall YAML configuration +type Config struct { + Version string `yaml:"version"` + RenderingConfig render.Config `yaml:"renderingConfig"` + Server struct { + Port string `yaml:"port"` + } `yaml:"server"` + Backend struct { + S3 backend.S3 `yaml:"s3"` + FileSystem backend.FileSystem `yaml:"fileSystem"` + } `yaml:"backend"` +} + +type renderAndCacheConfig struct { + backend *backend.Backend + render *render.Config } diff --git a/pilot/pilot.go b/pilot/pilot.go index 15a3432..7d29b9c 100644 --- a/pilot/pilot.go +++ b/pilot/pilot.go @@ -7,12 +7,19 @@ import ( "github.com/gin-gonic/gin" ) -func Grender() { +func (C *Config) Grender() { + router := gin.Default() router.GET("/render/*url", func(c *gin.Context) { url := c.Param("url") fmt.Println(url) - renderedHTML, err := GetPages(url) + + be, err := createBackendFromConfig(C.Backend.S3) + if err != nil { + fmt.Println(err) + } + rac := renderAndCacheConfig{backend: &be, render: &C.RenderingConfig} + renderedHTML, err := rac.RenderAndCache(url) // Page is rendered successfully if renderedHTML != nil { @@ -31,5 +38,5 @@ func Grender() { }) - router.Run(":8080") + router.Run(fmt.Sprintf(":%s", C.Server.Port)) } diff --git a/pilot/render-cache.go b/pilot/render-cache.go index f4765f2..bc139b5 100644 --- a/pilot/render-cache.go +++ b/pilot/render-cache.go @@ -2,27 +2,20 @@ package pilot import ( "github.com/krishanthisera/grender/backend" - "github.com/krishanthisera/grender/render" ) -func GetPages(url string) ([]byte, error) { - // GetPages logic - fs := backend.FileSystem{BaseDir: "/tmp/"} +func (c *renderAndCacheConfig) RenderAndCache(url string) ([]byte, error) { - res, err := backend.Backend.Get(fs, url) - - pageWaitCondition := `(function() { - return window.prerenderReady === true; - })()` + res, err := backend.Backend.Get(*c.backend, url) // If errored te app must render the page on the fly if err != nil { - page, err := render.RenderingConfigs{PageWailCondition: pageWaitCondition, PageWaitTime: 16}.Render(url) + page, err := c.render.Render(url) if err != nil { return nil, err } // If the page is rendered successfully, save it to the backend - if err := backend.Backend.Put(fs, url, []byte(*page)); err != nil { + if err := backend.Backend.Put(*c.backend, url, []byte(*page)); err != nil { return []byte(*page), err } return []byte(*page), nil diff --git a/pilot/utils.go b/pilot/utils.go new file mode 100644 index 0000000..4e1b316 --- /dev/null +++ b/pilot/utils.go @@ -0,0 +1,105 @@ +package pilot + +import ( + "context" + "fmt" + "os" + + aws "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/krishanthisera/grender/backend" + "gopkg.in/yaml.v2" +) + +func createBackendFromConfig(backendConfig interface{}) (backend.Backend, error) { + switch b := backendConfig.(type) { + case backend.S3: + cfg, err := aws.LoadDefaultConfig(context.TODO(), aws.WithRegion(fmt.Sprintf(b.Region))) + bucket := backend.S3{BucketName: "grender.io", S3Client: s3.NewFromConfig(cfg)} + return &bucket, err + case backend.FileSystem: + fs := backend.FileSystem{BaseDir: b.BaseDir} + return &fs, nil + default: + + panic(fmt.Sprintf("Unknown backend: %T", b)) + } +} + +// TO DO: Refactor this to use the backend package +// func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { +// var buf map[string]interface{} +// err := unmarshal(&buf) +// if err != nil { +// return err +// } + +// c.Version = buf["version"].(string) + +// renderingConfig, ok := buf["renderingConfig"].(map[interface{}]interface{}) +// if !ok { +// return errors.New("renderingConfig field not found or is not a map") +// } + +// renderingConfigData, err := yaml.Marshal(renderingConfig) +// if err != nil { +// return err +// } +// err = yaml.Unmarshal(renderingConfigData, &c.RenderingConfig) +// if err != nil { +// return err +// } + +// server, ok := buf["server"].(map[interface{}]interface{}) +// if !ok { +// return errors.New("server field not found or is not a map") +// } + +// c.Server.Port, ok = server["port"].(string) +// if !ok { +// return errors.New("port field not found or is not a string") +// } + +// backend, ok := buf["backend"].(map[interface{}]interface{}) +// if !ok { +// return errors.New("backend field not found or is not a map") +// } + +// if s3, ok := backend["s3"].(map[interface{}]interface{}); ok { +// s3Data, err := yaml.Marshal(s3) +// if err != nil { +// return err +// } +// err = yaml.Unmarshal(s3Data, &c.Backend.S3) +// if err != nil { +// return err +// } + +// } else if fs, ok := backend["fileSystem"].(map[interface{}]interface{}); ok { +// fsData, err := yaml.Marshal(fs) +// if err != nil { +// return err +// } +// err = yaml.Unmarshal(fsData, &c.Backend.FileSystem) +// if err != nil { +// return err +// } +// } else { +// return errors.New("unknown backend type") +// } + +// return nil +// } + +func GenerateConfig(path string) (*Config, error) { + yamlData, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var config Config + err = yaml.Unmarshal(yamlData, &config) + if err != nil { + return nil, fmt.Errorf("error unmarshalling YAML: %v", err) + } + return &config, nil +} diff --git a/render/models.go b/render/models.go index 6ead7b9..51ee2fa 100644 --- a/render/models.go +++ b/render/models.go @@ -1,6 +1,6 @@ package render -type RenderingConfigs struct { - PageWaitTime float32 // Seconds - PageWailCondition string +type Config struct { + PageWaitTime float32 `yaml:"pageWaitTime"` // Seconds + PageWailCondition string `yaml:"pageWaitCondition"` } diff --git a/render/render.go b/render/render.go index 31bc6d3..c47f067 100644 --- a/render/render.go +++ b/render/render.go @@ -11,7 +11,7 @@ import ( "github.com/chromedp/chromedp" ) -func (config RenderingConfigs) Render(webAddr string) (*string, error) { +func (config Config) Render(webAddr string) (*string, error) { // Validate the URL webAddr = func(val string) string { u, err := url.Parse(val) @@ -47,7 +47,6 @@ func pageRender(webAddr string, waitCondition string, pageWaitTime time.Duration var result bool startTime := time.Now() - for time.Since(startTime) < pageWaitTime { if err := chromedp.Evaluate(waitCondition, &result).Do(ctx); err != nil { return err @@ -61,7 +60,7 @@ func pageRender(webAddr string, waitCondition string, pageWaitTime time.Duration } if !result { - return fmt.Errorf("timeout waiting for window.prerenderReady to become true") + return fmt.Errorf("timeout [%v] waiting for window.prerenderReady to become true", pageWaitTime) } node, err := dom.GetDocument().Do(ctx)