From a64b1e53c0d7030e07c3d3b07e4cc14148d3953a Mon Sep 17 00:00:00 2001 From: Andres Date: Thu, 27 Jul 2017 15:57:40 +0200 Subject: [PATCH 1/2] Support different runtime versions --- .../{nodejs-6.10 => nodejs}/.eslintrc.js | 0 .../{nodejs-6.10 => nodejs}/.gitignore | 0 .../Dockerfile => nodejs/Dockerfile.6} | 2 +- docker/runtime/nodejs/Dockerfile.8 | 12 ++ .../event-trigger/kubeless.js | 0 .../{nodejs-6.10 => nodejs}/event-trigger/lib | 0 .../event-trigger/package.json | 0 .../http-trigger/kubeless.js | 0 .../{nodejs-6.10 => nodejs}/http-trigger/lib | 0 .../http-trigger/package.json | 0 .../{nodejs-6.10 => nodejs}/lib/helper.js | 0 pkg/utils/k8sutil.go | 135 ++++++++++++------ pkg/utils/k8sutil_test.go | 60 ++++++++ 13 files changed, 168 insertions(+), 41 deletions(-) rename docker/runtime/{nodejs-6.10 => nodejs}/.eslintrc.js (100%) rename docker/runtime/{nodejs-6.10 => nodejs}/.gitignore (100%) rename docker/runtime/{nodejs-6.10/Dockerfile => nodejs/Dockerfile.6} (89%) create mode 100644 docker/runtime/nodejs/Dockerfile.8 rename docker/runtime/{nodejs-6.10 => nodejs}/event-trigger/kubeless.js (100%) rename docker/runtime/{nodejs-6.10 => nodejs}/event-trigger/lib (100%) rename docker/runtime/{nodejs-6.10 => nodejs}/event-trigger/package.json (100%) rename docker/runtime/{nodejs-6.10 => nodejs}/http-trigger/kubeless.js (100%) rename docker/runtime/{nodejs-6.10 => nodejs}/http-trigger/lib (100%) rename docker/runtime/{nodejs-6.10 => nodejs}/http-trigger/package.json (100%) rename docker/runtime/{nodejs-6.10 => nodejs}/lib/helper.js (100%) diff --git a/docker/runtime/nodejs-6.10/.eslintrc.js b/docker/runtime/nodejs/.eslintrc.js similarity index 100% rename from docker/runtime/nodejs-6.10/.eslintrc.js rename to docker/runtime/nodejs/.eslintrc.js diff --git a/docker/runtime/nodejs-6.10/.gitignore b/docker/runtime/nodejs/.gitignore similarity index 100% rename from docker/runtime/nodejs-6.10/.gitignore rename to docker/runtime/nodejs/.gitignore diff --git a/docker/runtime/nodejs-6.10/Dockerfile b/docker/runtime/nodejs/Dockerfile.6 similarity index 89% rename from docker/runtime/nodejs-6.10/Dockerfile rename to docker/runtime/nodejs/Dockerfile.6 index 9adf80675..11df76a51 100644 --- a/docker/runtime/nodejs-6.10/Dockerfile +++ b/docker/runtime/nodejs/Dockerfile.6 @@ -1,4 +1,4 @@ -FROM node:6.10-alpine +FROM node:6-alpine ARG TARGET=http-trigger RUN apk add --no-cache git diff --git a/docker/runtime/nodejs/Dockerfile.8 b/docker/runtime/nodejs/Dockerfile.8 new file mode 100644 index 000000000..b8626b3ce --- /dev/null +++ b/docker/runtime/nodejs/Dockerfile.8 @@ -0,0 +1,12 @@ +FROM node:8-alpine + +ARG TARGET=http-trigger +RUN apk add --no-cache git + +ADD lib/helper.js /lib/ +ADD ${TARGET}/kubeless.js / +ADD ${TARGET}/package.json / + +RUN npm install + +CMD ["node", "/kubeless.js"] diff --git a/docker/runtime/nodejs-6.10/event-trigger/kubeless.js b/docker/runtime/nodejs/event-trigger/kubeless.js similarity index 100% rename from docker/runtime/nodejs-6.10/event-trigger/kubeless.js rename to docker/runtime/nodejs/event-trigger/kubeless.js diff --git a/docker/runtime/nodejs-6.10/event-trigger/lib b/docker/runtime/nodejs/event-trigger/lib similarity index 100% rename from docker/runtime/nodejs-6.10/event-trigger/lib rename to docker/runtime/nodejs/event-trigger/lib diff --git a/docker/runtime/nodejs-6.10/event-trigger/package.json b/docker/runtime/nodejs/event-trigger/package.json similarity index 100% rename from docker/runtime/nodejs-6.10/event-trigger/package.json rename to docker/runtime/nodejs/event-trigger/package.json diff --git a/docker/runtime/nodejs-6.10/http-trigger/kubeless.js b/docker/runtime/nodejs/http-trigger/kubeless.js similarity index 100% rename from docker/runtime/nodejs-6.10/http-trigger/kubeless.js rename to docker/runtime/nodejs/http-trigger/kubeless.js diff --git a/docker/runtime/nodejs-6.10/http-trigger/lib b/docker/runtime/nodejs/http-trigger/lib similarity index 100% rename from docker/runtime/nodejs-6.10/http-trigger/lib rename to docker/runtime/nodejs/http-trigger/lib diff --git a/docker/runtime/nodejs-6.10/http-trigger/package.json b/docker/runtime/nodejs/http-trigger/package.json similarity index 100% rename from docker/runtime/nodejs-6.10/http-trigger/package.json rename to docker/runtime/nodejs/http-trigger/package.json diff --git a/docker/runtime/nodejs-6.10/lib/helper.js b/docker/runtime/nodejs/lib/helper.js similarity index 100% rename from docker/runtime/nodejs-6.10/lib/helper.js rename to docker/runtime/nodejs/lib/helper.js diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go index ee08b2c50..cec7a4717 100644 --- a/pkg/utils/k8sutil.go +++ b/pkg/utils/k8sutil.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "log" "os" + "regexp" "strings" "github.com/Sirupsen/logrus" @@ -51,12 +52,7 @@ import ( ) const ( - pythonRuntime = "bitnami/kubeless-python@sha256:6789266df0c97333f76e23efd58cf9c7efe24fa3e83b5fc826fd5cc317699b55" - pythonPubsubRuntime = "bitnami/kubeless-event-consumer@sha256:5ce469529811acf49c4d20bcd8a675be7aa029b43cf5252a8c9375b170859d83" - nodejsRuntime = "bitnami/kubeless-nodejs@sha256:6e86ed023f25cbe410d6085ca16ee14a9fb4b1df10034c05375d5e24e0c59acb" - nodejsPubsubRuntime = "bitnami/kubeless-nodejs-event-consumer@sha256:b027bfef5f99c3be68772155a1feaf1f771ab9a3c7bb49bef2e939d6b766abec" - rubyRuntime = "jbianquettibitnami/kubeless-ruby@sha256:9ea43e4e1570b46ae272e9f81a0ea4736e4956ee2ee67d8def29287a1d7153fe" - pubsubFunc = "PubSub" + pubsubFunc = "PubSub" ) // GetClient returns a k8s clientset to the request from inside of cluster @@ -168,6 +164,95 @@ func GetFunction(funcName, ns string) (spec.Function, error) { return f, nil } +// GetFunctionData given a runtime returns an Image ID, the dependencies filename and the function filename +func GetFunctionData(runtime, ftype, modName string) (imageName, depName, fileName string, err error) { + err = nil + imageName = "" + depName = "" + fileName = "" + + type runtimeVersion struct { + version string + httpImage string + pubsubImage string + } + + python27 := runtimeVersion{ + version: "2.7", + httpImage: "bitnami/kubeless-python@sha256:6789266df0c97333f76e23efd58cf9c7efe24fa3e83b5fc826fd5cc317699b55", + pubsubImage: "bitnami/kubeless-event-consumer@sha256:5ce469529811acf49c4d20bcd8a675be7aa029b43cf5252a8c9375b170859d83", + } + node6 := runtimeVersion{ + version: "6", + httpImage: "bitnami/kubeless-nodejs@sha256:6e86ed023f25cbe410d6085ca16ee14a9fb4b1df10034c05375d5e24e0c59acb", + pubsubImage: "bitnami/kubeless-nodejs-event-consumer@sha256:b027bfef5f99c3be68772155a1feaf1f771ab9a3c7bb49bef2e939d6b766abec", + } + node8 := runtimeVersion{ + version: "8", + httpImage: "bitnami/kubeless-nodejs:8", + pubsubImage: "bitnami/kubeless-nodejs-event-consumer:8", + } + ruby24 := runtimeVersion{ + version: "2.4", + httpImage: "jbianquettibitnami/kubeless-ruby@sha256:9ea43e4e1570b46ae272e9f81a0ea4736e4956ee2ee67d8def29287a1d7153fe", + pubsubImage: "", + } + python := []runtimeVersion{python27} + node := []runtimeVersion{node6, node8} + ruby := []runtimeVersion{ruby24} + + runtimeID := regexp.MustCompile("[a-zA-Z]+").FindString(runtime) + version := regexp.MustCompile("[0-9.]+").FindString(runtime) + + var versionsDef []runtimeVersion + switch { + case runtimeID == "python": + fileName = modName + ".py" + versionsDef = python + depName = "requirements.txt" + case runtimeID == "nodejs": + fileName = modName + ".js" + versionsDef = node + depName = "package.json" + case runtimeID == "ruby": + fileName = modName + ".rb" + versionsDef = ruby + depName = "Gemfile" + default: + err = errors.New("The given runtime is not valid") + return + } + imageNameEnvVar := "" + if ftype == pubsubFunc { + imageNameEnvVar = strings.ToUpper(runtime) + "_PUBSUB_RUNTIME" + } else { + imageNameEnvVar = strings.ToUpper(runtime) + "_RUNTIME" + } + if imageName = os.Getenv(imageNameEnvVar); imageName == "" { + rVersion := runtimeVersion{"", "", ""} + for i := range versionsDef { + if versionsDef[i].version == version { + rVersion = versionsDef[i] + break + } + } + if ftype == pubsubFunc { + if rVersion.pubsubImage == "" { + err = errors.New("The given runtime and version does not have a valid image for event based functions") + } else { + imageName = rVersion.pubsubImage + } + } else { + if rVersion.httpImage == "" { + err = errors.New("The given runtime and version does not have a valid image for http based functions") + } else { + imageName = rVersion.httpImage + } + } + } + return +} + // EnsureK8sResources creates/updates k8s objects (deploy, svc, configmap) for the function func EnsureK8sResources(ns, name string, funcObj *spec.Function, client kubernetes.Interface) error { str := strings.Split(funcObj.Spec.Handler, ".") @@ -177,40 +262,10 @@ func EnsureK8sResources(ns, name string, funcObj *spec.Function, client kubernet funcHandler := str[1] modName := str[0] fileName := modName + imageName, depName, fileName, err := GetFunctionData(funcObj.Spec.Runtime, funcObj.Spec.Type, modName) - imageName := "" - depName := "" - switch { - case strings.Contains(funcObj.Spec.Runtime, "python"): - fileName = modName + ".py" - if imageName = os.Getenv("PYTHON_RUNTIME"); imageName == "" { - imageName = pythonRuntime - } - if funcObj.Spec.Type == pubsubFunc { - if imageName = os.Getenv("PYTHON_PUBSUB_RUNTIME"); imageName == "" { - imageName = pythonPubsubRuntime - } - } - depName = "requirements.txt" - case strings.Contains(funcObj.Spec.Runtime, "go"): - fileName = modName + ".go" - case strings.Contains(funcObj.Spec.Runtime, "nodejs"): - fileName = modName + ".js" - if imageName = os.Getenv("NODEJS_RUNTIME"); imageName == "" { - imageName = nodejsRuntime - } - if funcObj.Spec.Type == pubsubFunc { - if imageName = os.Getenv("NODEJS_PUBSUB_RUNTIME"); imageName == "" { - imageName = nodejsPubsubRuntime - } - } - depName = "package.json" - case strings.Contains(funcObj.Spec.Runtime, "ruby"): - fileName = modName + ".rb" - if imageName = os.Getenv("RUBY_RUNTIME"); imageName == "" { - imageName = rubyRuntime - } - depName = "Gemfile" + if err != nil { + return err } //add configmap @@ -252,7 +307,7 @@ func EnsureK8sResources(ns, name string, funcObj *spec.Function, client kubernet }, } - _, err := client.Core().ConfigMaps(ns).Create(configMap) + _, err = client.Core().ConfigMaps(ns).Create(configMap) if err != nil && k8sErrors.IsAlreadyExists(err) { data, _ := json.Marshal(configMap) _, err = client.Core().ConfigMaps(ns).Patch(configMap.Name, types.StrategicMergePatchType, data) diff --git a/pkg/utils/k8sutil_test.go b/pkg/utils/k8sutil_test.go index 6f8ebc81c..d5e37107b 100644 --- a/pkg/utils/k8sutil_test.go +++ b/pkg/utils/k8sutil_test.go @@ -1,6 +1,7 @@ package utils import ( + "os" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,3 +74,62 @@ func TestDeleteK8sResources(t *testing.T) { t.Errorf("failed to delete service") } } + +func check(runtime, ftype, fname string, values []string, t *testing.T) { + imageName, depName, fileName, err := GetFunctionData(runtime, ftype, fname) + if err != nil { + t.Fatalf("Retrieving the image returned err: %v", err) + } + if imageName == "" { + t.Fatalf("Retrieving the image returned an empty Image ID") + } + if depName != values[0] { + t.Fatalf("Retrieving the image returned a wrong dependencies file. Received " + depName + " while expecting " + values[0]) + } + if fileName != values[1] { + t.Fatalf("Retrieving the image returned a wrong file name. Received " + fileName + " while expecting " + values[1]) + } +} +func TestGetFunctionData(t *testing.T) { + + expectedValues := []string{"requirements.txt", "test.py"} + check("python2.7", "HTTP", "test", expectedValues, t) + check("python2.7", "PubSub", "test", expectedValues, t) + + expectedValues = []string{"package.json", "test.js"} + check("nodejs6", "HTTP", "test", expectedValues, t) + check("nodejs6", "PubSub", "test", expectedValues, t) + check("nodejs8", "HTTP", "test", expectedValues, t) + check("nodejs8", "PubSub", "test", expectedValues, t) + + expectedValues = []string{"Gemfile", "test.rb"} + check("ruby2.4", "HTTP", "test", expectedValues, t) + + _, _, _, err := GetFunctionData("unexistent", "HTTP", "test") + if err == nil { + t.Fatalf("Retrieving data for 'unexistent' should return an error") + } + + expectedImageName := "ruby-test-image" + os.Setenv("RUBY_RUNTIME", expectedImageName) + imageR, _, _, errR := GetFunctionData("ruby", "HTTP", "test") + if errR != nil { + t.Fatalf("Retrieving the image returned err: %v", err) + } + if imageR != expectedImageName { + t.Fatalf("Expecting " + imageR + " to be set to " + expectedImageName) + } + os.Unsetenv("RUBY_RUNTIME") + + expectedImageName = "ruby-pubsub-test-image" + os.Setenv("RUBY_PUBSUB_RUNTIME", "ruby-pubsub-test-image") + imageR, _, _, errR = GetFunctionData("ruby", "PubSub", "test") + if errR != nil { + t.Fatalf("Retrieving the image returned err: %v", err) + } + if imageR != expectedImageName { + t.Fatalf("Expecting " + imageR + " to be set to " + expectedImageName) + } + os.Unsetenv("RUBY_PUBSUB_RUNTIME") + +} From 6dab6fdae968022c2e39b01dab65bf5b3d27d9a0 Mon Sep 17 00:00:00 2001 From: Andres Date: Fri, 28 Jul 2017 10:53:25 +0200 Subject: [PATCH 2/2] Move image definitions to the top of the file. Set final reference --- pkg/utils/k8sutil.go | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go index cec7a4717..62246da29 100644 --- a/pkg/utils/k8sutil.go +++ b/pkg/utils/k8sutil.go @@ -52,7 +52,14 @@ import ( ) const ( - pubsubFunc = "PubSub" + python27Http = "bitnami/kubeless-python@sha256:6789266df0c97333f76e23efd58cf9c7efe24fa3e83b5fc826fd5cc317699b55" + python27Pubsub = "bitnami/kubeless-event-consumer@sha256:5ce469529811acf49c4d20bcd8a675be7aa029b43cf5252a8c9375b170859d83" + node6Http = "bitnami/kubeless-nodejs@sha256:6e86ed023f25cbe410d6085ca16ee14a9fb4b1df10034c05375d5e24e0c59acb" + node6Pubsub = "bitnami/kubeless-nodejs-event-consumer@sha256:b027bfef5f99c3be68772155a1feaf1f771ab9a3c7bb49bef2e939d6b766abec" + node8Http = "bitnami/kubeless-nodejs@sha256:29077c5131ab63c557507596958510e9d2011dc0e88b968371c531ba3b41c47f" + node8Pubsub = "bitnami/kubeless-nodejs-event-consumer@sha256:4d005c9c0b462750d9ab7f1305897e7a01143fe869d3b722ed3330560f9c7fb5" + ruby24Http = "jbianquettibitnami/kubeless-ruby@sha256:9ea43e4e1570b46ae272e9f81a0ea4736e4956ee2ee67d8def29287a1d7153fe" + pubsubFunc = "PubSub" ) // GetClient returns a k8s clientset to the request from inside of cluster @@ -177,28 +184,14 @@ func GetFunctionData(runtime, ftype, modName string) (imageName, depName, fileNa pubsubImage string } - python27 := runtimeVersion{ - version: "2.7", - httpImage: "bitnami/kubeless-python@sha256:6789266df0c97333f76e23efd58cf9c7efe24fa3e83b5fc826fd5cc317699b55", - pubsubImage: "bitnami/kubeless-event-consumer@sha256:5ce469529811acf49c4d20bcd8a675be7aa029b43cf5252a8c9375b170859d83", - } - node6 := runtimeVersion{ - version: "6", - httpImage: "bitnami/kubeless-nodejs@sha256:6e86ed023f25cbe410d6085ca16ee14a9fb4b1df10034c05375d5e24e0c59acb", - pubsubImage: "bitnami/kubeless-nodejs-event-consumer@sha256:b027bfef5f99c3be68772155a1feaf1f771ab9a3c7bb49bef2e939d6b766abec", - } - node8 := runtimeVersion{ - version: "8", - httpImage: "bitnami/kubeless-nodejs:8", - pubsubImage: "bitnami/kubeless-nodejs-event-consumer:8", - } - ruby24 := runtimeVersion{ - version: "2.4", - httpImage: "jbianquettibitnami/kubeless-ruby@sha256:9ea43e4e1570b46ae272e9f81a0ea4736e4956ee2ee67d8def29287a1d7153fe", - pubsubImage: "", - } + python27 := runtimeVersion{version: "2.7", httpImage: python27Http, pubsubImage: python27Pubsub} python := []runtimeVersion{python27} + + node6 := runtimeVersion{version: "6", httpImage: node6Http, pubsubImage: node6Pubsub} + node8 := runtimeVersion{version: "8", httpImage: node8Http, pubsubImage: node8Pubsub} node := []runtimeVersion{node6, node8} + + ruby24 := runtimeVersion{version: "2.4", httpImage: ruby24Http, pubsubImage: ""} ruby := []runtimeVersion{ruby24} runtimeID := regexp.MustCompile("[a-zA-Z]+").FindString(runtime)