Enable YAML conformance tests

Makefile

@@ -1,6 +1,8 @@
 PULUMI_TEST_ORG   ?= $(shell pulumi whoami)
 PULUMI_TEST_OWNER ?= ${PULUMI_TEST_ORG}
 PULUMI_LIVE_TEST  ?= false
+PULUMI_ROOT       ?= $$HOME/.pulumi-dev
+PULUMI_BIN        := $(PULUMI_ROOT)/bin
 export PULUMI_TEST_ORG
 export PULUMI_TEST_OWNER
 
@@ -43,6 +45,9 @@ update_plugin_docs::
 install::
 	${GO} install ./cmd/...
 
+install_plugin:
+	GOBIN=$(PULUMI_BIN) go install -C cmd/pulumi-language-yaml
+
 clean::
 	rm -f ./bin/*
 	rm -f pkg/pulumiyaml/testing/test/testdata/{aws,azure-native,azure,kubernetes,random,eks,aws-native,docker}.json
@@ -105,3 +110,13 @@ get_testdata:
 	rsync -avm --exclude='transpiled_examples' --include='*.pp' --include='*/' --exclude='*' --exclude='.*' \
 		'${PULUMI_DIR}/pkg/codegen/testing/test/testdata/' \
 		./pkg/pulumiyaml/testing/test/testdata/
+
+CMD_DIR := $(realpath ./cmd)
+# Build the pulumi-test-language binary
+# Assumes the main pulumi repo is a sibling of this repo
+sync-pulumi-test-language:
+	rm -rf ${CMD_DIR}/pulumi-test-language
+	mkdir -p ${CMD_DIR}/pulumi-test-language
+	${GO} build \
+	    -C ${PULUMI_DIR}/cmd/pulumi-test-language \
+		-o '${CMD_DIR}/pulumi-test-language/pulumi-test-language'

go.mod

@@ -12,8 +12,8 @@ require (
 	github.com/hexops/autogold v1.3.0
 	github.com/iancoleman/strcase v0.2.0
 	github.com/pkg/errors v0.9.1
-	github.com/pulumi/pulumi/pkg/v3 v3.99.0
-	github.com/pulumi/pulumi/sdk/v3 v3.99.0
+	github.com/pulumi/pulumi/pkg/v3 v3.104.1
+	github.com/pulumi/pulumi/sdk/v3 v3.104.1
 	github.com/spf13/afero v1.9.5
 	github.com/stretchr/testify v1.8.4
 	github.com/zclconf/go-cty v1.13.2

go.sum

@@ -107,10 +107,15 @@ github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw=
+github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM=
+github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw=
 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.0.2/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA=
 github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg=
@@ -156,6 +161,7 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
@@ -743,6 +749,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
 github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@@ -1382,6 +1389,7 @@ github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
 github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1458,10 +1466,10 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435
 github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE=
 github.com/pulumi/esc v0.6.2 h1:+z+l8cuwIauLSwXQS0uoI3rqB+YG4SzsZYtHfNoXBvw=
 github.com/pulumi/esc v0.6.2/go.mod h1:jNnYNjzsOgVTjCp0LL24NsCk8ZJxq4IoLQdCT0X7l8k=
-github.com/pulumi/pulumi/pkg/v3 v3.99.0 h1:Mal/ncVw16RBw+0M7jwX4mDlma/X324PXotuHJHDa7s=
-github.com/pulumi/pulumi/pkg/v3 v3.99.0/go.mod h1:Gk/N8qblqKI6+fTORH7oJi2SSWaOVTDhNwTrazoQPao=
-github.com/pulumi/pulumi/sdk/v3 v3.99.0 h1:vsFoEEdweYg3Hm6/Jlj1sE2cLtauzoqAdVbLMcC7Cw8=
-github.com/pulumi/pulumi/sdk/v3 v3.99.0/go.mod h1:wFM/6iAMlidgLDSF9QU+p3P+B+vg/xloFyVeZrVwA1w=
+github.com/pulumi/pulumi/pkg/v3 v3.104.1 h1:6VctJJPwQWm1eAksnf8ptEv0hf5510iiaHddRHS3ejs=
+github.com/pulumi/pulumi/pkg/v3 v3.104.1/go.mod h1:8HQDMJDAnLMyhzrMXygwq69qjwaD4Yyi0ElqTxj0UjA=
+github.com/pulumi/pulumi/sdk/v3 v3.104.1 h1:5IUzvUSfytLwk/4xWqLEf9dVsvvLy7y12iHwhx7aRww=
+github.com/pulumi/pulumi/sdk/v3 v3.104.1/go.mod h1:Ml3rpGfyZlI4zQCG7LN2XDSmH4XUNYdyBwJ3yEr/OpI=
 github.com/rakyll/embedmd v0.0.0-20171029212350-c8060a0752a2/go.mod h1:7jOTMgqac46PZcF54q6l2hkLEG8op93fZu61KmxWDV4=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=

pkg/pulumiyaml/codegen/gen_program.go

@@ -29,9 +29,21 @@ import (
 )
 
 // Generate a serializable YAML template.
-func GenerateProgram(program *pcl.Program) (map[string][]byte, hcl.Diagnostics, error) {
+func GenerateProgramWithLocalDependencies(
+	program *pcl.Program,
+	localDependencies map[string]string,
+) (map[string][]byte, hcl.Diagnostics, error) {
 	g := generator{}
 
+	for packageName, version := range localDependencies {
+		if packageName == "pulumi" {
+			// YAML runtime doesn't require a local dependency for pulumi
+			continue
+		}
+
+		g.genLocalDependency(packageName, version)
+	}
+
 	for _, n := range program.Nodes {
 		g.genNode(n)
 	}
@@ -55,8 +67,17 @@ func GenerateProgram(program *pcl.Program) (map[string][]byte, hcl.Diagnostics,
 	return map[string][]byte{"Main.yaml": w.Bytes()}, g.diags, err
 }
 
-func GenerateProject(directory string, project workspace.Project, program *pcl.Program) error {
-	files, diagnostics, err := GenerateProgram(program)
+func GenerateProgram(program *pcl.Program) (map[string][]byte, hcl.Diagnostics, error) {
+	return GenerateProgramWithLocalDependencies(program, nil)
+}
+
+func GenerateProject(
+	directory string,
+	project workspace.Project,
+	program *pcl.Program,
+	localDependencies map[string]string,
+) error {
+	files, diagnostics, err := GenerateProgramWithLocalDependencies(program, localDependencies)
 	if err != nil {
 		return err
 	}
@@ -209,6 +230,21 @@ func (g *generator) genResourceOpts(opts *pcl.ResourceOptions) *syn.ObjectNode {
 	return syn.Object(rOpts...)
 }
 
+func (g *generator) genLocalDependency(packageName string, version string) {
+	resourceOptions := syn.Object(
+		syn.ObjectProperty(syn.String("version"), syn.String(version)),
+	)
+
+	resourceDefinition := syn.Object(
+		g.TypeProperty(fmt.Sprintf("pulumi:providers:%s", packageName)),
+		syn.ObjectProperty(syn.String("defaultProvider"), syn.Boolean(true)),
+		syn.ObjectProperty(syn.String("options"), resourceOptions),
+	)
+
+	resourceName := fmt.Sprintf("default-%s-provider", packageName)
+	g.resources = append(g.resources, syn.ObjectProperty(syn.String(resourceName), resourceDefinition))
+}
+
 func (g *generator) genResource(n *pcl.Resource) {
 	properties := make([]syn.ObjectPropertyDef, len(n.Inputs))
 	for i, input := range n.Inputs {

pkg/pulumiyaml/run.go

@@ -42,8 +42,8 @@ import (
 const MainTemplate = "Main"
 
 // Load a template from the current working directory
-func Load() (*ast.TemplateDecl, syntax.Diagnostics, error) {
-	return LoadDir(".")
+func Load(rootDir string) (*ast.TemplateDecl, syntax.Diagnostics, error) {
+	return LoadDir(rootDir)
 }
 
 func LoadFromCompiler(compiler string, workingDirectory string) (*ast.TemplateDecl, syntax.Diagnostics, error) {
@@ -88,7 +88,7 @@ func LoadDir(cwd string) (*ast.TemplateDecl, syntax.Diagnostics, error) {
 	} else if b, err := os.ReadFile(filepath.Join(cwd, "Pulumi.yaml")); err == nil {
 		filename, bs = "Pulumi.yaml", b
 	} else {
-		return nil, nil, fmt.Errorf("reading template %s: %w", MainTemplate, err)
+		return nil, nil, fmt.Errorf("reading template %s from %s: %w", MainTemplate, cwd, err)
 	}
 
 	return LoadYAMLBytes(filename, bs)

pkg/server/server.go

@@ -18,8 +18,13 @@ package server
 
 import (
 	"context"
+	"encoding/json"
 	"os"
 
+	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/codegen"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
+
 	pbempty "github.com/golang/protobuf/ptypes/empty"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/version"
@@ -53,7 +58,7 @@ func NewLanguageHost(engineAddress, tracing string, compiler string) pulumirpc.L
 	}
 }
 
-func (host *yamlLanguageHost) loadTemplate() (*ast.TemplateDecl, syntax.Diagnostics, error) {
+func (host *yamlLanguageHost) loadTemplate(rootDir string) (*ast.TemplateDecl, syntax.Diagnostics, error) {
 	if host.template != nil {
 		return host.template, host.diags, nil
 	}
@@ -62,9 +67,9 @@ func (host *yamlLanguageHost) loadTemplate() (*ast.TemplateDecl, syntax.Diagnost
 	var diags syntax.Diagnostics
 	var err error
 	if host.compiler == "" {
-		template, diags, err = pulumiyaml.Load()
+		template, diags, err = pulumiyaml.Load(rootDir)
 	} else {
-		template, diags, err = pulumiyaml.LoadFromCompiler(host.compiler, "")
+		template, diags, err = pulumiyaml.LoadFromCompiler(host.compiler, rootDir)
 	}
 	if err != nil {
 		return nil, diags, err
@@ -82,7 +87,7 @@ func (host *yamlLanguageHost) loadTemplate() (*ast.TemplateDecl, syntax.Diagnost
 func (host *yamlLanguageHost) GetRequiredPlugins(ctx context.Context,
 	req *pulumirpc.GetRequiredPluginsRequest,
 ) (*pulumirpc.GetRequiredPluginsResponse, error) {
-	template, diags, err := host.loadTemplate()
+	template, diags, err := host.loadTemplate(req.Info.RootDirectory)
 	if err != nil {
 		return nil, err
 	}
@@ -114,14 +119,7 @@ func (host *yamlLanguageHost) GetRequiredPlugins(ctx context.Context,
 
 // RPC endpoint for LanguageRuntimeServer::Run. This actually evaluates the JSON-based project.
 func (host *yamlLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest) (*pulumirpc.RunResponse, error) {
-	if pwd := req.GetPwd(); pwd != "" {
-		err := os.Chdir(pwd)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	template, diags, err := host.loadTemplate()
+	template, diags, err := host.loadTemplate(req.Info.RootDirectory)
 	if err != nil {
 		return &pulumirpc.RunResponse{Error: err.Error()}, nil
 	}
@@ -208,3 +206,57 @@ func (host *yamlLanguageHost) GetProgramDependencies(ctx context.Context, req *p
 func (host *yamlLanguageHost) About(ctx context.Context, req *emptypb.Empty) (*pulumirpc.AboutResponse, error) {
 	return &pulumirpc.AboutResponse{}, nil
 }
+
+func (host *yamlLanguageHost) Pack(ctx context.Context, req *pulumirpc.PackRequest) (*pulumirpc.PackResponse, error) {
+	return &pulumirpc.PackResponse{ArtifactPath: req.Version}, nil
+}
+
+func (host *yamlLanguageHost) GenerateProject(
+	context context.Context, req *pulumirpc.GenerateProjectRequest,
+) (*pulumirpc.GenerateProjectResponse, error) {
+	loader, err := schema.NewLoaderClient(req.LoaderTarget)
+	if err != nil {
+		return nil, err
+	}
+
+	var extraOptions []pcl.BindOption
+	if !req.Strict {
+		extraOptions = append(extraOptions, pcl.NonStrictBindOptions()...)
+	}
+
+	// for nodejs, prefer output-versioned invokes
+	extraOptions = append(extraOptions, pcl.PreferOutputVersionedInvokes)
+
+	program, diags, err := pcl.BindDirectory(req.SourceDirectory, loader, extraOptions...)
+	if err != nil {
+		return nil, err
+	}
+	if diags.HasErrors() {
+		rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
+
+		return &pulumirpc.GenerateProjectResponse{
+			Diagnostics: rpcDiagnostics,
+		}, nil
+	}
+
+	var project workspace.Project
+	if err := json.Unmarshal([]byte(req.Project), &project); err != nil {
+		return nil, err
+	}
+
+	err = codegen.GenerateProject(req.TargetDirectory, project, program, req.LocalDependencies)
+	if err != nil {
+		return nil, err
+	}
+
+	rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
+
+	return &pulumirpc.GenerateProjectResponse{
+		Diagnostics: rpcDiagnostics,
+	}, nil
+}
+
+func (host *yamlLanguageHost) GeneratePackage(
+	context context.Context, req *pulumirpc.GeneratePackageRequest) (*pulumirpc.GeneratePackageResponse, error) {
+	return &pulumirpc.GeneratePackageResponse{}, nil
+}