Enable l2-resource-simple (#676)

This enables the l2-resource-simple test.

This required a few fixes. 
Firstly in `GetReferencedPlugins` we look at any package declarations to
get package versions. This is so we can return the plugin info with the
right version.

Secondly we add `GetReferencePackages` method to use from
`GetProgramDependencies` so that we can actually return something from
`GetProgramDependencies` rather than nothing. Conformance testing tests
this so we have to fill it in.

Thirdly we implement `Pack`, it doesn't do much just copies the
generated package declaration yaml file.

.changes/unreleased/Improvements-676.yaml

@@ -0,0 +1,6 @@
+component: runtime
+kind: Improvements
+body: '`GetProgramDependencies` now returns packages used to show in `pulumi about`'
+time: 2024-11-14T17:39:33.889564362Z
+custom:
+  PR: "676"

cmd/pulumi-language-yaml/language_test.go

@@ -187,7 +187,6 @@ var expectedFailures = map[string]string{
 	"l2-invoke-simple":                      "TODO",
 	"l2-plain":                              "TODO",
 	"l2-ref-ref":                            "TODO",
-	"l2-resource-simple":                    "TODO",
 	"l2-failed-create-continue-on-error":    "TODO",
 	"l2-invoke-variants":                    "TODO",
 	"l2-primitive-ref":                      "TODO",
@@ -212,7 +211,7 @@ func TestLanguage(t *testing.T) {
 	// Run the language plugin
 	handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
 		Init: func(srv *grpc.Server) error {
-			host := server.NewLanguageHost(engineAddress, "", "")
+			host := server.NewLanguageHost(engineAddress, "", "", true /* useRPCLoader */)
 			pulumirpc.RegisterLanguageRuntimeServer(srv, host)
 			return nil
 		},

cmd/pulumi-language-yaml/main.go

@@ -64,7 +64,7 @@ func main() {
 	// Fire up a gRPC server, letting the kernel choose a free port.
 	port, done, err := rpcutil.Serve(0, cancelChannel, []func(*grpc.Server) error{
 		func(srv *grpc.Server) error {
-			host := server.NewLanguageHost(engineAddress, tracing, compiler)
+			host := server.NewLanguageHost(engineAddress, tracing, compiler, false /* useRPCLoader */)
 			pulumirpc.RegisterLanguageRuntimeServer(srv, host)
 			return nil
 		},

cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Main.yaml

@@ -0,0 +1,5 @@
+resources:
+  res:
+    type: simple:Resource
+    properties:
+      value: true

cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Pulumi.yaml

@@ -0,0 +1,2 @@
+name: l2-resource-simple
+runtime: yaml

cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/sdks/simple.yaml

@@ -0,0 +1,3 @@
+packageDeclarationVersion: 1
+name: simple
+version: 2.0.0

cmd/pulumi-language-yaml/testdata/sdks/simple-2.0.0/simple-2.0.0.yaml

@@ -0,0 +1,3 @@
+packageDeclarationVersion: 1
+name: simple
+version: 2.0.0

pkg/pulumiyaml/codegen/gen_program.go

@@ -22,6 +22,7 @@ import (
 	"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
 	enc "github.com/pulumi/pulumi/sdk/v3/go/common/encoding"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
 
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/ast"
@@ -56,7 +57,7 @@ func GenerateProgram(program *pcl.Program) (map[string][]byte, hcl.Diagnostics,
 	return map[string][]byte{"Main.yaml": w.Bytes()}, g.diags, err
 }
 
-func GenerateProject(directory string, project workspace.Project, program *pcl.Program) error {
+func GenerateProject(directory string, project workspace.Project, program *pcl.Program, localDependencies map[string]string) error {
 	files, diagnostics, err := GenerateProgram(program)
 	if err != nil {
 		return err
@@ -94,6 +95,14 @@ func GenerateProject(directory string, project workspace.Project, program *pcl.P
 		}
 	}
 
+	for name, content := range localDependencies {
+		outPath := path.Join(directory, "sdks", name+".yaml")
+		err := fsutil.CopyFile(outPath, content, nil)
+		if err != nil {
+			return fmt.Errorf("copy local dependency: %w", err)
+		}
+	}
+
 	return nil
 }
 

pkg/pulumiyaml/packages.go

@@ -13,6 +13,7 @@ import (
 	"github.com/blang/semver"
 	"github.com/iancoleman/strcase"
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/ast"
+	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/packages"
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/syntax"
 	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
@@ -121,6 +122,30 @@ type pluginEntry struct {
 func GetReferencedPlugins(tmpl *ast.TemplateDecl) ([]Plugin, syntax.Diagnostics) {
 	pluginMap := map[string]*pluginEntry{}
 
+	// Iterate over the package declarations
+	for _, pkg := range tmpl.Packages {
+		name := pkg.Name
+		version := pkg.Version
+		if pkg.Parameterization != nil {
+			name = pkg.Parameterization.Name
+			version = pkg.Parameterization.Version
+		}
+
+		if entry, found := pluginMap[name]; found {
+			if entry.version == "" {
+				entry.version = version
+			}
+			if entry.pluginDownloadURL == "" {
+				entry.pluginDownloadURL = pkg.DownloadURL
+			}
+		} else {
+			pluginMap[name] = &pluginEntry{
+				version:           version,
+				pluginDownloadURL: pkg.DownloadURL,
+			}
+		}
+	}
+
 	acceptType := func(r *Runner, typeName string, version, pluginDownloadURL *ast.StringExpr) {
 		pkg := ResolvePkgName(typeName)
 		if entry, found := pluginMap[pkg]; found {
@@ -197,6 +222,118 @@ func GetReferencedPlugins(tmpl *ast.TemplateDecl) ([]Plugin, syntax.Diagnostics)
 	return plugins, nil
 }
 
+// GetReferencedPlugins returns the packages and (if provided) versions for each referenced package
+// used in the program.
+func GetReferencedPackages(tmpl *ast.TemplateDecl) ([]packages.PackageDecl, syntax.Diagnostics) {
+	packageMap := map[string]*packages.PackageDecl{}
+
+	// Iterate over the package declarations
+	for _, pkg := range tmpl.Packages {
+		pkg := pkg
+		name := pkg.Name
+		version := pkg.Version
+		if pkg.Parameterization != nil {
+			name = pkg.Parameterization.Name
+			version = pkg.Parameterization.Version
+		}
+
+		if entry, found := packageMap[name]; found {
+			if entry.Version == "" {
+				entry.Version = version
+			}
+			if entry.DownloadURL == "" {
+				entry.DownloadURL = pkg.DownloadURL
+			}
+		} else {
+			packageMap[name] = &pkg
+		}
+	}
+
+	acceptType := func(r *Runner, typeName string, version, pluginDownloadURL *ast.StringExpr) {
+		pkg := ResolvePkgName(typeName)
+		if entry, found := packageMap[pkg]; found {
+			if v := version.GetValue(); v != "" && entry.Version != v {
+				if entry.Version == "" {
+					entry.Version = v
+				} else {
+					r.sdiags.Extend(ast.ExprError(version, fmt.Sprintf("Package %v already declared with a conflicting version: %v", pkg, entry.Version), ""))
+				}
+			}
+			if url := pluginDownloadURL.GetValue(); url != "" && entry.DownloadURL != url {
+				if entry.DownloadURL == "" {
+					entry.DownloadURL = url
+				} else {
+					r.sdiags.Extend(ast.ExprError(pluginDownloadURL, fmt.Sprintf("Package %v already declared with a conflicting plugin download URL: %v", pkg, entry.DownloadURL), ""))
+				}
+			}
+		} else {
+			packageMap[pkg] = &packages.PackageDecl{
+				Name:        pkg,
+				Version:     version.GetValue(),
+				DownloadURL: pluginDownloadURL.GetValue(),
+			}
+		}
+	}
+
+	diags := newRunner(tmpl, nil).Run(walker{
+		VisitResource: func(r *Runner, node resourceNode) bool {
+			res := node.Value
+
+			if res.Type == nil {
+				r.sdiags.Extend(syntax.NodeError(node.Value.Syntax(), fmt.Sprintf("Resource declared without a 'type': %q", node.Key.Value), ""))
+				return true
+			}
+			acceptType(r, res.Type.Value, res.Options.Version, res.Options.PluginDownloadURL)
+
+			return true
+		},
+		VisitExpr: func(ctx *evalContext, expr ast.Expr) bool {
+			if expr, ok := expr.(*ast.InvokeExpr); ok {
+				if expr.Token == nil {
+					ctx.Runner.sdiags.Extend(syntax.NodeError(expr.Syntax(), "Invoke declared without a 'function' type", ""))
+					return true
+				}
+				acceptType(ctx.Runner, expr.Token.GetValue(), expr.CallOpts.Version, expr.CallOpts.PluginDownloadURL)
+			}
+			return true
+		},
+	})
+
+	if diags.HasErrors() {
+		return nil, diags
+	}
+
+	var packages []packages.PackageDecl
+	for _, pkg := range packageMap {
+		packages = append(packages, *pkg)
+	}
+
+	sort.Slice(packages, func(i, j int) bool {
+		pI, pJ := packages[i], packages[j]
+		if pI.Name != pJ.Name {
+			return pI.Name < pJ.Name
+		}
+		if pI.Version != pJ.Version {
+			return pI.Version < pJ.Version
+		}
+		if pI.Parameterization == nil && pJ.Parameterization == nil {
+			return pI.DownloadURL < pJ.DownloadURL
+		}
+		if pI.Parameterization == nil {
+			return true
+		}
+		if pJ.Parameterization == nil {
+			return false
+		}
+		if pI.Parameterization.Name != pJ.Parameterization.Name {
+			return pI.Parameterization.Name < pJ.Parameterization.Name
+		}
+		return pI.Parameterization.Version < pJ.Parameterization.Version
+	})
+
+	return packages, nil
+}
+
 func ResolvePkgName(typeString string) string {
 	typeParts := strings.Split(typeString, ":")
 

pkg/server/server.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -59,15 +60,17 @@ type yamlLanguageHost struct {
 	tracing       string
 	compiler      string
 
+	useRPCLoader  bool
 	templateCache map[string]templateCacheEntry
 }
 
-func NewLanguageHost(engineAddress, tracing string, compiler string) pulumirpc.LanguageRuntimeServer {
+func NewLanguageHost(engineAddress, tracing, compiler string, useRPCLoader bool) pulumirpc.LanguageRuntimeServer {
 	return &yamlLanguageHost{
 		engineAddress: engineAddress,
 		tracing:       tracing,
 		compiler:      compiler,
 
+		useRPCLoader:  useRPCLoader,
 		templateCache: make(map[string]templateCacheEntry),
 	}
 }
@@ -214,9 +217,19 @@ func (host *yamlLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest
 
 	// Because of async applies we may need the package loader to outlast the RunTemplate function. But by the
 	// time RunWithContext returns we should be done with all async work.
-	loader, err := pulumiyaml.NewPackageLoader(proj.Plugins)
-	if err != nil {
-		return &pulumirpc.RunResponse{Error: err.Error()}, nil
+	var loader pulumiyaml.PackageLoader
+	if host.useRPCLoader {
+		rpcLoader, err := schema.NewLoaderClient(req.LoaderTarget)
+		if err != nil {
+			return &pulumirpc.RunResponse{Error: err.Error()}, nil
+		}
+		loader = pulumiyaml.NewPackageLoaderFromSchemaLoader(
+			schema.NewCachedLoader(rpcLoader))
+	} else {
+		loader, err = pulumiyaml.NewPackageLoader(proj.Plugins)
+		if err != nil {
+			return &pulumirpc.RunResponse{Error: err.Error()}, nil
+		}
 	}
 	defer loader.Close()
 
@@ -254,7 +267,43 @@ func (host *yamlLanguageHost) InstallDependencies(req *pulumirpc.InstallDependen
 
 // GetProgramDependencies returns the set of dependencies required by the program.
 func (host *yamlLanguageHost) GetProgramDependencies(ctx context.Context, req *pulumirpc.GetProgramDependenciesRequest) (*pulumirpc.GetProgramDependenciesResponse, error) {
-	return &pulumirpc.GetProgramDependenciesResponse{}, nil
+	// YAML doesn't _really_ have dependencies per-se but we can list all the "packages" that are referenced
+	// in the program here. In the presesnce of parameterization this could differ to the set of plugins
+	// reported by GetRequiredPlugins.
+
+	template, diags, err := host.loadTemplate(req.Info.ProgramDirectory, nil)
+	if err != nil {
+		return nil, err
+	}
+	if diags.HasErrors() {
+		return nil, diags
+	}
+
+	pkgs, pluginDiags := pulumiyaml.GetReferencedPackages(template)
+	diags.Extend(pluginDiags...)
+	if diags.HasErrors() {
+		// We currently swallow the error to allow project config to evaluate
+		// Specifically, if one sets a config key via the CLI but not within the `config` block
+		// of their YAML program, it would error.
+		return &pulumirpc.GetProgramDependenciesResponse{}, nil
+	}
+	var dependencies []*pulumirpc.DependencyInfo
+	for _, pkg := range pkgs {
+		name := pkg.Name
+		version := pkg.Version
+		if pkg.Parameterization != nil {
+			name = pkg.Parameterization.Name
+			version = pkg.Parameterization.Version
+		}
+
+		dependencies = append(dependencies, &pulumirpc.DependencyInfo{
+			Name:    name,
+			Version: version,
+		})
+	}
+	return &pulumirpc.GetProgramDependenciesResponse{
+		Dependencies: dependencies,
+	}, nil
 }
 
 // RuntimeOptionsPrompts returns a list of additional prompts to ask during `pulumi new`.
@@ -298,7 +347,7 @@ func (host *yamlLanguageHost) GenerateProject(
 		return nil, err
 	}
 
-	err = codegen.GenerateProject(req.TargetDirectory, project, program)
+	err = codegen.GenerateProject(req.TargetDirectory, project, program, req.LocalDependencies)
 	if err != nil {
 		return nil, err
 	}
@@ -451,3 +500,51 @@ func (host *yamlLanguageHost) GeneratePackage(ctx context.Context, req *pulumirp
 		Diagnostics: rpcDiagnostics,
 	}, nil
 }
+
+func (host *yamlLanguageHost) Pack(ctx context.Context, req *pulumirpc.PackRequest) (*pulumirpc.PackResponse, error) {
+	// Yaml "SDKs" are just files, we can just copy the file
+	if err := os.MkdirAll(req.DestinationDirectory, 0700); err != nil {
+		return nil, err
+	}
+
+	files, err := os.ReadDir(req.PackageDirectory)
+	if err != nil {
+		return nil, fmt.Errorf("reading package directory: %w", err)
+	}
+
+	copyFile := func(src, dst string) error {
+		srcFile, err := os.Open(src)
+		if err != nil {
+			return fmt.Errorf("opening %s: %w", src, err)
+		}
+		defer srcFile.Close()
+		dstFile, err := os.Create(dst)
+		if err != nil {
+			return fmt.Errorf("creating %s: %w", dst, err)
+		}
+		defer dstFile.Close()
+		if _, err := io.Copy(dstFile, srcFile); err != nil {
+			return fmt.Errorf("copying %s to %s: %w", src, dst, err)
+		}
+		return nil
+	}
+
+	// We only expect one file in the package directory
+	var single string
+	for _, file := range files {
+		if single != "" {
+			return nil, fmt.Errorf("multiple files in package directory %s: %s and %s", req.PackageDirectory, single, file.Name())
+		}
+		single = file.Name()
+	}
+
+	src := filepath.Join(req.PackageDirectory, single)
+	dst := filepath.Join(req.DestinationDirectory, single)
+	if err := copyFile(src, dst); err != nil {
+		return nil, fmt.Errorf("copying %s to %s: %w", src, dst, err)
+	}
+
+	return &pulumirpc.PackResponse{
+		ArtifactPath: dst,
+	}, nil
+}