From 7eb01d63a7d89b86c3baf48d491c0032d8ca5669 Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Thu, 14 Nov 2024 15:29:03 +0000 Subject: [PATCH] Enable l2-simple --- .changes/unreleased/Improvements-676.yaml | 6 + cmd/pulumi-language-yaml/language_test.go | 3 +- cmd/pulumi-language-yaml/main.go | 2 +- .../projects/l2-resource-simple/Main.yaml | 5 + .../projects/l2-resource-simple/Pulumi.yaml | 2 + .../l2-resource-simple/sdks/simple.yaml | 3 + .../sdks/simple-2.0.0/simple-2.0.0.yaml | 3 + pkg/pulumiyaml/codegen/gen_program.go | 11 +- pkg/pulumiyaml/packages.go | 137 ++++++++++++++++++ pkg/server/server.go | 109 +++++++++++++- 10 files changed, 271 insertions(+), 10 deletions(-) create mode 100644 .changes/unreleased/Improvements-676.yaml create mode 100644 cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Main.yaml create mode 100644 cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Pulumi.yaml create mode 100644 cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/sdks/simple.yaml create mode 100644 cmd/pulumi-language-yaml/testdata/sdks/simple-2.0.0/simple-2.0.0.yaml diff --git a/.changes/unreleased/Improvements-676.yaml b/.changes/unreleased/Improvements-676.yaml new file mode 100644 index 00000000..bbf66be3 --- /dev/null +++ b/.changes/unreleased/Improvements-676.yaml @@ -0,0 +1,6 @@ +component: runtime +kind: Improvements +body: '`GetProgramDependencies` now returns packages used to show in `pulumi about`' +time: 2024-11-14T17:39:33.889564362Z +custom: + PR: "676" diff --git a/cmd/pulumi-language-yaml/language_test.go b/cmd/pulumi-language-yaml/language_test.go index 6f353362..de8c2ced 100644 --- a/cmd/pulumi-language-yaml/language_test.go +++ b/cmd/pulumi-language-yaml/language_test.go @@ -187,7 +187,6 @@ var expectedFailures = map[string]string{ "l2-invoke-simple": "TODO", "l2-plain": "TODO", "l2-ref-ref": "TODO", - "l2-resource-simple": "TODO", "l2-failed-create-continue-on-error": "TODO", "l2-invoke-variants": "TODO", "l2-primitive-ref": "TODO", @@ -212,7 +211,7 @@ func TestLanguage(t *testing.T) { // Run the language plugin handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{ Init: func(srv *grpc.Server) error { - host := server.NewLanguageHost(engineAddress, "", "") + host := server.NewLanguageHost(engineAddress, "", "", true /* useRPCLoader */) pulumirpc.RegisterLanguageRuntimeServer(srv, host) return nil }, diff --git a/cmd/pulumi-language-yaml/main.go b/cmd/pulumi-language-yaml/main.go index 482ea61c..f04c3ff8 100644 --- a/cmd/pulumi-language-yaml/main.go +++ b/cmd/pulumi-language-yaml/main.go @@ -64,7 +64,7 @@ func main() { // Fire up a gRPC server, letting the kernel choose a free port. port, done, err := rpcutil.Serve(0, cancelChannel, []func(*grpc.Server) error{ func(srv *grpc.Server) error { - host := server.NewLanguageHost(engineAddress, tracing, compiler) + host := server.NewLanguageHost(engineAddress, tracing, compiler, false /* useRPCLoader */) pulumirpc.RegisterLanguageRuntimeServer(srv, host) return nil }, diff --git a/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Main.yaml b/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Main.yaml new file mode 100644 index 00000000..7d9179a7 --- /dev/null +++ b/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Main.yaml @@ -0,0 +1,5 @@ +resources: + res: + type: simple:Resource + properties: + value: true diff --git a/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Pulumi.yaml b/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Pulumi.yaml new file mode 100644 index 00000000..93da58f6 --- /dev/null +++ b/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/Pulumi.yaml @@ -0,0 +1,2 @@ +name: l2-resource-simple +runtime: yaml diff --git a/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/sdks/simple.yaml b/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/sdks/simple.yaml new file mode 100644 index 00000000..f357f94b --- /dev/null +++ b/cmd/pulumi-language-yaml/testdata/projects/l2-resource-simple/sdks/simple.yaml @@ -0,0 +1,3 @@ +packageDeclarationVersion: 1 +name: simple +version: 2.0.0 diff --git a/cmd/pulumi-language-yaml/testdata/sdks/simple-2.0.0/simple-2.0.0.yaml b/cmd/pulumi-language-yaml/testdata/sdks/simple-2.0.0/simple-2.0.0.yaml new file mode 100644 index 00000000..f357f94b --- /dev/null +++ b/cmd/pulumi-language-yaml/testdata/sdks/simple-2.0.0/simple-2.0.0.yaml @@ -0,0 +1,3 @@ +packageDeclarationVersion: 1 +name: simple +version: 2.0.0 diff --git a/pkg/pulumiyaml/codegen/gen_program.go b/pkg/pulumiyaml/codegen/gen_program.go index 23f32e59..7ddcf946 100644 --- a/pkg/pulumiyaml/codegen/gen_program.go +++ b/pkg/pulumiyaml/codegen/gen_program.go @@ -22,6 +22,7 @@ import ( "github.com/pulumi/pulumi/pkg/v3/codegen/pcl" enc "github.com/pulumi/pulumi/sdk/v3/go/common/encoding" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil" "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/ast" @@ -56,7 +57,7 @@ func GenerateProgram(program *pcl.Program) (map[string][]byte, hcl.Diagnostics, return map[string][]byte{"Main.yaml": w.Bytes()}, g.diags, err } -func GenerateProject(directory string, project workspace.Project, program *pcl.Program) error { +func GenerateProject(directory string, project workspace.Project, program *pcl.Program, localDependencies map[string]string) error { files, diagnostics, err := GenerateProgram(program) if err != nil { return err @@ -94,6 +95,14 @@ func GenerateProject(directory string, project workspace.Project, program *pcl.P } } + for name, content := range localDependencies { + outPath := path.Join(directory, "sdks", name+".yaml") + err := fsutil.CopyFile(outPath, content, nil) + if err != nil { + return fmt.Errorf("copy local dependency: %w", err) + } + } + return nil } diff --git a/pkg/pulumiyaml/packages.go b/pkg/pulumiyaml/packages.go index fea18939..1c6a8caf 100644 --- a/pkg/pulumiyaml/packages.go +++ b/pkg/pulumiyaml/packages.go @@ -13,6 +13,7 @@ import ( "github.com/blang/semver" "github.com/iancoleman/strcase" "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/ast" + "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/packages" "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/syntax" "github.com/pulumi/pulumi/pkg/v3/codegen/schema" "github.com/pulumi/pulumi/sdk/v3/go/common/diag" @@ -121,6 +122,30 @@ type pluginEntry struct { func GetReferencedPlugins(tmpl *ast.TemplateDecl) ([]Plugin, syntax.Diagnostics) { pluginMap := map[string]*pluginEntry{} + // Iterate over the package declarations + for _, pkg := range tmpl.Packages { + name := pkg.Name + version := pkg.Version + if pkg.Parameterization != nil { + name = pkg.Parameterization.Name + version = pkg.Parameterization.Version + } + + if entry, found := pluginMap[name]; found { + if entry.version == "" { + entry.version = version + } + if entry.pluginDownloadURL == "" { + entry.pluginDownloadURL = pkg.DownloadURL + } + } else { + pluginMap[name] = &pluginEntry{ + version: version, + pluginDownloadURL: pkg.DownloadURL, + } + } + } + acceptType := func(r *Runner, typeName string, version, pluginDownloadURL *ast.StringExpr) { pkg := ResolvePkgName(typeName) if entry, found := pluginMap[pkg]; found { @@ -197,6 +222,118 @@ func GetReferencedPlugins(tmpl *ast.TemplateDecl) ([]Plugin, syntax.Diagnostics) return plugins, nil } +// GetReferencedPlugins returns the packages and (if provided) versions for each referenced package +// used in the program. +func GetReferencedPackages(tmpl *ast.TemplateDecl) ([]packages.PackageDecl, syntax.Diagnostics) { + packageMap := map[string]*packages.PackageDecl{} + + // Iterate over the package declarations + for _, pkg := range tmpl.Packages { + pkg := pkg + name := pkg.Name + version := pkg.Version + if pkg.Parameterization != nil { + name = pkg.Parameterization.Name + version = pkg.Parameterization.Version + } + + if entry, found := packageMap[name]; found { + if entry.Version == "" { + entry.Version = version + } + if entry.DownloadURL == "" { + entry.DownloadURL = pkg.DownloadURL + } + } else { + packageMap[name] = &pkg + } + } + + acceptType := func(r *Runner, typeName string, version, pluginDownloadURL *ast.StringExpr) { + pkg := ResolvePkgName(typeName) + if entry, found := packageMap[pkg]; found { + if v := version.GetValue(); v != "" && entry.Version != v { + if entry.Version == "" { + entry.Version = v + } else { + r.sdiags.Extend(ast.ExprError(version, fmt.Sprintf("Package %v already declared with a conflicting version: %v", pkg, entry.Version), "")) + } + } + if url := pluginDownloadURL.GetValue(); url != "" && entry.DownloadURL != url { + if entry.DownloadURL == "" { + entry.DownloadURL = url + } else { + r.sdiags.Extend(ast.ExprError(pluginDownloadURL, fmt.Sprintf("Package %v already declared with a conflicting plugin download URL: %v", pkg, entry.DownloadURL), "")) + } + } + } else { + packageMap[pkg] = &packages.PackageDecl{ + Name: pkg, + Version: version.GetValue(), + DownloadURL: pluginDownloadURL.GetValue(), + } + } + } + + diags := newRunner(tmpl, nil).Run(walker{ + VisitResource: func(r *Runner, node resourceNode) bool { + res := node.Value + + if res.Type == nil { + r.sdiags.Extend(syntax.NodeError(node.Value.Syntax(), fmt.Sprintf("Resource declared without a 'type': %q", node.Key.Value), "")) + return true + } + acceptType(r, res.Type.Value, res.Options.Version, res.Options.PluginDownloadURL) + + return true + }, + VisitExpr: func(ctx *evalContext, expr ast.Expr) bool { + if expr, ok := expr.(*ast.InvokeExpr); ok { + if expr.Token == nil { + ctx.Runner.sdiags.Extend(syntax.NodeError(expr.Syntax(), "Invoke declared without a 'function' type", "")) + return true + } + acceptType(ctx.Runner, expr.Token.GetValue(), expr.CallOpts.Version, expr.CallOpts.PluginDownloadURL) + } + return true + }, + }) + + if diags.HasErrors() { + return nil, diags + } + + var packages []packages.PackageDecl + for _, pkg := range packageMap { + packages = append(packages, *pkg) + } + + sort.Slice(packages, func(i, j int) bool { + pI, pJ := packages[i], packages[j] + if pI.Name != pJ.Name { + return pI.Name < pJ.Name + } + if pI.Version != pJ.Version { + return pI.Version < pJ.Version + } + if pI.Parameterization == nil && pJ.Parameterization == nil { + return pI.DownloadURL < pJ.DownloadURL + } + if pI.Parameterization == nil { + return true + } + if pJ.Parameterization == nil { + return false + } + if pI.Parameterization.Name != pJ.Parameterization.Name { + return pI.Parameterization.Name < pJ.Parameterization.Name + } + return pI.Parameterization.Version < pJ.Parameterization.Version + }) + + return packages, nil +} + func ResolvePkgName(typeString string) string { typeParts := strings.Split(typeString, ":") diff --git a/pkg/server/server.go b/pkg/server/server.go index f2395773..2b7518ec 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "strings" @@ -59,15 +60,17 @@ type yamlLanguageHost struct { tracing string compiler string + useRPCLoader bool templateCache map[string]templateCacheEntry } -func NewLanguageHost(engineAddress, tracing string, compiler string) pulumirpc.LanguageRuntimeServer { +func NewLanguageHost(engineAddress, tracing, compiler string, useRPCLoader bool) pulumirpc.LanguageRuntimeServer { return &yamlLanguageHost{ engineAddress: engineAddress, tracing: tracing, compiler: compiler, + useRPCLoader: useRPCLoader, templateCache: make(map[string]templateCacheEntry), } } @@ -214,9 +217,19 @@ func (host *yamlLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest // Because of async applies we may need the package loader to outlast the RunTemplate function. But by the // time RunWithContext returns we should be done with all async work. - loader, err := pulumiyaml.NewPackageLoader(proj.Plugins) - if err != nil { - return &pulumirpc.RunResponse{Error: err.Error()}, nil + var loader pulumiyaml.PackageLoader + if host.useRPCLoader { + rpcLoader, err := schema.NewLoaderClient(req.LoaderTarget) + if err != nil { + return &pulumirpc.RunResponse{Error: err.Error()}, nil + } + loader = pulumiyaml.NewPackageLoaderFromSchemaLoader( + schema.NewCachedLoader(rpcLoader)) + } else { + loader, err = pulumiyaml.NewPackageLoader(proj.Plugins) + if err != nil { + return &pulumirpc.RunResponse{Error: err.Error()}, nil + } } defer loader.Close() @@ -254,7 +267,43 @@ func (host *yamlLanguageHost) InstallDependencies(req *pulumirpc.InstallDependen // GetProgramDependencies returns the set of dependencies required by the program. func (host *yamlLanguageHost) GetProgramDependencies(ctx context.Context, req *pulumirpc.GetProgramDependenciesRequest) (*pulumirpc.GetProgramDependenciesResponse, error) { - return &pulumirpc.GetProgramDependenciesResponse{}, nil + // YAML doesn't _really_ have dependencies per-se but we can list all the "packages" that are referenced + // in the program here. In the presesnce of parameterization this could differ to the set of plugins + // reported by GetRequiredPlugins. + + template, diags, err := host.loadTemplate(req.Info.ProgramDirectory, nil) + if err != nil { + return nil, err + } + if diags.HasErrors() { + return nil, diags + } + + pkgs, pluginDiags := pulumiyaml.GetReferencedPackages(template) + diags.Extend(pluginDiags...) + if diags.HasErrors() { + // We currently swallow the error to allow project config to evaluate + // Specifically, if one sets a config key via the CLI but not within the `config` block + // of their YAML program, it would error. + return &pulumirpc.GetProgramDependenciesResponse{}, nil + } + var dependencies []*pulumirpc.DependencyInfo + for _, pkg := range pkgs { + name := pkg.Name + version := pkg.Version + if pkg.Parameterization != nil { + name = pkg.Parameterization.Name + version = pkg.Parameterization.Version + } + + dependencies = append(dependencies, &pulumirpc.DependencyInfo{ + Name: name, + Version: version, + }) + } + return &pulumirpc.GetProgramDependenciesResponse{ + Dependencies: dependencies, + }, nil } // RuntimeOptionsPrompts returns a list of additional prompts to ask during `pulumi new`. @@ -298,7 +347,7 @@ func (host *yamlLanguageHost) GenerateProject( return nil, err } - err = codegen.GenerateProject(req.TargetDirectory, project, program) + err = codegen.GenerateProject(req.TargetDirectory, project, program, req.LocalDependencies) if err != nil { return nil, err } @@ -451,3 +500,51 @@ func (host *yamlLanguageHost) GeneratePackage(ctx context.Context, req *pulumirp Diagnostics: rpcDiagnostics, }, nil } + +func (host *yamlLanguageHost) Pack(ctx context.Context, req *pulumirpc.PackRequest) (*pulumirpc.PackResponse, error) { + // Yaml "SDKs" are just files, we can just copy the file + if err := os.MkdirAll(req.DestinationDirectory, 0700); err != nil { + return nil, err + } + + files, err := os.ReadDir(req.PackageDirectory) + if err != nil { + return nil, fmt.Errorf("reading package directory: %w", err) + } + + copyFile := func(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening %s: %w", src, err) + } + defer srcFile.Close() + dstFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("creating %s: %w", dst, err) + } + defer dstFile.Close() + if _, err := io.Copy(dstFile, srcFile); err != nil { + return fmt.Errorf("copying %s to %s: %w", src, dst, err) + } + return nil + } + + // We only expect one file in the package directory + var single string + for _, file := range files { + if single != "" { + return nil, fmt.Errorf("multiple files in package directory %s: %s and %s", req.PackageDirectory, single, file.Name()) + } + single = file.Name() + } + + src := filepath.Join(req.PackageDirectory, single) + dst := filepath.Join(req.DestinationDirectory, single) + if err := copyFile(src, dst); err != nil { + return nil, fmt.Errorf("copying %s to %s: %w", src, dst, err) + } + + return &pulumirpc.PackResponse{ + ArtifactPath: dst, + }, nil +}