diff --git a/CHANGELOG.md b/CHANGELOG.md index 72fe1ad6f..7d10a35e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Add validation check that Kibana min/max are valid semver versions. [#99](https://github.com/elastic/integrations-registry/pull/99) * Adding Cache-Control max-age headers to all http responses set to 1h. [#101](https://github.com/elastic/integrations-registry/pull/101) * Validate packages to guarantee only predefined categories can be used. [#100](https://github.com/elastic/integrations-registry/pull/100) +* Cache all manifest on service startup for resource optimisation. [#103](https://github.com/elastic/integrations-registry/pull/103) ### Changed diff --git a/categories.go b/categories.go index e6ca1aff3..adf09d695 100644 --- a/categories.go +++ b/categories.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( @@ -9,8 +13,6 @@ import ( "github.com/elastic/integrations-registry/util" ) - - type Category struct { Id string `yaml:"id" json:"id"` Title string `yaml:"title" json:"title"` @@ -22,19 +24,14 @@ func categoriesHandler() func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { cacheHeaders(w) - packagePaths, err := util.GetPackagePaths(packagesBasePath) + packages, err := util.GetPackages(packagesBasePath) if err != nil { notFound(w, err) return } - - packageList := map[string]*util.Package{} + packageList := map[string]util.Package{} // Get unique list of newest packages - for _, i := range packagePaths { - p, err := util.NewPackage(packagesBasePath, i) - if err != nil { - return - } + for _, p := range packages { // Check if the version exists and if it should be added or not. if pp, ok := packageList[p.Name]; ok { diff --git a/magefile.go b/magefile.go index 830d0d8c1..aa21589bf 100644 --- a/magefile.go +++ b/magefile.go @@ -90,13 +90,13 @@ func BuildIntegrationPackages() error { packagesBasePath = publicDir + "/" + packageDir + "/" } - packagePaths, err := util.GetPackagePaths(packagesBasePath) + packages, err := util.GetPackages(packagesBasePath) if err != nil { return err } - for _, path := range packagePaths { - err = buildPackage(packagesBasePath, path) + for _, p := range packages { + err = buildPackage(packagesBasePath, p) if err != nil { return err } @@ -104,7 +104,7 @@ func BuildIntegrationPackages() error { return nil } -func buildPackage(packagesBasePath, path string) error { +func buildPackage(packagesBasePath string, p util.Package) error { // Change path to simplify tar command currentPath, err := os.Getwd() @@ -117,29 +117,23 @@ func buildPackage(packagesBasePath, path string) error { } defer os.Chdir(currentPath) - err = sh.RunV("tar", "cvzf", path+".tar.gz", filepath.Base(path)+"/") + err = sh.RunV("tar", "cvzf", p.GetPath()+".tar.gz", filepath.Base(p.GetPath())+"/") if err != nil { - return err - } - - // Build package endpoint - p, err := util.NewPackage(".", path) - if err != nil { - return fmt.Errorf("Error creating package: %s: %s", path, err) + return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err) } // Checks if the package is valid err = p.Validate() if err != nil { - return fmt.Errorf("Invalid package %s-%s: %s", p.Name, p.Version, err) + return fmt.Errorf("Invalid package: %s: %s", p.GetPath(), err) } - err = p.LoadAssets(path) + err = p.LoadAssets(p.GetPath()) if err != nil { return err } - err = writeJsonFile(p, path+"/index.json") + err = writeJsonFile(p, p.GetPath()+"/index.json") if err != nil { return err } diff --git a/main.go b/main.go index a882a139d..05d133552 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,8 @@ import ( "strconv" "syscall" + "github.com/elastic/integrations-registry/util" + ucfgYAML "github.com/elastic/go-ucfg/yaml" "github.com/gorilla/mux" @@ -46,6 +48,14 @@ func main() { } packagesBasePath = config.PackagesPath + // Prefill the package cache + packages, err := util.GetPackages(packagesBasePath) + if err != nil { + log.Print(err) + os.Exit(1) + } + log.Printf("%v package manifests loaded into memory.\n", len(packages)) + server := &http.Server{Addr: address, Handler: getRouter()} go func() { diff --git a/search.go b/search.go index 055c8130c..8e1f91cbf 100644 --- a/search.go +++ b/search.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( @@ -47,21 +51,15 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) { } } - packagePaths, err := util.GetPackagePaths(packagesBasePath) + packages, err := util.GetPackages(packagesBasePath) if err != nil { - notFound(w, err) + notFound(w, fmt.Errorf("problem fetching packages: %s", err)) return } - - packagesList := map[string]map[string]*util.Package{} + packagesList := map[string]map[string]util.Package{} // Checks that only the most recent version of an integration is added to the list - for _, path := range packagePaths { - p, err := util.NewPackage(packagesBasePath, path) - if err != nil { - notFound(w, err) - return - } + for _, p := range packages { // Filter by category first as this could heavily reduce the number of packages // It must happen before the version filtering as there only the newest version @@ -100,7 +98,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) { } if _, ok := packagesList[p.Name]; !ok { - packagesList[p.Name] = map[string]*util.Package{} + packagesList[p.Name] = map[string]util.Package{} } packagesList[p.Name][p.Version] = p } @@ -116,7 +114,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) { } } -func getPackageOutput(packagesList map[string]map[string]*util.Package) ([]byte, error) { +func getPackageOutput(packagesList map[string]map[string]util.Package) ([]byte, error) { separator := "@" // Packages need to be sorted to be always outputted in the same order diff --git a/util/package.go b/util/package.go index 6722254fd..16c0d7366 100644 --- a/util/package.go +++ b/util/package.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package util import ( @@ -8,10 +12,9 @@ import ( "strings" "github.com/pkg/errors" + yaml "gopkg.in/yaml.v2" "github.com/blang/semver" - - "gopkg.in/yaml.v2" ) const defaultType = "integration" @@ -137,7 +140,7 @@ func (p *Package) HasKibanaVersion(version *semver.Version) bool { return true } -func (p *Package) IsNewer(pp *Package) bool { +func (p *Package) IsNewer(pp Package) bool { return p.versionSemVer.GT(pp.versionSemVer) } @@ -230,3 +233,7 @@ func (p *Package) Validate() error { return nil } + +func (p *Package) GetPath() string { + return p.Name + "-" + p.Version +} diff --git a/util/package_test.go b/util/package_test.go index f271751b2..5b52e34c1 100644 --- a/util/package_test.go +++ b/util/package_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package util import ( diff --git a/util/packages.go b/util/packages.go index d6c2c5c5c..35633431b 100644 --- a/util/packages.go +++ b/util/packages.go @@ -1,9 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package util -import "io/ioutil" +import ( + "io/ioutil" +) + +var packageList []Package + +// GetPackages returns a slice with all existing packages. +// The list is stored in memory and on the second request directly +// served from memory. This assumes chnages to packages only happen on restart. +// Caching the packages request many file reads every time this method is called. +func GetPackages(packagesBasePath string) ([]Package, error) { + if packageList != nil { + return packageList, nil + } + + packagePaths, err := getPackagePaths(packagesBasePath) + if err != nil { + return nil, err + } + + for _, i := range packagePaths { + p, err := NewPackage(packagesBasePath, i) + if err != nil { + return nil, err + } + packageList = append(packageList, *p) + } + return packageList, nil +} -// GetPackagePaths returns list of available packages, one for each version. -func GetPackagePaths(packagesPath string) ([]string, error) { +// getPackagePaths returns list of available packages, one for each version. +func getPackagePaths(packagesPath string) ([]string, error) { files, err := ioutil.ReadDir(packagesPath) if err != nil {