From 75c06caed972fa7870a28360f3a3f6c18e7d0377 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Mon, 6 Jun 2022 18:13:19 -0700 Subject: [PATCH 01/22] first attempt at GC for oras artifacts Signed-off-by: Akash Singhal --- registry/extension/oras/artifactservice.go | 140 +++++++------------ registry/root.go | 20 ++- registry/storage/extension.go | 71 ++++++++++ registry/storage/garbagecollect.go | 154 ++++++++++++++++++++- registry/storage/paths.go | 9 ++ registry/storage/vacuum.go | 52 ++++++- 6 files changed, 350 insertions(+), 96 deletions(-) diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index bdc690067f4..caa27e5037e 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -10,6 +10,7 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" "github.com/distribution/distribution/v3/registry/extension" + "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" artifactv1 "github.com/oras-project/artifacts-spec/specs-go/v1" @@ -51,49 +52,64 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest blobStatter := h.extContext.Registry.BlobStatter() rootPath := path.Join(referrersLinkPath(repo.Named().Name()), revision.Algorithm().String(), revision.Hex()) - err = h.enumerateReferrerLinks(ctx, rootPath, func(referrerRevision digest.Digest) error { - man, err := manifests.Get(ctx, referrerRevision) - if err != nil { - return err - } - - ArtifactMan, ok := man.(*DeserializedManifest) - if !ok { - // The PUT handler would guard against this situation. Skip this manifest. - return nil - } - - extractedArtifactType := ArtifactMan.ArtifactType() - - // filtering by artifact type or bypass if no artifact type specified - if artifactType == "" || extractedArtifactType == artifactType { - desc, err := blobStatter.Stat(ctx, referrerRevision) + err = storage.EnumerateReferrerLinks(ctx, + rootPath, + h.storageDriver, + blobStatter, + manifests, + repo.Named().Name(), + map[digest.Digest]struct{}{}, + map[digest.Digest]storage.ArtifactManifestDel{}, + func(ctx context.Context, + referrerRevision digest.Digest, + manifestService distribution.ManifestService, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]storage.ArtifactManifestDel, + repoName string, + storageDriver driver.StorageDriver, + blobStatter distribution.BlobStatter) error { + man, err := manifests.Get(ctx, referrerRevision) if err != nil { return err } - desc.MediaType, _, _ = man.Payload() - artifactDesc := artifactv1.Descriptor{ - MediaType: desc.MediaType, - Size: desc.Size, - Digest: desc.Digest, - ArtifactType: extractedArtifactType, + + ArtifactMan, ok := man.(*DeserializedManifest) + if !ok { + // The PUT handler would guard against this situation. Skip this manifest. + return nil } - if annotation, ok := ArtifactMan.Annotations()[createAnnotationName]; !ok { - referrersUnsorted = append(referrersUnsorted, artifactDesc) - } else { - extractedTimestamp, err := time.Parse(createAnnotationTimestampFormat, annotation) + extractedArtifactType := ArtifactMan.ArtifactType() + + // filtering by artifact type or bypass if no artifact type specified + if artifactType == "" || extractedArtifactType == artifactType { + desc, err := blobStatter.Stat(ctx, referrerRevision) if err != nil { - return fmt.Errorf("failed to parse created annotation timestamp: %v", err) + return err + } + desc.MediaType, _, _ = man.Payload() + artifactDesc := artifactv1.Descriptor{ + MediaType: desc.MediaType, + Size: desc.Size, + Digest: desc.Digest, + ArtifactType: extractedArtifactType, + } + + if annotation, ok := ArtifactMan.Annotations()[createAnnotationName]; !ok { + referrersUnsorted = append(referrersUnsorted, artifactDesc) + } else { + extractedTimestamp, err := time.Parse(createAnnotationTimestampFormat, annotation) + if err != nil { + return fmt.Errorf("failed to parse created annotation timestamp: %v", err) + } + referrersWrappers = append(referrersWrappers, referrersSortedWrapper{ + createdAt: extractedTimestamp, + descriptor: artifactDesc, + }) } - referrersWrappers = append(referrersWrappers, referrersSortedWrapper{ - createdAt: extractedTimestamp, - descriptor: artifactDesc, - }) } - } - return nil - }) + return nil + }) if err != nil { switch err.(type) { @@ -116,57 +132,3 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest referrersSorted = append(referrersSorted, referrersUnsorted...) return referrersSorted, nil } -func (h *referrersHandler) enumerateReferrerLinks(ctx context.Context, rootPath string, ingestor func(digest.Digest) error) error { - blobStatter := h.extContext.Registry.BlobStatter() - - return h.storageDriver.Walk(ctx, rootPath, func(fileInfo driver.FileInfo) error { - // exit early if directory... - if fileInfo.IsDir() { - return nil - } - filePath := fileInfo.Path() - - // check if it's a link - _, fileName := path.Split(filePath) - if fileName != "link" { - return nil - } - - // read the digest found in link - digest, err := h.readlink(ctx, filePath) - if err != nil { - return err - } - - // ensure this conforms to the linkPathFns - _, err = blobStatter.Stat(ctx, digest) - if err != nil { - // we expect this error to occur so we move on - if err == distribution.ErrBlobUnknown { - return nil - } - return err - } - - err = ingestor(digest) - if err != nil { - return err - } - - return nil - }) -} - -func (h *referrersHandler) readlink(ctx context.Context, path string) (digest.Digest, error) { - content, err := h.storageDriver.GetContent(ctx, path) - if err != nil { - return "", err - } - - linked, err := digest.Parse(string(content)) - if err != nil { - return "", err - } - - return linked, nil -} diff --git a/registry/root.go b/registry/root.go index e32e0984cb2..769f689e3cc 100644 --- a/registry/root.go +++ b/registry/root.go @@ -5,6 +5,7 @@ import ( "os" dcontext "github.com/distribution/distribution/v3/context" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver/factory" "github.com/distribution/distribution/v3/version" @@ -71,7 +72,24 @@ var GCCmd = &cobra.Command{ os.Exit(1) } - registry, err := storage.NewRegistry(ctx, driver, storage.Schema1SigningKey(k)) + extensions := config.Extensions + extensionNamespaces := []extension.Namespace{} + for key, options := range extensions { + ns, err := extension.Get(ctx, key, driver, options) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to configure extension namespace (%s): %s", key, err) + os.Exit(1) + } + extensionNamespaces = append(extensionNamespaces, ns) + } + + options := []storage.RegistryOption{storage.Schema1SigningKey(k)} + // add the extended storage for every namespace to the new registry options + for _, ns := range extensionNamespaces { + options = append(options, storage.AddExtendedStorage(ns)) + } + + registry, err := storage.NewRegistry(ctx, driver, options...) if err != nil { fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err) os.Exit(1) diff --git a/registry/storage/extension.go b/registry/storage/extension.go index edd2af4ac0d..2a7f80c8b56 100644 --- a/registry/storage/extension.go +++ b/registry/storage/extension.go @@ -2,8 +2,10 @@ package storage import ( "context" + "path" "github.com/distribution/distribution/v3" + "github.com/distribution/distribution/v3/registry/storage/driver" storagedriver "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" ) @@ -115,3 +117,72 @@ func GetTagLinkReadOnlyBlobStore( }, } } + +func EnumerateReferrerLinks(ctx context.Context, + rootPath string, + stDriver storagedriver.StorageDriver, + blobStatter distribution.BlobStatter, + manifestService distribution.ManifestService, + repositoryName string, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]ArtifactManifestDel, + ingestor func(ctx context.Context, + digest digest.Digest, + manifestService distribution.ManifestService, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]ArtifactManifestDel, + repoName string, + storageDriver driver.StorageDriver, + blobStatter distribution.BlobStatter) error) error { + + return stDriver.Walk(ctx, rootPath, func(fileInfo driver.FileInfo) error { + // exit early if directory... + if fileInfo.IsDir() { + return nil + } + filePath := fileInfo.Path() + + // check if it's a link + _, fileName := path.Split(filePath) + if fileName != "link" { + return nil + } + + // read the digest found in link + digest, err := readlink(ctx, filePath, stDriver) + if err != nil { + return err + } + + // ensure this conforms to the linkPathFns + _, err = blobStatter.Stat(ctx, digest) + if err != nil { + // we expect this error to occur so we move on + if err == distribution.ErrBlobUnknown { + return nil + } + return err + } + + err = ingestor(ctx, digest, manifestService, markSet, artifactManifestIndex, repositoryName, stDriver, blobStatter) + if err != nil { + return err + } + + return nil + }) +} + +func readlink(ctx context.Context, path string, stDriver storagedriver.StorageDriver) (digest.Digest, error) { + content, err := stDriver.GetContent(ctx, path) + if err != nil { + return "", err + } + + linked, err := digest.Parse(string(content)) + if err != nil { + return "", err + } + + return linked, nil +} diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 13c9b180c89..4ecfcfe1e01 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -3,11 +3,13 @@ package storage import ( "context" "fmt" + "path" "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" + v1 "github.com/oras-project/artifacts-spec/specs-go/v1" ) func emit(format string, a ...interface{}) { @@ -27,6 +29,12 @@ type ManifestDel struct { Tags []string } +// ArtifactManifestDel contains artifact manifest structure which will be deleted +type ArtifactManifestDel struct { + Name string + ArtifactDigest digest.Digest +} + // MarkAndSweep performs a mark and sweep of registry data func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, opts GCOpts) error { repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) @@ -37,6 +45,7 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis // mark markSet := make(map[digest.Digest]struct{}) manifestArr := make([]ManifestDel, 0) + artifactManifestIndex := make(map[digest.Digest]ArtifactManifestDel) err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { emit(repoName) @@ -61,6 +70,28 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis } err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { + manifest, err := manifestService.Get(ctx, dgst) + if err != nil { + return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) + } + + mediaType, _, err := manifest.Payload() + if err != nil { + return err + } + + // if the manifest is an oras artifact, skip it + // the artifact marking occurs when walking the refs + if mediaType == v1.MediaTypeArtifactManifest { + return nil + } + + blobStatter := registry.BlobStatter() + referrerRootPath, err := pathFor(referrersRootPathSpec{name: repository.Named().Name()}) + if err != nil { + return err + } + if opts.RemoveUntagged { // fetch all tags where this manifest is the latest one tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst}) @@ -77,6 +108,26 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to retrieve tags %v", err) } manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) + + // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion + rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + err = EnumerateReferrerLinks(ctx, + rootPath, + storageDriver, + blobStatter, + manifestService, + repository.Named().Name(), + markSet, + artifactManifestIndex, + artifactSweepIngestor) + + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } return nil } } @@ -84,17 +135,31 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis emit("%s: marking manifest %s ", repoName, dgst) markSet[dgst] = struct{}{} - manifest, err := manifestService.Get(ctx, dgst) - if err != nil { - return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) - } - descriptors := manifest.References() for _, descriptor := range descriptors { markSet[descriptor.Digest] = struct{}{} emit("%s: marking blob %s", repoName, descriptor.Digest) } + // recurse child artifact as subject to find lower level referrers + rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + err = EnumerateReferrerLinks(ctx, + rootPath, + storageDriver, + blobStatter, + manifestService, + repository.Named().Name(), + markSet, + artifactManifestIndex, + artifactMarkIngestor) + + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } return nil }) @@ -123,6 +188,13 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) } } + // remove each artifact in the index + for artifactDigest, obj := range artifactManifestIndex { + err = vacuum.RemoveArtifactManifest(obj.Name, artifactDigest) + if err != nil { + return fmt.Errorf("failed to delete artifact manifest %s: %v", artifactDigest, err) + } + } } blobService := registry.Blobs() deleteSet := make(map[digest.Digest]struct{}) @@ -150,3 +222,75 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return err } + +// ingestor method used in EnumerateReferrerLinks +// marks each artifact manifest and associated blobs +func artifactMarkIngestor(ctx context.Context, + referrerRevision digest.Digest, + manifestService distribution.ManifestService, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]ArtifactManifestDel, + repoName string, + storageDriver driver.StorageDriver, + blobStatter distribution.BlobStatter) error { + man, err := manifestService.Get(ctx, referrerRevision) + if err != nil { + return err + } + + // mark the artifact manifest blob + emit("%s: marking artifact manifest %s ", repoName, referrerRevision.String()) + markSet[referrerRevision] = struct{}{} + + // mark the artifact blobs + descriptors := man.References() + for _, descriptor := range descriptors { + markSet[descriptor.Digest] = struct{}{} + emit("%s: marking blob %s", repoName, descriptor.Digest) + } + referrerRootPath, err := pathFor(referrersRootPathSpec{name: repoName}) + if err != nil { + return err + } + rootPath := path.Join(referrerRootPath, referrerRevision.Algorithm().String(), referrerRevision.Hex()) + _, err = storageDriver.Stat(ctx, rootPath) + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return EnumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactMarkIngestor) +} + +// ingestor method used in EnumerateReferrerLinks +// indexes each artifact manifest and adds ArtifactManifestDel struct to index +func artifactSweepIngestor(ctx context.Context, + referrerRevision digest.Digest, + manifestService distribution.ManifestService, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]ArtifactManifestDel, + repoName string, + storageDriver driver.StorageDriver, + blobStatter distribution.BlobStatter) error { + + // index the manifest + emit("%s: indexing artifact manifest %s ", repoName, referrerRevision.String()) + artifactManifestIndex[referrerRevision] = ArtifactManifestDel{Name: repoName, ArtifactDigest: referrerRevision} + + referrerRootPath, err := pathFor(referrersRootPathSpec{name: repoName}) + if err != nil { + return err + } + rootPath := path.Join(referrerRootPath, referrerRevision.Algorithm().String(), referrerRevision.Hex()) + _, err = storageDriver.Stat(ctx, rootPath) + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return EnumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactMarkIngestor) +} diff --git a/registry/storage/paths.go b/registry/storage/paths.go index 4e9ad0856e8..75d9fd40985 100644 --- a/registry/storage/paths.go +++ b/registry/storage/paths.go @@ -242,6 +242,8 @@ func pathFor(spec pathSpec) (string, error) { return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "hashstates", string(v.alg), offset)...), nil case repositoriesRootPathSpec: return path.Join(repoPrefix...), nil + case referrersRootPathSpec: + return path.Join("/docker/registry/", "v2", "repositories", v.name, "_refs", "subjects"), nil default: // TODO(sday): This is an internal error. Ensure it doesn't escape (panic?). return "", fmt.Errorf("unknown path spec: %#v", v) @@ -436,6 +438,13 @@ type repositoriesRootPathSpec struct { func (repositoriesRootPathSpec) pathSpec() {} +// referrersRootPathSpec returns the root of referrers links +type referrersRootPathSpec struct { + name string +} + +func (referrersRootPathSpec) pathSpec() {} + // digestPathComponents provides a consistent path breakdown for a given // digest. For a generic digest, it will be as follows: // diff --git a/registry/storage/vacuum.go b/registry/storage/vacuum.go index 749fb31906b..a3ad2fcc5e2 100644 --- a/registry/storage/vacuum.go +++ b/registry/storage/vacuum.go @@ -51,6 +51,7 @@ func (v Vacuum) RemoveBlob(dgst string) error { } // RemoveManifest removes a manifest from the filesystem +// Removes manifest's ref folder if it exists func (v Vacuum) RemoveManifest(name string, dgst digest.Digest, tags []string) error { // remove a tag manifest reference, in case of not found continue to next one for _, tag := range tags { @@ -81,7 +82,26 @@ func (v Vacuum) RemoveManifest(name string, dgst digest.Digest, tags []string) e return err } dcontext.GetLogger(v.ctx).Infof("deleting manifest: %s", manifestPath) - return v.driver.Delete(v.ctx, manifestPath) + err = v.driver.Delete(v.ctx, manifestPath) + if err != nil { + return err + } + + referrerRootPath, err := pathFor(referrersRootPathSpec{name: name}) + if err != nil { + return err + } + fullArtifactManifestPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + dcontext.GetLogger(v.ctx).Infof("deleting manifest ref folder: %s", fullArtifactManifestPath) + v.driver.Delete(v.ctx, fullArtifactManifestPath) + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return nil } // RemoveRepository removes a repository directory from the @@ -100,3 +120,33 @@ func (v Vacuum) RemoveRepository(repoName string) error { return nil } + +// RemoveArtifactManifest removes a artifact manifest from the filesystem +// Removes manifest revision file and manifest ref folder if it exists +func (v Vacuum) RemoveArtifactManifest(name string, artifactDgst digest.Digest) error { + manifestPath, err := pathFor(manifestRevisionPathSpec{name: name, revision: artifactDgst}) + if err != nil { + return err + } + dcontext.GetLogger(v.ctx).Infof("deleting artifact manifest: %s", manifestPath) + err = v.driver.Delete(v.ctx, manifestPath) + if err != nil { + return err + } + + referrerRootPath, err := pathFor(referrersRootPathSpec{name: name}) + if err != nil { + return err + } + fullArtifactManifestPath := path.Join(referrerRootPath, artifactDgst.Algorithm().String(), artifactDgst.Hex()) + dcontext.GetLogger(v.ctx).Infof("deleting artifact manifest ref: %s", fullArtifactManifestPath) + err = v.driver.Delete(v.ctx, fullArtifactManifestPath) + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return nil +} From 4cc8e6d2aa62dd20e3934833d5e36a31cca94760 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Tue, 7 Jun 2022 14:31:59 -0700 Subject: [PATCH 02/22] add documentation Signed-off-by: Akash Singhal --- docs/garbage-collection.md | 8 +++ docs/referrers.md | 97 ++++++++++++++++++++++++++++++ registry/storage/garbagecollect.go | 4 +- 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 docs/referrers.md diff --git a/docs/garbage-collection.md b/docs/garbage-collection.md index 928fab9aee8..c6b15542acd 100644 --- a/docs/garbage-collection.md +++ b/docs/garbage-collection.md @@ -122,3 +122,11 @@ blob eligible for deletion: sha256:87192bdbe00f8f2a62527f36bb4c7c7f4eaf9307e4b87 blob eligible for deletion: sha256:b549a9959a664038fc35c155a95742cf12297672ca0ae35735ec027d55bf4e97 blob eligible for deletion: sha256:f251d679a7c61455f06d793e43c06786d7766c88b8c24edf242b2c08e3c3f599 ``` + +## Garbage Collection With Referrers + +The life of a reference artifact is directly linked to it's subject. When a reference artifact's subject manifest is deleted, the attached artifacts and its descendants must be deleted. + +Manifest garbage collection is extended to include reference artifact collection. During the marking process, each manifest is queried for any reference artifacts by enumerating the link files at the path `repositories//_refs/subjects/sha256/`. For each artifact, the artifact manifest and its blobs are marked. Finally, collection recurses to look for further artifact descendants to mark in a similar fashion. + +If a manifest is indexed for deletion because it is untagged, the attached reference artifacts are also indexed. Similar to the marking process, the subject manifest's `_ref` folder is queried for reference artifact descendants. Each encountered descendant is indexed. Indexing recurses to the next levels of descendants until all successor artifacts are indexed. Finally, during manifest link deletion, the revision link files of the indexed artifact manifests as well as the corresponding `_refs` are removed from storage. \ No newline at end of file diff --git a/docs/referrers.md b/docs/referrers.md new file mode 100644 index 00000000000..164a97c6559 --- /dev/null +++ b/docs/referrers.md @@ -0,0 +1,97 @@ +[[__TOC__]] + +# ORAS Artifacts Distribution + +This document describes an experimental prototype that implements the +[ORAS Artifact Manifest](https://github.com/oras-project/artifacts-spec) spec. + +## Implementation + +To power the [/referrers](https://github.com/oras-project/artifacts-spec/blob/main/manifest-referrers-api.md) API, the +referrers of a manifest are indexed in the repository store. The following example illustrates the creation of this +index. + +The `nginx:v1` image is already persisted: + +- repository: `nginx` +- digest: `sha256:111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m` +- tag: `v1.0` + +The repository store layout is represented as: + +```bash + +└── v2 + └── repositories + └── nginx + └── _manifests + └── revisions + └── sha256 + └── 111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m + └── link +``` + +Push a signature as blob and an ORAS Artifact that contains a blobs property referencing the signature, with the +following properties: + +- digest: `sha256:222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i` +- `subjectManifest` digest: `sha256:111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m` +- `artifactType`: `application/vnd.example.artifact` + +On `PUT`, the artifact appears as a manifest revision. Additionally, an index entry is created under +the subject ref folder to facilitate a lookup to the referrer. The index path where the entry is added is +`/_refs/subjects/sha256/`, as shown below. + +``` + +└── v2 + └── repositories + └── nginx + ├── _manifests + │ └── _revisions + │ └── sha256 + │ ├── 111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m + │ │ └── link + │ └── 222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i + │ └── link + └── _refs + └── subjects + └── sha256 + └── 111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m + └── sha256 + └── 222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i + └── link +``` + +Push another ORAS artifact with the following properties: + +- digest: `sha256:333ic0c33ebc4a74a0a554c86ac2b28ddf3454a5ad9cf90ea8cea9f9e75c333i` +- `subjectManifest` digest: `sha256:111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m` +- `artifactType`: `application/vnd.another.example.artifact` + +This results in an addition to the index as shown below. + +``` + +└── v2 + └── repositories + └── nginx + ├── _manifests + │ └── _revisions + │ └── sha256 + │ ├── 111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m + │ │ └── link + │ ├── 222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i + │ │ └── link + │ └── 333ic0c33ebc4a74a0a554c86ac2b28ddf3454a5ad9cf90ea8cea9f9e75c333i + │ └── link + └── _refs + └── subjects + └── sha256 + └── 111ma2d22ae5ef400769fa51c84717264cd1520ac8d93dc071374c1be49a111m + └── sha256 + ├── 222ibbf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cb222i + │ └── link + └── 333ic0c33ebc4a74a0a554c86ac2b28ddf3454a5ad9cf90ea8cea9f9e75c333i + └── link +``` \ No newline at end of file diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 4ecfcfe1e01..40d38f2f319 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -208,7 +208,7 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("error enumerating blobs: %v", err) } - emit("\n%d blobs marked, %d blobs and %d manifests eligible for deletion", len(markSet), len(deleteSet), len(manifestArr)) + emit("\n%d blobs marked, %d blobs, %d manifests, and %d artifacts eligible for deletion", len(markSet), len(deleteSet), len(manifestArr), len(artifactManifestIndex)) for dgst := range deleteSet { emit("blob eligible for deletion: %s", dgst) if opts.DryRun { @@ -292,5 +292,5 @@ func artifactSweepIngestor(ctx context.Context, } return err } - return EnumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactMarkIngestor) + return EnumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactSweepIngestor) } From df10cedb8ce48d297124a9309a5fb7c00a6e6cbb Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Thu, 9 Jun 2022 15:30:05 -0700 Subject: [PATCH 03/22] adding artifact support on Delete path; address comments Signed-off-by: Akash Singhal --- registry/storage/garbagecollect_test.go | 17 ++++++++++++ registry/storage/linkedblobstore.go | 9 +++++++ registry/storage/manifeststore.go | 36 +++++++++++++++++++++++++ registry/storage/paths.go | 2 +- registry/storage/registry.go | 1 + 5 files changed, 64 insertions(+), 1 deletion(-) diff --git a/registry/storage/garbagecollect_test.go b/registry/storage/garbagecollect_test.go index 25c5e2f8fe0..14edc533ca1 100644 --- a/registry/storage/garbagecollect_test.go +++ b/registry/storage/garbagecollect_test.go @@ -500,3 +500,20 @@ func TestOrphanBlobDeleted(t *testing.T) { } } } + +// func TestReferrersBlobsDeleted(t *testing.T) { +// inmemoryDriver := inmemory.New() +// ctx := context.Background() +// // extConfig := configuration.ExtensionConfig{} +// // ns, err := extension.Get(ctx, "oras", inmemoryDriver, extConfig) +// // if err != nil { +// // fmt.Fprintf(os.Stderr, "unable to configure extension namespace oras: %s", err) +// // os.Exit(1) +// // } + +// options := []RegistryOption{AddExtendedStorage(ns)} +// // add the extended storage for every namespace to the new registry options + +// registry := createRegistry(t, inmemoryDriver, options...) +// repo := makeRepository(t, registry, "michael_z_doukas") +// } diff --git a/registry/storage/linkedblobstore.go b/registry/storage/linkedblobstore.go index 89573ddc79d..dde328bc709 100644 --- a/registry/storage/linkedblobstore.go +++ b/registry/storage/linkedblobstore.go @@ -463,3 +463,12 @@ func blobLinkPath(name string, dgst digest.Digest) (string, error) { func manifestRevisionLinkPath(name string, dgst digest.Digest) (string, error) { return pathFor(manifestRevisionLinkPathSpec{name: name, revision: dgst}) } + +// artifactRefPath provides the path to the manifest's _refs directory. +func artifactRefPath(name string, dgst digest.Digest) (string, error) { + rootPath, err := pathFor(referrersRootPathSpec{name: name}) + if err != nil { + return "", err + } + return path.Join(rootPath, dgst.Algorithm().String(), dgst.Hex()), nil +} diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index a4c7ec0c5e2..bbdf730484d 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "path" "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" @@ -12,6 +13,7 @@ import ( "github.com/distribution/distribution/v3/manifest/ocischema" "github.com/distribution/distribution/v3/manifest/schema1" "github.com/distribution/distribution/v3/manifest/schema2" + "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -171,6 +173,40 @@ func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest // Delete removes the revision of the specified manifest. func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error { dcontext.GetLogger(ms.ctx).Debug("(*manifestStore).Delete") + + // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion + artifactManifestIndex := make(map[digest.Digest]ArtifactManifestDel) + repositoryName := ms.repository.Named().Name() + referrerRootPath, err := pathFor(referrersRootPathSpec{name: repositoryName}) + if err != nil { + return err + } + rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + err = EnumerateReferrerLinks(ctx, + rootPath, + ms.blobStore.driver, + ms.repository.statter, + ms, + repositoryName, + map[digest.Digest]struct{}{}, + artifactManifestIndex, + artifactSweepIngestor) + + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + // delete the artifact manifest revision and the _refs directory for each artifact indexed + for key, _ := range artifactManifestIndex { + err := ms.blobStore.Delete(ctx, key) + if err != nil { + return err + } + } + // delete the manifest revision and the _refs directory for original manifest return ms.blobStore.Delete(ctx, dgst) } diff --git a/registry/storage/paths.go b/registry/storage/paths.go index 75d9fd40985..1d90241ee88 100644 --- a/registry/storage/paths.go +++ b/registry/storage/paths.go @@ -243,7 +243,7 @@ func pathFor(spec pathSpec) (string, error) { case repositoriesRootPathSpec: return path.Join(repoPrefix...), nil case referrersRootPathSpec: - return path.Join("/docker/registry/", "v2", "repositories", v.name, "_refs", "subjects"), nil + return path.Join(append(repoPrefix, v.name, "_refs", "subjects")...), nil default: // TODO(sday): This is an internal error. Ensure it doesn't escape (panic?). return "", fmt.Errorf("unknown path spec: %#v", v) diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 5eedb4433b1..d7b0d01a583 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -230,6 +230,7 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M // 2.1.0 unintentionally linked into _layers. manifestRevisionLinkPath, blobLinkPath, + artifactRefPath, } manifestDirectoryPathSpec := manifestRevisionsPathSpec{name: repo.name.Name()} From e73539000049e20a6350bc29ed2c4710a96a16d9 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Thu, 9 Jun 2022 15:35:14 -0700 Subject: [PATCH 04/22] address more comments Signed-off-by: Akash Singhal --- docs/garbage-collection.md | 2 +- docs/referrers.md | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/garbage-collection.md b/docs/garbage-collection.md index c6b15542acd..26be77a700b 100644 --- a/docs/garbage-collection.md +++ b/docs/garbage-collection.md @@ -125,7 +125,7 @@ blob eligible for deletion: sha256:f251d679a7c61455f06d793e43c06786d7766c88b8c24 ## Garbage Collection With Referrers -The life of a reference artifact is directly linked to it's subject. When a reference artifact's subject manifest is deleted, the attached artifacts and its descendants must be deleted. +The life of a reference artifact is directly linked to its subject. When a reference artifact's subject manifest is deleted, the attached artifacts and its descendants must be deleted. Manifest garbage collection is extended to include reference artifact collection. During the marking process, each manifest is queried for any reference artifacts by enumerating the link files at the path `repositories//_refs/subjects/sha256/`. For each artifact, the artifact manifest and its blobs are marked. Finally, collection recurses to look for further artifact descendants to mark in a similar fashion. diff --git a/docs/referrers.md b/docs/referrers.md index 164a97c6559..aba155032c8 100644 --- a/docs/referrers.md +++ b/docs/referrers.md @@ -1,5 +1,3 @@ -[[__TOC__]] - # ORAS Artifacts Distribution This document describes an experimental prototype that implements the From 98ea25a027672825de28f6239118c4181433c435 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Tue, 14 Jun 2022 11:12:54 -0700 Subject: [PATCH 05/22] move artifact manifest and handler to new location; move extension namespace to storage package; add garbage collect unit test; fix manifest storage unit test Signed-off-by: Akash Singhal --- .../orasartifact}/artifactmanifest.go | 48 +++---- registry/extension/distribution/manifests.go | 3 +- registry/extension/distribution/registry.go | 19 ++- registry/extension/distribution/taghistory.go | 3 +- registry/extension/oci/discover.go | 8 +- registry/extension/oci/oci.go | 19 ++- registry/extension/oras/artifactservice.go | 17 +-- registry/extension/oras/oras.go | 23 ++-- registry/handlers/app.go | 11 +- registry/root.go | 5 +- .../artifactmanifesthandler.go | 46 +++---- .../artifactmanifesthandler_test.go | 94 +++++--------- registry/storage/artifactnamespace_mock.go | 25 ++++ .../extensionnamespace.go} | 5 +- registry/storage/garbagecollect_test.go | 118 +++++++++++++++--- registry/storage/manifeststore.go | 6 +- 16 files changed, 261 insertions(+), 189 deletions(-) rename {registry/extension/oras => manifest/orasartifact}/artifactmanifest.go (74%) rename registry/{extension/oras => storage}/artifactmanifesthandler.go (73%) rename registry/{extension/oras => storage}/artifactmanifesthandler_test.go (64%) create mode 100644 registry/storage/artifactnamespace_mock.go rename registry/{extension/extension.go => storage/extensionnamespace.go} (97%) diff --git a/registry/extension/oras/artifactmanifest.go b/manifest/orasartifact/artifactmanifest.go similarity index 74% rename from registry/extension/oras/artifactmanifest.go rename to manifest/orasartifact/artifactmanifest.go index e883efd4afd..5ad113ff583 100644 --- a/registry/extension/oras/artifactmanifest.go +++ b/manifest/orasartifact/artifactmanifest.go @@ -1,15 +1,19 @@ -package oras +package orasartifact import ( "encoding/json" "errors" "fmt" + "time" "github.com/distribution/distribution/v3" "github.com/opencontainers/go-digest" v1 "github.com/oras-project/artifacts-spec/specs-go/v1" ) +const CreateAnnotationName = "io.cncf.oras.artifact.created" +const CreateAnnotationTimestampFormat = time.RFC3339 + func init() { unmarshalFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { d := new(DeserializedManifest) @@ -29,32 +33,32 @@ func init() { // Manifest describes ORAS artifact manifests. type Manifest struct { - inner v1.Manifest + Inner v1.Manifest } // ArtifactType returns the artifactType of this ORAS artifact. func (a Manifest) ArtifactType() string { - return a.inner.ArtifactType + return a.Inner.ArtifactType } // Annotations returns the annotations of this ORAS artifact. func (a Manifest) Annotations() map[string]string { - return a.inner.Annotations + return a.Inner.Annotations } // MediaType returns the media type of this ORAS artifact. func (a Manifest) MediaType() string { - return a.inner.MediaType + return a.Inner.MediaType } // References returns the distribution descriptors for the referenced blobs. func (a Manifest) References() []distribution.Descriptor { - blobs := make([]distribution.Descriptor, len(a.inner.Blobs)) - for i := range a.inner.Blobs { + blobs := make([]distribution.Descriptor, len(a.Inner.Blobs)) + for i := range a.Inner.Blobs { blobs[i] = distribution.Descriptor{ - MediaType: a.inner.Blobs[i].MediaType, - Digest: a.inner.Blobs[i].Digest, - Size: a.inner.Blobs[i].Size, + MediaType: a.Inner.Blobs[i].MediaType, + Digest: a.Inner.Blobs[i].Digest, + Size: a.Inner.Blobs[i].Size, } } return blobs @@ -63,9 +67,9 @@ func (a Manifest) References() []distribution.Descriptor { // Subject returns the the subject manifest this artifact references. func (a Manifest) Subject() distribution.Descriptor { return distribution.Descriptor{ - MediaType: a.inner.Subject.MediaType, - Digest: a.inner.Subject.Digest, - Size: a.inner.Subject.Size, + MediaType: a.Inner.Subject.MediaType, + Digest: a.Inner.Subject.Digest, + Size: a.Inner.Subject.Size, } } @@ -73,17 +77,17 @@ func (a Manifest) Subject() distribution.Descriptor { type DeserializedManifest struct { Manifest - // raw is the raw byte representation of the ORAS artifact. - raw []byte + // Raw is the Raw byte representation of the ORAS artifact. + Raw []byte } // UnmarshalJSON populates a new Manifest struct from JSON data. func (d *DeserializedManifest) UnmarshalJSON(b []byte) error { - d.raw = make([]byte, len(b)) - copy(d.raw, b) + d.Raw = make([]byte, len(b)) + copy(d.Raw, b) var man v1.Manifest - if err := json.Unmarshal(d.raw, &man); err != nil { + if err := json.Unmarshal(d.Raw, &man); err != nil { return err } if man.ArtifactType == "" { @@ -93,15 +97,15 @@ func (d *DeserializedManifest) UnmarshalJSON(b []byte) error { return errors.New("mediaType is invalid") } - d.inner = man + d.Inner = man return nil } // MarshalJSON returns the raw content. func (d *DeserializedManifest) MarshalJSON() ([]byte, error) { - if len(d.raw) > 0 { - return d.raw, nil + if len(d.Raw) > 0 { + return d.Raw, nil } return nil, errors.New("JSON representation not initialized in DeserializedManifest") @@ -111,5 +115,5 @@ func (d *DeserializedManifest) MarshalJSON() ([]byte, error) { // used to calculate the content identifier. func (d DeserializedManifest) Payload() (string, []byte, error) { // NOTE: This is a hack. The media type should be read from storage. - return v1.MediaTypeArtifactManifest, d.raw, nil + return v1.MediaTypeArtifactManifest, d.Raw, nil } diff --git a/registry/extension/distribution/manifests.go b/registry/extension/distribution/manifests.go index 51a52e8171f..2aa67d5126f 100644 --- a/registry/extension/distribution/manifests.go +++ b/registry/extension/distribution/manifests.go @@ -6,7 +6,6 @@ import ( "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -19,7 +18,7 @@ type manifestsGetAPIResponse struct { // manifestHandler handles requests for manifests under a manifest name. type manifestHandler struct { - *extension.Context + *storage.Context storageDriver driver.StorageDriver } diff --git a/registry/extension/distribution/registry.go b/registry/extension/distribution/registry.go index 8e6c5000814..7a8c71cbd09 100644 --- a/registry/extension/distribution/registry.go +++ b/registry/extension/distribution/registry.go @@ -7,7 +7,6 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" @@ -34,7 +33,7 @@ type distributionOptions struct { } // newDistNamespace creates a new extension namespace with the name "distribution" -func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { +func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (storage.Namespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { @@ -67,7 +66,7 @@ func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, o func init() { // register the extension namespace. - extension.Register(namespaceName, newDistNamespace) + storage.Register(namespaceName, newDistNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. @@ -77,11 +76,11 @@ func (o *distributionNamespace) GetManifestHandlers(repo distribution.Repository } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route { - var routes []extension.Route +func (d *distributionNamespace) GetRepositoryRoutes() []storage.Route { + var routes []storage.Route if d.manifestsEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, storage.Route{ Namespace: namespaceName, Extension: extensionName, Component: manifestsComponentName, @@ -99,7 +98,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route { } if d.tagHistoryEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, storage.Route{ Namespace: namespaceName, Extension: extensionName, Component: tagHistoryComponentName, @@ -132,7 +131,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route { // GetRegistryRoutes returns a list of extension routes scoped at a registry level // There are no registry scoped routes exposed by this namespace -func (d *distributionNamespace) GetRegistryRoutes() []extension.Route { +func (d *distributionNamespace) GetRegistryRoutes() []storage.Route { return nil } @@ -151,7 +150,7 @@ func (d *distributionNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (d *distributionNamespace) tagHistoryDispatcher(ctx *extension.Context, r *http.Request) http.Handler { +func (d *distributionNamespace) tagHistoryDispatcher(ctx *storage.Context, r *http.Request) http.Handler { tagHistoryHandler := &tagHistoryHandler{ Context: ctx, storageDriver: d.storageDriver, @@ -162,7 +161,7 @@ func (d *distributionNamespace) tagHistoryDispatcher(ctx *extension.Context, r * } } -func (d *distributionNamespace) manifestsDispatcher(ctx *extension.Context, r *http.Request) http.Handler { +func (d *distributionNamespace) manifestsDispatcher(ctx *storage.Context, r *http.Request) http.Handler { manifestsHandler := &manifestHandler{ Context: ctx, storageDriver: d.storageDriver, diff --git a/registry/extension/distribution/taghistory.go b/registry/extension/distribution/taghistory.go index 9cb957b87d8..2e560feb277 100644 --- a/registry/extension/distribution/taghistory.go +++ b/registry/extension/distribution/taghistory.go @@ -6,7 +6,6 @@ import ( "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -20,7 +19,7 @@ type tagHistoryAPIResponse struct { // manifestHandler handles requests for manifests under a manifest name. type tagHistoryHandler struct { - *extension.Context + *storage.Context storageDriver driver.StorageDriver } diff --git a/registry/extension/oci/discover.go b/registry/extension/oci/discover.go index 3d2ec3fce97..007e0007473 100644 --- a/registry/extension/oci/discover.go +++ b/registry/extension/oci/discover.go @@ -5,17 +5,17 @@ import ( "net/http" "github.com/distribution/distribution/v3/registry/api/errcode" - "github.com/distribution/distribution/v3/registry/extension" + "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" ) type discoverGetAPIResponse struct { - Extensions []extension.EnumerateExtension `json:"extensions"` + Extensions []storage.EnumerateExtension `json:"extensions"` } // extensionHandler handles requests for manifests under a manifest name. type extensionHandler struct { - *extension.Context + *storage.Context storageDriver driver.StorageDriver } @@ -25,7 +25,7 @@ func (eh *extensionHandler) getExtensions(w http.ResponseWriter, r *http.Request w.Header().Set("Content-Type", "application/json") // get list of extension information seperated at the namespace level - enumeratedExtensions := extension.EnumerateRegistered(*eh.Context) + enumeratedExtensions := storage.EnumerateRegistered(*eh.Context) // remove the oci extension so it's not returned by discover for i, e := range enumeratedExtensions { diff --git a/registry/extension/oci/oci.go b/registry/extension/oci/oci.go index d68e5b8cd8f..4c738845094 100644 --- a/registry/extension/oci/oci.go +++ b/registry/extension/oci/oci.go @@ -7,7 +7,6 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" @@ -32,7 +31,7 @@ type ociOptions struct { } // newOciNamespace creates a new extension namespace with the name "oci" -func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { +func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (storage.Namespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err @@ -60,7 +59,7 @@ func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, op func init() { // register the extension namespace. - extension.Register(namespaceName, newOciNamespace) + storage.Register(namespaceName, newOciNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. @@ -70,11 +69,11 @@ func (o *ociNamespace) GetManifestHandlers(repo distribution.Repository, blobSto } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (o *ociNamespace) GetRepositoryRoutes() []extension.Route { - var routes []extension.Route +func (o *ociNamespace) GetRepositoryRoutes() []storage.Route { + var routes []storage.Route if o.discoverEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, storage.Route{ Namespace: namespaceName, Extension: extensionName, Component: discoverComponentName, @@ -95,11 +94,11 @@ func (o *ociNamespace) GetRepositoryRoutes() []extension.Route { } // GetRegistryRoutes returns a list of extension routes scoped at a registry level -func (o *ociNamespace) GetRegistryRoutes() []extension.Route { - var routes []extension.Route +func (o *ociNamespace) GetRegistryRoutes() []storage.Route { + var routes []storage.Route if o.discoverEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, storage.Route{ Namespace: namespaceName, Extension: extensionName, Component: discoverComponentName, @@ -134,7 +133,7 @@ func (o *ociNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (o *ociNamespace) discoverDispatcher(ctx *extension.Context, r *http.Request) http.Handler { +func (o *ociNamespace) discoverDispatcher(ctx *storage.Context, r *http.Request) http.Handler { extensionHandler := &extensionHandler{ Context: ctx, storageDriver: o.storageDriver, diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index caa27e5037e..21cc814a83a 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -9,7 +9,7 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" - "github.com/distribution/distribution/v3/registry/extension" + "github.com/distribution/distribution/v3/manifest/orasartifact" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -22,7 +22,7 @@ type ArtifactService interface { // referrersHandler handles http operations on manifest referrers. type referrersHandler struct { - extContext *extension.Context + extContext *storage.Context storageDriver driver.StorageDriver // Digest is the target manifest's digest. @@ -34,9 +34,6 @@ type referrersSortedWrapper struct { descriptor artifactv1.Descriptor } -const createAnnotationName = "io.cncf.oras.artifact.created" -const createAnnotationTimestampFormat = time.RFC3339 - func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest, artifactType string) ([]artifactv1.Descriptor, error) { dcontext.GetLogger(ctx).Debug("(*manifestStore).Referrers") @@ -73,7 +70,7 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest return err } - ArtifactMan, ok := man.(*DeserializedManifest) + ArtifactMan, ok := man.(*orasartifact.DeserializedManifest) if !ok { // The PUT handler would guard against this situation. Skip this manifest. return nil @@ -95,10 +92,10 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest ArtifactType: extractedArtifactType, } - if annotation, ok := ArtifactMan.Annotations()[createAnnotationName]; !ok { + if annotation, ok := ArtifactMan.Annotations()[orasartifact.CreateAnnotationName]; !ok { referrersUnsorted = append(referrersUnsorted, artifactDesc) } else { - extractedTimestamp, err := time.Parse(createAnnotationTimestampFormat, annotation) + extractedTimestamp, err := time.Parse(orasartifact.CreateAnnotationTimestampFormat, annotation) if err != nil { return fmt.Errorf("failed to parse created annotation timestamp: %v", err) } @@ -132,3 +129,7 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest referrersSorted = append(referrersSorted, referrersUnsorted...) return referrersSorted, nil } + +func referrersLinkPath(name string) string { + return path.Join("/docker/registry/", "v2", "repositories", name, "_refs", "subjects") +} diff --git a/registry/extension/oras/oras.go b/registry/extension/oras/oras.go index 340085e6c97..5dd22bd4c48 100644 --- a/registry/extension/oras/oras.go +++ b/registry/extension/oras/oras.go @@ -8,7 +8,6 @@ import ( "github.com/distribution/distribution/v3/configuration" dcontext "github.com/distribution/distribution/v3/context" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" @@ -34,7 +33,7 @@ type OrasOptions struct { } // newOrasNamespace creates a new extension namespace with the name "oras" -func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { +func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (storage.Namespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err @@ -61,17 +60,17 @@ func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, o } func init() { - extension.Register(namespaceName, newOrasNamespace) + storage.Register(namespaceName, newOrasNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []storage.ManifestHandler { if o.referrersEnabled { return []storage.ManifestHandler{ - &artifactManifestHandler{ - repository: repo, - blobStore: blobStore, - storageDriver: o.storageDriver, + &storage.ArtifactManifestHandler{ + Repository: repo, + BlobStore: blobStore, + StorageDriver: o.storageDriver, }} } @@ -79,11 +78,11 @@ func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobSt } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (d *orasNamespace) GetRepositoryRoutes() []extension.Route { - var routes []extension.Route +func (d *orasNamespace) GetRepositoryRoutes() []storage.Route { + var routes []storage.Route if d.referrersEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, storage.Route{ Namespace: namespaceName, Extension: extensionName, Component: referrersComponentName, @@ -105,7 +104,7 @@ func (d *orasNamespace) GetRepositoryRoutes() []extension.Route { // GetRegistryRoutes returns a list of extension routes scoped at a registry level // There are no registry scoped routes exposed by this namespace -func (d *orasNamespace) GetRegistryRoutes() []extension.Route { +func (d *orasNamespace) GetRegistryRoutes() []storage.Route { return nil } @@ -124,7 +123,7 @@ func (d *orasNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (o *orasNamespace) referrersDispatcher(extCtx *extension.Context, r *http.Request) http.Handler { +func (o *orasNamespace) referrersDispatcher(extCtx *storage.Context, r *http.Request) http.Handler { handler := &referrersHandler{ storageDriver: o.storageDriver, diff --git a/registry/handlers/app.go b/registry/handlers/app.go index da337e5ed5a..dd627032d88 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -28,7 +28,6 @@ import ( "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" "github.com/distribution/distribution/v3/registry/auth" - "github.com/distribution/distribution/v3/registry/extension" registrymiddleware "github.com/distribution/distribution/v3/registry/middleware/registry" repositorymiddleware "github.com/distribution/distribution/v3/registry/middleware/repository" "github.com/distribution/distribution/v3/registry/proxy" @@ -98,7 +97,7 @@ type App struct { repositoryExtensions []string // extensionNamespaces is a list of namespaces that are configured as extensions to the distribution - extensionNamespaces []extension.Namespace + extensionNamespaces []storage.Namespace } // NewApp takes a configuration and returns a configured app, ready to serve @@ -927,9 +926,9 @@ func (app *App) nameRequired(r *http.Request) bool { func (app *App) initializeExtensionNamespaces(ctx context.Context, extensions map[string]configuration.ExtensionConfig) error { - extensionNamespaces := []extension.Namespace{} + extensionNamespaces := []storage.Namespace{} for key, options := range extensions { - ns, err := extension.Get(ctx, key, app.driver, options) + ns, err := storage.Get(ctx, key, app.driver, options) if err != nil { return fmt.Errorf("unable to configure extension namespace (%s): %s", key, err) } @@ -979,7 +978,7 @@ func (app *App) registerExtensionRoutes(ctx context.Context) error { return nil } -func (app *App) registerExtensionRoute(route extension.Route, nameRequired bool) error { +func (app *App) registerExtensionRoute(route storage.Route, nameRequired bool) error { if route.Dispatcher == nil { return nil } @@ -997,7 +996,7 @@ func (app *App) registerExtensionRoute(route extension.Route, nameRequired bool) dispatch := route.Dispatcher app.register(desc.Name, func(ctx *Context, r *http.Request) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - extCtx := &extension.Context{ + extCtx := &storage.Context{ Context: ctx.Context, Repository: ctx.Repository, Errors: ctx.Errors, diff --git a/registry/root.go b/registry/root.go index 769f689e3cc..b5c2b859e17 100644 --- a/registry/root.go +++ b/registry/root.go @@ -5,7 +5,6 @@ import ( "os" dcontext "github.com/distribution/distribution/v3/context" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver/factory" "github.com/distribution/distribution/v3/version" @@ -73,9 +72,9 @@ var GCCmd = &cobra.Command{ } extensions := config.Extensions - extensionNamespaces := []extension.Namespace{} + extensionNamespaces := []storage.Namespace{} for key, options := range extensions { - ns, err := extension.Get(ctx, key, driver, options) + ns, err := storage.Get(ctx, key, driver, options) if err != nil { fmt.Fprintf(os.Stderr, "unable to configure extension namespace (%s): %s", key, err) os.Exit(1) diff --git a/registry/extension/oras/artifactmanifesthandler.go b/registry/storage/artifactmanifesthandler.go similarity index 73% rename from registry/extension/oras/artifactmanifesthandler.go rename to registry/storage/artifactmanifesthandler.go index 7688868886e..5e689c8f8db 100644 --- a/registry/extension/oras/artifactmanifesthandler.go +++ b/registry/storage/artifactmanifesthandler.go @@ -1,4 +1,4 @@ -package oras +package storage import ( "context" @@ -9,6 +9,7 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" + "github.com/distribution/distribution/v3/manifest/orasartifact" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" v1 "github.com/oras-project/artifacts-spec/specs-go/v1" @@ -20,14 +21,14 @@ var ( errInvalidCreatedAnnotation = errors.New("failed to parse created time") ) -// artifactManifestHandler is a ManifestHandler that covers ORAS Artifacts. -type artifactManifestHandler struct { - repository distribution.Repository - blobStore distribution.BlobStore - storageDriver driver.StorageDriver +// ArtifactManifestHandler is a ManifestHandler that covers ORAS Artifacts. +type ArtifactManifestHandler struct { + Repository distribution.Repository + BlobStore distribution.BlobStore + StorageDriver driver.StorageDriver } -func (amh *artifactManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { +func (amh *ArtifactManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ctx).Debug("(*artifactManifestHandler).Unmarshal") var v json.RawMessage @@ -35,7 +36,7 @@ func (amh *artifactManifestHandler) Unmarshal(ctx context.Context, dgst digest.D return nil, distribution.ErrManifestFormatUnsupported } - dm := &DeserializedManifest{} + dm := &orasartifact.DeserializedManifest{} if err := dm.UnmarshalJSON(content); err != nil { return nil, distribution.ErrManifestFormatUnsupported } @@ -43,10 +44,10 @@ func (amh *artifactManifestHandler) Unmarshal(ctx context.Context, dgst digest.D return dm, nil } -func (ah *artifactManifestHandler) Put(ctx context.Context, man distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { +func (ah *ArtifactManifestHandler) Put(ctx context.Context, man distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { dcontext.GetLogger(ctx).Debug("(*artifactManifestHandler).Put") - da, ok := man.(*DeserializedManifest) + da, ok := man.(*orasartifact.DeserializedManifest) if !ok { return "", distribution.ErrManifestFormatUnsupported } @@ -60,7 +61,7 @@ func (ah *artifactManifestHandler) Put(ctx context.Context, man distribution.Man return "", err } - revision, err := ah.blobStore.Put(ctx, mt, payload) + revision, err := ah.BlobStore.Put(ctx, mt, payload) if err != nil { dcontext.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) return "", err @@ -79,7 +80,7 @@ func (ah *artifactManifestHandler) Put(ctx context.Context, man distribution.Man // perspective of the registry. As a policy, the registry only tries to // store valid content, leaving trust policies of that content up to // consumers. -func (amh *artifactManifestHandler) verifyManifest(ctx context.Context, dm DeserializedManifest, skipDependencyVerification bool) error { +func (amh *ArtifactManifestHandler) verifyManifest(ctx context.Context, dm orasartifact.DeserializedManifest, skipDependencyVerification bool) error { var errs distribution.ErrManifestVerification if dm.ArtifactType() == "" { @@ -90,7 +91,7 @@ func (amh *artifactManifestHandler) verifyManifest(ctx context.Context, dm Deser errs = append(errs, errInvalidMediaType) } - if createdAt, ok := dm.Annotations()[createAnnotationName]; ok { + if createdAt, ok := dm.Annotations()[orasartifact.CreateAnnotationName]; ok { _, err := time.Parse(time.RFC3339, createdAt) if err != nil { errs = append(errs, errInvalidCreatedAnnotation) @@ -98,7 +99,7 @@ func (amh *artifactManifestHandler) verifyManifest(ctx context.Context, dm Deser } if !skipDependencyVerification { - bs := amh.repository.Blobs(ctx) + bs := amh.Repository.Blobs(ctx) // All references must exist. for _, blobDesc := range dm.References() { @@ -112,7 +113,7 @@ func (amh *artifactManifestHandler) verifyManifest(ctx context.Context, dm Deser } } - ms, err := amh.repository.Manifests(ctx) + ms, err := amh.Repository.Manifests(ctx) if err != nil { return err } @@ -135,21 +136,20 @@ func (amh *artifactManifestHandler) verifyManifest(ctx context.Context, dm Deser } // indexReferrers indexes the subject of the given revision in its referrers index store. -func (amh *artifactManifestHandler) indexReferrers(ctx context.Context, dm DeserializedManifest, revision digest.Digest) error { +func (amh *ArtifactManifestHandler) indexReferrers(ctx context.Context, dm orasartifact.DeserializedManifest, revision digest.Digest) error { // [TODO] We can use artifact type in the link path to support filtering by artifact type // but need to consider the max path length in different os //artifactType := dm.ArtifactType() subjectRevision := dm.Subject().Digest - - rootPath := path.Join(referrersLinkPath(amh.repository.Named().Name()), subjectRevision.Algorithm().String(), subjectRevision.Hex()) + referrerRoot, err := pathFor(referrersRootPathSpec{name: amh.Repository.Named().Name()}) + if err != nil { + return err + } + rootPath := path.Join(referrerRoot, subjectRevision.Algorithm().String(), subjectRevision.Hex()) referenceLinkPath := path.Join(rootPath, revision.Algorithm().String(), revision.Hex(), "link") - if err := amh.storageDriver.PutContent(ctx, referenceLinkPath, []byte(revision.String())); err != nil { + if err := amh.StorageDriver.PutContent(ctx, referenceLinkPath, []byte(revision.String())); err != nil { return err } return nil } - -func referrersLinkPath(name string) string { - return path.Join("/docker/registry/", "v2", "repositories", name, "_refs", "subjects") -} diff --git a/registry/extension/oras/artifactmanifesthandler_test.go b/registry/storage/artifactmanifesthandler_test.go similarity index 64% rename from registry/extension/oras/artifactmanifesthandler_test.go rename to registry/storage/artifactmanifesthandler_test.go index 6212aadcaf0..ebffa3d1920 100644 --- a/registry/extension/oras/artifactmanifesthandler_test.go +++ b/registry/storage/artifactmanifesthandler_test.go @@ -1,4 +1,4 @@ -package oras +package storage import ( "context" @@ -7,62 +7,28 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/manifest" + "github.com/distribution/distribution/v3/manifest/orasartifact" "github.com/distribution/distribution/v3/manifest/schema2" - "github.com/distribution/distribution/v3/reference" - "github.com/distribution/distribution/v3/registry/extension" - storage "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" "github.com/opencontainers/go-digest" orasartifacts "github.com/oras-project/artifacts-spec/specs-go/v1" ) -func createRegistry(t *testing.T, driver driver.StorageDriver, options ...storage.RegistryOption) distribution.Namespace { +func createArtifactRegistry(t *testing.T, driver driver.StorageDriver, options ...RegistryOption) distribution.Namespace { ctx := context.Background() - options = append([]storage.RegistryOption{storage.EnableDelete}, options...) - extensionConfig := OrasOptions{ - ArtifactsExtComponents: []string{"referrers"}, - } - ns, err := extension.Get(ctx, "oras", driver, extensionConfig) - if err != nil { - t.Fatalf("unable to configure extension namespace (%s): %s", "oras", err) - } - options = append(options, storage.AddExtendedStorage(ns)) - registry, err := storage.NewRegistry(ctx, driver, options...) + options = append([]RegistryOption{EnableDelete, AddExtendedStorage(&MockNamespace{storageDriver: driver, referrersEnabled: true})}, options...) + registry, err := NewRegistry(ctx, driver, options...) if err != nil { t.Fatalf("failed to construct namespace") } return registry } -func makeRepository(t *testing.T, registry distribution.Namespace, name string) distribution.Repository { - ctx := context.Background() - named, err := reference.WithName(name) - if err != nil { - t.Fatalf("failed to parse name %s: %v", name, err) - } - - repo, err := registry.Repository(ctx, named) - if err != nil { - t.Fatalf("failed to construct repository: %v", err) - } - return repo -} - -func makeManifestService(t *testing.T, repository distribution.Repository) distribution.ManifestService { - ctx := context.Background() - - manifestService, err := repository.Manifests(ctx) - if err != nil { - t.Fatalf("failed to construct manifest store: %v", err) - } - return manifestService -} - func TestVerifyArtifactManifestPut(t *testing.T) { ctx := context.Background() inmemoryDriver := inmemory.New() - registry := createRegistry(t, inmemoryDriver) + registry := createArtifactRegistry(t, inmemoryDriver) repo := makeRepository(t, registry, "test") manifestService := makeManifestService(t, repo) @@ -112,8 +78,8 @@ func TestVerifyArtifactManifestPut(t *testing.T) { Size: artifactBlob.Size, } - template := Manifest{ - inner: orasartifacts.Manifest{ + template := orasartifact.Manifest{ + Inner: orasartifacts.Manifest{ MediaType: orasartifacts.MediaTypeArtifactManifest, ArtifactType: "test_artifactType", Blobs: []orasartifacts.Descriptor{ @@ -125,7 +91,7 @@ func TestVerifyArtifactManifestPut(t *testing.T) { Digest: dg, }, Annotations: map[string]string{ - createAnnotationName: "2022-04-22T17:03:05-07:00", + orasartifact.CreateAnnotationName: "2022-04-22T17:03:05-07:00", }, }, } @@ -142,18 +108,18 @@ func TestVerifyArtifactManifestPut(t *testing.T) { cases := []testcase{ { orasartifacts.MediaTypeArtifactManifest, - template.inner.ArtifactType, - template.inner.Blobs, - template.inner.Subject, + template.Inner.ArtifactType, + template.Inner.Blobs, + template.Inner.Subject, template.Annotations(), nil, }, // non oras artifact manifest media type { "wrongMediaType", - template.inner.ArtifactType, - template.inner.Blobs, - template.inner.Subject, + template.Inner.ArtifactType, + template.Inner.Blobs, + template.Inner.Subject, template.Annotations(), errInvalidMediaType, }, @@ -161,16 +127,16 @@ func TestVerifyArtifactManifestPut(t *testing.T) { { orasartifacts.MediaTypeArtifactManifest, "", - template.inner.Blobs, - template.inner.Subject, + template.Inner.Blobs, + template.Inner.Subject, template.Annotations(), errInvalidArtifactType, }, // invalid subject { orasartifacts.MediaTypeArtifactManifest, - template.inner.ArtifactType, - template.inner.Blobs, + template.Inner.ArtifactType, + template.Inner.Blobs, orasartifacts.Descriptor{ MediaType: dm.MediaType, Size: int64(len(dmPayload)), @@ -182,18 +148,18 @@ func TestVerifyArtifactManifestPut(t *testing.T) { // invalid created annotation { orasartifacts.MediaTypeArtifactManifest, - template.inner.ArtifactType, - template.inner.Blobs, - template.inner.Subject, + template.Inner.ArtifactType, + template.Inner.Blobs, + template.Inner.Subject, map[string]string{ - createAnnotationName: "invalid_timestamp", + orasartifact.CreateAnnotationName: "invalid_timestamp", }, errInvalidCreatedAnnotation, }, // invalid blob { orasartifacts.MediaTypeArtifactManifest, - template.inner.ArtifactType, + template.Inner.ArtifactType, []orasartifacts.Descriptor{ { MediaType: artifactBlob.MediaType, @@ -201,15 +167,15 @@ func TestVerifyArtifactManifestPut(t *testing.T) { Size: artifactBlob.Size, }, }, - template.inner.Subject, + template.Inner.Subject, template.Annotations(), distribution.ErrManifestBlobUnknown{Digest: digest.FromString("sha256:invalid_blob_digest")}, }, } for _, c := range cases { - manifest := Manifest{ - inner: orasartifacts.Manifest{ + manifest := orasartifact.Manifest{ + Inner: orasartifacts.Manifest{ MediaType: c.MediaType, ArtifactType: c.ArtifactType, Blobs: c.Blobs, @@ -218,14 +184,14 @@ func TestVerifyArtifactManifestPut(t *testing.T) { }, } - marshalledManifest, err := json.Marshal(manifest.inner) + marshalledManifest, err := json.Marshal(manifest.Inner) if err != nil { t.Fatalf("failed to marshal manifest: %v", err) } - _, err = manifestService.Put(ctx, &DeserializedManifest{ + _, err = manifestService.Put(ctx, &orasartifact.DeserializedManifest{ Manifest: manifest, - raw: marshalledManifest, + Raw: marshalledManifest, }) if verr, ok := err.(distribution.ErrManifestVerification); ok { err = verr[0] diff --git a/registry/storage/artifactnamespace_mock.go b/registry/storage/artifactnamespace_mock.go new file mode 100644 index 00000000000..c7b43b70b6c --- /dev/null +++ b/registry/storage/artifactnamespace_mock.go @@ -0,0 +1,25 @@ +package storage + +import ( + "github.com/distribution/distribution/v3" + "github.com/distribution/distribution/v3/registry/storage/driver" +) + +type MockNamespace struct { + storageDriver driver.StorageDriver + referrersEnabled bool +} + +// GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. +func (o *MockNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []ManifestHandler { + if o.referrersEnabled { + return []ManifestHandler{ + &ArtifactManifestHandler{ + Repository: repo, + BlobStore: blobStore, + StorageDriver: o.storageDriver, + }} + } + + return []ManifestHandler{} +} diff --git a/registry/extension/extension.go b/registry/storage/extensionnamespace.go similarity index 97% rename from registry/extension/extension.go rename to registry/storage/extensionnamespace.go index 673288c5a08..187df48ceac 100644 --- a/registry/extension/extension.go +++ b/registry/storage/extensionnamespace.go @@ -1,4 +1,4 @@ -package extension +package storage import ( c "context" @@ -9,7 +9,6 @@ import ( "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" ) @@ -44,7 +43,7 @@ type Route struct { // Namespace is the namespace that is used to define extensions to the distribution. type Namespace interface { - storage.ExtendedStorage + ExtendedStorage // GetRepositoryRoutes returns a list of extension routes scoped at a repository level GetRepositoryRoutes() []Route // GetRegistryRoutes returns a list of extension routes scoped at a registry level diff --git a/registry/storage/garbagecollect_test.go b/registry/storage/garbagecollect_test.go index 14edc533ca1..d3c4b38f50d 100644 --- a/registry/storage/garbagecollect_test.go +++ b/registry/storage/garbagecollect_test.go @@ -1,18 +1,23 @@ package storage import ( + "bytes" + "encoding/json" "io" "path" "testing" "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/context" + "github.com/distribution/distribution/v3/manifest/orasartifact" + "github.com/distribution/distribution/v3/manifest/schema2" "github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" "github.com/distribution/distribution/v3/testutil" "github.com/docker/libtrust" "github.com/opencontainers/go-digest" + orasartifacts "github.com/oras-project/artifacts-spec/specs-go/v1" ) type image struct { @@ -158,6 +163,56 @@ func uploadRandomSchema2Image(t *testing.T, repository distribution.Repository) } } +func uploadRandomArtifact(t *testing.T, repository distribution.Repository, subjImg image) (digest.Digest, *orasartifact.DeserializedManifest) { + // build artifact blob and push blob + artifactBlobDigest := digest.FromBytes([]byte{}) + + testutil.UploadBlobs(repository, map[digest.Digest]io.ReadSeeker{ + artifactBlobDigest: bytes.NewReader([]byte{}), + }) + + artifactBlobDescriptor := orasartifacts.Descriptor{ + MediaType: orasartifacts.MediaTypeDescriptor, + Digest: artifactBlobDigest, + Size: 0, + } + + _, subjImgPayload, err := subjImg.manifest.Payload() + if err != nil { + t.Fatalf("failed to get subject image payload: %v", err) + } + // build artifact manifest template + artifactManifest := orasartifacts.Manifest{ + MediaType: orasartifacts.MediaTypeArtifactManifest, + ArtifactType: "test_artifactType", + Blobs: []orasartifacts.Descriptor{ + artifactBlobDescriptor, + }, + Subject: orasartifacts.Descriptor{ + MediaType: schema2.MediaTypeManifest, + Size: int64(len(subjImgPayload)), + Digest: subjImg.manifestDigest, + }, + } + dm := new(orasartifact.DeserializedManifest) + marshalledMan, err := json.Marshal(artifactManifest) + if err != nil { + t.Fatalf("artifact manifest could not be serialized to byte array: %v", err) + } + err = dm.UnmarshalJSON(marshalledMan) + if err != nil { + t.Fatalf("artifact manifest could not be unmarshalled: %v", err) + } + // upload manifest + ctx := context.Background() + manifestService := makeManifestService(t, repository) + manifestDigest, err := manifestService.Put(ctx, dm) + if err != nil { + t.Fatalf("artifact manifest upload failed: %v", err) + } + return manifestDigest, dm +} + func TestNoDeletionNoEffect(t *testing.T) { ctx := context.Background() inmemoryDriver := inmemory.New() @@ -501,19 +556,50 @@ func TestOrphanBlobDeleted(t *testing.T) { } } -// func TestReferrersBlobsDeleted(t *testing.T) { -// inmemoryDriver := inmemory.New() -// ctx := context.Background() -// // extConfig := configuration.ExtensionConfig{} -// // ns, err := extension.Get(ctx, "oras", inmemoryDriver, extConfig) -// // if err != nil { -// // fmt.Fprintf(os.Stderr, "unable to configure extension namespace oras: %s", err) -// // os.Exit(1) -// // } - -// options := []RegistryOption{AddExtendedStorage(ns)} -// // add the extended storage for every namespace to the new registry options - -// registry := createRegistry(t, inmemoryDriver, options...) -// repo := makeRepository(t, registry, "michael_z_doukas") -// } +func TestReferrersBlobsDeleted(t *testing.T) { + inmemoryDriver := inmemory.New() + registry := createArtifactRegistry(t, inmemoryDriver) + repo := makeRepository(t, registry, "referrers_repo") + ms := makeManifestService(t, repo) + ctx := context.Background() + tagService := repo.Tags(ctx) + + subjImg := uploadRandomSchema2Image(t, repo) + artifactDigest, artifactManifest := uploadRandomArtifact(t, repo, subjImg) + + // the tags folder doesn't exist for this repo until a tag is added + // this leads to an error in Mark and Sweep if tags folder not found + err := tagService.Tag(ctx, "test", distribution.Descriptor{Digest: subjImg.manifestDigest}) + if err != nil { + t.Fatalf("failed to tag subject image: %v", err) + } + err = tagService.Untag(ctx, "test") + if err != nil { + t.Fatalf("failed to untag subject image: %v", err) + } + + // Run GC + err = MarkAndSweep(ctx, inmemoryDriver, registry, GCOpts{ + DryRun: false, + RemoveUntagged: true, + }) + if err != nil { + t.Fatalf("Failed mark and sweep: %v", err) + } + + manifests := allManifests(t, ms) + blobs := allBlobs(t, registry) + + if _, exists := manifests[artifactDigest]; exists { + t.Fatalf("artifact manifest with digest %s should have been deleted", artifactDigest.String()) + } + + if _, exists := blobs[artifactDigest]; exists { + t.Fatalf("artifact manifest blob with digest %s should have been deleted", artifactDigest.String()) + } + + blobDigest := artifactManifest.Inner.Blobs[0].Digest + if _, exists := blobs[blobDigest]; exists { + t.Fatalf("artifact blob with digest %s should have been deleted", blobDigest) + } +} diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index bbdf730484d..149f219c080 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -193,11 +193,9 @@ func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error { artifactSweepIngestor) if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil + if _, ok := err.(driver.PathNotFoundError); !ok { + return err } - return err } // delete the artifact manifest revision and the _refs directory for each artifact indexed for key, _ := range artifactManifestIndex { From 8317e7e56811994b6a4e71015b0fc10619a3fdb6 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Tue, 14 Jun 2022 14:55:37 -0700 Subject: [PATCH 06/22] Revert "move artifact manifest and handler to new location; move extension namespace to storage package; add garbage collect unit test; fix manifest storage unit test" This reverts commit 0ab8efb7b17c0fb9c97c7a019da183a616656b6d. Signed-off-by: Akash Singhal --- registry/extension/distribution/manifests.go | 3 +- registry/extension/distribution/registry.go | 19 +-- registry/extension/distribution/taghistory.go | 3 +- .../extension.go} | 5 +- registry/extension/oci/discover.go | 8 +- registry/extension/oci/oci.go | 19 +-- .../extension/oras}/artifactmanifest.go | 48 ++++--- .../oras}/artifactmanifesthandler.go | 46 +++---- .../oras}/artifactmanifesthandler_test.go | 94 +++++++++----- registry/extension/oras/artifactservice.go | 17 ++- registry/extension/oras/oras.go | 23 ++-- registry/handlers/app.go | 11 +- registry/root.go | 5 +- registry/storage/artifactnamespace_mock.go | 25 ---- registry/storage/garbagecollect_test.go | 118 +++--------------- registry/storage/manifeststore.go | 6 +- 16 files changed, 189 insertions(+), 261 deletions(-) rename registry/{storage/extensionnamespace.go => extension/extension.go} (97%) rename {manifest/orasartifact => registry/extension/oras}/artifactmanifest.go (74%) rename registry/{storage => extension/oras}/artifactmanifesthandler.go (73%) rename registry/{storage => extension/oras}/artifactmanifesthandler_test.go (64%) delete mode 100644 registry/storage/artifactnamespace_mock.go diff --git a/registry/extension/distribution/manifests.go b/registry/extension/distribution/manifests.go index 2aa67d5126f..51a52e8171f 100644 --- a/registry/extension/distribution/manifests.go +++ b/registry/extension/distribution/manifests.go @@ -6,6 +6,7 @@ import ( "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -18,7 +19,7 @@ type manifestsGetAPIResponse struct { // manifestHandler handles requests for manifests under a manifest name. type manifestHandler struct { - *storage.Context + *extension.Context storageDriver driver.StorageDriver } diff --git a/registry/extension/distribution/registry.go b/registry/extension/distribution/registry.go index 7a8c71cbd09..8e6c5000814 100644 --- a/registry/extension/distribution/registry.go +++ b/registry/extension/distribution/registry.go @@ -7,6 +7,7 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" @@ -33,7 +34,7 @@ type distributionOptions struct { } // newDistNamespace creates a new extension namespace with the name "distribution" -func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (storage.Namespace, error) { +func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { @@ -66,7 +67,7 @@ func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, o func init() { // register the extension namespace. - storage.Register(namespaceName, newDistNamespace) + extension.Register(namespaceName, newDistNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. @@ -76,11 +77,11 @@ func (o *distributionNamespace) GetManifestHandlers(repo distribution.Repository } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (d *distributionNamespace) GetRepositoryRoutes() []storage.Route { - var routes []storage.Route +func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route { + var routes []extension.Route if d.manifestsEnabled { - routes = append(routes, storage.Route{ + routes = append(routes, extension.Route{ Namespace: namespaceName, Extension: extensionName, Component: manifestsComponentName, @@ -98,7 +99,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []storage.Route { } if d.tagHistoryEnabled { - routes = append(routes, storage.Route{ + routes = append(routes, extension.Route{ Namespace: namespaceName, Extension: extensionName, Component: tagHistoryComponentName, @@ -131,7 +132,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []storage.Route { // GetRegistryRoutes returns a list of extension routes scoped at a registry level // There are no registry scoped routes exposed by this namespace -func (d *distributionNamespace) GetRegistryRoutes() []storage.Route { +func (d *distributionNamespace) GetRegistryRoutes() []extension.Route { return nil } @@ -150,7 +151,7 @@ func (d *distributionNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (d *distributionNamespace) tagHistoryDispatcher(ctx *storage.Context, r *http.Request) http.Handler { +func (d *distributionNamespace) tagHistoryDispatcher(ctx *extension.Context, r *http.Request) http.Handler { tagHistoryHandler := &tagHistoryHandler{ Context: ctx, storageDriver: d.storageDriver, @@ -161,7 +162,7 @@ func (d *distributionNamespace) tagHistoryDispatcher(ctx *storage.Context, r *ht } } -func (d *distributionNamespace) manifestsDispatcher(ctx *storage.Context, r *http.Request) http.Handler { +func (d *distributionNamespace) manifestsDispatcher(ctx *extension.Context, r *http.Request) http.Handler { manifestsHandler := &manifestHandler{ Context: ctx, storageDriver: d.storageDriver, diff --git a/registry/extension/distribution/taghistory.go b/registry/extension/distribution/taghistory.go index 2e560feb277..9cb957b87d8 100644 --- a/registry/extension/distribution/taghistory.go +++ b/registry/extension/distribution/taghistory.go @@ -6,6 +6,7 @@ import ( "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -19,7 +20,7 @@ type tagHistoryAPIResponse struct { // manifestHandler handles requests for manifests under a manifest name. type tagHistoryHandler struct { - *storage.Context + *extension.Context storageDriver driver.StorageDriver } diff --git a/registry/storage/extensionnamespace.go b/registry/extension/extension.go similarity index 97% rename from registry/storage/extensionnamespace.go rename to registry/extension/extension.go index 187df48ceac..673288c5a08 100644 --- a/registry/storage/extensionnamespace.go +++ b/registry/extension/extension.go @@ -1,4 +1,4 @@ -package storage +package extension import ( c "context" @@ -9,6 +9,7 @@ import ( "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" ) @@ -43,7 +44,7 @@ type Route struct { // Namespace is the namespace that is used to define extensions to the distribution. type Namespace interface { - ExtendedStorage + storage.ExtendedStorage // GetRepositoryRoutes returns a list of extension routes scoped at a repository level GetRepositoryRoutes() []Route // GetRegistryRoutes returns a list of extension routes scoped at a registry level diff --git a/registry/extension/oci/discover.go b/registry/extension/oci/discover.go index 007e0007473..3d2ec3fce97 100644 --- a/registry/extension/oci/discover.go +++ b/registry/extension/oci/discover.go @@ -5,17 +5,17 @@ import ( "net/http" "github.com/distribution/distribution/v3/registry/api/errcode" - "github.com/distribution/distribution/v3/registry/storage" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage/driver" ) type discoverGetAPIResponse struct { - Extensions []storage.EnumerateExtension `json:"extensions"` + Extensions []extension.EnumerateExtension `json:"extensions"` } // extensionHandler handles requests for manifests under a manifest name. type extensionHandler struct { - *storage.Context + *extension.Context storageDriver driver.StorageDriver } @@ -25,7 +25,7 @@ func (eh *extensionHandler) getExtensions(w http.ResponseWriter, r *http.Request w.Header().Set("Content-Type", "application/json") // get list of extension information seperated at the namespace level - enumeratedExtensions := storage.EnumerateRegistered(*eh.Context) + enumeratedExtensions := extension.EnumerateRegistered(*eh.Context) // remove the oci extension so it's not returned by discover for i, e := range enumeratedExtensions { diff --git a/registry/extension/oci/oci.go b/registry/extension/oci/oci.go index 4c738845094..d68e5b8cd8f 100644 --- a/registry/extension/oci/oci.go +++ b/registry/extension/oci/oci.go @@ -7,6 +7,7 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" @@ -31,7 +32,7 @@ type ociOptions struct { } // newOciNamespace creates a new extension namespace with the name "oci" -func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (storage.Namespace, error) { +func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err @@ -59,7 +60,7 @@ func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, op func init() { // register the extension namespace. - storage.Register(namespaceName, newOciNamespace) + extension.Register(namespaceName, newOciNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. @@ -69,11 +70,11 @@ func (o *ociNamespace) GetManifestHandlers(repo distribution.Repository, blobSto } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (o *ociNamespace) GetRepositoryRoutes() []storage.Route { - var routes []storage.Route +func (o *ociNamespace) GetRepositoryRoutes() []extension.Route { + var routes []extension.Route if o.discoverEnabled { - routes = append(routes, storage.Route{ + routes = append(routes, extension.Route{ Namespace: namespaceName, Extension: extensionName, Component: discoverComponentName, @@ -94,11 +95,11 @@ func (o *ociNamespace) GetRepositoryRoutes() []storage.Route { } // GetRegistryRoutes returns a list of extension routes scoped at a registry level -func (o *ociNamespace) GetRegistryRoutes() []storage.Route { - var routes []storage.Route +func (o *ociNamespace) GetRegistryRoutes() []extension.Route { + var routes []extension.Route if o.discoverEnabled { - routes = append(routes, storage.Route{ + routes = append(routes, extension.Route{ Namespace: namespaceName, Extension: extensionName, Component: discoverComponentName, @@ -133,7 +134,7 @@ func (o *ociNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (o *ociNamespace) discoverDispatcher(ctx *storage.Context, r *http.Request) http.Handler { +func (o *ociNamespace) discoverDispatcher(ctx *extension.Context, r *http.Request) http.Handler { extensionHandler := &extensionHandler{ Context: ctx, storageDriver: o.storageDriver, diff --git a/manifest/orasartifact/artifactmanifest.go b/registry/extension/oras/artifactmanifest.go similarity index 74% rename from manifest/orasartifact/artifactmanifest.go rename to registry/extension/oras/artifactmanifest.go index 5ad113ff583..e883efd4afd 100644 --- a/manifest/orasartifact/artifactmanifest.go +++ b/registry/extension/oras/artifactmanifest.go @@ -1,19 +1,15 @@ -package orasartifact +package oras import ( "encoding/json" "errors" "fmt" - "time" "github.com/distribution/distribution/v3" "github.com/opencontainers/go-digest" v1 "github.com/oras-project/artifacts-spec/specs-go/v1" ) -const CreateAnnotationName = "io.cncf.oras.artifact.created" -const CreateAnnotationTimestampFormat = time.RFC3339 - func init() { unmarshalFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { d := new(DeserializedManifest) @@ -33,32 +29,32 @@ func init() { // Manifest describes ORAS artifact manifests. type Manifest struct { - Inner v1.Manifest + inner v1.Manifest } // ArtifactType returns the artifactType of this ORAS artifact. func (a Manifest) ArtifactType() string { - return a.Inner.ArtifactType + return a.inner.ArtifactType } // Annotations returns the annotations of this ORAS artifact. func (a Manifest) Annotations() map[string]string { - return a.Inner.Annotations + return a.inner.Annotations } // MediaType returns the media type of this ORAS artifact. func (a Manifest) MediaType() string { - return a.Inner.MediaType + return a.inner.MediaType } // References returns the distribution descriptors for the referenced blobs. func (a Manifest) References() []distribution.Descriptor { - blobs := make([]distribution.Descriptor, len(a.Inner.Blobs)) - for i := range a.Inner.Blobs { + blobs := make([]distribution.Descriptor, len(a.inner.Blobs)) + for i := range a.inner.Blobs { blobs[i] = distribution.Descriptor{ - MediaType: a.Inner.Blobs[i].MediaType, - Digest: a.Inner.Blobs[i].Digest, - Size: a.Inner.Blobs[i].Size, + MediaType: a.inner.Blobs[i].MediaType, + Digest: a.inner.Blobs[i].Digest, + Size: a.inner.Blobs[i].Size, } } return blobs @@ -67,9 +63,9 @@ func (a Manifest) References() []distribution.Descriptor { // Subject returns the the subject manifest this artifact references. func (a Manifest) Subject() distribution.Descriptor { return distribution.Descriptor{ - MediaType: a.Inner.Subject.MediaType, - Digest: a.Inner.Subject.Digest, - Size: a.Inner.Subject.Size, + MediaType: a.inner.Subject.MediaType, + Digest: a.inner.Subject.Digest, + Size: a.inner.Subject.Size, } } @@ -77,17 +73,17 @@ func (a Manifest) Subject() distribution.Descriptor { type DeserializedManifest struct { Manifest - // Raw is the Raw byte representation of the ORAS artifact. - Raw []byte + // raw is the raw byte representation of the ORAS artifact. + raw []byte } // UnmarshalJSON populates a new Manifest struct from JSON data. func (d *DeserializedManifest) UnmarshalJSON(b []byte) error { - d.Raw = make([]byte, len(b)) - copy(d.Raw, b) + d.raw = make([]byte, len(b)) + copy(d.raw, b) var man v1.Manifest - if err := json.Unmarshal(d.Raw, &man); err != nil { + if err := json.Unmarshal(d.raw, &man); err != nil { return err } if man.ArtifactType == "" { @@ -97,15 +93,15 @@ func (d *DeserializedManifest) UnmarshalJSON(b []byte) error { return errors.New("mediaType is invalid") } - d.Inner = man + d.inner = man return nil } // MarshalJSON returns the raw content. func (d *DeserializedManifest) MarshalJSON() ([]byte, error) { - if len(d.Raw) > 0 { - return d.Raw, nil + if len(d.raw) > 0 { + return d.raw, nil } return nil, errors.New("JSON representation not initialized in DeserializedManifest") @@ -115,5 +111,5 @@ func (d *DeserializedManifest) MarshalJSON() ([]byte, error) { // used to calculate the content identifier. func (d DeserializedManifest) Payload() (string, []byte, error) { // NOTE: This is a hack. The media type should be read from storage. - return v1.MediaTypeArtifactManifest, d.Raw, nil + return v1.MediaTypeArtifactManifest, d.raw, nil } diff --git a/registry/storage/artifactmanifesthandler.go b/registry/extension/oras/artifactmanifesthandler.go similarity index 73% rename from registry/storage/artifactmanifesthandler.go rename to registry/extension/oras/artifactmanifesthandler.go index 5e689c8f8db..7688868886e 100644 --- a/registry/storage/artifactmanifesthandler.go +++ b/registry/extension/oras/artifactmanifesthandler.go @@ -1,4 +1,4 @@ -package storage +package oras import ( "context" @@ -9,7 +9,6 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" - "github.com/distribution/distribution/v3/manifest/orasartifact" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" v1 "github.com/oras-project/artifacts-spec/specs-go/v1" @@ -21,14 +20,14 @@ var ( errInvalidCreatedAnnotation = errors.New("failed to parse created time") ) -// ArtifactManifestHandler is a ManifestHandler that covers ORAS Artifacts. -type ArtifactManifestHandler struct { - Repository distribution.Repository - BlobStore distribution.BlobStore - StorageDriver driver.StorageDriver +// artifactManifestHandler is a ManifestHandler that covers ORAS Artifacts. +type artifactManifestHandler struct { + repository distribution.Repository + blobStore distribution.BlobStore + storageDriver driver.StorageDriver } -func (amh *ArtifactManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { +func (amh *artifactManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ctx).Debug("(*artifactManifestHandler).Unmarshal") var v json.RawMessage @@ -36,7 +35,7 @@ func (amh *ArtifactManifestHandler) Unmarshal(ctx context.Context, dgst digest.D return nil, distribution.ErrManifestFormatUnsupported } - dm := &orasartifact.DeserializedManifest{} + dm := &DeserializedManifest{} if err := dm.UnmarshalJSON(content); err != nil { return nil, distribution.ErrManifestFormatUnsupported } @@ -44,10 +43,10 @@ func (amh *ArtifactManifestHandler) Unmarshal(ctx context.Context, dgst digest.D return dm, nil } -func (ah *ArtifactManifestHandler) Put(ctx context.Context, man distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { +func (ah *artifactManifestHandler) Put(ctx context.Context, man distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { dcontext.GetLogger(ctx).Debug("(*artifactManifestHandler).Put") - da, ok := man.(*orasartifact.DeserializedManifest) + da, ok := man.(*DeserializedManifest) if !ok { return "", distribution.ErrManifestFormatUnsupported } @@ -61,7 +60,7 @@ func (ah *ArtifactManifestHandler) Put(ctx context.Context, man distribution.Man return "", err } - revision, err := ah.BlobStore.Put(ctx, mt, payload) + revision, err := ah.blobStore.Put(ctx, mt, payload) if err != nil { dcontext.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) return "", err @@ -80,7 +79,7 @@ func (ah *ArtifactManifestHandler) Put(ctx context.Context, man distribution.Man // perspective of the registry. As a policy, the registry only tries to // store valid content, leaving trust policies of that content up to // consumers. -func (amh *ArtifactManifestHandler) verifyManifest(ctx context.Context, dm orasartifact.DeserializedManifest, skipDependencyVerification bool) error { +func (amh *artifactManifestHandler) verifyManifest(ctx context.Context, dm DeserializedManifest, skipDependencyVerification bool) error { var errs distribution.ErrManifestVerification if dm.ArtifactType() == "" { @@ -91,7 +90,7 @@ func (amh *ArtifactManifestHandler) verifyManifest(ctx context.Context, dm orasa errs = append(errs, errInvalidMediaType) } - if createdAt, ok := dm.Annotations()[orasartifact.CreateAnnotationName]; ok { + if createdAt, ok := dm.Annotations()[createAnnotationName]; ok { _, err := time.Parse(time.RFC3339, createdAt) if err != nil { errs = append(errs, errInvalidCreatedAnnotation) @@ -99,7 +98,7 @@ func (amh *ArtifactManifestHandler) verifyManifest(ctx context.Context, dm orasa } if !skipDependencyVerification { - bs := amh.Repository.Blobs(ctx) + bs := amh.repository.Blobs(ctx) // All references must exist. for _, blobDesc := range dm.References() { @@ -113,7 +112,7 @@ func (amh *ArtifactManifestHandler) verifyManifest(ctx context.Context, dm orasa } } - ms, err := amh.Repository.Manifests(ctx) + ms, err := amh.repository.Manifests(ctx) if err != nil { return err } @@ -136,20 +135,21 @@ func (amh *ArtifactManifestHandler) verifyManifest(ctx context.Context, dm orasa } // indexReferrers indexes the subject of the given revision in its referrers index store. -func (amh *ArtifactManifestHandler) indexReferrers(ctx context.Context, dm orasartifact.DeserializedManifest, revision digest.Digest) error { +func (amh *artifactManifestHandler) indexReferrers(ctx context.Context, dm DeserializedManifest, revision digest.Digest) error { // [TODO] We can use artifact type in the link path to support filtering by artifact type // but need to consider the max path length in different os //artifactType := dm.ArtifactType() subjectRevision := dm.Subject().Digest - referrerRoot, err := pathFor(referrersRootPathSpec{name: amh.Repository.Named().Name()}) - if err != nil { - return err - } - rootPath := path.Join(referrerRoot, subjectRevision.Algorithm().String(), subjectRevision.Hex()) + + rootPath := path.Join(referrersLinkPath(amh.repository.Named().Name()), subjectRevision.Algorithm().String(), subjectRevision.Hex()) referenceLinkPath := path.Join(rootPath, revision.Algorithm().String(), revision.Hex(), "link") - if err := amh.StorageDriver.PutContent(ctx, referenceLinkPath, []byte(revision.String())); err != nil { + if err := amh.storageDriver.PutContent(ctx, referenceLinkPath, []byte(revision.String())); err != nil { return err } return nil } + +func referrersLinkPath(name string) string { + return path.Join("/docker/registry/", "v2", "repositories", name, "_refs", "subjects") +} diff --git a/registry/storage/artifactmanifesthandler_test.go b/registry/extension/oras/artifactmanifesthandler_test.go similarity index 64% rename from registry/storage/artifactmanifesthandler_test.go rename to registry/extension/oras/artifactmanifesthandler_test.go index ebffa3d1920..6212aadcaf0 100644 --- a/registry/storage/artifactmanifesthandler_test.go +++ b/registry/extension/oras/artifactmanifesthandler_test.go @@ -1,4 +1,4 @@ -package storage +package oras import ( "context" @@ -7,28 +7,62 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/manifest" - "github.com/distribution/distribution/v3/manifest/orasartifact" "github.com/distribution/distribution/v3/manifest/schema2" + "github.com/distribution/distribution/v3/reference" + "github.com/distribution/distribution/v3/registry/extension" + storage "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" "github.com/opencontainers/go-digest" orasartifacts "github.com/oras-project/artifacts-spec/specs-go/v1" ) -func createArtifactRegistry(t *testing.T, driver driver.StorageDriver, options ...RegistryOption) distribution.Namespace { +func createRegistry(t *testing.T, driver driver.StorageDriver, options ...storage.RegistryOption) distribution.Namespace { ctx := context.Background() - options = append([]RegistryOption{EnableDelete, AddExtendedStorage(&MockNamespace{storageDriver: driver, referrersEnabled: true})}, options...) - registry, err := NewRegistry(ctx, driver, options...) + options = append([]storage.RegistryOption{storage.EnableDelete}, options...) + extensionConfig := OrasOptions{ + ArtifactsExtComponents: []string{"referrers"}, + } + ns, err := extension.Get(ctx, "oras", driver, extensionConfig) + if err != nil { + t.Fatalf("unable to configure extension namespace (%s): %s", "oras", err) + } + options = append(options, storage.AddExtendedStorage(ns)) + registry, err := storage.NewRegistry(ctx, driver, options...) if err != nil { t.Fatalf("failed to construct namespace") } return registry } +func makeRepository(t *testing.T, registry distribution.Namespace, name string) distribution.Repository { + ctx := context.Background() + named, err := reference.WithName(name) + if err != nil { + t.Fatalf("failed to parse name %s: %v", name, err) + } + + repo, err := registry.Repository(ctx, named) + if err != nil { + t.Fatalf("failed to construct repository: %v", err) + } + return repo +} + +func makeManifestService(t *testing.T, repository distribution.Repository) distribution.ManifestService { + ctx := context.Background() + + manifestService, err := repository.Manifests(ctx) + if err != nil { + t.Fatalf("failed to construct manifest store: %v", err) + } + return manifestService +} + func TestVerifyArtifactManifestPut(t *testing.T) { ctx := context.Background() inmemoryDriver := inmemory.New() - registry := createArtifactRegistry(t, inmemoryDriver) + registry := createRegistry(t, inmemoryDriver) repo := makeRepository(t, registry, "test") manifestService := makeManifestService(t, repo) @@ -78,8 +112,8 @@ func TestVerifyArtifactManifestPut(t *testing.T) { Size: artifactBlob.Size, } - template := orasartifact.Manifest{ - Inner: orasartifacts.Manifest{ + template := Manifest{ + inner: orasartifacts.Manifest{ MediaType: orasartifacts.MediaTypeArtifactManifest, ArtifactType: "test_artifactType", Blobs: []orasartifacts.Descriptor{ @@ -91,7 +125,7 @@ func TestVerifyArtifactManifestPut(t *testing.T) { Digest: dg, }, Annotations: map[string]string{ - orasartifact.CreateAnnotationName: "2022-04-22T17:03:05-07:00", + createAnnotationName: "2022-04-22T17:03:05-07:00", }, }, } @@ -108,18 +142,18 @@ func TestVerifyArtifactManifestPut(t *testing.T) { cases := []testcase{ { orasartifacts.MediaTypeArtifactManifest, - template.Inner.ArtifactType, - template.Inner.Blobs, - template.Inner.Subject, + template.inner.ArtifactType, + template.inner.Blobs, + template.inner.Subject, template.Annotations(), nil, }, // non oras artifact manifest media type { "wrongMediaType", - template.Inner.ArtifactType, - template.Inner.Blobs, - template.Inner.Subject, + template.inner.ArtifactType, + template.inner.Blobs, + template.inner.Subject, template.Annotations(), errInvalidMediaType, }, @@ -127,16 +161,16 @@ func TestVerifyArtifactManifestPut(t *testing.T) { { orasartifacts.MediaTypeArtifactManifest, "", - template.Inner.Blobs, - template.Inner.Subject, + template.inner.Blobs, + template.inner.Subject, template.Annotations(), errInvalidArtifactType, }, // invalid subject { orasartifacts.MediaTypeArtifactManifest, - template.Inner.ArtifactType, - template.Inner.Blobs, + template.inner.ArtifactType, + template.inner.Blobs, orasartifacts.Descriptor{ MediaType: dm.MediaType, Size: int64(len(dmPayload)), @@ -148,18 +182,18 @@ func TestVerifyArtifactManifestPut(t *testing.T) { // invalid created annotation { orasartifacts.MediaTypeArtifactManifest, - template.Inner.ArtifactType, - template.Inner.Blobs, - template.Inner.Subject, + template.inner.ArtifactType, + template.inner.Blobs, + template.inner.Subject, map[string]string{ - orasartifact.CreateAnnotationName: "invalid_timestamp", + createAnnotationName: "invalid_timestamp", }, errInvalidCreatedAnnotation, }, // invalid blob { orasartifacts.MediaTypeArtifactManifest, - template.Inner.ArtifactType, + template.inner.ArtifactType, []orasartifacts.Descriptor{ { MediaType: artifactBlob.MediaType, @@ -167,15 +201,15 @@ func TestVerifyArtifactManifestPut(t *testing.T) { Size: artifactBlob.Size, }, }, - template.Inner.Subject, + template.inner.Subject, template.Annotations(), distribution.ErrManifestBlobUnknown{Digest: digest.FromString("sha256:invalid_blob_digest")}, }, } for _, c := range cases { - manifest := orasartifact.Manifest{ - Inner: orasartifacts.Manifest{ + manifest := Manifest{ + inner: orasartifacts.Manifest{ MediaType: c.MediaType, ArtifactType: c.ArtifactType, Blobs: c.Blobs, @@ -184,14 +218,14 @@ func TestVerifyArtifactManifestPut(t *testing.T) { }, } - marshalledManifest, err := json.Marshal(manifest.Inner) + marshalledManifest, err := json.Marshal(manifest.inner) if err != nil { t.Fatalf("failed to marshal manifest: %v", err) } - _, err = manifestService.Put(ctx, &orasartifact.DeserializedManifest{ + _, err = manifestService.Put(ctx, &DeserializedManifest{ Manifest: manifest, - Raw: marshalledManifest, + raw: marshalledManifest, }) if verr, ok := err.(distribution.ErrManifestVerification); ok { err = verr[0] diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index 21cc814a83a..caa27e5037e 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -9,7 +9,7 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" - "github.com/distribution/distribution/v3/manifest/orasartifact" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -22,7 +22,7 @@ type ArtifactService interface { // referrersHandler handles http operations on manifest referrers. type referrersHandler struct { - extContext *storage.Context + extContext *extension.Context storageDriver driver.StorageDriver // Digest is the target manifest's digest. @@ -34,6 +34,9 @@ type referrersSortedWrapper struct { descriptor artifactv1.Descriptor } +const createAnnotationName = "io.cncf.oras.artifact.created" +const createAnnotationTimestampFormat = time.RFC3339 + func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest, artifactType string) ([]artifactv1.Descriptor, error) { dcontext.GetLogger(ctx).Debug("(*manifestStore).Referrers") @@ -70,7 +73,7 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest return err } - ArtifactMan, ok := man.(*orasartifact.DeserializedManifest) + ArtifactMan, ok := man.(*DeserializedManifest) if !ok { // The PUT handler would guard against this situation. Skip this manifest. return nil @@ -92,10 +95,10 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest ArtifactType: extractedArtifactType, } - if annotation, ok := ArtifactMan.Annotations()[orasartifact.CreateAnnotationName]; !ok { + if annotation, ok := ArtifactMan.Annotations()[createAnnotationName]; !ok { referrersUnsorted = append(referrersUnsorted, artifactDesc) } else { - extractedTimestamp, err := time.Parse(orasartifact.CreateAnnotationTimestampFormat, annotation) + extractedTimestamp, err := time.Parse(createAnnotationTimestampFormat, annotation) if err != nil { return fmt.Errorf("failed to parse created annotation timestamp: %v", err) } @@ -129,7 +132,3 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest referrersSorted = append(referrersSorted, referrersUnsorted...) return referrersSorted, nil } - -func referrersLinkPath(name string) string { - return path.Join("/docker/registry/", "v2", "repositories", name, "_refs", "subjects") -} diff --git a/registry/extension/oras/oras.go b/registry/extension/oras/oras.go index 5dd22bd4c48..340085e6c97 100644 --- a/registry/extension/oras/oras.go +++ b/registry/extension/oras/oras.go @@ -8,6 +8,7 @@ import ( "github.com/distribution/distribution/v3/configuration" dcontext "github.com/distribution/distribution/v3/context" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" @@ -33,7 +34,7 @@ type OrasOptions struct { } // newOrasNamespace creates a new extension namespace with the name "oras" -func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (storage.Namespace, error) { +func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err @@ -60,17 +61,17 @@ func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, o } func init() { - storage.Register(namespaceName, newOrasNamespace) + extension.Register(namespaceName, newOrasNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []storage.ManifestHandler { if o.referrersEnabled { return []storage.ManifestHandler{ - &storage.ArtifactManifestHandler{ - Repository: repo, - BlobStore: blobStore, - StorageDriver: o.storageDriver, + &artifactManifestHandler{ + repository: repo, + blobStore: blobStore, + storageDriver: o.storageDriver, }} } @@ -78,11 +79,11 @@ func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobSt } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (d *orasNamespace) GetRepositoryRoutes() []storage.Route { - var routes []storage.Route +func (d *orasNamespace) GetRepositoryRoutes() []extension.Route { + var routes []extension.Route if d.referrersEnabled { - routes = append(routes, storage.Route{ + routes = append(routes, extension.Route{ Namespace: namespaceName, Extension: extensionName, Component: referrersComponentName, @@ -104,7 +105,7 @@ func (d *orasNamespace) GetRepositoryRoutes() []storage.Route { // GetRegistryRoutes returns a list of extension routes scoped at a registry level // There are no registry scoped routes exposed by this namespace -func (d *orasNamespace) GetRegistryRoutes() []storage.Route { +func (d *orasNamespace) GetRegistryRoutes() []extension.Route { return nil } @@ -123,7 +124,7 @@ func (d *orasNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (o *orasNamespace) referrersDispatcher(extCtx *storage.Context, r *http.Request) http.Handler { +func (o *orasNamespace) referrersDispatcher(extCtx *extension.Context, r *http.Request) http.Handler { handler := &referrersHandler{ storageDriver: o.storageDriver, diff --git a/registry/handlers/app.go b/registry/handlers/app.go index dd627032d88..da337e5ed5a 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -28,6 +28,7 @@ import ( "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" "github.com/distribution/distribution/v3/registry/auth" + "github.com/distribution/distribution/v3/registry/extension" registrymiddleware "github.com/distribution/distribution/v3/registry/middleware/registry" repositorymiddleware "github.com/distribution/distribution/v3/registry/middleware/repository" "github.com/distribution/distribution/v3/registry/proxy" @@ -97,7 +98,7 @@ type App struct { repositoryExtensions []string // extensionNamespaces is a list of namespaces that are configured as extensions to the distribution - extensionNamespaces []storage.Namespace + extensionNamespaces []extension.Namespace } // NewApp takes a configuration and returns a configured app, ready to serve @@ -926,9 +927,9 @@ func (app *App) nameRequired(r *http.Request) bool { func (app *App) initializeExtensionNamespaces(ctx context.Context, extensions map[string]configuration.ExtensionConfig) error { - extensionNamespaces := []storage.Namespace{} + extensionNamespaces := []extension.Namespace{} for key, options := range extensions { - ns, err := storage.Get(ctx, key, app.driver, options) + ns, err := extension.Get(ctx, key, app.driver, options) if err != nil { return fmt.Errorf("unable to configure extension namespace (%s): %s", key, err) } @@ -978,7 +979,7 @@ func (app *App) registerExtensionRoutes(ctx context.Context) error { return nil } -func (app *App) registerExtensionRoute(route storage.Route, nameRequired bool) error { +func (app *App) registerExtensionRoute(route extension.Route, nameRequired bool) error { if route.Dispatcher == nil { return nil } @@ -996,7 +997,7 @@ func (app *App) registerExtensionRoute(route storage.Route, nameRequired bool) e dispatch := route.Dispatcher app.register(desc.Name, func(ctx *Context, r *http.Request) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - extCtx := &storage.Context{ + extCtx := &extension.Context{ Context: ctx.Context, Repository: ctx.Repository, Errors: ctx.Errors, diff --git a/registry/root.go b/registry/root.go index b5c2b859e17..769f689e3cc 100644 --- a/registry/root.go +++ b/registry/root.go @@ -5,6 +5,7 @@ import ( "os" dcontext "github.com/distribution/distribution/v3/context" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver/factory" "github.com/distribution/distribution/v3/version" @@ -72,9 +73,9 @@ var GCCmd = &cobra.Command{ } extensions := config.Extensions - extensionNamespaces := []storage.Namespace{} + extensionNamespaces := []extension.Namespace{} for key, options := range extensions { - ns, err := storage.Get(ctx, key, driver, options) + ns, err := extension.Get(ctx, key, driver, options) if err != nil { fmt.Fprintf(os.Stderr, "unable to configure extension namespace (%s): %s", key, err) os.Exit(1) diff --git a/registry/storage/artifactnamespace_mock.go b/registry/storage/artifactnamespace_mock.go deleted file mode 100644 index c7b43b70b6c..00000000000 --- a/registry/storage/artifactnamespace_mock.go +++ /dev/null @@ -1,25 +0,0 @@ -package storage - -import ( - "github.com/distribution/distribution/v3" - "github.com/distribution/distribution/v3/registry/storage/driver" -) - -type MockNamespace struct { - storageDriver driver.StorageDriver - referrersEnabled bool -} - -// GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. -func (o *MockNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []ManifestHandler { - if o.referrersEnabled { - return []ManifestHandler{ - &ArtifactManifestHandler{ - Repository: repo, - BlobStore: blobStore, - StorageDriver: o.storageDriver, - }} - } - - return []ManifestHandler{} -} diff --git a/registry/storage/garbagecollect_test.go b/registry/storage/garbagecollect_test.go index d3c4b38f50d..14edc533ca1 100644 --- a/registry/storage/garbagecollect_test.go +++ b/registry/storage/garbagecollect_test.go @@ -1,23 +1,18 @@ package storage import ( - "bytes" - "encoding/json" "io" "path" "testing" "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/context" - "github.com/distribution/distribution/v3/manifest/orasartifact" - "github.com/distribution/distribution/v3/manifest/schema2" "github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" "github.com/distribution/distribution/v3/testutil" "github.com/docker/libtrust" "github.com/opencontainers/go-digest" - orasartifacts "github.com/oras-project/artifacts-spec/specs-go/v1" ) type image struct { @@ -163,56 +158,6 @@ func uploadRandomSchema2Image(t *testing.T, repository distribution.Repository) } } -func uploadRandomArtifact(t *testing.T, repository distribution.Repository, subjImg image) (digest.Digest, *orasartifact.DeserializedManifest) { - // build artifact blob and push blob - artifactBlobDigest := digest.FromBytes([]byte{}) - - testutil.UploadBlobs(repository, map[digest.Digest]io.ReadSeeker{ - artifactBlobDigest: bytes.NewReader([]byte{}), - }) - - artifactBlobDescriptor := orasartifacts.Descriptor{ - MediaType: orasartifacts.MediaTypeDescriptor, - Digest: artifactBlobDigest, - Size: 0, - } - - _, subjImgPayload, err := subjImg.manifest.Payload() - if err != nil { - t.Fatalf("failed to get subject image payload: %v", err) - } - // build artifact manifest template - artifactManifest := orasartifacts.Manifest{ - MediaType: orasartifacts.MediaTypeArtifactManifest, - ArtifactType: "test_artifactType", - Blobs: []orasartifacts.Descriptor{ - artifactBlobDescriptor, - }, - Subject: orasartifacts.Descriptor{ - MediaType: schema2.MediaTypeManifest, - Size: int64(len(subjImgPayload)), - Digest: subjImg.manifestDigest, - }, - } - dm := new(orasartifact.DeserializedManifest) - marshalledMan, err := json.Marshal(artifactManifest) - if err != nil { - t.Fatalf("artifact manifest could not be serialized to byte array: %v", err) - } - err = dm.UnmarshalJSON(marshalledMan) - if err != nil { - t.Fatalf("artifact manifest could not be unmarshalled: %v", err) - } - // upload manifest - ctx := context.Background() - manifestService := makeManifestService(t, repository) - manifestDigest, err := manifestService.Put(ctx, dm) - if err != nil { - t.Fatalf("artifact manifest upload failed: %v", err) - } - return manifestDigest, dm -} - func TestNoDeletionNoEffect(t *testing.T) { ctx := context.Background() inmemoryDriver := inmemory.New() @@ -556,50 +501,19 @@ func TestOrphanBlobDeleted(t *testing.T) { } } -func TestReferrersBlobsDeleted(t *testing.T) { - inmemoryDriver := inmemory.New() - registry := createArtifactRegistry(t, inmemoryDriver) - repo := makeRepository(t, registry, "referrers_repo") - ms := makeManifestService(t, repo) - ctx := context.Background() - tagService := repo.Tags(ctx) - - subjImg := uploadRandomSchema2Image(t, repo) - artifactDigest, artifactManifest := uploadRandomArtifact(t, repo, subjImg) - - // the tags folder doesn't exist for this repo until a tag is added - // this leads to an error in Mark and Sweep if tags folder not found - err := tagService.Tag(ctx, "test", distribution.Descriptor{Digest: subjImg.manifestDigest}) - if err != nil { - t.Fatalf("failed to tag subject image: %v", err) - } - err = tagService.Untag(ctx, "test") - if err != nil { - t.Fatalf("failed to untag subject image: %v", err) - } - - // Run GC - err = MarkAndSweep(ctx, inmemoryDriver, registry, GCOpts{ - DryRun: false, - RemoveUntagged: true, - }) - if err != nil { - t.Fatalf("Failed mark and sweep: %v", err) - } - - manifests := allManifests(t, ms) - blobs := allBlobs(t, registry) - - if _, exists := manifests[artifactDigest]; exists { - t.Fatalf("artifact manifest with digest %s should have been deleted", artifactDigest.String()) - } - - if _, exists := blobs[artifactDigest]; exists { - t.Fatalf("artifact manifest blob with digest %s should have been deleted", artifactDigest.String()) - } - - blobDigest := artifactManifest.Inner.Blobs[0].Digest - if _, exists := blobs[blobDigest]; exists { - t.Fatalf("artifact blob with digest %s should have been deleted", blobDigest) - } -} +// func TestReferrersBlobsDeleted(t *testing.T) { +// inmemoryDriver := inmemory.New() +// ctx := context.Background() +// // extConfig := configuration.ExtensionConfig{} +// // ns, err := extension.Get(ctx, "oras", inmemoryDriver, extConfig) +// // if err != nil { +// // fmt.Fprintf(os.Stderr, "unable to configure extension namespace oras: %s", err) +// // os.Exit(1) +// // } + +// options := []RegistryOption{AddExtendedStorage(ns)} +// // add the extended storage for every namespace to the new registry options + +// registry := createRegistry(t, inmemoryDriver, options...) +// repo := makeRepository(t, registry, "michael_z_doukas") +// } diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 149f219c080..bbdf730484d 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -193,9 +193,11 @@ func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error { artifactSweepIngestor) if err != nil { - if _, ok := err.(driver.PathNotFoundError); !ok { - return err + switch err.(type) { + case driver.PathNotFoundError: + return nil } + return err } // delete the artifact manifest revision and the _refs directory for each artifact indexed for key, _ := range artifactManifestIndex { From 2bba5b737acc9eb2a9e8760136452c26f1a11906 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Tue, 14 Jun 2022 15:12:11 -0700 Subject: [PATCH 07/22] fix failing manifest test Signed-off-by: Akash Singhal --- registry/storage/manifeststore.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index bbdf730484d..83c7da45663 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -193,14 +193,12 @@ func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error { artifactSweepIngestor) if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil + if _, ok := err.(driver.PathNotFoundError); !ok { + return err } - return err } // delete the artifact manifest revision and the _refs directory for each artifact indexed - for key, _ := range artifactManifestIndex { + for key := range artifactManifestIndex { err := ms.blobStore.Delete(ctx, key) if err != nil { return err From 1d94df2ca6d5f7eb073a580f2296ff3e4def9bb7 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Thu, 16 Jun 2022 11:34:25 -0700 Subject: [PATCH 08/22] introducing extensions interface in distribution package Signed-off-by: Akash Singhal --- extension.go | 161 +++++++++++ manifests.go | 9 + registry.go | 2 + registry/extension/distribution/manifests.go | 4 +- registry/extension/distribution/registry.go | 37 +-- registry/extension/distribution/taghistory.go | 4 +- registry/extension/extension.go | 255 +++++++++--------- registry/extension/oci/discover.go | 8 +- registry/extension/oci/oci.go | 33 +-- .../oras/artifactgarbagecollectionhandler.go | 236 ++++++++++++++++ .../oras/artifactmanifesthandler_test.go | 5 +- registry/extension/oras/artifactservice.go | 79 +++++- registry/extension/oras/oras.go | 36 ++- registry/handlers/app.go | 13 +- registry/proxy/proxyregistry.go | 4 + registry/root.go | 10 +- registry/storage/extension.go | 79 ------ registry/storage/garbagecollect.go | 178 +++--------- registry/storage/garbagecollect_test.go | 17 -- registry/storage/manifestlisthandler.go | 2 +- registry/storage/manifeststore.go | 58 +--- registry/storage/ocimanifesthandler.go | 2 +- registry/storage/registry.go | 20 +- registry/storage/schema2manifesthandler.go | 2 +- registry/storage/signedmanifesthandler.go | 2 +- registry/storage/v1unsupportedhandler.go | 4 +- 26 files changed, 749 insertions(+), 511 deletions(-) create mode 100644 extension.go create mode 100644 registry/extension/oras/artifactgarbagecollectionhandler.go diff --git a/extension.go b/extension.go new file mode 100644 index 00000000000..b497206d993 --- /dev/null +++ b/extension.go @@ -0,0 +1,161 @@ +package distribution + +import ( + "context" + "fmt" + "net/http" + + "github.com/distribution/distribution/v3/configuration" + "github.com/distribution/distribution/v3/registry/api/errcode" + v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/storage/driver" + "github.com/opencontainers/go-digest" +) + +type Extension interface { + ExtendedNamespace +} + +// ExtensionContext contains the request specific context for use in across handlers. +type ExtensionContext struct { + context.Context + + // Registry is the base namespace that is used by all extension namespaces + Registry Namespace + // Repository is a reference to a named repository + Repository Repository + // Errors are the set of errors that occurred within this request context + Errors errcode.Errors +} + +// RouteDispatchFunc is the http route dispatcher used by the extension route handlers +type RouteDispatchFunc func(extContext *ExtensionContext, r *http.Request) http.Handler + +// ExtensionRoute describes an extension route. +type ExtensionRoute struct { + // Namespace is the name of the extension namespace + Namespace string + // Extension is the name of the extension under the namespace + Extension string + // Component is the name of the component under the extension + Component string + // Descriptor is the route descriptor that gives its path + Descriptor v2.RouteDescriptor + // Dispatcher if present signifies that the route is http route with a dispatcher + Dispatcher RouteDispatchFunc +} + +type GCExtensionHandler interface { + Mark(ctx context.Context, + storageDriver driver.StorageDriver, + registry Namespace, + dryRun bool, + removeUntagged bool) (map[digest.Digest]struct{}, error) + Sweep(ctx context.Context, + storageDriver driver.StorageDriver, + registry Namespace, + dryRun bool, + removeUntagged bool) error +} + +// ExtendedStorage defines extensions to store operations like manifest for example. +type ExtendedStorage interface { + // GetManifestHandlers returns the list of manifest handlers that handle custom manifest formats supported by the extensions. + GetManifestHandlers( + repo Repository, + blobStore BlobStore) []ManifestHandler + GetGarbageCollectionHandlers() []GCExtensionHandler +} + +//Namespace is the namespace that is used to define extensions to the distribution. +type ExtendedNamespace interface { + ExtendedStorage + // GetRepositoryRoutes returns a list of extension routes scoped at a repository level + GetRepositoryRoutes() []ExtensionRoute + // GetRegistryRoutes returns a list of extension routes scoped at a registry level + GetRegistryRoutes() []ExtensionRoute + // GetNamespaceName returns the name associated with the namespace + GetNamespaceName() string + // GetNamespaceUrl returns the url link to the documentation where the namespace's extension and endpoints are defined + GetNamespaceUrl() string + // GetNamespaceDescription returns the description associated with the namespace + GetNamespaceDescription() string +} + +// // InitExtensionNamespace is the initialize function for creating the extension namespace +type InitExtensionNamespace func(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (ExtendedNamespace, error) + +// EnumerateExtension specifies extension information at the namespace level +type EnumerateExtension struct { + Name string `json:"name"` + Url string `json:"url"` + Description string `json:"description,omitempty"` + Endpoints []string `json:"endpoints"` +} + +var extensions map[string]InitExtensionNamespace +var extensionsNamespaces map[string]ExtendedNamespace + +func EnumerateRegistered(ctx ExtensionContext) (enumeratedExtensions []EnumerateExtension) { + for _, namespace := range extensionsNamespaces { + enumerateExtension := EnumerateExtension{ + Name: namespace.GetNamespaceName(), + Url: namespace.GetNamespaceUrl(), + Description: namespace.GetNamespaceDescription(), + Endpoints: []string{}, + } + + scopedRoutes := namespace.GetRepositoryRoutes() + + // if the repository is not set in the context, scope is registry wide + if ctx.Repository == nil { + scopedRoutes = namespace.GetRegistryRoutes() + } + + for _, route := range scopedRoutes { + path := fmt.Sprintf("_%s/%s/%s", route.Namespace, route.Extension, route.Component) + enumerateExtension.Endpoints = append(enumerateExtension.Endpoints, path) + } + + // add extension to list if endpoints exist + if len(enumerateExtension.Endpoints) > 0 { + enumeratedExtensions = append(enumeratedExtensions, enumerateExtension) + } + } + + return enumeratedExtensions +} + +// RegisterExtension is used to register an InitExtensionNamespace for +// an extension namespace with the given name. +func RegisterExtension(name string, initFunc InitExtensionNamespace) { + if extensions == nil { + extensions = make(map[string]InitExtensionNamespace) + } + + if _, exists := extensions[name]; exists { + panic(fmt.Sprintf("namespace name already registered: %s", name)) + } + + extensions[name] = initFunc +} + +// GetExtension constructs an extension namespace with the given options using the given name. +func GetExtension(ctx context.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (ExtendedNamespace, error) { + if extensions != nil { + if extensionsNamespaces == nil { + extensionsNamespaces = make(map[string]ExtendedNamespace) + } + + if initFunc, exists := extensions[name]; exists { + namespace, err := initFunc(ctx, storageDriver, options) + if err == nil { + // adds the initialized namespace to map for simple access to namespaces by EnumerateRegistered + extensionsNamespaces[name] = namespace + } + return namespace, err + } + } + + return nil, fmt.Errorf("no extension registered with name: %s", name) +} diff --git a/manifests.go b/manifests.go index 8f84a220a97..85d2f47a910 100644 --- a/manifests.go +++ b/manifests.go @@ -74,6 +74,15 @@ type Describable interface { Descriptor() Descriptor } +// A ManifestHandler gets and puts manifests of a particular type. +type ManifestHandler interface { + // Unmarshal unmarshals the manifest from a byte slice. + Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (Manifest, error) + + // Put creates or updates the given manifest returning the manifest digest. + Put(ctx context.Context, manifest Manifest, skipDependencyVerification bool) (digest.Digest, error) +} + // ManifestMediaTypes returns the supported media types for manifests. func ManifestMediaTypes() (mediaTypes []string) { for t := range mappings { diff --git a/registry.go b/registry.go index 658f2df0825..97e74f277eb 100644 --- a/registry.go +++ b/registry.go @@ -47,6 +47,8 @@ type Namespace interface { // BlobStatter returns a BlobStatter to control BlobStatter() BlobStatter + + Extensions() []Extension } // RepositoryEnumerator describes an operation to enumerate repositories diff --git a/registry/extension/distribution/manifests.go b/registry/extension/distribution/manifests.go index 51a52e8171f..8440b6daa4a 100644 --- a/registry/extension/distribution/manifests.go +++ b/registry/extension/distribution/manifests.go @@ -4,9 +4,9 @@ import ( "encoding/json" "net/http" + "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -19,7 +19,7 @@ type manifestsGetAPIResponse struct { // manifestHandler handles requests for manifests under a manifest name. type manifestHandler struct { - *extension.Context + *distribution.ExtensionContext storageDriver driver.StorageDriver } diff --git a/registry/extension/distribution/registry.go b/registry/extension/distribution/registry.go index 8e6c5000814..a1dde25c278 100644 --- a/registry/extension/distribution/registry.go +++ b/registry/extension/distribution/registry.go @@ -7,8 +7,6 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" - "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" "gopkg.in/yaml.v2" @@ -34,7 +32,7 @@ type distributionOptions struct { } // newDistNamespace creates a new extension namespace with the name "distribution" -func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { +func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.ExtendedNamespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { @@ -67,21 +65,26 @@ func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, o func init() { // register the extension namespace. - extension.Register(namespaceName, newDistNamespace) + distribution.RegisterExtension(namespaceName, newDistNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. -func (o *distributionNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []storage.ManifestHandler { +func (o *distributionNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []distribution.ManifestHandler { // This extension doesn't extend any manifest store operations. - return []storage.ManifestHandler{} + return []distribution.ManifestHandler{} +} + +func (o *distributionNamespace) GetGarbageCollectionHandlers() []distribution.GCExtensionHandler { + // This extension doesn't extend any garbage collection operations. + return []distribution.GCExtensionHandler{} } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route { - var routes []extension.Route +func (d *distributionNamespace) GetRepositoryRoutes() []distribution.ExtensionRoute { + var routes []distribution.ExtensionRoute if d.manifestsEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, distribution.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: manifestsComponentName, @@ -99,7 +102,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route { } if d.tagHistoryEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, distribution.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: tagHistoryComponentName, @@ -132,7 +135,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []extension.Route { // GetRegistryRoutes returns a list of extension routes scoped at a registry level // There are no registry scoped routes exposed by this namespace -func (d *distributionNamespace) GetRegistryRoutes() []extension.Route { +func (d *distributionNamespace) GetRegistryRoutes() []distribution.ExtensionRoute { return nil } @@ -151,10 +154,10 @@ func (d *distributionNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (d *distributionNamespace) tagHistoryDispatcher(ctx *extension.Context, r *http.Request) http.Handler { +func (d *distributionNamespace) tagHistoryDispatcher(ctx *distribution.ExtensionContext, r *http.Request) http.Handler { tagHistoryHandler := &tagHistoryHandler{ - Context: ctx, - storageDriver: d.storageDriver, + ExtensionContext: ctx, + storageDriver: d.storageDriver, } return handlers.MethodHandler{ @@ -162,10 +165,10 @@ func (d *distributionNamespace) tagHistoryDispatcher(ctx *extension.Context, r * } } -func (d *distributionNamespace) manifestsDispatcher(ctx *extension.Context, r *http.Request) http.Handler { +func (d *distributionNamespace) manifestsDispatcher(ctx *distribution.ExtensionContext, r *http.Request) http.Handler { manifestsHandler := &manifestHandler{ - Context: ctx, - storageDriver: d.storageDriver, + ExtensionContext: ctx, + storageDriver: d.storageDriver, } return handlers.MethodHandler{ diff --git a/registry/extension/distribution/taghistory.go b/registry/extension/distribution/taghistory.go index 9cb957b87d8..8caf8dce564 100644 --- a/registry/extension/distribution/taghistory.go +++ b/registry/extension/distribution/taghistory.go @@ -4,9 +4,9 @@ import ( "encoding/json" "net/http" + "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -20,7 +20,7 @@ type tagHistoryAPIResponse struct { // manifestHandler handles requests for manifests under a manifest name. type tagHistoryHandler struct { - *extension.Context + *distribution.ExtensionContext storageDriver driver.StorageDriver } diff --git a/registry/extension/extension.go b/registry/extension/extension.go index 673288c5a08..ddd2e4a6c80 100644 --- a/registry/extension/extension.go +++ b/registry/extension/extension.go @@ -1,136 +1,123 @@ package extension -import ( - c "context" - "fmt" - "net/http" - - "github.com/distribution/distribution/v3" - "github.com/distribution/distribution/v3/configuration" - "github.com/distribution/distribution/v3/registry/api/errcode" - v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/storage" - "github.com/distribution/distribution/v3/registry/storage/driver" -) - -// Context contains the request specific context for use in across handlers. -type Context struct { - c.Context - - // Registry is the base namespace that is used by all extension namespaces - Registry distribution.Namespace - // Repository is a reference to a named repository - Repository distribution.Repository - // Errors are the set of errors that occurred within this request context - Errors errcode.Errors -} - -// RouteDispatchFunc is the http route dispatcher used by the extension route handlers -type RouteDispatchFunc func(extContext *Context, r *http.Request) http.Handler - -// Route describes an extension route. -type Route struct { - // Namespace is the name of the extension namespace - Namespace string - // Extension is the name of the extension under the namespace - Extension string - // Component is the name of the component under the extension - Component string - // Descriptor is the route descriptor that gives its path - Descriptor v2.RouteDescriptor - // Dispatcher if present signifies that the route is http route with a dispatcher - Dispatcher RouteDispatchFunc -} - -// Namespace is the namespace that is used to define extensions to the distribution. -type Namespace interface { - storage.ExtendedStorage - // GetRepositoryRoutes returns a list of extension routes scoped at a repository level - GetRepositoryRoutes() []Route - // GetRegistryRoutes returns a list of extension routes scoped at a registry level - GetRegistryRoutes() []Route - // GetNamespaceName returns the name associated with the namespace - GetNamespaceName() string - // GetNamespaceUrl returns the url link to the documentation where the namespace's extension and endpoints are defined - GetNamespaceUrl() string - // GetNamespaceDescription returns the description associated with the namespace - GetNamespaceDescription() string -} - -// InitExtensionNamespace is the initialize function for creating the extension namespace -type InitExtensionNamespace func(ctx c.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Namespace, error) - -// EnumerateExtension specifies extension information at the namespace level -type EnumerateExtension struct { - Name string `json:"name"` - Url string `json:"url"` - Description string `json:"description,omitempty"` - Endpoints []string `json:"endpoints"` -} - -var extensions map[string]InitExtensionNamespace -var extensionsNamespaces map[string]Namespace - -func EnumerateRegistered(ctx Context) (enumeratedExtensions []EnumerateExtension) { - for _, namespace := range extensionsNamespaces { - enumerateExtension := EnumerateExtension{ - Name: namespace.GetNamespaceName(), - Url: namespace.GetNamespaceUrl(), - Description: namespace.GetNamespaceDescription(), - Endpoints: []string{}, - } - - scopedRoutes := namespace.GetRepositoryRoutes() - - // if the repository is not set in the context, scope is registry wide - if ctx.Repository == nil { - scopedRoutes = namespace.GetRegistryRoutes() - } - - for _, route := range scopedRoutes { - path := fmt.Sprintf("_%s/%s/%s", route.Namespace, route.Extension, route.Component) - enumerateExtension.Endpoints = append(enumerateExtension.Endpoints, path) - } - - // add extension to list if endpoints exist - if len(enumerateExtension.Endpoints) > 0 { - enumeratedExtensions = append(enumeratedExtensions, enumerateExtension) - } - } - - return enumeratedExtensions -} - -// Register is used to register an InitExtensionNamespace for -// an extension namespace with the given name. -func Register(name string, initFunc InitExtensionNamespace) { - if extensions == nil { - extensions = make(map[string]InitExtensionNamespace) - } - - if _, exists := extensions[name]; exists { - panic(fmt.Sprintf("namespace name already registered: %s", name)) - } - - extensions[name] = initFunc -} - -// Get constructs an extension namespace with the given options using the given name. -func Get(ctx c.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Namespace, error) { - if extensions != nil { - if extensionsNamespaces == nil { - extensionsNamespaces = make(map[string]Namespace) - } - - if initFunc, exists := extensions[name]; exists { - namespace, err := initFunc(ctx, storageDriver, options) - if err == nil { - // adds the initialized namespace to map for simple access to namespaces by EnumerateRegistered - extensionsNamespaces[name] = namespace - } - return namespace, err - } - } - - return nil, fmt.Errorf("no extension registered with name: %s", name) -} +// // Context contains the request specific context for use in across handlers. +// type Context struct { +// c.Context + +// // Registry is the base namespace that is used by all extension namespaces +// Registry distribution.Namespace +// // Repository is a reference to a named repository +// Repository distribution.Repository +// // Errors are the set of errors that occurred within this request context +// Errors errcode.Errors +// } + +// // RouteDispatchFunc is the http route dispatcher used by the extension route handlers +// type RouteDispatchFunc func(extContext *Context, r *http.Request) http.Handler + +// // Route describes an extension route. +// type Route struct { +// // Namespace is the name of the extension namespace +// Namespace string +// // Extension is the name of the extension under the namespace +// Extension string +// // Component is the name of the component under the extension +// Component string +// // Descriptor is the route descriptor that gives its path +// Descriptor v2.RouteDescriptor +// // Dispatcher if present signifies that the route is http route with a dispatcher +// Dispatcher RouteDispatchFunc +// } + +// // Namespace is the namespace that is used to define extensions to the distribution. +// type Namespace interface { +// distribution.ExtendedStorage +// // GetRepositoryRoutes returns a list of extension routes scoped at a repository level +// GetRepositoryRoutes() []Route +// // GetRegistryRoutes returns a list of extension routes scoped at a registry level +// GetRegistryRoutes() []Route +// // GetNamespaceName returns the name associated with the namespace +// GetNamespaceName() string +// // GetNamespaceUrl returns the url link to the documentation where the namespace's extension and endpoints are defined +// GetNamespaceUrl() string +// // GetNamespaceDescription returns the description associated with the namespace +// GetNamespaceDescription() string +// } + +// // InitExtensionNamespace is the initialize function for creating the extension namespace +// type InitExtensionNamespace func(ctx c.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Namespace, error) + +// // EnumerateExtension specifies extension information at the namespace level +// type EnumerateExtension struct { +// Name string `json:"name"` +// Url string `json:"url"` +// Description string `json:"description,omitempty"` +// Endpoints []string `json:"endpoints"` +// } + +// var extensions map[string]InitExtensionNamespace +// var extensionsNamespaces map[string]Namespace + +// func EnumerateRegistered(ctx Context) (enumeratedExtensions []EnumerateExtension) { +// for _, namespace := range extensionsNamespaces { +// enumerateExtension := EnumerateExtension{ +// Name: namespace.GetNamespaceName(), +// Url: namespace.GetNamespaceUrl(), +// Description: namespace.GetNamespaceDescription(), +// Endpoints: []string{}, +// } + +// scopedRoutes := namespace.GetRepositoryRoutes() + +// // if the repository is not set in the context, scope is registry wide +// if ctx.Repository == nil { +// scopedRoutes = namespace.GetRegistryRoutes() +// } + +// for _, route := range scopedRoutes { +// path := fmt.Sprintf("_%s/%s/%s", route.Namespace, route.Extension, route.Component) +// enumerateExtension.Endpoints = append(enumerateExtension.Endpoints, path) +// } + +// // add extension to list if endpoints exist +// if len(enumerateExtension.Endpoints) > 0 { +// enumeratedExtensions = append(enumeratedExtensions, enumerateExtension) +// } +// } + +// return enumeratedExtensions +// } + +// // Register is used to register an InitExtensionNamespace for +// // an extension namespace with the given name. +// func Register(name string, initFunc InitExtensionNamespace) { +// if extensions == nil { +// extensions = make(map[string]InitExtensionNamespace) +// } + +// if _, exists := extensions[name]; exists { +// panic(fmt.Sprintf("namespace name already registered: %s", name)) +// } + +// extensions[name] = initFunc +// } + +// // Get constructs an extension namespace with the given options using the given name. +// func Get(ctx c.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Namespace, error) { +// if extensions != nil { +// if extensionsNamespaces == nil { +// extensionsNamespaces = make(map[string]Namespace) +// } + +// if initFunc, exists := extensions[name]; exists { +// namespace, err := initFunc(ctx, storageDriver, options) +// if err == nil { +// // adds the initialized namespace to map for simple access to namespaces by EnumerateRegistered +// extensionsNamespaces[name] = namespace +// } +// return namespace, err +// } +// } + +// return nil, fmt.Errorf("no extension registered with name: %s", name) +// } diff --git a/registry/extension/oci/discover.go b/registry/extension/oci/discover.go index 3d2ec3fce97..d1d76762986 100644 --- a/registry/extension/oci/discover.go +++ b/registry/extension/oci/discover.go @@ -4,18 +4,18 @@ import ( "encoding/json" "net/http" + "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/registry/api/errcode" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage/driver" ) type discoverGetAPIResponse struct { - Extensions []extension.EnumerateExtension `json:"extensions"` + Extensions []distribution.EnumerateExtension `json:"extensions"` } // extensionHandler handles requests for manifests under a manifest name. type extensionHandler struct { - *extension.Context + *distribution.ExtensionContext storageDriver driver.StorageDriver } @@ -25,7 +25,7 @@ func (eh *extensionHandler) getExtensions(w http.ResponseWriter, r *http.Request w.Header().Set("Content-Type", "application/json") // get list of extension information seperated at the namespace level - enumeratedExtensions := extension.EnumerateRegistered(*eh.Context) + enumeratedExtensions := distribution.EnumerateRegistered(*eh.ExtensionContext) // remove the oci extension so it's not returned by discover for i, e := range enumeratedExtensions { diff --git a/registry/extension/oci/oci.go b/registry/extension/oci/oci.go index d68e5b8cd8f..4f40cc054da 100644 --- a/registry/extension/oci/oci.go +++ b/registry/extension/oci/oci.go @@ -7,8 +7,6 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" - "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" "gopkg.in/yaml.v2" @@ -32,7 +30,7 @@ type ociOptions struct { } // newOciNamespace creates a new extension namespace with the name "oci" -func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { +func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.ExtendedNamespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err @@ -60,21 +58,26 @@ func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, op func init() { // register the extension namespace. - extension.Register(namespaceName, newOciNamespace) + distribution.RegisterExtension(namespaceName, newOciNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. -func (o *ociNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []storage.ManifestHandler { +func (o *ociNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []distribution.ManifestHandler { // This extension doesn't extend any manifest store operations. - return []storage.ManifestHandler{} + return []distribution.ManifestHandler{} +} + +func (o *ociNamespace) GetGarbageCollectionHandlers() []distribution.GCExtensionHandler { + // This extension doesn't extend any garbage collection operations. + return []distribution.GCExtensionHandler{} } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (o *ociNamespace) GetRepositoryRoutes() []extension.Route { - var routes []extension.Route +func (o *ociNamespace) GetRepositoryRoutes() []distribution.ExtensionRoute { + var routes []distribution.ExtensionRoute if o.discoverEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, distribution.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: discoverComponentName, @@ -95,11 +98,11 @@ func (o *ociNamespace) GetRepositoryRoutes() []extension.Route { } // GetRegistryRoutes returns a list of extension routes scoped at a registry level -func (o *ociNamespace) GetRegistryRoutes() []extension.Route { - var routes []extension.Route +func (o *ociNamespace) GetRegistryRoutes() []distribution.ExtensionRoute { + var routes []distribution.ExtensionRoute if o.discoverEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, distribution.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: discoverComponentName, @@ -134,10 +137,10 @@ func (o *ociNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (o *ociNamespace) discoverDispatcher(ctx *extension.Context, r *http.Request) http.Handler { +func (o *ociNamespace) discoverDispatcher(ctx *distribution.ExtensionContext, r *http.Request) http.Handler { extensionHandler := &extensionHandler{ - Context: ctx, - storageDriver: o.storageDriver, + ExtensionContext: ctx, + storageDriver: o.storageDriver, } return handlers.MethodHandler{ diff --git a/registry/extension/oras/artifactgarbagecollectionhandler.go b/registry/extension/oras/artifactgarbagecollectionhandler.go new file mode 100644 index 00000000000..3ad5e33c601 --- /dev/null +++ b/registry/extension/oras/artifactgarbagecollectionhandler.go @@ -0,0 +1,236 @@ +package oras + +import ( + "context" + "fmt" + "path" + + "github.com/distribution/distribution/v3" + "github.com/distribution/distribution/v3/reference" + "github.com/distribution/distribution/v3/registry/storage" + "github.com/distribution/distribution/v3/registry/storage/driver" + "github.com/opencontainers/go-digest" + v1 "github.com/oras-project/artifacts-spec/specs-go/v1" +) + +type orasGCHandler struct { + artifactManifestIndex map[digest.Digest]artifactManifestDel +} + +type artifactManifestDel struct { + name string + artifactDigest digest.Digest +} + +func (gc *orasGCHandler) Mark(ctx context.Context, + storageDriver driver.StorageDriver, + registry distribution.Namespace, + dryRun bool, + removeUntagged bool) (map[digest.Digest]struct{}, error) { + repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) + if !ok { + return nil, fmt.Errorf("unable to convert Namespace to RepositoryEnumerator") + } + + gc.artifactManifestIndex = make(map[digest.Digest]artifactManifestDel) + // mark + markSet := make(map[digest.Digest]struct{}) + err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { + fmt.Printf(repoName + "\n") + + var err error + named, err := reference.WithName(repoName) + if err != nil { + return fmt.Errorf("failed to parse repo name %s: %v", repoName, err) + } + repository, err := registry.Repository(ctx, named) + if err != nil { + return fmt.Errorf("failed to construct repository: %v", err) + } + + manifestService, err := repository.Manifests(ctx) + if err != nil { + return fmt.Errorf("failed to construct manifest service: %v", err) + } + + manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator) + if !ok { + return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator") + } + + err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { + manifest, err := manifestService.Get(ctx, dgst) + if err != nil { + return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) + } + + mediaType, _, err := manifest.Payload() + if err != nil { + return err + } + + // if the manifest is an oras artifact, skip it + // the artifact marking occurs when walking the refs + if mediaType == v1.MediaTypeArtifactManifest { + return nil + } + + blobStatter := registry.BlobStatter() + referrerRootPath := referrersLinkPath(repoName) + if removeUntagged { + // fetch all tags where this manifest is the latest one + tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst}) + if err != nil { + return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err) + } + if len(tags) == 0 { + + // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion + rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + err = enumerateReferrerLinks(ctx, + rootPath, + storageDriver, + blobStatter, + manifestService, + repository.Named().Name(), + markSet, + gc.artifactManifestIndex, + artifactSweepIngestor) + + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return nil + } + } + + // recurse child artifact as subject to find lower level referrers + rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + err = enumerateReferrerLinks(ctx, + rootPath, + storageDriver, + blobStatter, + manifestService, + repository.Named().Name(), + markSet, + gc.artifactManifestIndex, + artifactMarkIngestor) + + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return nil + }) + + // In certain situations such as unfinished uploads, deleting all + // tags in S3 or removing the _manifests folder manually, this + // error may be of type PathNotFound. + // + // In these cases we can continue marking other manifests safely. + if _, ok := err.(driver.PathNotFoundError); ok { + return nil + } + + return err + }) + + if err != nil { + return nil, fmt.Errorf("failed to mark: %v", err) + } + return markSet, nil +} + +func (gc *orasGCHandler) Sweep(ctx context.Context, + storageDriver driver.StorageDriver, + registry distribution.Namespace, + dryRun bool, + removeUntagged bool) error { + vacuum := storage.NewVacuum(ctx, storageDriver) + if !dryRun { + // remove each artifact in the index + for artifactDigest, obj := range gc.artifactManifestIndex { + err := vacuum.RemoveArtifactManifest(obj.name, artifactDigest) + if err != nil { + return fmt.Errorf("failed to delete artifact manifest %s: %v", artifactDigest, err) + } + } + } + + return nil +} + +// ingestor method used in EnumerateReferrerLinks +// marks each artifact manifest and associated blobs +func artifactMarkIngestor(ctx context.Context, + referrerRevision digest.Digest, + manifestService distribution.ManifestService, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]artifactManifestDel, + repoName string, + storageDriver driver.StorageDriver, + blobStatter distribution.BlobStatter) error { + man, err := manifestService.Get(ctx, referrerRevision) + if err != nil { + return err + } + + // mark the artifact manifest blob + fmt.Printf("%s: marking artifact manifest %s\n", repoName, referrerRevision.String()) + markSet[referrerRevision] = struct{}{} + + // mark the artifact blobs + descriptors := man.References() + for _, descriptor := range descriptors { + markSet[descriptor.Digest] = struct{}{} + fmt.Printf("%s: marking blob %s\n", repoName, descriptor.Digest) + } + referrerRootPath := referrersLinkPath(repoName) + + rootPath := path.Join(referrerRootPath, referrerRevision.Algorithm().String(), referrerRevision.Hex()) + _, err = storageDriver.Stat(ctx, rootPath) + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return enumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactMarkIngestor) +} + +// ingestor method used in EnumerateReferrerLinks +// indexes each artifact manifest and adds ArtifactManifestDel struct to index +func artifactSweepIngestor(ctx context.Context, + referrerRevision digest.Digest, + manifestService distribution.ManifestService, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]artifactManifestDel, + repoName string, + storageDriver driver.StorageDriver, + blobStatter distribution.BlobStatter) error { + + // index the manifest + fmt.Printf("%s: indexing artifact manifest %s\n", repoName, referrerRevision.String()) + artifactManifestIndex[referrerRevision] = artifactManifestDel{name: repoName, artifactDigest: referrerRevision} + + referrerRootPath := referrersLinkPath(repoName) + + rootPath := path.Join(referrerRootPath, referrerRevision.Algorithm().String(), referrerRevision.Hex()) + _, err := storageDriver.Stat(ctx, rootPath) + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return enumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactSweepIngestor) +} diff --git a/registry/extension/oras/artifactmanifesthandler_test.go b/registry/extension/oras/artifactmanifesthandler_test.go index 6212aadcaf0..0ac8ed93ce6 100644 --- a/registry/extension/oras/artifactmanifesthandler_test.go +++ b/registry/extension/oras/artifactmanifesthandler_test.go @@ -9,7 +9,6 @@ import ( "github.com/distribution/distribution/v3/manifest" "github.com/distribution/distribution/v3/manifest/schema2" "github.com/distribution/distribution/v3/reference" - "github.com/distribution/distribution/v3/registry/extension" storage "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" @@ -23,11 +22,11 @@ func createRegistry(t *testing.T, driver driver.StorageDriver, options ...storag extensionConfig := OrasOptions{ ArtifactsExtComponents: []string{"referrers"}, } - ns, err := extension.Get(ctx, "oras", driver, extensionConfig) + ns, err := distribution.GetExtension(ctx, "oras", driver, extensionConfig) if err != nil { t.Fatalf("unable to configure extension namespace (%s): %s", "oras", err) } - options = append(options, storage.AddExtendedStorage(ns)) + options = append(options, storage.AddExtendedNamespace(ns)) registry, err := storage.NewRegistry(ctx, driver, options...) if err != nil { t.Fatalf("failed to construct namespace") diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index caa27e5037e..8d3f9c62c1b 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -9,8 +9,6 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" - "github.com/distribution/distribution/v3/registry/extension" - "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" artifactv1 "github.com/oras-project/artifacts-spec/specs-go/v1" @@ -22,7 +20,7 @@ type ArtifactService interface { // referrersHandler handles http operations on manifest referrers. type referrersHandler struct { - extContext *extension.Context + extContext *distribution.ExtensionContext storageDriver driver.StorageDriver // Digest is the target manifest's digest. @@ -52,19 +50,19 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest blobStatter := h.extContext.Registry.BlobStatter() rootPath := path.Join(referrersLinkPath(repo.Named().Name()), revision.Algorithm().String(), revision.Hex()) - err = storage.EnumerateReferrerLinks(ctx, + err = enumerateReferrerLinks(ctx, rootPath, h.storageDriver, blobStatter, manifests, repo.Named().Name(), map[digest.Digest]struct{}{}, - map[digest.Digest]storage.ArtifactManifestDel{}, + map[digest.Digest]artifactManifestDel{}, func(ctx context.Context, referrerRevision digest.Digest, manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]storage.ArtifactManifestDel, + artifactManifestIndex map[digest.Digest]artifactManifestDel, repoName string, storageDriver driver.StorageDriver, blobStatter distribution.BlobStatter) error { @@ -132,3 +130,72 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest referrersSorted = append(referrersSorted, referrersUnsorted...) return referrersSorted, nil } + +func enumerateReferrerLinks(ctx context.Context, + rootPath string, + stDriver driver.StorageDriver, + blobStatter distribution.BlobStatter, + manifestService distribution.ManifestService, + repositoryName string, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]artifactManifestDel, + ingestor func(ctx context.Context, + digest digest.Digest, + manifestService distribution.ManifestService, + markSet map[digest.Digest]struct{}, + artifactManifestIndex map[digest.Digest]artifactManifestDel, + repoName string, + storageDriver driver.StorageDriver, + blobStatter distribution.BlobStatter) error) error { + + return stDriver.Walk(ctx, rootPath, func(fileInfo driver.FileInfo) error { + // exit early if directory... + if fileInfo.IsDir() { + return nil + } + filePath := fileInfo.Path() + + // check if it's a link + _, fileName := path.Split(filePath) + if fileName != "link" { + return nil + } + + // read the digest found in link + digest, err := readlink(ctx, filePath, stDriver) + if err != nil { + return err + } + + // ensure this conforms to the linkPathFns + _, err = blobStatter.Stat(ctx, digest) + if err != nil { + // we expect this error to occur so we move on + if err == distribution.ErrBlobUnknown { + return nil + } + return err + } + + err = ingestor(ctx, digest, manifestService, markSet, artifactManifestIndex, repositoryName, stDriver, blobStatter) + if err != nil { + return err + } + + return nil + }) +} + +func readlink(ctx context.Context, path string, stDriver driver.StorageDriver) (digest.Digest, error) { + content, err := stDriver.GetContent(ctx, path) + if err != nil { + return "", err + } + + linked, err := digest.Parse(string(content)) + if err != nil { + return "", err + } + + return linked, nil +} diff --git a/registry/extension/oras/oras.go b/registry/extension/oras/oras.go index 340085e6c97..563bb4f8968 100644 --- a/registry/extension/oras/oras.go +++ b/registry/extension/oras/oras.go @@ -8,8 +8,6 @@ import ( "github.com/distribution/distribution/v3/configuration" dcontext "github.com/distribution/distribution/v3/context" v2 "github.com/distribution/distribution/v3/registry/api/v2" - "github.com/distribution/distribution/v3/registry/extension" - "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" "github.com/opencontainers/go-digest" @@ -27,6 +25,7 @@ const ( type orasNamespace struct { storageDriver driver.StorageDriver referrersEnabled bool + gcHandler orasGCHandler } type OrasOptions struct { @@ -34,7 +33,7 @@ type OrasOptions struct { } // newOrasNamespace creates a new extension namespace with the name "oras" -func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Namespace, error) { +func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.ExtendedNamespace, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err @@ -54,20 +53,23 @@ func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, o } } + orasGCHandler := orasGCHandler{} + return &orasNamespace{ referrersEnabled: referrersEnabled, storageDriver: storageDriver, + gcHandler: orasGCHandler, }, nil } func init() { - extension.Register(namespaceName, newOrasNamespace) + distribution.RegisterExtension(namespaceName, newOrasNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. -func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []storage.ManifestHandler { +func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []distribution.ManifestHandler { if o.referrersEnabled { - return []storage.ManifestHandler{ + return []distribution.ManifestHandler{ &artifactManifestHandler{ repository: repo, blobStore: blobStore, @@ -75,15 +77,25 @@ func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobSt }} } - return []storage.ManifestHandler{} + return []distribution.ManifestHandler{} +} + +func (o *orasNamespace) GetGarbageCollectionHandlers() []distribution.GCExtensionHandler { + if o.referrersEnabled { + return []distribution.GCExtensionHandler{ + &o.gcHandler, + } + } + + return []distribution.GCExtensionHandler{} } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (d *orasNamespace) GetRepositoryRoutes() []extension.Route { - var routes []extension.Route +func (d *orasNamespace) GetRepositoryRoutes() []distribution.ExtensionRoute { + var routes []distribution.ExtensionRoute if d.referrersEnabled { - routes = append(routes, extension.Route{ + routes = append(routes, distribution.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: referrersComponentName, @@ -105,7 +117,7 @@ func (d *orasNamespace) GetRepositoryRoutes() []extension.Route { // GetRegistryRoutes returns a list of extension routes scoped at a registry level // There are no registry scoped routes exposed by this namespace -func (d *orasNamespace) GetRegistryRoutes() []extension.Route { +func (d *orasNamespace) GetRegistryRoutes() []distribution.ExtensionRoute { return nil } @@ -124,7 +136,7 @@ func (d *orasNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (o *orasNamespace) referrersDispatcher(extCtx *extension.Context, r *http.Request) http.Handler { +func (o *orasNamespace) referrersDispatcher(extCtx *distribution.ExtensionContext, r *http.Request) http.Handler { handler := &referrersHandler{ storageDriver: o.storageDriver, diff --git a/registry/handlers/app.go b/registry/handlers/app.go index da337e5ed5a..b88f289c2bd 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -28,7 +28,6 @@ import ( "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" "github.com/distribution/distribution/v3/registry/auth" - "github.com/distribution/distribution/v3/registry/extension" registrymiddleware "github.com/distribution/distribution/v3/registry/middleware/registry" repositorymiddleware "github.com/distribution/distribution/v3/registry/middleware/repository" "github.com/distribution/distribution/v3/registry/proxy" @@ -98,7 +97,7 @@ type App struct { repositoryExtensions []string // extensionNamespaces is a list of namespaces that are configured as extensions to the distribution - extensionNamespaces []extension.Namespace + extensionNamespaces []distribution.ExtendedNamespace } // NewApp takes a configuration and returns a configured app, ready to serve @@ -280,7 +279,7 @@ func NewApp(ctx context.Context, config *configuration.Configuration) *App { // add the extended storage for every namespace to the new registry options for _, ns := range app.extensionNamespaces { - options = append(options, storage.AddExtendedStorage(ns)) + options = append(options, storage.AddExtendedNamespace(ns)) } // configure storage caches @@ -927,9 +926,9 @@ func (app *App) nameRequired(r *http.Request) bool { func (app *App) initializeExtensionNamespaces(ctx context.Context, extensions map[string]configuration.ExtensionConfig) error { - extensionNamespaces := []extension.Namespace{} + extensionNamespaces := []distribution.ExtendedNamespace{} for key, options := range extensions { - ns, err := extension.Get(ctx, key, app.driver, options) + ns, err := distribution.GetExtension(ctx, key, app.driver, options) if err != nil { return fmt.Errorf("unable to configure extension namespace (%s): %s", key, err) } @@ -979,7 +978,7 @@ func (app *App) registerExtensionRoutes(ctx context.Context) error { return nil } -func (app *App) registerExtensionRoute(route extension.Route, nameRequired bool) error { +func (app *App) registerExtensionRoute(route distribution.ExtensionRoute, nameRequired bool) error { if route.Dispatcher == nil { return nil } @@ -997,7 +996,7 @@ func (app *App) registerExtensionRoute(route extension.Route, nameRequired bool) dispatch := route.Dispatcher app.register(desc.Name, func(ctx *Context, r *http.Request) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - extCtx := &extension.Context{ + extCtx := &distribution.ExtensionContext{ Context: ctx.Context, Repository: ctx.Repository, Errors: ctx.Errors, diff --git a/registry/proxy/proxyregistry.go b/registry/proxy/proxyregistry.go index 00f560daa1b..b813bc8f51f 100644 --- a/registry/proxy/proxyregistry.go +++ b/registry/proxy/proxyregistry.go @@ -189,6 +189,10 @@ func (pr *proxyingRegistry) BlobStatter() distribution.BlobStatter { return pr.embedded.BlobStatter() } +func (pr *proxyingRegistry) Extensions() []distribution.Extension { + return []distribution.Extension{} +} + // authChallenger encapsulates a request to the upstream to establish credential challenges type authChallenger interface { tryEstablishChallenges(context.Context) error diff --git a/registry/root.go b/registry/root.go index 769f689e3cc..b452eb162c2 100644 --- a/registry/root.go +++ b/registry/root.go @@ -4,8 +4,8 @@ import ( "fmt" "os" + "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" - "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver/factory" "github.com/distribution/distribution/v3/version" @@ -73,9 +73,9 @@ var GCCmd = &cobra.Command{ } extensions := config.Extensions - extensionNamespaces := []extension.Namespace{} + extensionNamespaces := []distribution.ExtendedNamespace{} for key, options := range extensions { - ns, err := extension.Get(ctx, key, driver, options) + ns, err := distribution.GetExtension(ctx, key, driver, options) if err != nil { fmt.Fprintf(os.Stderr, "unable to configure extension namespace (%s): %s", key, err) os.Exit(1) @@ -84,9 +84,9 @@ var GCCmd = &cobra.Command{ } options := []storage.RegistryOption{storage.Schema1SigningKey(k)} - // add the extended storage for every namespace to the new registry options + // add all the extended namespaces to the new registry options for _, ns := range extensionNamespaces { - options = append(options, storage.AddExtendedStorage(ns)) + options = append(options, storage.AddExtendedNamespace(ns)) } registry, err := storage.NewRegistry(ctx, driver, options...) diff --git a/registry/storage/extension.go b/registry/storage/extension.go index 2a7f80c8b56..f2a78c3ac0d 100644 --- a/registry/storage/extension.go +++ b/registry/storage/extension.go @@ -2,10 +2,8 @@ package storage import ( "context" - "path" "github.com/distribution/distribution/v3" - "github.com/distribution/distribution/v3/registry/storage/driver" storagedriver "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" ) @@ -17,14 +15,6 @@ type ReadOnlyBlobStore interface { distribution.BlobProvider } -// ExtendedStorage defines extensions to store operations like manifest for example. -type ExtendedStorage interface { - // GetManifestHandlers returns the list of manifest handlers that handle custom manifest formats supported by the extensions. - GetManifestHandlers( - repo distribution.Repository, - blobStore distribution.BlobStore) []ManifestHandler -} - // GetManifestLinkReadOnlyBlobStore will enable extensions to access the underlying linked blob store for readonly operations. // This blob store is scoped only to manifest link paths. Manifest link paths doesn't use blob cache func GetManifestLinkReadOnlyBlobStore( @@ -117,72 +107,3 @@ func GetTagLinkReadOnlyBlobStore( }, } } - -func EnumerateReferrerLinks(ctx context.Context, - rootPath string, - stDriver storagedriver.StorageDriver, - blobStatter distribution.BlobStatter, - manifestService distribution.ManifestService, - repositoryName string, - markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]ArtifactManifestDel, - ingestor func(ctx context.Context, - digest digest.Digest, - manifestService distribution.ManifestService, - markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]ArtifactManifestDel, - repoName string, - storageDriver driver.StorageDriver, - blobStatter distribution.BlobStatter) error) error { - - return stDriver.Walk(ctx, rootPath, func(fileInfo driver.FileInfo) error { - // exit early if directory... - if fileInfo.IsDir() { - return nil - } - filePath := fileInfo.Path() - - // check if it's a link - _, fileName := path.Split(filePath) - if fileName != "link" { - return nil - } - - // read the digest found in link - digest, err := readlink(ctx, filePath, stDriver) - if err != nil { - return err - } - - // ensure this conforms to the linkPathFns - _, err = blobStatter.Stat(ctx, digest) - if err != nil { - // we expect this error to occur so we move on - if err == distribution.ErrBlobUnknown { - return nil - } - return err - } - - err = ingestor(ctx, digest, manifestService, markSet, artifactManifestIndex, repositoryName, stDriver, blobStatter) - if err != nil { - return err - } - - return nil - }) -} - -func readlink(ctx context.Context, path string, stDriver storagedriver.StorageDriver) (digest.Digest, error) { - content, err := stDriver.GetContent(ctx, path) - if err != nil { - return "", err - } - - linked, err := digest.Parse(string(content)) - if err != nil { - return "", err - } - - return linked, nil -} diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 40d38f2f319..16e348aa5f6 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -3,13 +3,11 @@ package storage import ( "context" "fmt" - "path" "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" - v1 "github.com/oras-project/artifacts-spec/specs-go/v1" ) func emit(format string, a ...interface{}) { @@ -29,12 +27,6 @@ type ManifestDel struct { Tags []string } -// ArtifactManifestDel contains artifact manifest structure which will be deleted -type ArtifactManifestDel struct { - Name string - ArtifactDigest digest.Digest -} - // MarkAndSweep performs a mark and sweep of registry data func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, opts GCOpts) error { repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) @@ -45,7 +37,6 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis // mark markSet := make(map[digest.Digest]struct{}) manifestArr := make([]ManifestDel, 0) - artifactManifestIndex := make(map[digest.Digest]ArtifactManifestDel) err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { emit(repoName) @@ -70,28 +61,6 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis } err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { - manifest, err := manifestService.Get(ctx, dgst) - if err != nil { - return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) - } - - mediaType, _, err := manifest.Payload() - if err != nil { - return err - } - - // if the manifest is an oras artifact, skip it - // the artifact marking occurs when walking the refs - if mediaType == v1.MediaTypeArtifactManifest { - return nil - } - - blobStatter := registry.BlobStatter() - referrerRootPath, err := pathFor(referrersRootPathSpec{name: repository.Named().Name()}) - if err != nil { - return err - } - if opts.RemoveUntagged { // fetch all tags where this manifest is the latest one tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst}) @@ -108,26 +77,6 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to retrieve tags %v", err) } manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) - - // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion - rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) - err = EnumerateReferrerLinks(ctx, - rootPath, - storageDriver, - blobStatter, - manifestService, - repository.Named().Name(), - markSet, - artifactManifestIndex, - artifactSweepIngestor) - - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil - } - return err - } return nil } } @@ -135,31 +84,17 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis emit("%s: marking manifest %s ", repoName, dgst) markSet[dgst] = struct{}{} + manifest, err := manifestService.Get(ctx, dgst) + if err != nil { + return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) + } + descriptors := manifest.References() for _, descriptor := range descriptors { markSet[descriptor.Digest] = struct{}{} emit("%s: marking blob %s", repoName, descriptor.Digest) } - // recurse child artifact as subject to find lower level referrers - rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) - err = EnumerateReferrerLinks(ctx, - rootPath, - storageDriver, - blobStatter, - manifestService, - repository.Named().Name(), - markSet, - artifactManifestIndex, - artifactMarkIngestor) - - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil - } - return err - } return nil }) @@ -179,6 +114,21 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to mark: %v", err) } + // call GC extension handlers' mark + for _, extNamespace := range registry.Extensions() { + handlers := extNamespace.GetGarbageCollectionHandlers() + for _, gcHandler := range handlers { + extensionMarkSet, err := gcHandler.Mark(ctx, storageDriver, registry, opts.DryRun, opts.RemoveUntagged) + if err != nil { + return fmt.Errorf("failed to mark using extension handler: %v", err) + } + + for k, _ := range extensionMarkSet { + markSet[k] = struct{}{} + } + } + } + // sweep vacuum := NewVacuum(ctx, storageDriver) if !opts.DryRun { @@ -188,13 +138,6 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) } } - // remove each artifact in the index - for artifactDigest, obj := range artifactManifestIndex { - err = vacuum.RemoveArtifactManifest(obj.Name, artifactDigest) - if err != nil { - return fmt.Errorf("failed to delete artifact manifest %s: %v", artifactDigest, err) - } - } } blobService := registry.Blobs() deleteSet := make(map[digest.Digest]struct{}) @@ -208,7 +151,7 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("error enumerating blobs: %v", err) } - emit("\n%d blobs marked, %d blobs, %d manifests, and %d artifacts eligible for deletion", len(markSet), len(deleteSet), len(manifestArr), len(artifactManifestIndex)) + emit("\n%d blobs marked, %d blobs and %d manifests eligible for deletion", len(markSet), len(deleteSet), len(manifestArr)) for dgst := range deleteSet { emit("blob eligible for deletion: %s", dgst) if opts.DryRun { @@ -220,77 +163,16 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis } } - return err -} - -// ingestor method used in EnumerateReferrerLinks -// marks each artifact manifest and associated blobs -func artifactMarkIngestor(ctx context.Context, - referrerRevision digest.Digest, - manifestService distribution.ManifestService, - markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]ArtifactManifestDel, - repoName string, - storageDriver driver.StorageDriver, - blobStatter distribution.BlobStatter) error { - man, err := manifestService.Get(ctx, referrerRevision) - if err != nil { - return err - } - - // mark the artifact manifest blob - emit("%s: marking artifact manifest %s ", repoName, referrerRevision.String()) - markSet[referrerRevision] = struct{}{} - - // mark the artifact blobs - descriptors := man.References() - for _, descriptor := range descriptors { - markSet[descriptor.Digest] = struct{}{} - emit("%s: marking blob %s", repoName, descriptor.Digest) - } - referrerRootPath, err := pathFor(referrersRootPathSpec{name: repoName}) - if err != nil { - return err - } - rootPath := path.Join(referrerRootPath, referrerRevision.Algorithm().String(), referrerRevision.Hex()) - _, err = storageDriver.Stat(ctx, rootPath) - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil + // call GC extension handlers' sweep + for _, extNamespace := range registry.Extensions() { + handlers := extNamespace.GetGarbageCollectionHandlers() + for _, gcHandler := range handlers { + err := gcHandler.Sweep(ctx, storageDriver, registry, opts.DryRun, opts.RemoveUntagged) + if err != nil { + return fmt.Errorf("failed to sweep using extension handler: %v", err) + } } - return err } - return EnumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactMarkIngestor) -} - -// ingestor method used in EnumerateReferrerLinks -// indexes each artifact manifest and adds ArtifactManifestDel struct to index -func artifactSweepIngestor(ctx context.Context, - referrerRevision digest.Digest, - manifestService distribution.ManifestService, - markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]ArtifactManifestDel, - repoName string, - storageDriver driver.StorageDriver, - blobStatter distribution.BlobStatter) error { - // index the manifest - emit("%s: indexing artifact manifest %s ", repoName, referrerRevision.String()) - artifactManifestIndex[referrerRevision] = ArtifactManifestDel{Name: repoName, ArtifactDigest: referrerRevision} - - referrerRootPath, err := pathFor(referrersRootPathSpec{name: repoName}) - if err != nil { - return err - } - rootPath := path.Join(referrerRootPath, referrerRevision.Algorithm().String(), referrerRevision.Hex()) - _, err = storageDriver.Stat(ctx, rootPath) - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil - } - return err - } - return EnumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactSweepIngestor) + return err } diff --git a/registry/storage/garbagecollect_test.go b/registry/storage/garbagecollect_test.go index 14edc533ca1..25c5e2f8fe0 100644 --- a/registry/storage/garbagecollect_test.go +++ b/registry/storage/garbagecollect_test.go @@ -500,20 +500,3 @@ func TestOrphanBlobDeleted(t *testing.T) { } } } - -// func TestReferrersBlobsDeleted(t *testing.T) { -// inmemoryDriver := inmemory.New() -// ctx := context.Background() -// // extConfig := configuration.ExtensionConfig{} -// // ns, err := extension.Get(ctx, "oras", inmemoryDriver, extConfig) -// // if err != nil { -// // fmt.Fprintf(os.Stderr, "unable to configure extension namespace oras: %s", err) -// // os.Exit(1) -// // } - -// options := []RegistryOption{AddExtendedStorage(ns)} -// // add the extended storage for every namespace to the new registry options - -// registry := createRegistry(t, inmemoryDriver, options...) -// repo := makeRepository(t, registry, "michael_z_doukas") -// } diff --git a/registry/storage/manifestlisthandler.go b/registry/storage/manifestlisthandler.go index e9c71d4c0d9..eca20a69e5c 100644 --- a/registry/storage/manifestlisthandler.go +++ b/registry/storage/manifestlisthandler.go @@ -17,7 +17,7 @@ type manifestListHandler struct { ctx context.Context } -var _ ManifestHandler = &manifestListHandler{} +var _ distribution.ManifestHandler = &manifestListHandler{} func (ms *manifestListHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ms.ctx).Debug("(*manifestListHandler).Unmarshal") diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 83c7da45663..c070dfcea51 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "path" "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" @@ -13,19 +12,18 @@ import ( "github.com/distribution/distribution/v3/manifest/ocischema" "github.com/distribution/distribution/v3/manifest/schema1" "github.com/distribution/distribution/v3/manifest/schema2" - "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) -// A ManifestHandler gets and puts manifests of a particular type. -type ManifestHandler interface { - // Unmarshal unmarshals the manifest from a byte slice. - Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) +// // A ManifestHandler gets and puts manifests of a particular type. +// type ManifestHandler interface { +// // Unmarshal unmarshals the manifest from a byte slice. +// Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) - // Put creates or updates the given manifest returning the manifest digest. - Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) -} +// // Put creates or updates the given manifest returning the manifest digest. +// Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) +// } // SkipLayerVerification allows a manifest to be Put before its // layers are on the filesystem @@ -50,12 +48,12 @@ type manifestStore struct { skipDependencyVerification bool - schema1Handler ManifestHandler - schema2Handler ManifestHandler - ocischemaHandler ManifestHandler - manifestListHandler ManifestHandler + schema1Handler distribution.ManifestHandler + schema2Handler distribution.ManifestHandler + ocischemaHandler distribution.ManifestHandler + manifestListHandler distribution.ManifestHandler - extensionManifestHandlers []ManifestHandler + extensionManifestHandlers []distribution.ManifestHandler } var _ distribution.ManifestService = &manifestStore{} @@ -173,38 +171,6 @@ func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest // Delete removes the revision of the specified manifest. func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error { dcontext.GetLogger(ms.ctx).Debug("(*manifestStore).Delete") - - // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion - artifactManifestIndex := make(map[digest.Digest]ArtifactManifestDel) - repositoryName := ms.repository.Named().Name() - referrerRootPath, err := pathFor(referrersRootPathSpec{name: repositoryName}) - if err != nil { - return err - } - rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) - err = EnumerateReferrerLinks(ctx, - rootPath, - ms.blobStore.driver, - ms.repository.statter, - ms, - repositoryName, - map[digest.Digest]struct{}{}, - artifactManifestIndex, - artifactSweepIngestor) - - if err != nil { - if _, ok := err.(driver.PathNotFoundError); !ok { - return err - } - } - // delete the artifact manifest revision and the _refs directory for each artifact indexed - for key := range artifactManifestIndex { - err := ms.blobStore.Delete(ctx, key) - if err != nil { - return err - } - } - // delete the manifest revision and the _refs directory for original manifest return ms.blobStore.Delete(ctx, dgst) } diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index 2735a03eca7..661480f4121 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -20,7 +20,7 @@ type ocischemaManifestHandler struct { manifestURLs manifestURLs } -var _ ManifestHandler = &ocischemaManifestHandler{} +var _ distribution.ManifestHandler = &ocischemaManifestHandler{} func (ms *ocischemaManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Unmarshal") diff --git a/registry/storage/registry.go b/registry/storage/registry.go index d7b0d01a583..03168830e17 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -25,7 +25,7 @@ type registry struct { blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory manifestURLs manifestURLs driver storagedriver.StorageDriver - extendedStorages []ExtendedStorage + extendedNamespaces []distribution.Extension } // manifestURLs holds regular expressions for controlling manifest URL whitelisting @@ -37,11 +37,11 @@ type manifestURLs struct { // RegistryOption is the type used for functional options for NewRegistry. type RegistryOption func(*registry) error -// AddExtendedStorage is a functional option for NewRegistry. It adds the given -// extended storage to the list of extended storages in the registry. -func AddExtendedStorage(extendedStorage ExtendedStorage) RegistryOption { +// AddExtendedNamespace is a functional option for NewRegistry. It adds the given +// extended namespace to the list of extended namespaces in the registry. +func AddExtendedNamespace(extendedNamespace distribution.Extension) RegistryOption { return func(registry *registry) error { - registry.extendedStorages = append(registry.extendedStorages, extendedStorage) + registry.extendedNamespaces = append(registry.extendedNamespaces, extendedNamespace) return nil } } @@ -199,6 +199,10 @@ func (reg *registry) BlobStatter() distribution.BlobStatter { return reg.statter } +func (reg *registry) Extensions() []distribution.Extension { + return reg.extendedNamespaces +} + // repository provides name-scoped access to various services. type repository struct { *registry @@ -258,7 +262,7 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M linkDirectoryPathSpec: manifestDirectoryPathSpec, } - var v1Handler ManifestHandler + var v1Handler distribution.ManifestHandler if repo.schema1Enabled { v1Handler = &signedManifestHandler{ ctx: ctx, @@ -277,8 +281,8 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M } } - var extensionManifestHandlers []ManifestHandler - for _, ext := range repo.registry.extendedStorages { + var extensionManifestHandlers []distribution.ManifestHandler + for _, ext := range repo.registry.extendedNamespaces { handlers := ext.GetManifestHandlers(repo, blobStore) if len(handlers) > 0 { extensionManifestHandlers = append(extensionManifestHandlers, handlers...) diff --git a/registry/storage/schema2manifesthandler.go b/registry/storage/schema2manifesthandler.go index 023427c1328..94b3215c85e 100644 --- a/registry/storage/schema2manifesthandler.go +++ b/registry/storage/schema2manifesthandler.go @@ -26,7 +26,7 @@ type schema2ManifestHandler struct { manifestURLs manifestURLs } -var _ ManifestHandler = &schema2ManifestHandler{} +var _ distribution.ManifestHandler = &schema2ManifestHandler{} func (ms *schema2ManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ms.ctx).Debug("(*schema2ManifestHandler).Unmarshal") diff --git a/registry/storage/signedmanifesthandler.go b/registry/storage/signedmanifesthandler.go index eb5b5742adb..9c8f9ca366e 100644 --- a/registry/storage/signedmanifesthandler.go +++ b/registry/storage/signedmanifesthandler.go @@ -22,7 +22,7 @@ type signedManifestHandler struct { ctx context.Context } -var _ ManifestHandler = &signedManifestHandler{} +var _ distribution.ManifestHandler = &signedManifestHandler{} func (ms *signedManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Unmarshal") diff --git a/registry/storage/v1unsupportedhandler.go b/registry/storage/v1unsupportedhandler.go index 8170646bc5d..9aef69497c9 100644 --- a/registry/storage/v1unsupportedhandler.go +++ b/registry/storage/v1unsupportedhandler.go @@ -10,10 +10,10 @@ import ( // signedManifestHandler is a ManifestHandler that unmarshals v1 manifests but // refuses to Put v1 manifests type v1UnsupportedHandler struct { - innerHandler ManifestHandler + innerHandler distribution.ManifestHandler } -var _ ManifestHandler = &v1UnsupportedHandler{} +var _ distribution.ManifestHandler = &v1UnsupportedHandler{} func (v *v1UnsupportedHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { return v.innerHandler.Unmarshal(ctx, dgst, content) From 7ba9ca296af1ff7341c19a8eb6fd0fc26772a49a Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Thu, 16 Jun 2022 11:45:09 -0700 Subject: [PATCH 09/22] remove all delete path additions Signed-off-by: Akash Singhal --- registry/storage/linkedblobstore.go | 9 --------- registry/storage/manifeststore.go | 9 --------- registry/storage/registry.go | 1 - 3 files changed, 19 deletions(-) diff --git a/registry/storage/linkedblobstore.go b/registry/storage/linkedblobstore.go index dde328bc709..89573ddc79d 100644 --- a/registry/storage/linkedblobstore.go +++ b/registry/storage/linkedblobstore.go @@ -463,12 +463,3 @@ func blobLinkPath(name string, dgst digest.Digest) (string, error) { func manifestRevisionLinkPath(name string, dgst digest.Digest) (string, error) { return pathFor(manifestRevisionLinkPathSpec{name: name, revision: dgst}) } - -// artifactRefPath provides the path to the manifest's _refs directory. -func artifactRefPath(name string, dgst digest.Digest) (string, error) { - rootPath, err := pathFor(referrersRootPathSpec{name: name}) - if err != nil { - return "", err - } - return path.Join(rootPath, dgst.Algorithm().String(), dgst.Hex()), nil -} diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index c070dfcea51..e529bdd4c6f 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -16,15 +16,6 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" ) -// // A ManifestHandler gets and puts manifests of a particular type. -// type ManifestHandler interface { -// // Unmarshal unmarshals the manifest from a byte slice. -// Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) - -// // Put creates or updates the given manifest returning the manifest digest. -// Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) -// } - // SkipLayerVerification allows a manifest to be Put before its // layers are on the filesystem func SkipLayerVerification() distribution.ManifestServiceOption { diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 03168830e17..b82bcea707d 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -234,7 +234,6 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M // 2.1.0 unintentionally linked into _layers. manifestRevisionLinkPath, blobLinkPath, - artifactRefPath, } manifestDirectoryPathSpec := manifestRevisionsPathSpec{name: repo.name.Name()} From a5ed412de60892b0b06e28c8efafcf6c1d2209c0 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Thu, 16 Jun 2022 16:21:13 -0700 Subject: [PATCH 10/22] adding artifact garbage collection test; removing referrer specific code from other packages Signed-off-by: Akash Singhal --- extension.go | 4 + registry/extension/extension.go | 123 ------------- .../oras/artifactgarbagecollectionhandler.go | 20 ++- .../artifactgarbagecollectionhandler_test.go | 165 ++++++++++++++++++ registry/proxy/proxyregistry.go | 2 +- registry/storage/garbagecollect.go | 2 +- registry/storage/paths.go | 9 - registry/storage/vacuum.go | 68 +++----- 8 files changed, 209 insertions(+), 184 deletions(-) delete mode 100644 registry/extension/extension.go create mode 100644 registry/extension/oras/artifactgarbagecollectionhandler_test.go diff --git a/extension.go b/extension.go index b497206d993..2a54e206793 100644 --- a/extension.go +++ b/extension.go @@ -56,6 +56,10 @@ type GCExtensionHandler interface { registry Namespace, dryRun bool, removeUntagged bool) error + RemoveManifestVacuum(ctx context.Context, + storageDriver driver.StorageDriver, + dgst digest.Digest, + repositoryName string) error } // ExtendedStorage defines extensions to store operations like manifest for example. diff --git a/registry/extension/extension.go b/registry/extension/extension.go deleted file mode 100644 index ddd2e4a6c80..00000000000 --- a/registry/extension/extension.go +++ /dev/null @@ -1,123 +0,0 @@ -package extension - -// // Context contains the request specific context for use in across handlers. -// type Context struct { -// c.Context - -// // Registry is the base namespace that is used by all extension namespaces -// Registry distribution.Namespace -// // Repository is a reference to a named repository -// Repository distribution.Repository -// // Errors are the set of errors that occurred within this request context -// Errors errcode.Errors -// } - -// // RouteDispatchFunc is the http route dispatcher used by the extension route handlers -// type RouteDispatchFunc func(extContext *Context, r *http.Request) http.Handler - -// // Route describes an extension route. -// type Route struct { -// // Namespace is the name of the extension namespace -// Namespace string -// // Extension is the name of the extension under the namespace -// Extension string -// // Component is the name of the component under the extension -// Component string -// // Descriptor is the route descriptor that gives its path -// Descriptor v2.RouteDescriptor -// // Dispatcher if present signifies that the route is http route with a dispatcher -// Dispatcher RouteDispatchFunc -// } - -// // Namespace is the namespace that is used to define extensions to the distribution. -// type Namespace interface { -// distribution.ExtendedStorage -// // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -// GetRepositoryRoutes() []Route -// // GetRegistryRoutes returns a list of extension routes scoped at a registry level -// GetRegistryRoutes() []Route -// // GetNamespaceName returns the name associated with the namespace -// GetNamespaceName() string -// // GetNamespaceUrl returns the url link to the documentation where the namespace's extension and endpoints are defined -// GetNamespaceUrl() string -// // GetNamespaceDescription returns the description associated with the namespace -// GetNamespaceDescription() string -// } - -// // InitExtensionNamespace is the initialize function for creating the extension namespace -// type InitExtensionNamespace func(ctx c.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Namespace, error) - -// // EnumerateExtension specifies extension information at the namespace level -// type EnumerateExtension struct { -// Name string `json:"name"` -// Url string `json:"url"` -// Description string `json:"description,omitempty"` -// Endpoints []string `json:"endpoints"` -// } - -// var extensions map[string]InitExtensionNamespace -// var extensionsNamespaces map[string]Namespace - -// func EnumerateRegistered(ctx Context) (enumeratedExtensions []EnumerateExtension) { -// for _, namespace := range extensionsNamespaces { -// enumerateExtension := EnumerateExtension{ -// Name: namespace.GetNamespaceName(), -// Url: namespace.GetNamespaceUrl(), -// Description: namespace.GetNamespaceDescription(), -// Endpoints: []string{}, -// } - -// scopedRoutes := namespace.GetRepositoryRoutes() - -// // if the repository is not set in the context, scope is registry wide -// if ctx.Repository == nil { -// scopedRoutes = namespace.GetRegistryRoutes() -// } - -// for _, route := range scopedRoutes { -// path := fmt.Sprintf("_%s/%s/%s", route.Namespace, route.Extension, route.Component) -// enumerateExtension.Endpoints = append(enumerateExtension.Endpoints, path) -// } - -// // add extension to list if endpoints exist -// if len(enumerateExtension.Endpoints) > 0 { -// enumeratedExtensions = append(enumeratedExtensions, enumerateExtension) -// } -// } - -// return enumeratedExtensions -// } - -// // Register is used to register an InitExtensionNamespace for -// // an extension namespace with the given name. -// func Register(name string, initFunc InitExtensionNamespace) { -// if extensions == nil { -// extensions = make(map[string]InitExtensionNamespace) -// } - -// if _, exists := extensions[name]; exists { -// panic(fmt.Sprintf("namespace name already registered: %s", name)) -// } - -// extensions[name] = initFunc -// } - -// // Get constructs an extension namespace with the given options using the given name. -// func Get(ctx c.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Namespace, error) { -// if extensions != nil { -// if extensionsNamespaces == nil { -// extensionsNamespaces = make(map[string]Namespace) -// } - -// if initFunc, exists := extensions[name]; exists { -// namespace, err := initFunc(ctx, storageDriver, options) -// if err == nil { -// // adds the initialized namespace to map for simple access to namespaces by EnumerateRegistered -// extensionsNamespaces[name] = namespace -// } -// return namespace, err -// } -// } - -// return nil, fmt.Errorf("no extension registered with name: %s", name) -// } diff --git a/registry/extension/oras/artifactgarbagecollectionhandler.go b/registry/extension/oras/artifactgarbagecollectionhandler.go index 3ad5e33c601..9456d3cfeee 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler.go @@ -6,6 +6,7 @@ import ( "path" "github.com/distribution/distribution/v3" + dcontext "github.com/distribution/distribution/v3/context" "github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" @@ -153,11 +154,11 @@ func (gc *orasGCHandler) Sweep(ctx context.Context, registry distribution.Namespace, dryRun bool, removeUntagged bool) error { - vacuum := storage.NewVacuum(ctx, storageDriver) + vacuum := storage.NewVacuum(ctx, storageDriver, registry) if !dryRun { // remove each artifact in the index for artifactDigest, obj := range gc.artifactManifestIndex { - err := vacuum.RemoveArtifactManifest(obj.name, artifactDigest) + err := vacuum.RemoveManifest(obj.name, artifactDigest, []string{}) if err != nil { return fmt.Errorf("failed to delete artifact manifest %s: %v", artifactDigest, err) } @@ -167,6 +168,21 @@ func (gc *orasGCHandler) Sweep(ctx context.Context, return nil } +func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver driver.StorageDriver, dgst digest.Digest, repositoryName string) error { + referrerRootPath := referrersLinkPath(repositoryName) + fullArtifactManifestPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + dcontext.GetLogger(ctx).Infof("deleting manifest ref folder: %s", fullArtifactManifestPath) + err := storageDriver.Delete(ctx, fullArtifactManifestPath) + if err != nil { + switch err.(type) { + case driver.PathNotFoundError: + return nil + } + return err + } + return nil +} + // ingestor method used in EnumerateReferrerLinks // marks each artifact manifest and associated blobs func artifactMarkIngestor(ctx context.Context, diff --git a/registry/extension/oras/artifactgarbagecollectionhandler_test.go b/registry/extension/oras/artifactgarbagecollectionhandler_test.go new file mode 100644 index 00000000000..2357904f001 --- /dev/null +++ b/registry/extension/oras/artifactgarbagecollectionhandler_test.go @@ -0,0 +1,165 @@ +package oras + +import ( + "context" + "encoding/json" + "testing" + + "github.com/distribution/distribution/v3" + "github.com/distribution/distribution/v3/manifest" + "github.com/distribution/distribution/v3/manifest/schema2" + "github.com/distribution/distribution/v3/registry/storage" + "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" + "github.com/opencontainers/go-digest" + orasartifacts "github.com/oras-project/artifacts-spec/specs-go/v1" +) + +func allManifests(t *testing.T, manifestService distribution.ManifestService) map[digest.Digest]struct{} { + ctx := context.Background() + allManMap := make(map[digest.Digest]struct{}) + manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator) + if !ok { + t.Fatalf("unable to convert ManifestService into ManifestEnumerator") + } + err := manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { + allManMap[dgst] = struct{}{} + return nil + }) + if err != nil { + t.Fatalf("Error getting all manifests: %v", err) + } + return allManMap +} + +func allBlobs(t *testing.T, registry distribution.Namespace) map[digest.Digest]struct{} { + ctx := context.Background() + blobService := registry.Blobs() + allBlobsMap := make(map[digest.Digest]struct{}) + err := blobService.Enumerate(ctx, func(dgst digest.Digest) error { + allBlobsMap[dgst] = struct{}{} + return nil + }) + if err != nil { + t.Fatalf("Error getting all blobs: %v", err) + } + return allBlobsMap +} + +func TestReferrersBlobsDeleted(t *testing.T) { + ctx := context.Background() + inmemoryDriver := inmemory.New() + registry := createRegistry(t, inmemoryDriver) + repo := makeRepository(t, registry, "test") + manifestService := makeManifestService(t, repo) + tagService := repo.Tags(ctx) + + artifactBlob, err := repo.Blobs(ctx).Put(ctx, orasartifacts.MediaTypeDescriptor, nil) + if err != nil { + t.Fatal(err) + } + + config, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeImageConfig, nil) + if err != nil { + t.Fatal(err) + } + + layer, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeLayer, nil) + if err != nil { + t.Fatal(err) + } + + subjectManifest := schema2.Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: schema2.MediaTypeManifest, + }, + Config: config, + Layers: []distribution.Descriptor{ + layer, + }, + } + + dm, err := schema2.FromStruct(subjectManifest) + if err != nil { + t.Fatalf("failed to marshal subject manifest: %v", err) + } + _, dmPayload, err := dm.Payload() + if err != nil { + t.Fatalf("failed to get subject manifest payload: %v", err) + } + + dg, err := manifestService.Put(ctx, dm) + if err != nil { + t.Fatalf("failed to put subject manifest with err: %v", err) + } + + artifactBlobDescriptor := orasartifacts.Descriptor{ + MediaType: artifactBlob.MediaType, + Digest: artifactBlob.Digest, + Size: artifactBlob.Size, + } + + artifactManifest := orasartifacts.Manifest{ + MediaType: orasartifacts.MediaTypeArtifactManifest, + ArtifactType: "test_artifactType", + Blobs: []orasartifacts.Descriptor{ + artifactBlobDescriptor, + }, + Subject: orasartifacts.Descriptor{ + MediaType: schema2.MediaTypeManifest, + Size: int64(len(dmPayload)), + Digest: dg, + }, + } + + marshalledMan, err := json.Marshal(artifactManifest) + if err != nil { + t.Fatalf("artifact manifest could not be serialized to byte array: %v", err) + } + // upload manifest + artifactManifestDigest, err := manifestService.Put(ctx, &DeserializedManifest{ + Manifest: Manifest{ + inner: artifactManifest, + }, + raw: marshalledMan, + }) + if err != nil { + t.Fatalf("artifact manifest upload failed: %v", err) + } + + // the tags folder doesn't exist for this repo until a tag is added + // this leads to an error in Mark and Sweep if tags folder not found + err = tagService.Tag(ctx, "test", distribution.Descriptor{Digest: dg}) + if err != nil { + t.Fatalf("failed to tag subject image: %v", err) + } + err = tagService.Untag(ctx, "test") + if err != nil { + t.Fatalf("failed to untag subject image: %v", err) + } + + // Run GC + err = storage.MarkAndSweep(ctx, inmemoryDriver, registry, storage.GCOpts{ + DryRun: false, + RemoveUntagged: true, + }) + if err != nil { + t.Fatalf("Failed mark and sweep: %v", err) + } + + manifests := allManifests(t, manifestService) + blobs := allBlobs(t, registry) + + if _, exists := manifests[artifactManifestDigest]; exists { + t.Fatalf("artifact manifest with digest %s should have been deleted", artifactManifestDigest.String()) + } + + if _, exists := blobs[artifactManifestDigest]; exists { + t.Fatalf("artifact manifest blob with digest %s should have been deleted", artifactManifestDigest.String()) + } + + blobDigest := artifactManifest.Blobs[0].Digest + if _, exists := blobs[blobDigest]; exists { + t.Fatalf("artifact blob with digest %s should have been deleted", blobDigest) + } +} diff --git a/registry/proxy/proxyregistry.go b/registry/proxy/proxyregistry.go index b813bc8f51f..e84ff9a2707 100644 --- a/registry/proxy/proxyregistry.go +++ b/registry/proxy/proxyregistry.go @@ -35,7 +35,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name return nil, err } - v := storage.NewVacuum(ctx, driver) + v := storage.NewVacuum(ctx, driver, registry) s := scheduler.New(ctx, driver, "/scheduler-state.json") s.OnBlobExpire(func(ref reference.Reference) error { var r reference.Canonical diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 16e348aa5f6..ae8d44adaf0 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -130,7 +130,7 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis } // sweep - vacuum := NewVacuum(ctx, storageDriver) + vacuum := NewVacuum(ctx, storageDriver, registry) if !opts.DryRun { for _, obj := range manifestArr { err = vacuum.RemoveManifest(obj.Name, obj.Digest, obj.Tags) diff --git a/registry/storage/paths.go b/registry/storage/paths.go index 1d90241ee88..4e9ad0856e8 100644 --- a/registry/storage/paths.go +++ b/registry/storage/paths.go @@ -242,8 +242,6 @@ func pathFor(spec pathSpec) (string, error) { return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "hashstates", string(v.alg), offset)...), nil case repositoriesRootPathSpec: return path.Join(repoPrefix...), nil - case referrersRootPathSpec: - return path.Join(append(repoPrefix, v.name, "_refs", "subjects")...), nil default: // TODO(sday): This is an internal error. Ensure it doesn't escape (panic?). return "", fmt.Errorf("unknown path spec: %#v", v) @@ -438,13 +436,6 @@ type repositoriesRootPathSpec struct { func (repositoriesRootPathSpec) pathSpec() {} -// referrersRootPathSpec returns the root of referrers links -type referrersRootPathSpec struct { - name string -} - -func (referrersRootPathSpec) pathSpec() {} - // digestPathComponents provides a consistent path breakdown for a given // digest. For a generic digest, it will be as follows: // diff --git a/registry/storage/vacuum.go b/registry/storage/vacuum.go index a3ad2fcc5e2..45df0d2d1ec 100644 --- a/registry/storage/vacuum.go +++ b/registry/storage/vacuum.go @@ -2,8 +2,10 @@ package storage import ( "context" + "fmt" "path" + "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -15,17 +17,19 @@ import ( // https://en.wikipedia.org/wiki/Consistency_model // NewVacuum creates a new Vacuum -func NewVacuum(ctx context.Context, driver driver.StorageDriver) Vacuum { +func NewVacuum(ctx context.Context, driver driver.StorageDriver, registry distribution.Namespace) Vacuum { return Vacuum{ - ctx: ctx, - driver: driver, + ctx: ctx, + driver: driver, + registry: registry, } } // Vacuum removes content from the filesystem type Vacuum struct { - driver driver.StorageDriver - ctx context.Context + driver driver.StorageDriver + ctx context.Context + registry distribution.Namespace } // RemoveBlob removes a blob from the filesystem @@ -84,23 +88,21 @@ func (v Vacuum) RemoveManifest(name string, dgst digest.Digest, tags []string) e dcontext.GetLogger(v.ctx).Infof("deleting manifest: %s", manifestPath) err = v.driver.Delete(v.ctx, manifestPath) if err != nil { - return err + if _, ok := err.(driver.PathNotFoundError); !ok { + return err + } } - referrerRootPath, err := pathFor(referrersRootPathSpec{name: name}) - if err != nil { - return err - } - fullArtifactManifestPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) - dcontext.GetLogger(v.ctx).Infof("deleting manifest ref folder: %s", fullArtifactManifestPath) - v.driver.Delete(v.ctx, fullArtifactManifestPath) - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil + for _, extNamespace := range v.registry.Extensions() { + handlers := extNamespace.GetGarbageCollectionHandlers() + for _, gcHandler := range handlers { + err := gcHandler.RemoveManifestVacuum(v.ctx, v.driver, dgst, name) + if err != nil { + return fmt.Errorf("failed to call remove manifest extension handler: %v", err) + } } - return err } + return nil } @@ -120,33 +122,3 @@ func (v Vacuum) RemoveRepository(repoName string) error { return nil } - -// RemoveArtifactManifest removes a artifact manifest from the filesystem -// Removes manifest revision file and manifest ref folder if it exists -func (v Vacuum) RemoveArtifactManifest(name string, artifactDgst digest.Digest) error { - manifestPath, err := pathFor(manifestRevisionPathSpec{name: name, revision: artifactDgst}) - if err != nil { - return err - } - dcontext.GetLogger(v.ctx).Infof("deleting artifact manifest: %s", manifestPath) - err = v.driver.Delete(v.ctx, manifestPath) - if err != nil { - return err - } - - referrerRootPath, err := pathFor(referrersRootPathSpec{name: name}) - if err != nil { - return err - } - fullArtifactManifestPath := path.Join(referrerRootPath, artifactDgst.Algorithm().String(), artifactDgst.Hex()) - dcontext.GetLogger(v.ctx).Infof("deleting artifact manifest ref: %s", fullArtifactManifestPath) - err = v.driver.Delete(v.ctx, fullArtifactManifestPath) - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil - } - return err - } - return nil -} From 565a1f83246833eb693760ae086aa0a4c0c80139 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Mon, 20 Jun 2022 13:41:03 -0700 Subject: [PATCH 11/22] updates to gc handler interface Signed-off-by: Akash Singhal --- extension.go | 8 +- .../oras/artifactgarbagecollectionhandler.go | 105 ++++++++++++------ .../extension/oras/artifactmanifesthandler.go | 12 +- registry/extension/oras/artifactservice.go | 22 +++- registry/storage/garbagecollect.go | 30 ++--- 5 files changed, 117 insertions(+), 60 deletions(-) diff --git a/extension.go b/extension.go index 2a54e206793..73358580d39 100644 --- a/extension.go +++ b/extension.go @@ -51,15 +51,13 @@ type GCExtensionHandler interface { registry Namespace, dryRun bool, removeUntagged bool) (map[digest.Digest]struct{}, error) - Sweep(ctx context.Context, - storageDriver driver.StorageDriver, - registry Namespace, - dryRun bool, - removeUntagged bool) error RemoveManifestVacuum(ctx context.Context, storageDriver driver.StorageDriver, dgst digest.Digest, repositoryName string) error + IsEligibleForDeletion(ctx context.Context, + dgst digest.Digest, + manifestService ManifestService) (bool, error) } // ExtendedStorage defines extensions to store operations like manifest for example. diff --git a/registry/extension/oras/artifactgarbagecollectionhandler.go b/registry/extension/oras/artifactgarbagecollectionhandler.go index 9456d3cfeee..7361a938f71 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler.go @@ -8,19 +8,13 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" "github.com/distribution/distribution/v3/reference" - "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" v1 "github.com/oras-project/artifacts-spec/specs-go/v1" ) type orasGCHandler struct { - artifactManifestIndex map[digest.Digest]artifactManifestDel -} - -type artifactManifestDel struct { - name string - artifactDigest digest.Digest + artifactManifestIndex map[digest.Digest][]digest.Digest } func (gc *orasGCHandler) Mark(ctx context.Context, @@ -33,7 +27,7 @@ func (gc *orasGCHandler) Mark(ctx context.Context, return nil, fmt.Errorf("unable to convert Namespace to RepositoryEnumerator") } - gc.artifactManifestIndex = make(map[digest.Digest]artifactManifestDel) + gc.artifactManifestIndex = make(map[digest.Digest][]digest.Digest) // mark markSet := make(map[digest.Digest]struct{}) err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { @@ -87,6 +81,7 @@ func (gc *orasGCHandler) Mark(ctx context.Context, if len(tags) == 0 { // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion + gc.artifactManifestIndex[dgst] = make([]digest.Digest, 0) rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) err = enumerateReferrerLinks(ctx, rootPath, @@ -95,6 +90,7 @@ func (gc *orasGCHandler) Mark(ctx context.Context, manifestService, repository.Named().Name(), markSet, + dgst, gc.artifactManifestIndex, artifactSweepIngestor) @@ -118,6 +114,7 @@ func (gc *orasGCHandler) Mark(ctx context.Context, manifestService, repository.Named().Name(), markSet, + dgst, gc.artifactManifestIndex, artifactMarkIngestor) @@ -149,18 +146,37 @@ func (gc *orasGCHandler) Mark(ctx context.Context, return markSet, nil } -func (gc *orasGCHandler) Sweep(ctx context.Context, - storageDriver driver.StorageDriver, - registry distribution.Namespace, - dryRun bool, - removeUntagged bool) error { - vacuum := storage.NewVacuum(ctx, storageDriver, registry) - if !dryRun { - // remove each artifact in the index - for artifactDigest, obj := range gc.artifactManifestIndex { - err := vacuum.RemoveManifest(obj.name, artifactDigest, []string{}) +func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver driver.StorageDriver, dgst digest.Digest, repositoryName string) error { + referrerRootPath := referrersLinkPath(repositoryName) + fullArtifactManifestPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + dcontext.GetLogger(ctx).Infof("deleting manifest ref folder: %s", fullArtifactManifestPath) + err := storageDriver.Delete(ctx, fullArtifactManifestPath) + if err != nil { + if _, ok := err.(driver.PathNotFoundError); !ok { + return err + } + } + + subjectLinkedArtifacts, ok := gc.artifactManifestIndex[dgst] + if ok { + for _, artifactDigest := range subjectLinkedArtifacts { + // delete each artifact manifest's revision + manifestPath := referrersRepositoriesManifestRevisionPath(repositoryName, artifactDigest) + dcontext.GetLogger(ctx).Infof("deleting artifact manifest revision: %s", manifestPath) + err = storageDriver.Delete(ctx, manifestPath) + if err != nil { + if _, ok := err.(driver.PathNotFoundError); !ok { + return err + } + } + // delete each artifact manifest's ref folder + fullArtifactManifestPath = path.Join(referrerRootPath, artifactDigest.Algorithm().String(), artifactDigest.Hex()) + dcontext.GetLogger(ctx).Infof("deleting artifact manifest ref folder: %s", fullArtifactManifestPath) + err = storageDriver.Delete(ctx, fullArtifactManifestPath) if err != nil { - return fmt.Errorf("failed to delete artifact manifest %s: %v", artifactDigest, err) + if _, ok := err.(driver.PathNotFoundError); !ok { + return err + } } } } @@ -168,19 +184,17 @@ func (gc *orasGCHandler) Sweep(ctx context.Context, return nil } -func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver driver.StorageDriver, dgst digest.Digest, repositoryName string) error { - referrerRootPath := referrersLinkPath(repositoryName) - fullArtifactManifestPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) - dcontext.GetLogger(ctx).Infof("deleting manifest ref folder: %s", fullArtifactManifestPath) - err := storageDriver.Delete(ctx, fullArtifactManifestPath) +func (gc *orasGCHandler) IsEligibleForDeletion(ctx context.Context, dgst digest.Digest, manifestService distribution.ManifestService) (bool, error) { + manifest, err := manifestService.Get(ctx, dgst) if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil - } - return err + return false, fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) } - return nil + + mediaType, _, err := manifest.Payload() + if err != nil { + return false, err + } + return mediaType != v1.MediaTypeArtifactManifest, nil } // ingestor method used in EnumerateReferrerLinks @@ -189,7 +203,8 @@ func artifactMarkIngestor(ctx context.Context, referrerRevision digest.Digest, manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]artifactManifestDel, + subjectRevision digest.Digest, + artifactManifestIndex map[digest.Digest][]digest.Digest, repoName string, storageDriver driver.StorageDriver, blobStatter distribution.BlobStatter) error { @@ -219,7 +234,16 @@ func artifactMarkIngestor(ctx context.Context, } return err } - return enumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactMarkIngestor) + return enumerateReferrerLinks(ctx, + rootPath, + storageDriver, + blobStatter, + manifestService, + repoName, + markSet, + subjectRevision, + artifactManifestIndex, + artifactMarkIngestor) } // ingestor method used in EnumerateReferrerLinks @@ -228,14 +252,16 @@ func artifactSweepIngestor(ctx context.Context, referrerRevision digest.Digest, manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]artifactManifestDel, + subjectRevision digest.Digest, + artifactManifestIndex map[digest.Digest][]digest.Digest, repoName string, storageDriver driver.StorageDriver, blobStatter distribution.BlobStatter) error { // index the manifest fmt.Printf("%s: indexing artifact manifest %s\n", repoName, referrerRevision.String()) - artifactManifestIndex[referrerRevision] = artifactManifestDel{name: repoName, artifactDigest: referrerRevision} + // TODO: check if the artifact is tagged or not + artifactManifestIndex[subjectRevision] = append(artifactManifestIndex[subjectRevision], referrerRevision) referrerRootPath := referrersLinkPath(repoName) @@ -248,5 +274,14 @@ func artifactSweepIngestor(ctx context.Context, } return err } - return enumerateReferrerLinks(ctx, rootPath, storageDriver, blobStatter, manifestService, repoName, markSet, artifactManifestIndex, artifactSweepIngestor) + return enumerateReferrerLinks(ctx, + rootPath, + storageDriver, + blobStatter, + manifestService, + repoName, + markSet, + subjectRevision, + artifactManifestIndex, + artifactSweepIngestor) } diff --git a/registry/extension/oras/artifactmanifesthandler.go b/registry/extension/oras/artifactmanifesthandler.go index 7688868886e..876b0fdee09 100644 --- a/registry/extension/oras/artifactmanifesthandler.go +++ b/registry/extension/oras/artifactmanifesthandler.go @@ -20,6 +20,8 @@ var ( errInvalidCreatedAnnotation = errors.New("failed to parse created time") ) +const rootPath = "/docker/registry/v2" + // artifactManifestHandler is a ManifestHandler that covers ORAS Artifacts. type artifactManifestHandler struct { repository distribution.Repository @@ -150,6 +152,14 @@ func (amh *artifactManifestHandler) indexReferrers(ctx context.Context, dm Deser return nil } +func referrersRepositoriesRootPath(name string) string { + return path.Join(rootPath, "repositories", name) +} + +func referrersRepositoriesManifestRevisionPath(name string, dgst digest.Digest) string { + return path.Join(referrersRepositoriesRootPath(name), "_manifests", "revisions", dgst.Algorithm().String(), dgst.Hex()) +} + func referrersLinkPath(name string) string { - return path.Join("/docker/registry/", "v2", "repositories", name, "_refs", "subjects") + return path.Join(referrersRepositoriesRootPath(name), "_refs", "subjects") } diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index 8d3f9c62c1b..2b42bef1636 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -57,12 +57,14 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest manifests, repo.Named().Name(), map[digest.Digest]struct{}{}, - map[digest.Digest]artifactManifestDel{}, + revision, + map[digest.Digest][]digest.Digest{}, func(ctx context.Context, referrerRevision digest.Digest, manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]artifactManifestDel, + subjectRevision digest.Digest, + artifactManifestIndex map[digest.Digest][]digest.Digest, repoName string, storageDriver driver.StorageDriver, blobStatter distribution.BlobStatter) error { @@ -138,12 +140,14 @@ func enumerateReferrerLinks(ctx context.Context, manifestService distribution.ManifestService, repositoryName string, markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]artifactManifestDel, + subjectRevision digest.Digest, + artifactManifestIndex map[digest.Digest][]digest.Digest, ingestor func(ctx context.Context, digest digest.Digest, manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, - artifactManifestIndex map[digest.Digest]artifactManifestDel, + subjectRevision digest.Digest, + artifactManifestIndex map[digest.Digest][]digest.Digest, repoName string, storageDriver driver.StorageDriver, blobStatter distribution.BlobStatter) error) error { @@ -177,7 +181,15 @@ func enumerateReferrerLinks(ctx context.Context, return err } - err = ingestor(ctx, digest, manifestService, markSet, artifactManifestIndex, repositoryName, stDriver, blobStatter) + err = ingestor(ctx, + digest, + manifestService, + markSet, + subjectRevision, + artifactManifestIndex, + repositoryName, + stDriver, + blobStatter) if err != nil { return err } diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index ae8d44adaf0..a6367ae0b71 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -68,7 +68,6 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err) } if len(tags) == 0 { - emit("manifest eligible for deletion: %s", dgst) // fetch all tags from repository // all of these tags could contain manifest in history // which means that we need check (and delete) those references when deleting manifest @@ -76,7 +75,22 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("failed to retrieve tags %v", err) } - manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) + isEligibleForDelete := true + for _, extNamespace := range registry.Extensions() { + handlers := extNamespace.GetGarbageCollectionHandlers() + for _, gcHandler := range handlers { + extensionDeleteEligible, err := gcHandler.IsEligibleForDeletion(ctx, dgst, manifestService) + if err != nil { + return fmt.Errorf("failed to determine deletion eligibility using extension handler: %v", err) + } + + isEligibleForDelete = isEligibleForDelete && extensionDeleteEligible + } + } + if isEligibleForDelete { + manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) + emit("manifest eligible for deletion: %s", dgst) + } return nil } } @@ -162,17 +176,5 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to delete blob %s: %v", dgst, err) } } - - // call GC extension handlers' sweep - for _, extNamespace := range registry.Extensions() { - handlers := extNamespace.GetGarbageCollectionHandlers() - for _, gcHandler := range handlers { - err := gcHandler.Sweep(ctx, storageDriver, registry, opts.DryRun, opts.RemoveUntagged) - if err != nil { - return fmt.Errorf("failed to sweep using extension handler: %v", err) - } - } - } - return err } From 7b3d7b97b19acb8435c7df26b6b1c317349a3bb0 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Mon, 20 Jun 2022 16:07:55 -0700 Subject: [PATCH 12/22] add support for keeping tagged artifacts Signed-off-by: Akash Singhal --- .../oras/artifactgarbagecollectionhandler.go | 49 +++++++++++-------- registry/extension/oras/artifactservice.go | 31 +++++------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/registry/extension/oras/artifactgarbagecollectionhandler.go b/registry/extension/oras/artifactgarbagecollectionhandler.go index 7361a938f71..5d54ca3ca7c 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler.go @@ -86,9 +86,8 @@ func (gc *orasGCHandler) Mark(ctx context.Context, err = enumerateReferrerLinks(ctx, rootPath, storageDriver, + repository, blobStatter, - manifestService, - repository.Named().Name(), markSet, dgst, gc.artifactManifestIndex, @@ -110,9 +109,8 @@ func (gc *orasGCHandler) Mark(ctx context.Context, err = enumerateReferrerLinks(ctx, rootPath, storageDriver, + repository, blobStatter, - manifestService, - repository.Named().Name(), markSet, dgst, gc.artifactManifestIndex, @@ -201,18 +199,23 @@ func (gc *orasGCHandler) IsEligibleForDeletion(ctx context.Context, dgst digest. // marks each artifact manifest and associated blobs func artifactMarkIngestor(ctx context.Context, referrerRevision digest.Digest, - manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, - repoName string, - storageDriver driver.StorageDriver, - blobStatter distribution.BlobStatter) error { + repository distribution.Repository, + blobstatter distribution.BlobStatter, + storageDriver driver.StorageDriver) error { + manifestService, err := repository.Manifests(ctx) + if err != nil { + return fmt.Errorf("failed to construct manifest service: %v", err) + } + man, err := manifestService.Get(ctx, referrerRevision) if err != nil { return err } + repoName := repository.Named().Name() // mark the artifact manifest blob fmt.Printf("%s: marking artifact manifest %s\n", repoName, referrerRevision.String()) markSet[referrerRevision] = struct{}{} @@ -237,9 +240,8 @@ func artifactMarkIngestor(ctx context.Context, return enumerateReferrerLinks(ctx, rootPath, storageDriver, - blobStatter, - manifestService, - repoName, + repository, + blobstatter, markSet, subjectRevision, artifactManifestIndex, @@ -250,23 +252,29 @@ func artifactMarkIngestor(ctx context.Context, // indexes each artifact manifest and adds ArtifactManifestDel struct to index func artifactSweepIngestor(ctx context.Context, referrerRevision digest.Digest, - manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, - repoName string, - storageDriver driver.StorageDriver, - blobStatter distribution.BlobStatter) error { - + repository distribution.Repository, + blobstatter distribution.BlobStatter, + storageDriver driver.StorageDriver) error { + repoName := repository.Named().Name() // index the manifest fmt.Printf("%s: indexing artifact manifest %s\n", repoName, referrerRevision.String()) - // TODO: check if the artifact is tagged or not + // if artifact is tagged, we don't add artifact and descendants to artifact manifest index + tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: referrerRevision}) + if err != nil { + return fmt.Errorf("failed to retrieve tags for artifact digest %v: %v", referrerRevision, err) + } + if len(tags) > 0 { + return nil + } artifactManifestIndex[subjectRevision] = append(artifactManifestIndex[subjectRevision], referrerRevision) referrerRootPath := referrersLinkPath(repoName) rootPath := path.Join(referrerRootPath, referrerRevision.Algorithm().String(), referrerRevision.Hex()) - _, err := storageDriver.Stat(ctx, rootPath) + _, err = storageDriver.Stat(ctx, rootPath) if err != nil { switch err.(type) { case driver.PathNotFoundError: @@ -277,9 +285,8 @@ func artifactSweepIngestor(ctx context.Context, return enumerateReferrerLinks(ctx, rootPath, storageDriver, - blobStatter, - manifestService, - repoName, + repository, + blobstatter, markSet, subjectRevision, artifactManifestIndex, diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index 2b42bef1636..f48fc917112 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -53,21 +53,19 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest err = enumerateReferrerLinks(ctx, rootPath, h.storageDriver, + repo, blobStatter, - manifests, - repo.Named().Name(), map[digest.Digest]struct{}{}, revision, map[digest.Digest][]digest.Digest{}, func(ctx context.Context, referrerRevision digest.Digest, - manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, - repoName string, - storageDriver driver.StorageDriver, - blobStatter distribution.BlobStatter) error { + repository distribution.Repository, + blobstatter distribution.BlobStatter, + storageDriver driver.StorageDriver) error { man, err := manifests.Get(ctx, referrerRevision) if err != nil { return err @@ -136,21 +134,19 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest func enumerateReferrerLinks(ctx context.Context, rootPath string, stDriver driver.StorageDriver, - blobStatter distribution.BlobStatter, - manifestService distribution.ManifestService, - repositoryName string, + repository distribution.Repository, + blobstatter distribution.BlobStatter, markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, ingestor func(ctx context.Context, digest digest.Digest, - manifestService distribution.ManifestService, markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, - repoName string, - storageDriver driver.StorageDriver, - blobStatter distribution.BlobStatter) error) error { + repository distribution.Repository, + blobstatter distribution.BlobStatter, + storageDriver driver.StorageDriver) error) error { return stDriver.Walk(ctx, rootPath, func(fileInfo driver.FileInfo) error { // exit early if directory... @@ -172,7 +168,7 @@ func enumerateReferrerLinks(ctx context.Context, } // ensure this conforms to the linkPathFns - _, err = blobStatter.Stat(ctx, digest) + _, err = blobstatter.Stat(ctx, digest) if err != nil { // we expect this error to occur so we move on if err == distribution.ErrBlobUnknown { @@ -183,13 +179,12 @@ func enumerateReferrerLinks(ctx context.Context, err = ingestor(ctx, digest, - manifestService, markSet, subjectRevision, artifactManifestIndex, - repositoryName, - stDriver, - blobStatter) + repository, + blobstatter, + stDriver) if err != nil { return err } From 3388b17db210f9bbbc0f8be906d3aeccff3a0e8e Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Tue, 21 Jun 2022 09:36:46 -0700 Subject: [PATCH 13/22] update docs Signed-off-by: Akash Singhal --- docs/garbage-collection.md | 10 +--------- {docs => registry/extension/oras}/referrers.md | 10 +++++++++- 2 files changed, 10 insertions(+), 10 deletions(-) rename {docs => registry/extension/oras}/referrers.md (78%) diff --git a/docs/garbage-collection.md b/docs/garbage-collection.md index 26be77a700b..4196b09f203 100644 --- a/docs/garbage-collection.md +++ b/docs/garbage-collection.md @@ -121,12 +121,4 @@ blob eligible for deletion: sha256:7e15ce58ccb2181a8fced7709e9893206f0937cc9543b blob eligible for deletion: sha256:87192bdbe00f8f2a62527f36bb4c7c7f4eaf9307e4b87e8334fb6abec1765bcb blob eligible for deletion: sha256:b549a9959a664038fc35c155a95742cf12297672ca0ae35735ec027d55bf4e97 blob eligible for deletion: sha256:f251d679a7c61455f06d793e43c06786d7766c88b8c24edf242b2c08e3c3f599 -``` - -## Garbage Collection With Referrers - -The life of a reference artifact is directly linked to its subject. When a reference artifact's subject manifest is deleted, the attached artifacts and its descendants must be deleted. - -Manifest garbage collection is extended to include reference artifact collection. During the marking process, each manifest is queried for any reference artifacts by enumerating the link files at the path `repositories//_refs/subjects/sha256/`. For each artifact, the artifact manifest and its blobs are marked. Finally, collection recurses to look for further artifact descendants to mark in a similar fashion. - -If a manifest is indexed for deletion because it is untagged, the attached reference artifacts are also indexed. Similar to the marking process, the subject manifest's `_ref` folder is queried for reference artifact descendants. Each encountered descendant is indexed. Indexing recurses to the next levels of descendants until all successor artifacts are indexed. Finally, during manifest link deletion, the revision link files of the indexed artifact manifests as well as the corresponding `_refs` are removed from storage. \ No newline at end of file +``` \ No newline at end of file diff --git a/docs/referrers.md b/registry/extension/oras/referrers.md similarity index 78% rename from docs/referrers.md rename to registry/extension/oras/referrers.md index aba155032c8..a402d70404c 100644 --- a/docs/referrers.md +++ b/registry/extension/oras/referrers.md @@ -92,4 +92,12 @@ This results in an addition to the index as shown below. │ └── link └── 333ic0c33ebc4a74a0a554c86ac2b28ddf3454a5ad9cf90ea8cea9f9e75c333i └── link -``` \ No newline at end of file +``` + +## Garbage Collection With Referrers + +The life of a referrer artifact is directly linked to its subject. When a referrer artifact's subject manifest is deleted, the artifact's referrers are also deleted. + +Manifest garbage collection is extended to include referrer artifact collection. During the marking process, each manifest is queried for any referrer artifacts by enumerating the link files at the path `repositories//_refs/subjects/sha256/`. For each artifact, the artifact manifest and its blobs are marked. Finally, collection recurses to look for further referrers to mark in a similar fashion. + +If a manifest is indexed for deletion because it is untagged, the attached reference artifacts are also indexed. Similar to the marking process, the subject manifest's `_ref` folder is queried for referrers. Each encountered referrer is indexed. Indexing recurses to the next levels referrers until all successive referrers are indexed. Finally, during manifest link deletion, the revision link files of the indexed artifact manifests as well as the corresponding `_refs` are removed from storage. \ No newline at end of file From e5ade00c067d33a324672b91bd51f745c5b680f4 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Tue, 21 Jun 2022 09:51:25 -0700 Subject: [PATCH 14/22] remove extra extension interface and combine with extensionnamespace interface Signed-off-by: Akash Singhal --- extension.go | 28 +++++++++------------ registry/extension/distribution/registry.go | 2 +- registry/extension/oci/oci.go | 2 +- registry/extension/oras/oras.go | 2 +- registry/handlers/app.go | 4 +-- registry/root.go | 2 +- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/extension.go b/extension.go index 73358580d39..afa002badf1 100644 --- a/extension.go +++ b/extension.go @@ -12,10 +12,6 @@ import ( "github.com/opencontainers/go-digest" ) -type Extension interface { - ExtendedNamespace -} - // ExtensionContext contains the request specific context for use in across handlers. type ExtensionContext struct { context.Context @@ -70,7 +66,7 @@ type ExtendedStorage interface { } //Namespace is the namespace that is used to define extensions to the distribution. -type ExtendedNamespace interface { +type Extension interface { ExtendedStorage // GetRepositoryRoutes returns a list of extension routes scoped at a repository level GetRepositoryRoutes() []ExtensionRoute @@ -84,8 +80,8 @@ type ExtendedNamespace interface { GetNamespaceDescription() string } -// // InitExtensionNamespace is the initialize function for creating the extension namespace -type InitExtensionNamespace func(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (ExtendedNamespace, error) +// InitExtension is the initialize function for creating the extension namespace +type InitExtension func(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error) // EnumerateExtension specifies extension information at the namespace level type EnumerateExtension struct { @@ -95,8 +91,8 @@ type EnumerateExtension struct { Endpoints []string `json:"endpoints"` } -var extensions map[string]InitExtensionNamespace -var extensionsNamespaces map[string]ExtendedNamespace +var extensions map[string]InitExtension +var extensionsNamespaces map[string]Extension func EnumerateRegistered(ctx ExtensionContext) (enumeratedExtensions []EnumerateExtension) { for _, namespace := range extensionsNamespaces { @@ -128,11 +124,11 @@ func EnumerateRegistered(ctx ExtensionContext) (enumeratedExtensions []Enumerate return enumeratedExtensions } -// RegisterExtension is used to register an InitExtensionNamespace for -// an extension namespace with the given name. -func RegisterExtension(name string, initFunc InitExtensionNamespace) { +// RegisterExtension is used to register an InitExtension for +// an extension with the given name. +func RegisterExtension(name string, initFunc InitExtension) { if extensions == nil { - extensions = make(map[string]InitExtensionNamespace) + extensions = make(map[string]InitExtension) } if _, exists := extensions[name]; exists { @@ -142,11 +138,11 @@ func RegisterExtension(name string, initFunc InitExtensionNamespace) { extensions[name] = initFunc } -// GetExtension constructs an extension namespace with the given options using the given name. -func GetExtension(ctx context.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (ExtendedNamespace, error) { +// GetExtension constructs an extension with the given options using the given name. +func GetExtension(ctx context.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error) { if extensions != nil { if extensionsNamespaces == nil { - extensionsNamespaces = make(map[string]ExtendedNamespace) + extensionsNamespaces = make(map[string]Extension) } if initFunc, exists := extensions[name]; exists { diff --git a/registry/extension/distribution/registry.go b/registry/extension/distribution/registry.go index a1dde25c278..33c2c0c25de 100644 --- a/registry/extension/distribution/registry.go +++ b/registry/extension/distribution/registry.go @@ -32,7 +32,7 @@ type distributionOptions struct { } // newDistNamespace creates a new extension namespace with the name "distribution" -func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.ExtendedNamespace, error) { +func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.Extension, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { diff --git a/registry/extension/oci/oci.go b/registry/extension/oci/oci.go index 4f40cc054da..01375626704 100644 --- a/registry/extension/oci/oci.go +++ b/registry/extension/oci/oci.go @@ -30,7 +30,7 @@ type ociOptions struct { } // newOciNamespace creates a new extension namespace with the name "oci" -func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.ExtendedNamespace, error) { +func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.Extension, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err diff --git a/registry/extension/oras/oras.go b/registry/extension/oras/oras.go index 563bb4f8968..8cdc6deefae 100644 --- a/registry/extension/oras/oras.go +++ b/registry/extension/oras/oras.go @@ -33,7 +33,7 @@ type OrasOptions struct { } // newOrasNamespace creates a new extension namespace with the name "oras" -func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.ExtendedNamespace, error) { +func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.Extension, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err diff --git a/registry/handlers/app.go b/registry/handlers/app.go index b88f289c2bd..9622b4f156c 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -97,7 +97,7 @@ type App struct { repositoryExtensions []string // extensionNamespaces is a list of namespaces that are configured as extensions to the distribution - extensionNamespaces []distribution.ExtendedNamespace + extensionNamespaces []distribution.Extension } // NewApp takes a configuration and returns a configured app, ready to serve @@ -926,7 +926,7 @@ func (app *App) nameRequired(r *http.Request) bool { func (app *App) initializeExtensionNamespaces(ctx context.Context, extensions map[string]configuration.ExtensionConfig) error { - extensionNamespaces := []distribution.ExtendedNamespace{} + extensionNamespaces := []distribution.Extension{} for key, options := range extensions { ns, err := distribution.GetExtension(ctx, key, app.driver, options) if err != nil { diff --git a/registry/root.go b/registry/root.go index b452eb162c2..b4ba205f32e 100644 --- a/registry/root.go +++ b/registry/root.go @@ -73,7 +73,7 @@ var GCCmd = &cobra.Command{ } extensions := config.Extensions - extensionNamespaces := []distribution.ExtendedNamespace{} + extensionNamespaces := []distribution.Extension{} for key, options := range extensions { ns, err := distribution.GetExtension(ctx, key, driver, options) if err != nil { From a0df7b5b3b8e6e62b9db7abac6bae3d89a153046 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Wed, 22 Jun 2022 12:30:44 -0700 Subject: [PATCH 15/22] small changes; add extensions documentation Signed-off-by: Akash Singhal --- docs/extensions.md | 145 +++++++++++++++++++++ extension.go | 3 +- registry.go | 1 + registry/extension/oras/artifactservice.go | 6 +- 4 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 docs/extensions.md diff --git a/docs/extensions.md b/docs/extensions.md new file mode 100644 index 00000000000..26944c01bce --- /dev/null +++ b/docs/extensions.md @@ -0,0 +1,145 @@ +--- +description: High level discussion of extensions +keywords: registry, extension, handlers, repository, distribution, artifacts +title: Extensions +--- + +This document serves as a high level discussion of the implementation of the extensions framework defined in the [OCI Distribution spec](https://github.com/opencontainers/distribution-spec/tree/main/extensions). + +## Extension Interface + +The `Extension` interface is introduced in the `distribution` package. It defines methods to access the extension's namespace specific attributes such as the Name, Url defining the extension namespace, and the Description of the namespace. It defines route enumeration at the Registry and Repository level. It also encases the `ExtendedStorage` interface which defines the methods requires to extend the underlying storage functionality of the registry. + +``` +type Extension interface { + ExtendedStorage + // GetRepositoryRoutes returns a list of extension routes scoped at a repository level + GetRepositoryRoutes() []ExtensionRoute + // GetRegistryRoutes returns a list of extension routes scoped at a registry level + GetRegistryRoutes() []ExtensionRoute + // GetNamespaceName returns the name associated with the namespace + GetNamespaceName() string + // GetNamespaceUrl returns the url link to the documentation where the namespace's extension and endpoints are defined + GetNamespaceUrl() string + // GetNamespaceDescription returns the description associated with the namespace + GetNamespaceDescription() string +} +``` + +The `Namespace` interface in the `distrubtion` package is modified to return a list of `Extensions` registered to the `Namespace` + +``` +type Namespace interface { + // Scope describes the names that can be used with this Namespace. The + // global namespace will have a scope that matches all names. The scope + // effectively provides an identity for the namespace. + Scope() Scope + + // Repository should return a reference to the named repository. The + // registry may or may not have the repository but should always return a + // reference. + Repository(ctx context.Context, name reference.Named) (Repository, error) + + // Repositories fills 'repos' with a lexicographically sorted catalog of repositories + // up to the size of 'repos' and returns the value 'n' for the number of entries + // which were filled. 'last' contains an offset in the catalog, and 'err' will be + // set to io.EOF if there are no more entries to obtain. + Repositories(ctx context.Context, repos []string, last string) (n int, err error) + + // Blobs returns a blob enumerator to access all blobs + Blobs() BlobEnumerator + + // BlobStatter returns a BlobStatter to control + BlobStatter() BlobStatter + + // Extensions returns a list of Extension registered to the Namespace + Extensions() []Extension +} +``` + +The `ExtendedStorage` interface defines methods that specify storage-specific handlers. Each extension will implement a handler extending the functionality. The interface can be expanded in the future to consider new handler types. +`GetManifestHandlers` is used to return new `ManifestHandlers` defined by each of the extensions. (Note: To support this interface in the `distribution` package, the `ManifestHandlers` interface has been moved to the `distribution` package) +`GetGarbageCollectionHandlers` is used to return `GCExtensionHandler` implemented by each extension. + +``` +type ExtendedStorage interface { + // GetManifestHandlers returns the list of manifest handlers that handle custom manifest formats supported by the extensions. + GetManifestHandlers( + repo Repository, + blobStore BlobStore) []ManifestHandler + // GetGarbageCollectHandlers returns the list of GC handlers that handle custom garbage collection behavior for the extensions + GetGarbageCollectionHandlers() []GCExtensionHandler +} +``` + +The `GCExtensionHandler` interface defines three methods that are used in the garbage colection mark and sweep process. The `Mark` method is invoked for each `GCExtensionHandler` after the existing mark process finishes in `MarkAndSweep`. `IsEligibleForDeletion` is used to define if a specific manifest set for deletion in `MarkAndSweep` should be eligible for deletion. Extensions may choose to special case certain manifest types in manifest deletion. `RemoveManifestVacuum` is invoked to extend the `RemoveManifest` functionality for the `Vacuum`. New or special-cased manifests may require custom manifest deletion which can be defined with this method. + +``` +type GCExtensionHandler interface { + Mark(ctx context.Context, + storageDriver driver.StorageDriver, + registry Namespace, + dryRun bool, + removeUntagged bool) (map[digest.Digest]struct{}, error) + RemoveManifestVacuum(ctx context.Context, + storageDriver driver.StorageDriver, + dgst digest.Digest, + repositoryName string) error + IsEligibleForDeletion(ctx context.Context, + dgst digest.Digest, + manifestService ManifestService) (bool, error) +} +``` + +## Registering Extensions + +Extensions are defined in the configuration yaml. + +### Sample Extension Configuration YAML +``` +# Configuration for extensions. It follows the below schema +# extensions +# namespace: +# configuration for the extension and its components in any schema specific to that namespace +extensions: + oci: + ext: + - discover # enable the discovery extension +``` + +Each `Extension` defined must call the `RegisterExtension` method to register an extension initialization function with the extension namespace name. The registered extension list is then used during configuration parsing to get and initialize the specified extension. (`GetExtension`) + +``` +// InitExtension is the initialize function for creating the extension namespace +type InitExtension func(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error) + +// RegisterExtension is used to register an InitExtension for +// an extension with the given name. +func RegisterExtension(name string, initFunc InitExtension) + +// GetExtension constructs an extension with the given options using the given name. +func GetExtension(ctx context.Context, name string, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error) +``` + +Each `Extension` defines an `ExtensionRoute` which contains the new `//` route attributes. Furthermore, the route `Descriptor` and `Dispatcher` are used to register the new route to the application. + +``` +type ExtensionRoute struct { + // Namespace is the name of the extension namespace + Namespace string + // Extension is the name of the extension under the namespace + Extension string + // Component is the name of the component under the extension + Component string + // Descriptor is the route descriptor that gives its path + Descriptor v2.RouteDescriptor + // Dispatcher if present signifies that the route is http route with a dispatcher + Dispatcher RouteDispatchFunc +} + +// RouteDispatchFunc is the http route dispatcher used by the extension route handlers +type RouteDispatchFunc func(extContext *ExtensionContext, r *http.Request) http.Handler +``` + + + diff --git a/extension.go b/extension.go index afa002badf1..8234d4c02ab 100644 --- a/extension.go +++ b/extension.go @@ -62,10 +62,11 @@ type ExtendedStorage interface { GetManifestHandlers( repo Repository, blobStore BlobStore) []ManifestHandler + // GetGarbageCollectHandlers returns the list of GC handlers that handle custom garbage collection behavior for the extensions GetGarbageCollectionHandlers() []GCExtensionHandler } -//Namespace is the namespace that is used to define extensions to the distribution. +// Extension is the interface that is used to define extensions to the distribution. type Extension interface { ExtendedStorage // GetRepositoryRoutes returns a list of extension routes scoped at a repository level diff --git a/registry.go b/registry.go index 97e74f277eb..1b27a5ed51e 100644 --- a/registry.go +++ b/registry.go @@ -48,6 +48,7 @@ type Namespace interface { // BlobStatter returns a BlobStatter to control BlobStatter() BlobStatter + // Extensions returns a list of Extension registered to the Namespace Extensions() []Extension } diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index f48fc917112..48441d14ffd 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -71,13 +71,13 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest return err } - ArtifactMan, ok := man.(*DeserializedManifest) + artifactManifest, ok := man.(*DeserializedManifest) if !ok { // The PUT handler would guard against this situation. Skip this manifest. return nil } - extractedArtifactType := ArtifactMan.ArtifactType() + extractedArtifactType := artifactManifest.ArtifactType() // filtering by artifact type or bypass if no artifact type specified if artifactType == "" || extractedArtifactType == artifactType { @@ -93,7 +93,7 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest ArtifactType: extractedArtifactType, } - if annotation, ok := ArtifactMan.Annotations()[createAnnotationName]; !ok { + if annotation, ok := artifactManifest.Annotations()[createAnnotationName]; !ok { referrersUnsorted = append(referrersUnsorted, artifactDesc) } else { extractedTimestamp, err := time.Parse(createAnnotationTimestampFormat, annotation) From 9e40a2d2e5c4c991efa8883e94070db93fe51caa Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Mon, 27 Jun 2022 13:23:29 -0700 Subject: [PATCH 16/22] addressing more comments Signed-off-by: Akash Singhal --- docs/extensions.md | 10 +++++----- registry/storage/garbagecollect.go | 12 +++++------- registry/storage/vacuum.go | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index 26944c01bce..ddd3701107a 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -8,7 +8,7 @@ This document serves as a high level discussion of the implementation of the ext ## Extension Interface -The `Extension` interface is introduced in the `distribution` package. It defines methods to access the extension's namespace specific attributes such as the Name, Url defining the extension namespace, and the Description of the namespace. It defines route enumeration at the Registry and Repository level. It also encases the `ExtendedStorage` interface which defines the methods requires to extend the underlying storage functionality of the registry. +The `Extension` interface is introduced in the `distribution` package. It defines methods to access the extension's namespace-specific attributes such as the Name, Url defining the extension namespace, and the Description of the namespace. It defines route enumeration at the Registry and Repository level. It also encases the `ExtendedStorage` interface which defines the methods requires to extend the underlying storage functionality of the registry. ``` type Extension interface { @@ -26,7 +26,7 @@ type Extension interface { } ``` -The `Namespace` interface in the `distrubtion` package is modified to return a list of `Extensions` registered to the `Namespace` +The `Namespace` interface in the `distribution` package is modified to return a list of `Extensions` registered to the `Namespace` ``` type Namespace interface { @@ -63,12 +63,12 @@ The `ExtendedStorage` interface defines methods that specify storage-specific ha ``` type ExtendedStorage interface { - // GetManifestHandlers returns the list of manifest handlers that handle custom manifest formats supported by the extensions. + // GetManifestHandlers returns the list of manifest handlers that handle custom manifest formats supported by the extension GetManifestHandlers( repo Repository, blobStore BlobStore) []ManifestHandler - // GetGarbageCollectHandlers returns the list of GC handlers that handle custom garbage collection behavior for the extensions - GetGarbageCollectionHandlers() []GCExtensionHandler + // GetGarbageCollectHandler returns the GCExtensionHandler that handles custom garbage collection behavior for the extension. + GetGarbageCollectionHandler() GCExtensionHandler } ``` diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index a6367ae0b71..4ca289bfe92 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -75,7 +75,6 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("failed to retrieve tags %v", err) } - isEligibleForDelete := true for _, extNamespace := range registry.Extensions() { handlers := extNamespace.GetGarbageCollectionHandlers() for _, gcHandler := range handlers { @@ -83,14 +82,13 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("failed to determine deletion eligibility using extension handler: %v", err) } - - isEligibleForDelete = isEligibleForDelete && extensionDeleteEligible + if !extensionDeleteEligible { + return nil + } } } - if isEligibleForDelete { - manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) - emit("manifest eligible for deletion: %s", dgst) - } + manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) + emit("manifest eligible for deletion: %s", dgst) return nil } } diff --git a/registry/storage/vacuum.go b/registry/storage/vacuum.go index 45df0d2d1ec..408835238f8 100644 --- a/registry/storage/vacuum.go +++ b/registry/storage/vacuum.go @@ -55,7 +55,7 @@ func (v Vacuum) RemoveBlob(dgst string) error { } // RemoveManifest removes a manifest from the filesystem -// Removes manifest's ref folder if it exists +// Invokes each GCExtensionHandler's RemoveManifestVacuum func (v Vacuum) RemoveManifest(name string, dgst digest.Digest, tags []string) error { // remove a tag manifest reference, in case of not found continue to next one for _, tag := range tags { From 35185e315da37eb299c360e64d1477829b1ed2c4 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Sat, 16 Jul 2022 16:55:11 -0700 Subject: [PATCH 17/22] next iteration of prototype Signed-off-by: Akash Singhal --- extension.go | 12 +- .../oras/artifactgarbagecollectionhandler.go | 262 +++++------------- registry/extension/oras/oras.go | 2 +- registry/extension/oras/referrers.md | 4 +- registry/storage/garbagecollect.go | 39 +-- registry/storage/vacuum.go | 11 - 6 files changed, 107 insertions(+), 223 deletions(-) diff --git a/extension.go b/extension.go index 8234d4c02ab..c0472233bd3 100644 --- a/extension.go +++ b/extension.go @@ -43,17 +43,19 @@ type ExtensionRoute struct { type GCExtensionHandler interface { Mark(ctx context.Context, + repository Repository, storageDriver driver.StorageDriver, registry Namespace, + manifest Manifest, + manifestDigest digest.Digest, dryRun bool, - removeUntagged bool) (map[digest.Digest]struct{}, error) + removeUntagged bool) (map[digest.Digest]struct{}, bool, error) RemoveManifestVacuum(ctx context.Context, storageDriver driver.StorageDriver, + registry Namespace, dgst digest.Digest, - repositoryName string) error - IsEligibleForDeletion(ctx context.Context, - dgst digest.Digest, - manifestService ManifestService) (bool, error) + markSet map[digest.Digest]struct{}, + repositoryName string) (map[digest.Digest]struct{}, error) } // ExtendedStorage defines extensions to store operations like manifest for example. diff --git a/registry/extension/oras/artifactgarbagecollectionhandler.go b/registry/extension/oras/artifactgarbagecollectionhandler.go index 5d54ca3ca7c..b52d052035a 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler.go @@ -10,7 +10,7 @@ import ( "github.com/distribution/distribution/v3/reference" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" - v1 "github.com/oras-project/artifacts-spec/specs-go/v1" + artifactv1 "github.com/oras-project/artifacts-spec/specs-go/v1" ) type orasGCHandler struct { @@ -18,153 +18,109 @@ type orasGCHandler struct { } func (gc *orasGCHandler) Mark(ctx context.Context, + repository distribution.Repository, storageDriver driver.StorageDriver, registry distribution.Namespace, + manifest distribution.Manifest, + dgst digest.Digest, dryRun bool, - removeUntagged bool) (map[digest.Digest]struct{}, error) { - repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) - if !ok { - return nil, fmt.Errorf("unable to convert Namespace to RepositoryEnumerator") - } - - gc.artifactManifestIndex = make(map[digest.Digest][]digest.Digest) - // mark + removeUntagged bool) (map[digest.Digest]struct{}, bool, error) { markSet := make(map[digest.Digest]struct{}) - err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { - fmt.Printf(repoName + "\n") - - var err error - named, err := reference.WithName(repoName) - if err != nil { - return fmt.Errorf("failed to parse repo name %s: %v", repoName, err) - } - repository, err := registry.Repository(ctx, named) - if err != nil { - return fmt.Errorf("failed to construct repository: %v", err) + blobStatter := registry.BlobStatter() + mediaType, _, err := manifest.Payload() + if err != nil { + return markSet, false, err + } + referrerRootPath := referrersLinkPath(repository.Named().Name()) + rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) + + if mediaType == artifactv1.MediaTypeArtifactManifest { + // if the manifest passed is an artifact -> mark the manifest and blobs for now + fmt.Printf("%s: marking artifact manifest %s\n", repository.Named().Name(), dgst.String()) + markSet[dgst] = struct{}{} + + // mark the artifact blobs + descriptors := manifest.References() + for _, descriptor := range descriptors { + markSet[descriptor.Digest] = struct{}{} + fmt.Printf("%s: marking blob %s\n", repository.Named().Name(), descriptor.Digest) } + return markSet, false, nil + } else { + // if the manifest passed isn't an an artifact -> call the sweep ingestor + // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion + gc.artifactManifestIndex[dgst] = make([]digest.Digest, 0) + err := enumerateReferrerLinks(ctx, + rootPath, + storageDriver, + repository, + blobStatter, + markSet, + dgst, + gc.artifactManifestIndex, + artifactSweepIngestor) - manifestService, err := repository.Manifests(ctx) if err != nil { - return fmt.Errorf("failed to construct manifest service: %v", err) - } - - manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator) - if !ok { - return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator") - } - - err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { - manifest, err := manifestService.Get(ctx, dgst) - if err != nil { - return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) - } - - mediaType, _, err := manifest.Payload() - if err != nil { - return err - } - - // if the manifest is an oras artifact, skip it - // the artifact marking occurs when walking the refs - if mediaType == v1.MediaTypeArtifactManifest { - return nil - } - - blobStatter := registry.BlobStatter() - referrerRootPath := referrersLinkPath(repoName) - if removeUntagged { - // fetch all tags where this manifest is the latest one - tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst}) - if err != nil { - return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err) - } - if len(tags) == 0 { - - // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion - gc.artifactManifestIndex[dgst] = make([]digest.Digest, 0) - rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) - err = enumerateReferrerLinks(ctx, - rootPath, - storageDriver, - repository, - blobStatter, - markSet, - dgst, - gc.artifactManifestIndex, - artifactSweepIngestor) - - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil - } - return err - } - return nil - } + switch err.(type) { + case driver.PathNotFoundError: + return markSet, true, nil } - - // recurse child artifact as subject to find lower level referrers - rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) - err = enumerateReferrerLinks(ctx, - rootPath, - storageDriver, - repository, - blobStatter, - markSet, - dgst, - gc.artifactManifestIndex, - artifactMarkIngestor) - - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil - } - return err - } - return nil - }) - - // In certain situations such as unfinished uploads, deleting all - // tags in S3 or removing the _manifests folder manually, this - // error may be of type PathNotFound. - // - // In these cases we can continue marking other manifests safely. - if _, ok := err.(driver.PathNotFoundError); ok { - return nil + return markSet, true, err } - - return err - }) - - if err != nil { - return nil, fmt.Errorf("failed to mark: %v", err) + return markSet, true, nil } - return markSet, nil } -func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver driver.StorageDriver, dgst digest.Digest, repositoryName string) error { +func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dgst digest.Digest, markSet map[digest.Digest]struct{}, repositoryName string) (map[digest.Digest]struct{}, error) { referrerRootPath := referrersLinkPath(repositoryName) fullArtifactManifestPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) dcontext.GetLogger(ctx).Infof("deleting manifest ref folder: %s", fullArtifactManifestPath) err := storageDriver.Delete(ctx, fullArtifactManifestPath) if err != nil { if _, ok := err.(driver.PathNotFoundError); !ok { - return err + return markSet, err } } subjectLinkedArtifacts, ok := gc.artifactManifestIndex[dgst] if ok { for _, artifactDigest := range subjectLinkedArtifacts { + // get the artifact manifest + named, err := reference.WithName(repositoryName) + if err != nil { + return markSet, fmt.Errorf("failed to parse repo name %s: %v", repositoryName, err) + } + repository, err := registry.Repository(ctx, named) + if err != nil { + return markSet, fmt.Errorf("failed to construct repository: %v", err) + } + + manifestService, err := repository.Manifests(ctx) + if err != nil { + return markSet, fmt.Errorf("failed to construct manifest service: %v", err) + } + artifactManifest, err := manifestService.Get(ctx, artifactDigest) + if err != nil { + return markSet, fmt.Errorf("failed to get artifact manifest: %v", err) + } + + // extract the reference + blobs := artifactManifest.References() + + // remove the blobs digests' and the manifest digest from the markset + delete(markSet, dgst) + fmt.Printf("%s: unmarking artifact manifest %s\n", repositoryName, dgst) + for _, descriptor := range blobs { + delete(markSet, descriptor.Digest) + fmt.Printf("%s: unmarking blob %s\n", repositoryName, descriptor.Digest) + } // delete each artifact manifest's revision manifestPath := referrersRepositoriesManifestRevisionPath(repositoryName, artifactDigest) dcontext.GetLogger(ctx).Infof("deleting artifact manifest revision: %s", manifestPath) err = storageDriver.Delete(ctx, manifestPath) if err != nil { if _, ok := err.(driver.PathNotFoundError); !ok { - return err + return markSet, err } } // delete each artifact manifest's ref folder @@ -173,79 +129,13 @@ func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver err = storageDriver.Delete(ctx, fullArtifactManifestPath) if err != nil { if _, ok := err.(driver.PathNotFoundError); !ok { - return err + return markSet, err } } } } - return nil -} - -func (gc *orasGCHandler) IsEligibleForDeletion(ctx context.Context, dgst digest.Digest, manifestService distribution.ManifestService) (bool, error) { - manifest, err := manifestService.Get(ctx, dgst) - if err != nil { - return false, fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) - } - - mediaType, _, err := manifest.Payload() - if err != nil { - return false, err - } - return mediaType != v1.MediaTypeArtifactManifest, nil -} - -// ingestor method used in EnumerateReferrerLinks -// marks each artifact manifest and associated blobs -func artifactMarkIngestor(ctx context.Context, - referrerRevision digest.Digest, - markSet map[digest.Digest]struct{}, - subjectRevision digest.Digest, - artifactManifestIndex map[digest.Digest][]digest.Digest, - repository distribution.Repository, - blobstatter distribution.BlobStatter, - storageDriver driver.StorageDriver) error { - manifestService, err := repository.Manifests(ctx) - if err != nil { - return fmt.Errorf("failed to construct manifest service: %v", err) - } - - man, err := manifestService.Get(ctx, referrerRevision) - if err != nil { - return err - } - - repoName := repository.Named().Name() - // mark the artifact manifest blob - fmt.Printf("%s: marking artifact manifest %s\n", repoName, referrerRevision.String()) - markSet[referrerRevision] = struct{}{} - - // mark the artifact blobs - descriptors := man.References() - for _, descriptor := range descriptors { - markSet[descriptor.Digest] = struct{}{} - fmt.Printf("%s: marking blob %s\n", repoName, descriptor.Digest) - } - referrerRootPath := referrersLinkPath(repoName) - - rootPath := path.Join(referrerRootPath, referrerRevision.Algorithm().String(), referrerRevision.Hex()) - _, err = storageDriver.Stat(ctx, rootPath) - if err != nil { - switch err.(type) { - case driver.PathNotFoundError: - return nil - } - return err - } - return enumerateReferrerLinks(ctx, - rootPath, - storageDriver, - repository, - blobstatter, - markSet, - subjectRevision, - artifactManifestIndex, - artifactMarkIngestor) + return markSet, nil } // ingestor method used in EnumerateReferrerLinks diff --git a/registry/extension/oras/oras.go b/registry/extension/oras/oras.go index 8cdc6deefae..30f0fd7b5c4 100644 --- a/registry/extension/oras/oras.go +++ b/registry/extension/oras/oras.go @@ -53,7 +53,7 @@ func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, o } } - orasGCHandler := orasGCHandler{} + orasGCHandler := orasGCHandler{artifactManifestIndex: make(map[digest.Digest][]digest.Digest)} return &orasNamespace{ referrersEnabled: referrersEnabled, diff --git a/registry/extension/oras/referrers.md b/registry/extension/oras/referrers.md index a402d70404c..1ee5e0bca61 100644 --- a/registry/extension/oras/referrers.md +++ b/registry/extension/oras/referrers.md @@ -98,6 +98,6 @@ This results in an addition to the index as shown below. The life of a referrer artifact is directly linked to its subject. When a referrer artifact's subject manifest is deleted, the artifact's referrers are also deleted. -Manifest garbage collection is extended to include referrer artifact collection. During the marking process, each manifest is queried for any referrer artifacts by enumerating the link files at the path `repositories//_refs/subjects/sha256/`. For each artifact, the artifact manifest and its blobs are marked. Finally, collection recurses to look for further referrers to mark in a similar fashion. +Manifest garbage collection is extended to include referrer artifact collection. During the marking process, each manifest and its blobs, regardless of it being an ORAS artifact manifest, are marked. If a non-ORAS manifest is found to be untagged and untag deletion is enabled, the manifest is queried for any referrer artifacts by enumerating the link files at the path `repositories//_refs/subjects/sha256/`. For each artifact, the artifact manifest and its blobs are un-marked, so that they are deleted. -If a manifest is indexed for deletion because it is untagged, the attached reference artifacts are also indexed. Similar to the marking process, the subject manifest's `_ref` folder is queried for referrers. Each encountered referrer is indexed. Indexing recurses to the next levels referrers until all successive referrers are indexed. Finally, during manifest link deletion, the revision link files of the indexed artifact manifests as well as the corresponding `_refs` are removed from storage. \ No newline at end of file +During manifest link deletion, the revision link files of the indexed artifact manifests as well as the corresponding `_refs` are removed from storage. \ No newline at end of file diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 4ca289bfe92..68e5ff9e450 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -75,16 +75,24 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("failed to retrieve tags %v", err) } + // call GC extension handlers' mark + manifest, err := manifestService.Get(ctx, dgst) + if err != nil { + return fmt.Errorf("failed to retrieve manifest: %v", err) + } for _, extNamespace := range registry.Extensions() { handlers := extNamespace.GetGarbageCollectionHandlers() for _, gcHandler := range handlers { - extensionDeleteEligible, err := gcHandler.IsEligibleForDeletion(ctx, dgst, manifestService) + extensionMarkSet, deleteEligible, err := gcHandler.Mark(ctx, repository, storageDriver, registry, manifest, dgst, opts.DryRun, opts.RemoveUntagged) if err != nil { - return fmt.Errorf("failed to determine deletion eligibility using extension handler: %v", err) + return fmt.Errorf("failed to mark using extension handler: %v", err) } - if !extensionDeleteEligible { + if !deleteEligible { return nil } + for k, _ := range extensionMarkSet { + markSet[k] = struct{}{} + } } } manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) @@ -126,21 +134,6 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to mark: %v", err) } - // call GC extension handlers' mark - for _, extNamespace := range registry.Extensions() { - handlers := extNamespace.GetGarbageCollectionHandlers() - for _, gcHandler := range handlers { - extensionMarkSet, err := gcHandler.Mark(ctx, storageDriver, registry, opts.DryRun, opts.RemoveUntagged) - if err != nil { - return fmt.Errorf("failed to mark using extension handler: %v", err) - } - - for k, _ := range extensionMarkSet { - markSet[k] = struct{}{} - } - } - } - // sweep vacuum := NewVacuum(ctx, storageDriver, registry) if !opts.DryRun { @@ -149,6 +142,16 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) } + for _, extNamespace := range registry.Extensions() { + handlers := extNamespace.GetGarbageCollectionHandlers() + for _, gcHandler := range handlers { + newMarkSet, err := gcHandler.RemoveManifestVacuum(ctx, storageDriver, registry, obj.Digest, markSet, obj.Name) + if err != nil { + return fmt.Errorf("failed to call remove manifest extension handler: %v", err) + } + markSet = newMarkSet + } + } } } blobService := registry.Blobs() diff --git a/registry/storage/vacuum.go b/registry/storage/vacuum.go index 408835238f8..604f2dc9c16 100644 --- a/registry/storage/vacuum.go +++ b/registry/storage/vacuum.go @@ -2,7 +2,6 @@ package storage import ( "context" - "fmt" "path" "github.com/distribution/distribution/v3" @@ -93,16 +92,6 @@ func (v Vacuum) RemoveManifest(name string, dgst digest.Digest, tags []string) e } } - for _, extNamespace := range v.registry.Extensions() { - handlers := extNamespace.GetGarbageCollectionHandlers() - for _, gcHandler := range handlers { - err := gcHandler.RemoveManifestVacuum(v.ctx, v.driver, dgst, name) - if err != nil { - return fmt.Errorf("failed to call remove manifest extension handler: %v", err) - } - } - } - return nil } From 2ae2738b5f1163fe796dbca7bee50385c687a708 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Wed, 3 Aug 2022 11:49:28 -0700 Subject: [PATCH 18/22] adding blob ref count map Signed-off-by: Akash Singhal --- extension.go | 13 ++-- manifests.go | 1 + .../oras/artifactgarbagecollectionhandler.go | 63 ++++++++++--------- registry/extension/oras/artifactservice.go | 5 -- registry/extension/oras/oras.go | 5 +- registry/storage/garbagecollect.go | 15 +++-- 6 files changed, 58 insertions(+), 44 deletions(-) diff --git a/extension.go b/extension.go index c0472233bd3..1019a30e4fa 100644 --- a/extension.go +++ b/extension.go @@ -49,13 +49,14 @@ type GCExtensionHandler interface { manifest Manifest, manifestDigest digest.Digest, dryRun bool, - removeUntagged bool) (map[digest.Digest]struct{}, bool, error) - RemoveManifestVacuum(ctx context.Context, + removeUntagged bool) (bool, error) + RemoveManifest(ctx context.Context, storageDriver driver.StorageDriver, registry Namespace, dgst digest.Digest, - markSet map[digest.Digest]struct{}, - repositoryName string) (map[digest.Digest]struct{}, error) + repositoryName string) error + SweepBlobs(ctx context.Context, + markSet map[digest.Digest]struct{}) map[digest.Digest]struct{} } // ExtendedStorage defines extensions to store operations like manifest for example. @@ -71,6 +72,7 @@ type ExtendedStorage interface { // Extension is the interface that is used to define extensions to the distribution. type Extension interface { ExtendedStorage + // ExtensionService // GetRepositoryRoutes returns a list of extension routes scoped at a repository level GetRepositoryRoutes() []ExtensionRoute // GetRegistryRoutes returns a list of extension routes scoped at a registry level @@ -83,6 +85,9 @@ type Extension interface { GetNamespaceDescription() string } +type ExtensionService interface { +} + // InitExtension is the initialize function for creating the extension namespace type InitExtension func(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error) diff --git a/manifests.go b/manifests.go index 85d2f47a910..74394f16128 100644 --- a/manifests.go +++ b/manifests.go @@ -74,6 +74,7 @@ type Describable interface { Descriptor() Descriptor } +// TODO: This interface should not live here in the future // A ManifestHandler gets and puts manifests of a particular type. type ManifestHandler interface { // Unmarshal unmarshals the manifest from a byte slice. diff --git a/registry/extension/oras/artifactgarbagecollectionhandler.go b/registry/extension/oras/artifactgarbagecollectionhandler.go index b52d052035a..3c5536dfc49 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler.go @@ -15,6 +15,7 @@ import ( type orasGCHandler struct { artifactManifestIndex map[digest.Digest][]digest.Digest + artifactMarkSet map[digest.Digest]int } func (gc *orasGCHandler) Mark(ctx context.Context, @@ -24,28 +25,28 @@ func (gc *orasGCHandler) Mark(ctx context.Context, manifest distribution.Manifest, dgst digest.Digest, dryRun bool, - removeUntagged bool) (map[digest.Digest]struct{}, bool, error) { - markSet := make(map[digest.Digest]struct{}) + removeUntagged bool) (bool, error) { + //markSet := make(map[digest.Digest]struct{}) blobStatter := registry.BlobStatter() mediaType, _, err := manifest.Payload() if err != nil { - return markSet, false, err + return false, err } referrerRootPath := referrersLinkPath(repository.Named().Name()) rootPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) if mediaType == artifactv1.MediaTypeArtifactManifest { // if the manifest passed is an artifact -> mark the manifest and blobs for now - fmt.Printf("%s: marking artifact manifest %s\n", repository.Named().Name(), dgst.String()) - markSet[dgst] = struct{}{} + fmt.Printf("%s: incrementing artifact manifest ref count %s\n", repository.Named().Name(), dgst.String()) + gc.artifactMarkSet[dgst] += 1 // mark the artifact blobs descriptors := manifest.References() for _, descriptor := range descriptors { - markSet[descriptor.Digest] = struct{}{} - fmt.Printf("%s: marking blob %s\n", repository.Named().Name(), descriptor.Digest) + gc.artifactMarkSet[descriptor.Digest] += 1 + fmt.Printf("%s: incrementing artifact blob ref count %s\n", repository.Named().Name(), descriptor.Digest) } - return markSet, false, nil + return false, nil } else { // if the manifest passed isn't an an artifact -> call the sweep ingestor // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion @@ -55,7 +56,6 @@ func (gc *orasGCHandler) Mark(ctx context.Context, storageDriver, repository, blobStatter, - markSet, dgst, gc.artifactManifestIndex, artifactSweepIngestor) @@ -63,22 +63,22 @@ func (gc *orasGCHandler) Mark(ctx context.Context, if err != nil { switch err.(type) { case driver.PathNotFoundError: - return markSet, true, nil + return true, nil } - return markSet, true, err + return true, err } - return markSet, true, nil + return true, nil } } -func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dgst digest.Digest, markSet map[digest.Digest]struct{}, repositoryName string) (map[digest.Digest]struct{}, error) { +func (gc *orasGCHandler) RemoveManifest(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dgst digest.Digest, repositoryName string) error { referrerRootPath := referrersLinkPath(repositoryName) fullArtifactManifestPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) dcontext.GetLogger(ctx).Infof("deleting manifest ref folder: %s", fullArtifactManifestPath) err := storageDriver.Delete(ctx, fullArtifactManifestPath) if err != nil { if _, ok := err.(driver.PathNotFoundError); !ok { - return markSet, err + return err } } @@ -88,31 +88,31 @@ func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver // get the artifact manifest named, err := reference.WithName(repositoryName) if err != nil { - return markSet, fmt.Errorf("failed to parse repo name %s: %v", repositoryName, err) + return fmt.Errorf("failed to parse repo name %s: %v", repositoryName, err) } repository, err := registry.Repository(ctx, named) if err != nil { - return markSet, fmt.Errorf("failed to construct repository: %v", err) + return fmt.Errorf("failed to construct repository: %v", err) } manifestService, err := repository.Manifests(ctx) if err != nil { - return markSet, fmt.Errorf("failed to construct manifest service: %v", err) + return fmt.Errorf("failed to construct manifest service: %v", err) } artifactManifest, err := manifestService.Get(ctx, artifactDigest) if err != nil { - return markSet, fmt.Errorf("failed to get artifact manifest: %v", err) + return fmt.Errorf("failed to get artifact manifest: %v", err) } // extract the reference blobs := artifactManifest.References() - // remove the blobs digests' and the manifest digest from the markset - delete(markSet, dgst) - fmt.Printf("%s: unmarking artifact manifest %s\n", repositoryName, dgst) + // decrement refcount for the blobs digests' and the manifest digest + gc.artifactMarkSet[artifactDigest] -= 1 + fmt.Printf("%s: decrementing artifact manifest ref count %s\n", repositoryName, dgst) for _, descriptor := range blobs { - delete(markSet, descriptor.Digest) - fmt.Printf("%s: unmarking blob %s\n", repositoryName, descriptor.Digest) + gc.artifactMarkSet[descriptor.Digest] -= 1 + fmt.Printf("%s: decrementing artifact blob ref count %s\n", repositoryName, descriptor.Digest) } // delete each artifact manifest's revision manifestPath := referrersRepositoriesManifestRevisionPath(repositoryName, artifactDigest) @@ -120,7 +120,7 @@ func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver err = storageDriver.Delete(ctx, manifestPath) if err != nil { if _, ok := err.(driver.PathNotFoundError); !ok { - return markSet, err + return err } } // delete each artifact manifest's ref folder @@ -129,20 +129,28 @@ func (gc *orasGCHandler) RemoveManifestVacuum(ctx context.Context, storageDriver err = storageDriver.Delete(ctx, fullArtifactManifestPath) if err != nil { if _, ok := err.(driver.PathNotFoundError); !ok { - return markSet, err + return err } } } } - return markSet, nil + return nil +} + +func (gc *orasGCHandler) SweepBlobs(ctx context.Context, markSet map[digest.Digest]struct{}) map[digest.Digest]struct{} { + for key, refCount := range gc.artifactMarkSet { + if refCount > 0 { + markSet[key] = struct{}{} + } + } + return markSet } // ingestor method used in EnumerateReferrerLinks // indexes each artifact manifest and adds ArtifactManifestDel struct to index func artifactSweepIngestor(ctx context.Context, referrerRevision digest.Digest, - markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, repository distribution.Repository, @@ -177,7 +185,6 @@ func artifactSweepIngestor(ctx context.Context, storageDriver, repository, blobstatter, - markSet, subjectRevision, artifactManifestIndex, artifactSweepIngestor) diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index 48441d14ffd..7df402e1a06 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -55,12 +55,10 @@ func (h *referrersHandler) Referrers(ctx context.Context, revision digest.Digest h.storageDriver, repo, blobStatter, - map[digest.Digest]struct{}{}, revision, map[digest.Digest][]digest.Digest{}, func(ctx context.Context, referrerRevision digest.Digest, - markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, repository distribution.Repository, @@ -136,12 +134,10 @@ func enumerateReferrerLinks(ctx context.Context, stDriver driver.StorageDriver, repository distribution.Repository, blobstatter distribution.BlobStatter, - markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, ingestor func(ctx context.Context, digest digest.Digest, - markSet map[digest.Digest]struct{}, subjectRevision digest.Digest, artifactManifestIndex map[digest.Digest][]digest.Digest, repository distribution.Repository, @@ -179,7 +175,6 @@ func enumerateReferrerLinks(ctx context.Context, err = ingestor(ctx, digest, - markSet, subjectRevision, artifactManifestIndex, repository, diff --git a/registry/extension/oras/oras.go b/registry/extension/oras/oras.go index 30f0fd7b5c4..7259e6448ad 100644 --- a/registry/extension/oras/oras.go +++ b/registry/extension/oras/oras.go @@ -53,7 +53,10 @@ func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, o } } - orasGCHandler := orasGCHandler{artifactManifestIndex: make(map[digest.Digest][]digest.Digest)} + orasGCHandler := orasGCHandler{ + artifactManifestIndex: make(map[digest.Digest][]digest.Digest), + artifactMarkSet: make(map[digest.Digest]int), + } return &orasNamespace{ referrersEnabled: referrersEnabled, diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 68e5ff9e450..91da5a915a3 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -83,16 +83,13 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis for _, extNamespace := range registry.Extensions() { handlers := extNamespace.GetGarbageCollectionHandlers() for _, gcHandler := range handlers { - extensionMarkSet, deleteEligible, err := gcHandler.Mark(ctx, repository, storageDriver, registry, manifest, dgst, opts.DryRun, opts.RemoveUntagged) + deleteEligible, err := gcHandler.Mark(ctx, repository, storageDriver, registry, manifest, dgst, opts.DryRun, opts.RemoveUntagged) if err != nil { return fmt.Errorf("failed to mark using extension handler: %v", err) } if !deleteEligible { return nil } - for k, _ := range extensionMarkSet { - markSet[k] = struct{}{} - } } } manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) @@ -145,15 +142,21 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis for _, extNamespace := range registry.Extensions() { handlers := extNamespace.GetGarbageCollectionHandlers() for _, gcHandler := range handlers { - newMarkSet, err := gcHandler.RemoveManifestVacuum(ctx, storageDriver, registry, obj.Digest, markSet, obj.Name) + err := gcHandler.RemoveManifest(ctx, storageDriver, registry, obj.Digest, obj.Name) if err != nil { return fmt.Errorf("failed to call remove manifest extension handler: %v", err) } - markSet = newMarkSet } } } } + // GC extension will add final saved blobs into markset for retention + for _, extNamespace := range registry.Extensions() { + handlers := extNamespace.GetGarbageCollectionHandlers() + for _, gcHandler := range handlers { + markSet = gcHandler.SweepBlobs(ctx, markSet) + } + } blobService := registry.Blobs() deleteSet := make(map[digest.Digest]struct{}) err = blobService.Enumerate(ctx, func(dgst digest.Digest) error { From 8610e623954a13825601513c9dca3cc8a99d7519 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Wed, 3 Aug 2022 14:59:32 -0700 Subject: [PATCH 19/22] reverting interfaces changes; introducing gcextensionhandlers list in GC Opts Signed-off-by: Akash Singhal --- manifests.go | 10 ----- registry.go | 3 -- registry/extension/distribution/manifests.go | 4 +- registry/extension/distribution/registry.go | 28 +++++++------ registry/extension/distribution/taghistory.go | 4 +- .../extension/extension.go | 39 +++-------------- registry/extension/oci/discover.go | 8 ++-- registry/extension/oci/oci.go | 28 +++++++------ .../artifactgarbagecollectionhandler_test.go | 7 ++-- .../oras/artifactmanifesthandler_test.go | 9 ++-- registry/extension/oras/artifactservice.go | 3 +- registry/extension/oras/oras.go | 28 +++++++------ registry/handlers/app.go | 11 ++--- registry/proxy/proxyregistry.go | 6 +-- registry/root.go | 13 +++--- registry/storage/extension.go | 29 +++++++++++++ registry/storage/garbagecollect.go | 42 ++++++++----------- registry/storage/manifestlisthandler.go | 2 +- registry/storage/manifeststore.go | 19 ++++++--- registry/storage/ocimanifesthandler.go | 2 +- registry/storage/registry.go | 12 ++---- registry/storage/schema2manifesthandler.go | 2 +- registry/storage/signedmanifesthandler.go | 2 +- registry/storage/v1unsupportedhandler.go | 4 +- registry/storage/vacuum.go | 13 +++--- 25 files changed, 160 insertions(+), 168 deletions(-) rename extension.go => registry/extension/extension.go (80%) diff --git a/manifests.go b/manifests.go index 74394f16128..8f84a220a97 100644 --- a/manifests.go +++ b/manifests.go @@ -74,16 +74,6 @@ type Describable interface { Descriptor() Descriptor } -// TODO: This interface should not live here in the future -// A ManifestHandler gets and puts manifests of a particular type. -type ManifestHandler interface { - // Unmarshal unmarshals the manifest from a byte slice. - Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (Manifest, error) - - // Put creates or updates the given manifest returning the manifest digest. - Put(ctx context.Context, manifest Manifest, skipDependencyVerification bool) (digest.Digest, error) -} - // ManifestMediaTypes returns the supported media types for manifests. func ManifestMediaTypes() (mediaTypes []string) { for t := range mappings { diff --git a/registry.go b/registry.go index 1b27a5ed51e..658f2df0825 100644 --- a/registry.go +++ b/registry.go @@ -47,9 +47,6 @@ type Namespace interface { // BlobStatter returns a BlobStatter to control BlobStatter() BlobStatter - - // Extensions returns a list of Extension registered to the Namespace - Extensions() []Extension } // RepositoryEnumerator describes an operation to enumerate repositories diff --git a/registry/extension/distribution/manifests.go b/registry/extension/distribution/manifests.go index 8440b6daa4a..d1974e1c83d 100644 --- a/registry/extension/distribution/manifests.go +++ b/registry/extension/distribution/manifests.go @@ -4,9 +4,9 @@ import ( "encoding/json" "net/http" - "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -19,7 +19,7 @@ type manifestsGetAPIResponse struct { // manifestHandler handles requests for manifests under a manifest name. type manifestHandler struct { - *distribution.ExtensionContext + *extension.ExtensionContext storageDriver driver.StorageDriver } diff --git a/registry/extension/distribution/registry.go b/registry/extension/distribution/registry.go index 33c2c0c25de..c8dae77bafe 100644 --- a/registry/extension/distribution/registry.go +++ b/registry/extension/distribution/registry.go @@ -7,6 +7,8 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" + "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" "gopkg.in/yaml.v2" @@ -32,7 +34,7 @@ type distributionOptions struct { } // newDistNamespace creates a new extension namespace with the name "distribution" -func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.Extension, error) { +func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Extension, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { @@ -65,26 +67,26 @@ func newDistNamespace(ctx context.Context, storageDriver driver.StorageDriver, o func init() { // register the extension namespace. - distribution.RegisterExtension(namespaceName, newDistNamespace) + extension.RegisterExtension(namespaceName, newDistNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. -func (o *distributionNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []distribution.ManifestHandler { +func (o *distributionNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []storage.ManifestHandler { // This extension doesn't extend any manifest store operations. - return []distribution.ManifestHandler{} + return []storage.ManifestHandler{} } -func (o *distributionNamespace) GetGarbageCollectionHandlers() []distribution.GCExtensionHandler { +func (o *distributionNamespace) GetGarbageCollectionHandlers() []storage.GCExtensionHandler { // This extension doesn't extend any garbage collection operations. - return []distribution.GCExtensionHandler{} + return []storage.GCExtensionHandler{} } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (d *distributionNamespace) GetRepositoryRoutes() []distribution.ExtensionRoute { - var routes []distribution.ExtensionRoute +func (d *distributionNamespace) GetRepositoryRoutes() []extension.ExtensionRoute { + var routes []extension.ExtensionRoute if d.manifestsEnabled { - routes = append(routes, distribution.ExtensionRoute{ + routes = append(routes, extension.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: manifestsComponentName, @@ -102,7 +104,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []distribution.ExtensionRo } if d.tagHistoryEnabled { - routes = append(routes, distribution.ExtensionRoute{ + routes = append(routes, extension.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: tagHistoryComponentName, @@ -135,7 +137,7 @@ func (d *distributionNamespace) GetRepositoryRoutes() []distribution.ExtensionRo // GetRegistryRoutes returns a list of extension routes scoped at a registry level // There are no registry scoped routes exposed by this namespace -func (d *distributionNamespace) GetRegistryRoutes() []distribution.ExtensionRoute { +func (d *distributionNamespace) GetRegistryRoutes() []extension.ExtensionRoute { return nil } @@ -154,7 +156,7 @@ func (d *distributionNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (d *distributionNamespace) tagHistoryDispatcher(ctx *distribution.ExtensionContext, r *http.Request) http.Handler { +func (d *distributionNamespace) tagHistoryDispatcher(ctx *extension.ExtensionContext, r *http.Request) http.Handler { tagHistoryHandler := &tagHistoryHandler{ ExtensionContext: ctx, storageDriver: d.storageDriver, @@ -165,7 +167,7 @@ func (d *distributionNamespace) tagHistoryDispatcher(ctx *distribution.Extension } } -func (d *distributionNamespace) manifestsDispatcher(ctx *distribution.ExtensionContext, r *http.Request) http.Handler { +func (d *distributionNamespace) manifestsDispatcher(ctx *extension.ExtensionContext, r *http.Request) http.Handler { manifestsHandler := &manifestHandler{ ExtensionContext: ctx, storageDriver: d.storageDriver, diff --git a/registry/extension/distribution/taghistory.go b/registry/extension/distribution/taghistory.go index 8caf8dce564..09c24bb6b9c 100644 --- a/registry/extension/distribution/taghistory.go +++ b/registry/extension/distribution/taghistory.go @@ -4,9 +4,9 @@ import ( "encoding/json" "net/http" - "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -20,7 +20,7 @@ type tagHistoryAPIResponse struct { // manifestHandler handles requests for manifests under a manifest name. type tagHistoryHandler struct { - *distribution.ExtensionContext + *extension.ExtensionContext storageDriver driver.StorageDriver } diff --git a/extension.go b/registry/extension/extension.go similarity index 80% rename from extension.go rename to registry/extension/extension.go index 1019a30e4fa..09fd2915b56 100644 --- a/extension.go +++ b/registry/extension/extension.go @@ -1,15 +1,16 @@ -package distribution +package extension import ( "context" "fmt" "net/http" + "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" - "github.com/opencontainers/go-digest" ) // ExtensionContext contains the request specific context for use in across handlers. @@ -17,9 +18,9 @@ type ExtensionContext struct { context.Context // Registry is the base namespace that is used by all extension namespaces - Registry Namespace + Registry distribution.Namespace // Repository is a reference to a named repository - Repository Repository + Repository distribution.Repository // Errors are the set of errors that occurred within this request context Errors errcode.Errors } @@ -41,37 +42,9 @@ type ExtensionRoute struct { Dispatcher RouteDispatchFunc } -type GCExtensionHandler interface { - Mark(ctx context.Context, - repository Repository, - storageDriver driver.StorageDriver, - registry Namespace, - manifest Manifest, - manifestDigest digest.Digest, - dryRun bool, - removeUntagged bool) (bool, error) - RemoveManifest(ctx context.Context, - storageDriver driver.StorageDriver, - registry Namespace, - dgst digest.Digest, - repositoryName string) error - SweepBlobs(ctx context.Context, - markSet map[digest.Digest]struct{}) map[digest.Digest]struct{} -} - -// ExtendedStorage defines extensions to store operations like manifest for example. -type ExtendedStorage interface { - // GetManifestHandlers returns the list of manifest handlers that handle custom manifest formats supported by the extensions. - GetManifestHandlers( - repo Repository, - blobStore BlobStore) []ManifestHandler - // GetGarbageCollectHandlers returns the list of GC handlers that handle custom garbage collection behavior for the extensions - GetGarbageCollectionHandlers() []GCExtensionHandler -} - // Extension is the interface that is used to define extensions to the distribution. type Extension interface { - ExtendedStorage + storage.ExtendedStorage // ExtensionService // GetRepositoryRoutes returns a list of extension routes scoped at a repository level GetRepositoryRoutes() []ExtensionRoute diff --git a/registry/extension/oci/discover.go b/registry/extension/oci/discover.go index d1d76762986..1d7b34e36cc 100644 --- a/registry/extension/oci/discover.go +++ b/registry/extension/oci/discover.go @@ -4,18 +4,18 @@ import ( "encoding/json" "net/http" - "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/registry/api/errcode" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage/driver" ) type discoverGetAPIResponse struct { - Extensions []distribution.EnumerateExtension `json:"extensions"` + Extensions []extension.EnumerateExtension `json:"extensions"` } // extensionHandler handles requests for manifests under a manifest name. type extensionHandler struct { - *distribution.ExtensionContext + *extension.ExtensionContext storageDriver driver.StorageDriver } @@ -25,7 +25,7 @@ func (eh *extensionHandler) getExtensions(w http.ResponseWriter, r *http.Request w.Header().Set("Content-Type", "application/json") // get list of extension information seperated at the namespace level - enumeratedExtensions := distribution.EnumerateRegistered(*eh.ExtensionContext) + enumeratedExtensions := extension.EnumerateRegistered(*eh.ExtensionContext) // remove the oci extension so it's not returned by discover for i, e := range enumeratedExtensions { diff --git a/registry/extension/oci/oci.go b/registry/extension/oci/oci.go index 01375626704..76312d9c288 100644 --- a/registry/extension/oci/oci.go +++ b/registry/extension/oci/oci.go @@ -7,6 +7,8 @@ import ( "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" + "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" "gopkg.in/yaml.v2" @@ -30,7 +32,7 @@ type ociOptions struct { } // newOciNamespace creates a new extension namespace with the name "oci" -func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.Extension, error) { +func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Extension, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err @@ -58,26 +60,26 @@ func newOciNamespace(ctx context.Context, storageDriver driver.StorageDriver, op func init() { // register the extension namespace. - distribution.RegisterExtension(namespaceName, newOciNamespace) + extension.RegisterExtension(namespaceName, newOciNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. -func (o *ociNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []distribution.ManifestHandler { +func (o *ociNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []storage.ManifestHandler { // This extension doesn't extend any manifest store operations. - return []distribution.ManifestHandler{} + return []storage.ManifestHandler{} } -func (o *ociNamespace) GetGarbageCollectionHandlers() []distribution.GCExtensionHandler { +func (o *ociNamespace) GetGarbageCollectionHandlers() []storage.GCExtensionHandler { // This extension doesn't extend any garbage collection operations. - return []distribution.GCExtensionHandler{} + return []storage.GCExtensionHandler{} } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (o *ociNamespace) GetRepositoryRoutes() []distribution.ExtensionRoute { - var routes []distribution.ExtensionRoute +func (o *ociNamespace) GetRepositoryRoutes() []extension.ExtensionRoute { + var routes []extension.ExtensionRoute if o.discoverEnabled { - routes = append(routes, distribution.ExtensionRoute{ + routes = append(routes, extension.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: discoverComponentName, @@ -98,11 +100,11 @@ func (o *ociNamespace) GetRepositoryRoutes() []distribution.ExtensionRoute { } // GetRegistryRoutes returns a list of extension routes scoped at a registry level -func (o *ociNamespace) GetRegistryRoutes() []distribution.ExtensionRoute { - var routes []distribution.ExtensionRoute +func (o *ociNamespace) GetRegistryRoutes() []extension.ExtensionRoute { + var routes []extension.ExtensionRoute if o.discoverEnabled { - routes = append(routes, distribution.ExtensionRoute{ + routes = append(routes, extension.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: discoverComponentName, @@ -137,7 +139,7 @@ func (o *ociNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (o *ociNamespace) discoverDispatcher(ctx *distribution.ExtensionContext, r *http.Request) http.Handler { +func (o *ociNamespace) discoverDispatcher(ctx *extension.ExtensionContext, r *http.Request) http.Handler { extensionHandler := &extensionHandler{ ExtensionContext: ctx, storageDriver: o.storageDriver, diff --git a/registry/extension/oras/artifactgarbagecollectionhandler_test.go b/registry/extension/oras/artifactgarbagecollectionhandler_test.go index 2357904f001..26ef3e47380 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler_test.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler_test.go @@ -48,7 +48,7 @@ func allBlobs(t *testing.T, registry distribution.Namespace) map[digest.Digest]s func TestReferrersBlobsDeleted(t *testing.T) { ctx := context.Background() inmemoryDriver := inmemory.New() - registry := createRegistry(t, inmemoryDriver) + registry, orasExtension := createRegistry(t, inmemoryDriver) repo := makeRepository(t, registry, "test") manifestService := makeManifestService(t, repo) tagService := repo.Tags(ctx) @@ -140,8 +140,9 @@ func TestReferrersBlobsDeleted(t *testing.T) { // Run GC err = storage.MarkAndSweep(ctx, inmemoryDriver, registry, storage.GCOpts{ - DryRun: false, - RemoveUntagged: true, + DryRun: false, + RemoveUntagged: true, + GCExtensionHandlers: orasExtension.GetGarbageCollectionHandlers(), }) if err != nil { t.Fatalf("Failed mark and sweep: %v", err) diff --git a/registry/extension/oras/artifactmanifesthandler_test.go b/registry/extension/oras/artifactmanifesthandler_test.go index 0ac8ed93ce6..076104270fd 100644 --- a/registry/extension/oras/artifactmanifesthandler_test.go +++ b/registry/extension/oras/artifactmanifesthandler_test.go @@ -9,6 +9,7 @@ import ( "github.com/distribution/distribution/v3/manifest" "github.com/distribution/distribution/v3/manifest/schema2" "github.com/distribution/distribution/v3/reference" + "github.com/distribution/distribution/v3/registry/extension" storage "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" @@ -16,13 +17,13 @@ import ( orasartifacts "github.com/oras-project/artifacts-spec/specs-go/v1" ) -func createRegistry(t *testing.T, driver driver.StorageDriver, options ...storage.RegistryOption) distribution.Namespace { +func createRegistry(t *testing.T, driver driver.StorageDriver, options ...storage.RegistryOption) (distribution.Namespace, extension.Extension) { ctx := context.Background() options = append([]storage.RegistryOption{storage.EnableDelete}, options...) extensionConfig := OrasOptions{ ArtifactsExtComponents: []string{"referrers"}, } - ns, err := distribution.GetExtension(ctx, "oras", driver, extensionConfig) + ns, err := extension.GetExtension(ctx, "oras", driver, extensionConfig) if err != nil { t.Fatalf("unable to configure extension namespace (%s): %s", "oras", err) } @@ -31,7 +32,7 @@ func createRegistry(t *testing.T, driver driver.StorageDriver, options ...storag if err != nil { t.Fatalf("failed to construct namespace") } - return registry + return registry, ns } func makeRepository(t *testing.T, registry distribution.Namespace, name string) distribution.Repository { @@ -61,7 +62,7 @@ func makeManifestService(t *testing.T, repository distribution.Repository) distr func TestVerifyArtifactManifestPut(t *testing.T) { ctx := context.Background() inmemoryDriver := inmemory.New() - registry := createRegistry(t, inmemoryDriver) + registry, _ := createRegistry(t, inmemoryDriver) repo := makeRepository(t, registry, "test") manifestService := makeManifestService(t, repo) diff --git a/registry/extension/oras/artifactservice.go b/registry/extension/oras/artifactservice.go index 7df402e1a06..cf4ede0922a 100644 --- a/registry/extension/oras/artifactservice.go +++ b/registry/extension/oras/artifactservice.go @@ -9,6 +9,7 @@ import ( "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" artifactv1 "github.com/oras-project/artifacts-spec/specs-go/v1" @@ -20,7 +21,7 @@ type ArtifactService interface { // referrersHandler handles http operations on manifest referrers. type referrersHandler struct { - extContext *distribution.ExtensionContext + extContext *extension.ExtensionContext storageDriver driver.StorageDriver // Digest is the target manifest's digest. diff --git a/registry/extension/oras/oras.go b/registry/extension/oras/oras.go index 7259e6448ad..5c54c7d272f 100644 --- a/registry/extension/oras/oras.go +++ b/registry/extension/oras/oras.go @@ -8,6 +8,8 @@ import ( "github.com/distribution/distribution/v3/configuration" dcontext "github.com/distribution/distribution/v3/context" v2 "github.com/distribution/distribution/v3/registry/api/v2" + "github.com/distribution/distribution/v3/registry/extension" + "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/gorilla/handlers" "github.com/opencontainers/go-digest" @@ -33,7 +35,7 @@ type OrasOptions struct { } // newOrasNamespace creates a new extension namespace with the name "oras" -func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (distribution.Extension, error) { +func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (extension.Extension, error) { optionsYaml, err := yaml.Marshal(options) if err != nil { return nil, err @@ -66,13 +68,13 @@ func newOrasNamespace(ctx context.Context, storageDriver driver.StorageDriver, o } func init() { - distribution.RegisterExtension(namespaceName, newOrasNamespace) + extension.RegisterExtension(namespaceName, newOrasNamespace) } // GetManifestHandlers returns a list of manifest handlers that will be registered in the manifest store. -func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []distribution.ManifestHandler { +func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobStore distribution.BlobStore) []storage.ManifestHandler { if o.referrersEnabled { - return []distribution.ManifestHandler{ + return []storage.ManifestHandler{ &artifactManifestHandler{ repository: repo, blobStore: blobStore, @@ -80,25 +82,25 @@ func (o *orasNamespace) GetManifestHandlers(repo distribution.Repository, blobSt }} } - return []distribution.ManifestHandler{} + return []storage.ManifestHandler{} } -func (o *orasNamespace) GetGarbageCollectionHandlers() []distribution.GCExtensionHandler { +func (o *orasNamespace) GetGarbageCollectionHandlers() []storage.GCExtensionHandler { if o.referrersEnabled { - return []distribution.GCExtensionHandler{ + return []storage.GCExtensionHandler{ &o.gcHandler, } } - return []distribution.GCExtensionHandler{} + return []storage.GCExtensionHandler{} } // GetRepositoryRoutes returns a list of extension routes scoped at a repository level -func (d *orasNamespace) GetRepositoryRoutes() []distribution.ExtensionRoute { - var routes []distribution.ExtensionRoute +func (d *orasNamespace) GetRepositoryRoutes() []extension.ExtensionRoute { + var routes []extension.ExtensionRoute if d.referrersEnabled { - routes = append(routes, distribution.ExtensionRoute{ + routes = append(routes, extension.ExtensionRoute{ Namespace: namespaceName, Extension: extensionName, Component: referrersComponentName, @@ -120,7 +122,7 @@ func (d *orasNamespace) GetRepositoryRoutes() []distribution.ExtensionRoute { // GetRegistryRoutes returns a list of extension routes scoped at a registry level // There are no registry scoped routes exposed by this namespace -func (d *orasNamespace) GetRegistryRoutes() []distribution.ExtensionRoute { +func (d *orasNamespace) GetRegistryRoutes() []extension.ExtensionRoute { return nil } @@ -139,7 +141,7 @@ func (d *orasNamespace) GetNamespaceDescription() string { return namespaceDescription } -func (o *orasNamespace) referrersDispatcher(extCtx *distribution.ExtensionContext, r *http.Request) http.Handler { +func (o *orasNamespace) referrersDispatcher(extCtx *extension.ExtensionContext, r *http.Request) http.Handler { handler := &referrersHandler{ storageDriver: o.storageDriver, diff --git a/registry/handlers/app.go b/registry/handlers/app.go index 9622b4f156c..0e37e14bc60 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -28,6 +28,7 @@ import ( "github.com/distribution/distribution/v3/registry/api/errcode" v2 "github.com/distribution/distribution/v3/registry/api/v2" "github.com/distribution/distribution/v3/registry/auth" + "github.com/distribution/distribution/v3/registry/extension" registrymiddleware "github.com/distribution/distribution/v3/registry/middleware/registry" repositorymiddleware "github.com/distribution/distribution/v3/registry/middleware/repository" "github.com/distribution/distribution/v3/registry/proxy" @@ -97,7 +98,7 @@ type App struct { repositoryExtensions []string // extensionNamespaces is a list of namespaces that are configured as extensions to the distribution - extensionNamespaces []distribution.Extension + extensionNamespaces []extension.Extension } // NewApp takes a configuration and returns a configured app, ready to serve @@ -926,9 +927,9 @@ func (app *App) nameRequired(r *http.Request) bool { func (app *App) initializeExtensionNamespaces(ctx context.Context, extensions map[string]configuration.ExtensionConfig) error { - extensionNamespaces := []distribution.Extension{} + extensionNamespaces := []extension.Extension{} for key, options := range extensions { - ns, err := distribution.GetExtension(ctx, key, app.driver, options) + ns, err := extension.GetExtension(ctx, key, app.driver, options) if err != nil { return fmt.Errorf("unable to configure extension namespace (%s): %s", key, err) } @@ -978,7 +979,7 @@ func (app *App) registerExtensionRoutes(ctx context.Context) error { return nil } -func (app *App) registerExtensionRoute(route distribution.ExtensionRoute, nameRequired bool) error { +func (app *App) registerExtensionRoute(route extension.ExtensionRoute, nameRequired bool) error { if route.Dispatcher == nil { return nil } @@ -996,7 +997,7 @@ func (app *App) registerExtensionRoute(route distribution.ExtensionRoute, nameRe dispatch := route.Dispatcher app.register(desc.Name, func(ctx *Context, r *http.Request) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - extCtx := &distribution.ExtensionContext{ + extCtx := &extension.ExtensionContext{ Context: ctx.Context, Repository: ctx.Repository, Errors: ctx.Errors, diff --git a/registry/proxy/proxyregistry.go b/registry/proxy/proxyregistry.go index e84ff9a2707..00f560daa1b 100644 --- a/registry/proxy/proxyregistry.go +++ b/registry/proxy/proxyregistry.go @@ -35,7 +35,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name return nil, err } - v := storage.NewVacuum(ctx, driver, registry) + v := storage.NewVacuum(ctx, driver) s := scheduler.New(ctx, driver, "/scheduler-state.json") s.OnBlobExpire(func(ref reference.Reference) error { var r reference.Canonical @@ -189,10 +189,6 @@ func (pr *proxyingRegistry) BlobStatter() distribution.BlobStatter { return pr.embedded.BlobStatter() } -func (pr *proxyingRegistry) Extensions() []distribution.Extension { - return []distribution.Extension{} -} - // authChallenger encapsulates a request to the upstream to establish credential challenges type authChallenger interface { tryEstablishChallenges(context.Context) error diff --git a/registry/root.go b/registry/root.go index b4ba205f32e..141366b294f 100644 --- a/registry/root.go +++ b/registry/root.go @@ -4,8 +4,8 @@ import ( "fmt" "os" - "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" + "github.com/distribution/distribution/v3/registry/extension" "github.com/distribution/distribution/v3/registry/storage" "github.com/distribution/distribution/v3/registry/storage/driver/factory" "github.com/distribution/distribution/v3/version" @@ -73,14 +73,16 @@ var GCCmd = &cobra.Command{ } extensions := config.Extensions - extensionNamespaces := []distribution.Extension{} + extensionNamespaces := []extension.Extension{} + gcExtensionhandlers := []storage.GCExtensionHandler{} for key, options := range extensions { - ns, err := distribution.GetExtension(ctx, key, driver, options) + ns, err := extension.GetExtension(ctx, key, driver, options) if err != nil { fmt.Fprintf(os.Stderr, "unable to configure extension namespace (%s): %s", key, err) os.Exit(1) } extensionNamespaces = append(extensionNamespaces, ns) + gcExtensionhandlers = append(gcExtensionhandlers, ns.GetGarbageCollectionHandlers()...) } options := []storage.RegistryOption{storage.Schema1SigningKey(k)} @@ -96,8 +98,9 @@ var GCCmd = &cobra.Command{ } err = storage.MarkAndSweep(ctx, driver, registry, storage.GCOpts{ - DryRun: dryRun, - RemoveUntagged: removeUntagged, + DryRun: dryRun, + RemoveUntagged: removeUntagged, + GCExtensionHandlers: gcExtensionhandlers, }) if err != nil { fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err) diff --git a/registry/storage/extension.go b/registry/storage/extension.go index f2a78c3ac0d..ae85b7dcc98 100644 --- a/registry/storage/extension.go +++ b/registry/storage/extension.go @@ -4,6 +4,7 @@ import ( "context" "github.com/distribution/distribution/v3" + "github.com/distribution/distribution/v3/registry/storage/driver" storagedriver "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" ) @@ -15,6 +16,34 @@ type ReadOnlyBlobStore interface { distribution.BlobProvider } +type GCExtensionHandler interface { + Mark(ctx context.Context, + repository distribution.Repository, + storageDriver driver.StorageDriver, + registry distribution.Namespace, + manifest distribution.Manifest, + manifestDigest digest.Digest, + dryRun bool, + removeUntagged bool) (bool, error) + RemoveManifest(ctx context.Context, + storageDriver driver.StorageDriver, + registry distribution.Namespace, + dgst digest.Digest, + repositoryName string) error + SweepBlobs(ctx context.Context, + markSet map[digest.Digest]struct{}) map[digest.Digest]struct{} +} + +// ExtendedStorage defines extensions to store operations like manifest for example. +type ExtendedStorage interface { + // GetManifestHandlers returns the list of manifest handlers that handle custom manifest formats supported by the extensions. + GetManifestHandlers( + repo distribution.Repository, + blobStore distribution.BlobStore) []ManifestHandler + // GetGarbageCollectHandlers returns the list of GC handlers that handle custom garbage collection behavior for the extensions + GetGarbageCollectionHandlers() []GCExtensionHandler +} + // GetManifestLinkReadOnlyBlobStore will enable extensions to access the underlying linked blob store for readonly operations. // This blob store is scoped only to manifest link paths. Manifest link paths doesn't use blob cache func GetManifestLinkReadOnlyBlobStore( diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 91da5a915a3..6e61707e4ce 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -16,8 +16,9 @@ func emit(format string, a ...interface{}) { // GCOpts contains options for garbage collector type GCOpts struct { - DryRun bool - RemoveUntagged bool + DryRun bool + RemoveUntagged bool + GCExtensionHandlers []GCExtensionHandler } // ManifestDel contains manifest structure which will be deleted @@ -80,16 +81,13 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("failed to retrieve manifest: %v", err) } - for _, extNamespace := range registry.Extensions() { - handlers := extNamespace.GetGarbageCollectionHandlers() - for _, gcHandler := range handlers { - deleteEligible, err := gcHandler.Mark(ctx, repository, storageDriver, registry, manifest, dgst, opts.DryRun, opts.RemoveUntagged) - if err != nil { - return fmt.Errorf("failed to mark using extension handler: %v", err) - } - if !deleteEligible { - return nil - } + for _, gcHandler := range opts.GCExtensionHandlers { + deleteEligible, err := gcHandler.Mark(ctx, repository, storageDriver, registry, manifest, dgst, opts.DryRun, opts.RemoveUntagged) + if err != nil { + return fmt.Errorf("failed to mark using extension handler: %v", err) + } + if !deleteEligible { + return nil } } manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) @@ -132,30 +130,24 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis } // sweep - vacuum := NewVacuum(ctx, storageDriver, registry) + vacuum := NewVacuum(ctx, storageDriver) if !opts.DryRun { for _, obj := range manifestArr { err = vacuum.RemoveManifest(obj.Name, obj.Digest, obj.Tags) if err != nil { return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) } - for _, extNamespace := range registry.Extensions() { - handlers := extNamespace.GetGarbageCollectionHandlers() - for _, gcHandler := range handlers { - err := gcHandler.RemoveManifest(ctx, storageDriver, registry, obj.Digest, obj.Name) - if err != nil { - return fmt.Errorf("failed to call remove manifest extension handler: %v", err) - } + for _, gcHandler := range opts.GCExtensionHandlers { + err := gcHandler.RemoveManifest(ctx, storageDriver, registry, obj.Digest, obj.Name) + if err != nil { + return fmt.Errorf("failed to call remove manifest extension handler: %v", err) } } } } // GC extension will add final saved blobs into markset for retention - for _, extNamespace := range registry.Extensions() { - handlers := extNamespace.GetGarbageCollectionHandlers() - for _, gcHandler := range handlers { - markSet = gcHandler.SweepBlobs(ctx, markSet) - } + for _, gcHandler := range opts.GCExtensionHandlers { + markSet = gcHandler.SweepBlobs(ctx, markSet) } blobService := registry.Blobs() deleteSet := make(map[digest.Digest]struct{}) diff --git a/registry/storage/manifestlisthandler.go b/registry/storage/manifestlisthandler.go index eca20a69e5c..e9c71d4c0d9 100644 --- a/registry/storage/manifestlisthandler.go +++ b/registry/storage/manifestlisthandler.go @@ -17,7 +17,7 @@ type manifestListHandler struct { ctx context.Context } -var _ distribution.ManifestHandler = &manifestListHandler{} +var _ ManifestHandler = &manifestListHandler{} func (ms *manifestListHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ms.ctx).Debug("(*manifestListHandler).Unmarshal") diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index e529bdd4c6f..a4c7ec0c5e2 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -16,6 +16,15 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" ) +// A ManifestHandler gets and puts manifests of a particular type. +type ManifestHandler interface { + // Unmarshal unmarshals the manifest from a byte slice. + Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) + + // Put creates or updates the given manifest returning the manifest digest. + Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) +} + // SkipLayerVerification allows a manifest to be Put before its // layers are on the filesystem func SkipLayerVerification() distribution.ManifestServiceOption { @@ -39,12 +48,12 @@ type manifestStore struct { skipDependencyVerification bool - schema1Handler distribution.ManifestHandler - schema2Handler distribution.ManifestHandler - ocischemaHandler distribution.ManifestHandler - manifestListHandler distribution.ManifestHandler + schema1Handler ManifestHandler + schema2Handler ManifestHandler + ocischemaHandler ManifestHandler + manifestListHandler ManifestHandler - extensionManifestHandlers []distribution.ManifestHandler + extensionManifestHandlers []ManifestHandler } var _ distribution.ManifestService = &manifestStore{} diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index 661480f4121..2735a03eca7 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -20,7 +20,7 @@ type ocischemaManifestHandler struct { manifestURLs manifestURLs } -var _ distribution.ManifestHandler = &ocischemaManifestHandler{} +var _ ManifestHandler = &ocischemaManifestHandler{} func (ms *ocischemaManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Unmarshal") diff --git a/registry/storage/registry.go b/registry/storage/registry.go index b82bcea707d..0e31827e466 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -25,7 +25,7 @@ type registry struct { blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory manifestURLs manifestURLs driver storagedriver.StorageDriver - extendedNamespaces []distribution.Extension + extendedNamespaces []ExtendedStorage } // manifestURLs holds regular expressions for controlling manifest URL whitelisting @@ -39,7 +39,7 @@ type RegistryOption func(*registry) error // AddExtendedNamespace is a functional option for NewRegistry. It adds the given // extended namespace to the list of extended namespaces in the registry. -func AddExtendedNamespace(extendedNamespace distribution.Extension) RegistryOption { +func AddExtendedNamespace(extendedNamespace ExtendedStorage) RegistryOption { return func(registry *registry) error { registry.extendedNamespaces = append(registry.extendedNamespaces, extendedNamespace) return nil @@ -199,10 +199,6 @@ func (reg *registry) BlobStatter() distribution.BlobStatter { return reg.statter } -func (reg *registry) Extensions() []distribution.Extension { - return reg.extendedNamespaces -} - // repository provides name-scoped access to various services. type repository struct { *registry @@ -261,7 +257,7 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M linkDirectoryPathSpec: manifestDirectoryPathSpec, } - var v1Handler distribution.ManifestHandler + var v1Handler ManifestHandler if repo.schema1Enabled { v1Handler = &signedManifestHandler{ ctx: ctx, @@ -280,7 +276,7 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M } } - var extensionManifestHandlers []distribution.ManifestHandler + var extensionManifestHandlers []ManifestHandler for _, ext := range repo.registry.extendedNamespaces { handlers := ext.GetManifestHandlers(repo, blobStore) if len(handlers) > 0 { diff --git a/registry/storage/schema2manifesthandler.go b/registry/storage/schema2manifesthandler.go index 94b3215c85e..023427c1328 100644 --- a/registry/storage/schema2manifesthandler.go +++ b/registry/storage/schema2manifesthandler.go @@ -26,7 +26,7 @@ type schema2ManifestHandler struct { manifestURLs manifestURLs } -var _ distribution.ManifestHandler = &schema2ManifestHandler{} +var _ ManifestHandler = &schema2ManifestHandler{} func (ms *schema2ManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ms.ctx).Debug("(*schema2ManifestHandler).Unmarshal") diff --git a/registry/storage/signedmanifesthandler.go b/registry/storage/signedmanifesthandler.go index 9c8f9ca366e..eb5b5742adb 100644 --- a/registry/storage/signedmanifesthandler.go +++ b/registry/storage/signedmanifesthandler.go @@ -22,7 +22,7 @@ type signedManifestHandler struct { ctx context.Context } -var _ distribution.ManifestHandler = &signedManifestHandler{} +var _ ManifestHandler = &signedManifestHandler{} func (ms *signedManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { dcontext.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Unmarshal") diff --git a/registry/storage/v1unsupportedhandler.go b/registry/storage/v1unsupportedhandler.go index 9aef69497c9..8170646bc5d 100644 --- a/registry/storage/v1unsupportedhandler.go +++ b/registry/storage/v1unsupportedhandler.go @@ -10,10 +10,10 @@ import ( // signedManifestHandler is a ManifestHandler that unmarshals v1 manifests but // refuses to Put v1 manifests type v1UnsupportedHandler struct { - innerHandler distribution.ManifestHandler + innerHandler ManifestHandler } -var _ distribution.ManifestHandler = &v1UnsupportedHandler{} +var _ ManifestHandler = &v1UnsupportedHandler{} func (v *v1UnsupportedHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { return v.innerHandler.Unmarshal(ctx, dgst, content) diff --git a/registry/storage/vacuum.go b/registry/storage/vacuum.go index 604f2dc9c16..112237754e3 100644 --- a/registry/storage/vacuum.go +++ b/registry/storage/vacuum.go @@ -4,7 +4,6 @@ import ( "context" "path" - "github.com/distribution/distribution/v3" dcontext "github.com/distribution/distribution/v3/context" "github.com/distribution/distribution/v3/registry/storage/driver" "github.com/opencontainers/go-digest" @@ -16,19 +15,17 @@ import ( // https://en.wikipedia.org/wiki/Consistency_model // NewVacuum creates a new Vacuum -func NewVacuum(ctx context.Context, driver driver.StorageDriver, registry distribution.Namespace) Vacuum { +func NewVacuum(ctx context.Context, driver driver.StorageDriver) Vacuum { return Vacuum{ - ctx: ctx, - driver: driver, - registry: registry, + ctx: ctx, + driver: driver, } } // Vacuum removes content from the filesystem type Vacuum struct { - driver driver.StorageDriver - ctx context.Context - registry distribution.Namespace + driver driver.StorageDriver + ctx context.Context } // RemoveBlob removes a blob from the filesystem From 36277103fc449ac10c8c9f615d64d59cbf875b06 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Wed, 3 Aug 2022 15:54:07 -0700 Subject: [PATCH 20/22] updating docs; merging latest changes Signed-off-by: Akash Singhal --- docs/extensions.md | 58 +++++-------------- .../oras/artifactgarbagecollectionhandler.go | 2 +- .../artifactgarbagecollectionhandler_test.go | 2 +- registry/extension/oras/referrers.md | 4 +- 4 files changed, 19 insertions(+), 47 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index ddd3701107a..e3b680885f9 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -8,11 +8,11 @@ This document serves as a high level discussion of the implementation of the ext ## Extension Interface -The `Extension` interface is introduced in the `distribution` package. It defines methods to access the extension's namespace-specific attributes such as the Name, Url defining the extension namespace, and the Description of the namespace. It defines route enumeration at the Registry and Repository level. It also encases the `ExtendedStorage` interface which defines the methods requires to extend the underlying storage functionality of the registry. +The `Extension` interface is introduced in the new `extension` package. It defines methods to access the extension's namespace-specific attributes such as the Name, Url defining the extension namespace, and the Description of the namespace. It defines route enumeration at the Registry and Repository level. It also encases the `ExtendedStorage` interface which defines the methods requires to extend the underlying storage functionality of the registry. ``` type Extension interface { - ExtendedStorage + storage.ExtendedStorage // GetRepositoryRoutes returns a list of extension routes scoped at a repository level GetRepositoryRoutes() []ExtensionRoute // GetRegistryRoutes returns a list of extension routes scoped at a registry level @@ -26,39 +26,8 @@ type Extension interface { } ``` -The `Namespace` interface in the `distribution` package is modified to return a list of `Extensions` registered to the `Namespace` - -``` -type Namespace interface { - // Scope describes the names that can be used with this Namespace. The - // global namespace will have a scope that matches all names. The scope - // effectively provides an identity for the namespace. - Scope() Scope - - // Repository should return a reference to the named repository. The - // registry may or may not have the repository but should always return a - // reference. - Repository(ctx context.Context, name reference.Named) (Repository, error) - - // Repositories fills 'repos' with a lexicographically sorted catalog of repositories - // up to the size of 'repos' and returns the value 'n' for the number of entries - // which were filled. 'last' contains an offset in the catalog, and 'err' will be - // set to io.EOF if there are no more entries to obtain. - Repositories(ctx context.Context, repos []string, last string) (n int, err error) - - // Blobs returns a blob enumerator to access all blobs - Blobs() BlobEnumerator - - // BlobStatter returns a BlobStatter to control - BlobStatter() BlobStatter - - // Extensions returns a list of Extension registered to the Namespace - Extensions() []Extension -} -``` - The `ExtendedStorage` interface defines methods that specify storage-specific handlers. Each extension will implement a handler extending the functionality. The interface can be expanded in the future to consider new handler types. -`GetManifestHandlers` is used to return new `ManifestHandlers` defined by each of the extensions. (Note: To support this interface in the `distribution` package, the `ManifestHandlers` interface has been moved to the `distribution` package) +`GetManifestHandlers` is used to return new `ManifestHandlers` defined by each of the extensions. `GetGarbageCollectionHandlers` is used to return `GCExtensionHandler` implemented by each extension. ``` @@ -67,27 +36,30 @@ type ExtendedStorage interface { GetManifestHandlers( repo Repository, blobStore BlobStore) []ManifestHandler - // GetGarbageCollectHandler returns the GCExtensionHandler that handles custom garbage collection behavior for the extension. - GetGarbageCollectionHandler() GCExtensionHandler + // GetGarbageCollectHandlers returns the GCExtensionHandlers that handles custom garbage collection behavior for the extension. + GetGarbageCollectionHandlers() []GCExtensionHandler } ``` -The `GCExtensionHandler` interface defines three methods that are used in the garbage colection mark and sweep process. The `Mark` method is invoked for each `GCExtensionHandler` after the existing mark process finishes in `MarkAndSweep`. `IsEligibleForDeletion` is used to define if a specific manifest set for deletion in `MarkAndSweep` should be eligible for deletion. Extensions may choose to special case certain manifest types in manifest deletion. `RemoveManifestVacuum` is invoked to extend the `RemoveManifest` functionality for the `Vacuum`. New or special-cased manifests may require custom manifest deletion which can be defined with this method. +The `GCExtensionHandler` interface defines three methods that are used in the garbage colection mark and sweep process. The `Mark` method is invoked for each `GCExtensionHandler` after the existing mark process finishes in `MarkAndSweep`. It is used to determine if the manifest and blobs should have their temporary ref count incremented in the case of an artifact manifest, or if the manifest and it's referrers should be recursively indexed for deletion in the case of a non-artifact manifest. `RemoveManifest` is invoked to extend the `RemoveManifest` functionality for the `Vacuum`. New or special-cased manifests may require custom manifest deletion which can be defined with this method. `SweepBlobs` is used to add artifact manifest/blobs to the original `markSet`. These blobs are retained after determining their ref count is still positive. ``` type GCExtensionHandler interface { Mark(ctx context.Context, + repository distribution.Repository, storageDriver driver.StorageDriver, - registry Namespace, + registry distribution.Namespace, + manifest distribution.Manifest, + manifestDigest digest.Digest, dryRun bool, - removeUntagged bool) (map[digest.Digest]struct{}, error) - RemoveManifestVacuum(ctx context.Context, + removeUntagged bool) (bool, error) + RemoveManifest(ctx context.Context, storageDriver driver.StorageDriver, + registry distribution.Namespace, dgst digest.Digest, repositoryName string) error - IsEligibleForDeletion(ctx context.Context, - dgst digest.Digest, - manifestService ManifestService) (bool, error) + SweepBlobs(ctx context.Context, + markSet map[digest.Digest]struct{}) map[digest.Digest]struct{} } ``` diff --git a/registry/extension/oras/artifactgarbagecollectionhandler.go b/registry/extension/oras/artifactgarbagecollectionhandler.go index 3c5536dfc49..c7410985218 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler.go @@ -107,7 +107,7 @@ func (gc *orasGCHandler) RemoveManifest(ctx context.Context, storageDriver drive // extract the reference blobs := artifactManifest.References() - // decrement refcount for the blobs digests' and the manifest digest + // decrement refcount for the blobs' digests and the manifest digest gc.artifactMarkSet[artifactDigest] -= 1 fmt.Printf("%s: decrementing artifact manifest ref count %s\n", repositoryName, dgst) for _, descriptor := range blobs { diff --git a/registry/extension/oras/artifactgarbagecollectionhandler_test.go b/registry/extension/oras/artifactgarbagecollectionhandler_test.go index 26ef3e47380..aaca7bffbe4 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler_test.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler_test.go @@ -105,7 +105,7 @@ func TestReferrersBlobsDeleted(t *testing.T) { Blobs: []orasartifacts.Descriptor{ artifactBlobDescriptor, }, - Subject: orasartifacts.Descriptor{ + Subject: &orasartifacts.Descriptor{ MediaType: schema2.MediaTypeManifest, Size: int64(len(dmPayload)), Digest: dg, diff --git a/registry/extension/oras/referrers.md b/registry/extension/oras/referrers.md index 1ee5e0bca61..194baf14a0d 100644 --- a/registry/extension/oras/referrers.md +++ b/registry/extension/oras/referrers.md @@ -98,6 +98,6 @@ This results in an addition to the index as shown below. The life of a referrer artifact is directly linked to its subject. When a referrer artifact's subject manifest is deleted, the artifact's referrers are also deleted. -Manifest garbage collection is extended to include referrer artifact collection. During the marking process, each manifest and its blobs, regardless of it being an ORAS artifact manifest, are marked. If a non-ORAS manifest is found to be untagged and untag deletion is enabled, the manifest is queried for any referrer artifacts by enumerating the link files at the path `repositories//_refs/subjects/sha256/`. For each artifact, the artifact manifest and its blobs are un-marked, so that they are deleted. +Manifest garbage collection is extended to include referrer artifact collection. The marking process begins with the normal marking behavior which consists of enumerating every manifest in every repository. If the manifest is untagged, we must consider the manifest for deletion. As we cannot guarantee that artifact manifests (tagged or untagged) will be traversed before their subjects, we must temporarily mark all artifact manifests and their blobs using a separate reference count map. If we encounter an untagged non-artifact manifest, then we proceed by adding the manifest to a deletion list, traversing it's referrers, and then indexing each artifact manifest for deletion. -During manifest link deletion, the revision link files of the indexed artifact manifests as well as the corresponding `_refs` are removed from storage. \ No newline at end of file +During the Sweep phase, each manifest in the deletion list has its contents and link files deleted. Then each of the indexed artifact manifests referring to the deleted subject will have its corresponding manifest and blobs' ref counts decremented. Furthermore, the artifact manifest revision, and `_refs` directories are removed. The final step is the vacuum of the blobs. Based on the final ref count map, we add each blob with a positive ref count back to the original `markSet` map. All unmarked blobs are then safely deleted. \ No newline at end of file From 7985eeaeb4be4499b48aacb733671cc58c930905 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Tue, 9 Aug 2022 13:11:46 -0700 Subject: [PATCH 21/22] address commits Signed-off-by: Akash Singhal --- registry/extension/extension.go | 3 --- registry/extension/oras/artifactgarbagecollectionhandler.go | 6 ++++-- registry/extension/oras/artifactmanifesthandler.go | 4 ++++ registry/storage/extension.go | 2 +- registry/storage/garbagecollect.go | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/registry/extension/extension.go b/registry/extension/extension.go index 671d0da71d5..a589b5bea26 100644 --- a/registry/extension/extension.go +++ b/registry/extension/extension.go @@ -58,9 +58,6 @@ type Extension interface { GetNamespaceDescription() string } -type ExtensionService interface { -} - // InitExtension is the initialize function for creating the extension namespace type InitExtension func(ctx context.Context, storageDriver driver.StorageDriver, options configuration.ExtensionConfig) (Extension, error) diff --git a/registry/extension/oras/artifactgarbagecollectionhandler.go b/registry/extension/oras/artifactgarbagecollectionhandler.go index c7410985218..dfeff97ba17 100644 --- a/registry/extension/oras/artifactgarbagecollectionhandler.go +++ b/registry/extension/oras/artifactgarbagecollectionhandler.go @@ -26,7 +26,6 @@ func (gc *orasGCHandler) Mark(ctx context.Context, dgst digest.Digest, dryRun bool, removeUntagged bool) (bool, error) { - //markSet := make(map[digest.Digest]struct{}) blobStatter := registry.BlobStatter() mediaType, _, err := manifest.Payload() if err != nil { @@ -48,6 +47,9 @@ func (gc *orasGCHandler) Mark(ctx context.Context, } return false, nil } else { + // TODO: Add support for untagged root artifact manifest (no subject specified) + // Subsequently, extend GC support for sweeping all referrers to untagged root artifact manifest + // if the manifest passed isn't an an artifact -> call the sweep ingestor // find all artifacts linked to manifest and add to artifactManifestIndex for subsequent deletion gc.artifactManifestIndex[dgst] = make([]digest.Digest, 0) @@ -71,7 +73,7 @@ func (gc *orasGCHandler) Mark(ctx context.Context, } } -func (gc *orasGCHandler) RemoveManifest(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dgst digest.Digest, repositoryName string) error { +func (gc *orasGCHandler) OnManifestDelete(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dgst digest.Digest, repositoryName string) error { referrerRootPath := referrersLinkPath(repositoryName) fullArtifactManifestPath := path.Join(referrerRootPath, dgst.Algorithm().String(), dgst.Hex()) dcontext.GetLogger(ctx).Infof("deleting manifest ref folder: %s", fullArtifactManifestPath) diff --git a/registry/extension/oras/artifactmanifesthandler.go b/registry/extension/oras/artifactmanifesthandler.go index 98605ddff74..27119c340a6 100644 --- a/registry/extension/oras/artifactmanifesthandler.go +++ b/registry/extension/oras/artifactmanifesthandler.go @@ -158,14 +158,18 @@ func (amh *artifactManifestHandler) indexReferrers(ctx context.Context, dm Deser return nil } +// TODO: Should be removed and paths package used func referrersRepositoriesRootPath(name string) string { return path.Join(rootPath, "repositories", name) } +// TODO: Should be removed and paths package used func referrersRepositoriesManifestRevisionPath(name string, dgst digest.Digest) string { return path.Join(referrersRepositoriesRootPath(name), "_manifests", "revisions", dgst.Algorithm().String(), dgst.Hex()) } +// TODO: Should be removed and defined instead in paths package +// Requires paths package to be exported func referrersLinkPath(name string) string { return path.Join(referrersRepositoriesRootPath(name), "_refs", "subjects") } diff --git a/registry/storage/extension.go b/registry/storage/extension.go index ae85b7dcc98..2a7d06152eb 100644 --- a/registry/storage/extension.go +++ b/registry/storage/extension.go @@ -25,7 +25,7 @@ type GCExtensionHandler interface { manifestDigest digest.Digest, dryRun bool, removeUntagged bool) (bool, error) - RemoveManifest(ctx context.Context, + OnManifestDelete(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dgst digest.Digest, diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 6e61707e4ce..06c425e88c4 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -138,7 +138,7 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) } for _, gcHandler := range opts.GCExtensionHandlers { - err := gcHandler.RemoveManifest(ctx, storageDriver, registry, obj.Digest, obj.Name) + err := gcHandler.OnManifestDelete(ctx, storageDriver, registry, obj.Digest, obj.Name) if err != nil { return fmt.Errorf("failed to call remove manifest extension handler: %v", err) } From 8e19c16701e70a604fa5ff1804a896a5d015e5e7 Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Tue, 9 Aug 2022 13:23:36 -0700 Subject: [PATCH 22/22] update release tags Signed-off-by: Akash Singhal --- .github/workflows/oras-release.yml | 2 +- docs/extensions.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/oras-release.yml b/.github/workflows/oras-release.yml index 33de2ff48b4..94e2411c10a 100644 --- a/.github/workflows/oras-release.yml +++ b/.github/workflows/oras-release.yml @@ -4,7 +4,7 @@ on: push: tags: - "v[0-9]+.[0-9]+.[0-9]+-alpha" - - "v[0-9]+.[0-9]+.[0-9]+-rc" + - "v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+" jobs: publish: diff --git a/docs/extensions.md b/docs/extensions.md index e3b680885f9..966017e4dc7 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -41,7 +41,7 @@ type ExtendedStorage interface { } ``` -The `GCExtensionHandler` interface defines three methods that are used in the garbage colection mark and sweep process. The `Mark` method is invoked for each `GCExtensionHandler` after the existing mark process finishes in `MarkAndSweep`. It is used to determine if the manifest and blobs should have their temporary ref count incremented in the case of an artifact manifest, or if the manifest and it's referrers should be recursively indexed for deletion in the case of a non-artifact manifest. `RemoveManifest` is invoked to extend the `RemoveManifest` functionality for the `Vacuum`. New or special-cased manifests may require custom manifest deletion which can be defined with this method. `SweepBlobs` is used to add artifact manifest/blobs to the original `markSet`. These blobs are retained after determining their ref count is still positive. +The `GCExtensionHandler` interface defines three methods that are used in the garbage colection mark and sweep process. The `Mark` method is invoked for each `GCExtensionHandler` after the existing mark process finishes in `MarkAndSweep`. It is used to determine if the manifest and blobs should have their temporary ref count incremented in the case of an artifact manifest, or if the manifest and it's referrers should be recursively indexed for deletion in the case of a non-artifact manifest. `OnManifestDelete` is invoked to extend the `RemoveManifest` functionality for the `Vacuum`. New or special-cased manifests may require custom manifest deletion which can be defined with this method. `SweepBlobs` is used to add artifact manifest/blobs to the original `markSet`. These blobs are retained after determining their ref count is still positive. ``` type GCExtensionHandler interface { @@ -53,7 +53,7 @@ type GCExtensionHandler interface { manifestDigest digest.Digest, dryRun bool, removeUntagged bool) (bool, error) - RemoveManifest(ctx context.Context, + OnManifestDelete(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dgst digest.Digest,