Skip to content

Commit

Permalink
feat: index referrers when pushing manifests (#35)
Browse files Browse the repository at this point in the history 
Signed-off-by: wangxiaoxuan273 <wangxiaoxuan119@gmail.com>
  • Loading branch information
@wangxiaoxuan273
wangxiaoxuan273 authored Nov 17, 2022
1 parent 6f0e3ea commit 7c54a46
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 48 deletions.
30 changes: 27 additions & 3 deletions registry/storage/ociartifactmanifesthandler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,17 @@ import (
"github.com/distribution/distribution/v3"
dcontext "github.com/distribution/distribution/v3/context"
"github.com/distribution/distribution/v3/manifest/ociartifact"
"github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/opencontainers/go-digest"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
)

// ociArtifactManifestHandler is a ManifestHandler that covers oci artifact manifests.
type ociArtifactManifestHandler struct {
repository distribution.Repository
blobStore distribution.BlobStore
ctx context.Context
repository distribution.Repository
blobStore distribution.BlobStore
ctx context.Context
storageDriver driver.StorageDriver
}

var _ ManifestHandler = &ociArtifactManifestHandler{}
Expand Down Expand Up @@ -54,6 +56,11 @@ func (ms *ociArtifactManifestHandler) Put(ctx context.Context, manifest distribu
return "", err
}

err = ms.indexReferrers(ctx, m, revision.Digest)
if err != nil {
dcontext.GetLogger(ctx).Errorf("error indexing referrers: %v", err)
return "", err
}
return revision.Digest, nil
}

Expand Down Expand Up @@ -112,3 +119,20 @@ func (ms *ociArtifactManifestHandler) verifyArtifactManifest(ctx context.Context

return nil
}

// indexReferrers indexes the subject of the given revision in its referrers index store.
func (ms *ociArtifactManifestHandler) indexReferrers(ctx context.Context, dm *ociartifact.DeserializedManifest, revision digest.Digest) error {
if dm.Subject == nil {
return nil
}

// [TODO] We can use artifact type in the link path to support filtering by artifact type
// but need to consider the max path length in different os
subjectRevision := dm.Subject.Digest

referrersLinkPath, err := pathFor(referrersLinkPathSpec{name: ms.repository.Named().Name(), revision: revision, subjectRevision: subjectRevision})
if err != nil {
return fmt.Errorf("failed to generate referrers link path for %v", revision)
}
return ms.storageDriver.PutContent(ctx, referrersLinkPath, []byte(revision.String()))
}
114 changes: 72 additions & 42 deletions registry/storage/paths.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,29 @@ const (
//
// The path layout in the storage backend is roughly as follows:
//
// <root>/v2
// -> repositories/
// -><name>/
// -> _manifests/
// revisions
// -> <manifest digest path>
// -> link
// tags/<tag>
// -> current/link
// -> index
// -> <algorithm>/<hex digest>/link
// -> _layers/
// <layer links to blob store>
// -> _uploads/<id>
// data
// startedat
// hashstates/<algorithm>/<offset>
// -> blob/<algorithm>
// <split directory content addressable storage>
// <root>/v2
// -> repositories/
// -><name>/
// -> _manifests/
// revisions
// -> <manifest digest path>
// -> link
// tags/<tag>
// -> current/link
// -> index
// -> <algorithm>/<hex digest>/link
// -> _layers/
// <layer links to blob store>
// -> _uploads/<id>
// data
// startedat
// hashstates/<algorithm>/<offset>
// -> _referrers/subjects
// -> <revision digest path>
// -> <subject digest path>/link
//
// -> blob/<algorithm>
// <split directory content addressable storage>
//
// The storage backend layout is broken up into a content-addressable blob
// store and repositories. The content-addressable blob store holds most data
Expand Down Expand Up @@ -71,36 +75,40 @@ const (
//
// Manifests:
//
// manifestRevisionsPathSpec: <root>/v2/repositories/<name>/_manifests/revisions/
// manifestRevisionPathSpec: <root>/v2/repositories/<name>/_manifests/revisions/<algorithm>/<hex digest>/
// manifestRevisionLinkPathSpec: <root>/v2/repositories/<name>/_manifests/revisions/<algorithm>/<hex digest>/link
// manifestRevisionsPathSpec: <root>/v2/repositories/<name>/_manifests/revisions/
// manifestRevisionPathSpec: <root>/v2/repositories/<name>/_manifests/revisions/<algorithm>/<hex digest>/
// manifestRevisionLinkPathSpec: <root>/v2/repositories/<name>/_manifests/revisions/<algorithm>/<hex digest>/link
//
// Tags:
//
// manifestTagsPathSpec: <root>/v2/repositories/<name>/_manifests/tags/
// manifestTagPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/
// manifestTagCurrentPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/current/link
// manifestTagIndexPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/
// manifestTagIndexEntryPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/
// manifestTagIndexEntryLinkPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/link
// manifestTagsPathSpec: <root>/v2/repositories/<name>/_manifests/tags/
// manifestTagPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/
// manifestTagCurrentPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/current/link
// manifestTagIndexPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/
// manifestTagIndexEntryPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/
// manifestTagIndexEntryLinkPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/link
//
// Blobs:
// Blobs:
//
// layerLinkPathSpec: <root>/v2/repositories/<name>/_layers/<algorithm>/<hex digest>/link
// layersPathSpec: <root>/v2/repositories/<name>/_layers
// layerLinkPathSpec: <root>/v2/repositories/<name>/_layers/<algorithm>/<hex digest>/link
// layersPathSpec: <root>/v2/repositories/<name>/_layers
//
// Uploads:
//
// uploadDataPathSpec: <root>/v2/repositories/<name>/_uploads/<id>/data
// uploadStartedAtPathSpec: <root>/v2/repositories/<name>/_uploads/<id>/startedat
// uploadHashStatePathSpec: <root>/v2/repositories/<name>/_uploads/<id>/hashstates/<algorithm>/<offset>
// uploadDataPathSpec: <root>/v2/repositories/<name>/_uploads/<id>/data
// uploadStartedAtPathSpec: <root>/v2/repositories/<name>/_uploads/<id>/startedat
// uploadHashStatePathSpec: <root>/v2/repositories/<name>/_uploads/<id>/hashstates/<algorithm>/<offset>
//
// Referrers:
//
// referrersLinkPathSpec: <root>/v2/repositories/<name>/_referrers/subjects/<algorithm>/<hex digest>/<subject algorithm>/<subject hex digest>/link
//
// Blob Store:
//
// blobsPathSpec: <root>/v2/blobs/
// blobPathSpec: <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>
// blobDataPathSpec: <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>/data
// blobMediaTypePathSpec: <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>/data
// blobPathSpec: <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>
// blobDataPathSpec: <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>/data
// blobMediaTypePathSpec: <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>/data
//
// For more information on the semantic meaning of each path and their
// contents, please see the path spec documentation.
Expand Down Expand Up @@ -242,6 +250,20 @@ func pathFor(spec pathSpec) (string, error) {
return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "hashstates", string(v.alg), offset)...), nil
case repositoriesRootPathSpec:
return path.Join(repoPrefix...), nil
case referrersLinkPathSpec:

revisionComponents, err := digestPathComponents(v.revision, false)
if err != nil {
return "", err
}

subjectComponents, err := digestPathComponents(v.subjectRevision, false)
if err != nil {
return "", err
}
referrersRootPath := append(repoPrefix, v.name, "_referrers", "subjects")
referrersComponentPath := append(append(referrersRootPath, revisionComponents...), subjectComponents...)
return path.Join(append(referrersComponentPath, "link")...), nil
default:
// TODO(sday): This is an internal error. Ensure it doesn't escape (panic?).
return "", fmt.Errorf("unknown path spec: %#v", v)
Expand Down Expand Up @@ -349,11 +371,11 @@ func (layersPathSpec) pathSpec() {}
// blob id. The blob link will contain a content addressable blob id reference
// into the blob store. The format of the contents is as follows:
//
// <algorithm>:<hex digest of layer data>
// <algorithm>:<hex digest of layer data>
//
// The following example of the file contents is more illustrative:
//
// sha256:96443a84ce518ac22acb2e985eda402b58ac19ce6f91980bde63726a79d80b36
// sha256:96443a84ce518ac22acb2e985eda402b58ac19ce6f91980bde63726a79d80b36
//
// This indicates that there is a blob with the id/digest, calculated via
// sha256 that can be fetched from the blob store.
Expand Down Expand Up @@ -436,16 +458,24 @@ type repositoriesRootPathSpec struct {

func (repositoriesRootPathSpec) pathSpec() {}

// referrersLinkPathSpec defines the link path of a referrer.
type referrersLinkPathSpec struct {
name string
revision digest.Digest
subjectRevision digest.Digest
}

func (referrersLinkPathSpec) pathSpec() {}

// digestPathComponents provides a consistent path breakdown for a given
// digest. For a generic digest, it will be as follows:
//
// <algorithm>/<hex digest>
// <algorithm>/<hex digest>
//
// If multilevel is true, the first two bytes of the digest will separate
// groups of digest folder. It will be as follows:
//
// <algorithm>/<first two bytes of digest>/<full digest>
//
// <algorithm>/<first two bytes of digest>/<full digest>
func digestPathComponents(dgst digest.Digest, multilevel bool) ([]string, error) {
if err := dgst.Validate(); err != nil {
return nil, err
Expand Down
7 changes: 7 additions & 0 deletions registry/storage/paths_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,13 @@ func TestPathMapper(t *testing.T) {
spec: layersPathSpec{name: "foo/bar"},
expected: "/docker/registry/v2/repositories/foo/bar/_layers",
},
{
spec: referrersLinkPathSpec{
name: "bar",
revision: "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
subjectRevision: "sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b"},
expected: "/docker/registry/v2/repositories/bar/_referrers/subjects/sha256/abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789/sha256/6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b/link",
},
} {
p, err := pathFor(testcase.spec)
if err != nil {
Expand Down
7 changes: 4 additions & 3 deletions registry/storage/registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,9 +289,10 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
manifestURLs: repo.registry.manifestURLs,
},
ociartifactHandler: &ociArtifactManifestHandler{
repository: repo,
blobStore: blobStore,
ctx: ctx,
repository: repo,
blobStore: blobStore,
ctx: ctx,
storageDriver: repo.driver,
},
}

Expand Down

0 comments on commit 7c54a46

Please sign in to comment.