diff --git a/errors.go b/errors.go
index 8e0b788d6c5..ccbf5fb841a 100644
--- a/errors.go
+++ b/errors.go
@@ -24,6 +24,10 @@ var ErrUnsupported = errors.New("operation unsupported")
 // manifest but the registry is configured to reject it
 var ErrSchemaV1Unsupported = errors.New("manifest schema v1 unsupported")
 
+// ErrInvalidSubjectMediaType is returned when a manifest has a subject that is
+// not a manifest
+var ErrInvalidSubjectMediaType = errors.New("subject is not a manifest")
+
 // ErrTagUnknown is returned if the given tag is not known by the tag service
 type ErrTagUnknown struct {
 	Tag string
diff --git a/manifest/ociartifact/manifest.go b/manifest/ociartifact/manifest.go
index fa96302ff51..8ba067ec32a 100644
--- a/manifest/ociartifact/manifest.go
+++ b/manifest/ociartifact/manifest.go
@@ -52,9 +52,6 @@ type Manifest struct {
 func (m Manifest) References() []distribution.Descriptor {
 	var references []distribution.Descriptor
 	references = append(references, m.Blobs...)
-	if m.Subject != nil {
-		references = append(references, *m.Subject)
-	}
 	return references
 }
 
diff --git a/manifest/ociartifact/manifest_test.go b/manifest/ociartifact/manifest_test.go
index 9652f373b58..ce853b9b601 100644
--- a/manifest/ociartifact/manifest_test.go
+++ b/manifest/ociartifact/manifest_test.go
@@ -93,7 +93,7 @@ func TestArtifactManifest(t *testing.T) {
 
 	// Test DeserializedManifest.References()
 	references := deserialized.References()
-	if len(references) != 2 {
+	if len(references) != 1 {
 		t.Fatalf("unexpected number of references: %d", len(references))
 	}
 }
diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go
index 31c20455bde..8f5b9b6c6b0 100644
--- a/manifest/ocischema/manifest.go
+++ b/manifest/ocischema/manifest.go
@@ -64,9 +64,6 @@ func (m Manifest) References() []distribution.Descriptor {
 	references := make([]distribution.Descriptor, 0, 1+len(m.Layers))
 	references = append(references, m.Config)
 	references = append(references, m.Layers...)
-	if m.Subject != nil {
-		references = append(references, *m.Subject)
-	}
 	return references
 }
 
diff --git a/registry/storage/ociartifactmanifesthandler.go b/registry/storage/ociartifactmanifesthandler.go
index 52752006760..27b17f6bbd1 100644
--- a/registry/storage/ociartifactmanifesthandler.go
+++ b/registry/storage/ociartifactmanifesthandler.go
@@ -6,7 +6,9 @@ import (
 
 	"github.com/distribution/distribution/v3"
 	dcontext "github.com/distribution/distribution/v3/context"
+	"github.com/distribution/distribution/v3/manifest/manifestlist"
 	"github.com/distribution/distribution/v3/manifest/ociartifact"
+	"github.com/distribution/distribution/v3/manifest/schema2"
 	"github.com/distribution/distribution/v3/registry/storage/driver"
 	"github.com/opencontainers/go-digest"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -78,22 +80,21 @@ func (ms *ociArtifactManifestHandler) verifyArtifactManifest(ctx context.Context
 		return nil
 	}
 
-	// validate the subject
+	// For subject, we need to verify that:
+	// First, its digest is valid. Second, it is a manifest.
+	// No need to check its existence.
 	if mnfst.Subject != nil {
 		// check if the digest is valid
 		err := mnfst.Subject.Digest.Validate()
 		if err != nil {
 			errs = append(errs, err, distribution.ErrManifestBlobUnknown{Digest: mnfst.Subject.Digest})
-		} else {
-			// check the presence
-			manifestService, err := ms.repository.Manifests(ctx)
-			if err != nil {
-				return err
-			}
-			exists, err := manifestService.Exists(ctx, mnfst.Subject.Digest)
-			if err != nil || !exists {
-				errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: mnfst.Subject.Digest})
-			}
+		}
+		// check the media type of subject
+		switch mnfst.Subject.MediaType {
+		case v1.MediaTypeImageManifest, v1.MediaTypeArtifactManifest, v1.MediaTypeImageIndex, schema2.MediaTypeManifest, manifestlist.MediaTypeManifestList:
+			// no operations for known manifest media types
+		default:
+			errs = append(errs, distribution.ErrInvalidSubjectMediaType)
 		}
 	}
 
diff --git a/registry/storage/ociartifactmanifesthandler_test.go b/registry/storage/ociartifactmanifesthandler_test.go
index 4a6024b0139..935cbf956f0 100644
--- a/registry/storage/ociartifactmanifesthandler_test.go
+++ b/registry/storage/ociartifactmanifesthandler_test.go
@@ -19,9 +19,9 @@ func TestVerifyOCIArtifactManifestBlobsAndSubject(t *testing.T) {
 	repo := makeRepository(t, registry, strings.ToLower(t.Name()))
 	manifestService := makeManifestService(t, repo)
 
-	subject, err := repo.Blobs(ctx).Put(ctx, v1.MediaTypeImageManifest, nil)
-	if err != nil {
-		t.Fatal(err)
+	subject := distribution.Descriptor{
+		Digest:    digest.Digest("sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b"),
+		MediaType: v1.MediaTypeImageManifest,
 	}
 
 	blob, err := repo.Blobs(ctx).Put(ctx, v1.MediaTypeImageLayer, nil)
@@ -98,9 +98,20 @@ func TestVerifyOCIArtifactManifestBlobsAndSubject(t *testing.T) {
 		},
 		// subject with invalid digest
 		{
-			distribution.Descriptor{Digest: digest.Digest("invalid")},
+			distribution.Descriptor{
+				Digest:    digest.Digest("invalid"),
+				MediaType: v1.MediaTypeImageManifest,
+			},
 			digest.ErrDigestInvalidFormat,
 		},
+		// subject with a non-manifest mediatype
+		{
+			distribution.Descriptor{
+				Digest:    digest.Digest("sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b"),
+				MediaType: v1.MediaTypeImageConfig,
+			},
+			distribution.ErrInvalidSubjectMediaType,
+		},
 	}
 
 	for _, c := range subjectcases {
diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go
index 05706719c46..60db47d990f 100644
--- a/registry/storage/ocimanifesthandler.go
+++ b/registry/storage/ocimanifesthandler.go
@@ -88,6 +88,24 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc
 		return err
 	}
 
+	// For subject, we need to verify that:
+	// First, its digest is valid. Second, it is a manifest.
+	// No need to check its existence.
+	if mnfst.Subject != nil {
+		// check if the digest is valid
+		err := mnfst.Subject.Digest.Validate()
+		if err != nil {
+			errs = append(errs, err, distribution.ErrManifestBlobUnknown{Digest: mnfst.Subject.Digest})
+		}
+		// check the media type of subject
+		switch mnfst.Subject.MediaType {
+		case v1.MediaTypeImageManifest, v1.MediaTypeArtifactManifest, v1.MediaTypeImageIndex, schema2.MediaTypeManifest, manifestlist.MediaTypeManifestList:
+			// no operations for known manifest media types
+		default:
+			errs = append(errs, distribution.ErrInvalidSubjectMediaType)
+		}
+	}
+
 	blobsService := ms.repository.Blobs(ctx)
 
 	for _, descriptor := range mnfst.References() {