diff --git a/core/engine/engine.go b/core/engine/engine.go index c88d4a58..cf7f80cf 100644 --- a/core/engine/engine.go +++ b/core/engine/engine.go @@ -64,7 +64,6 @@ func New(ctx context.Context, // NewFromConfig creates a reference provider engine with the corresponding config. func NewFromConfig(ctx context.Context, cfg config.Config, ds datastore.Batching, host host.Host) (*Engine, error) { - log.Debugw("Starting new reference provider engine") privKey, err := cfg.Identity.DecodePrivateKey("") if err != nil { diff --git a/core/engine/engine_test.go b/core/engine/engine_test.go index c88e8ce7..9f1924ec 100644 --- a/core/engine/engine_test.go +++ b/core/engine/engine_test.go @@ -52,7 +52,6 @@ func mkEngine(t *testing.T) (*Engine, error) { store := dssync.MutexWrap(datastore.NewMapDatastore()) return New(context.Background(), priv, h, store, testTopic) - } func connectHosts(t *testing.T, srcHost, dstHost host.Host) { @@ -64,7 +63,7 @@ func connectHosts(t *testing.T, srcHost, dstHost host.Host) { } func RandomCids(n int) ([]cid.Cid, error) { - var prng = rand.New(rand.NewSource(time.Now().UnixNano())) + prng := rand.New(rand.NewSource(time.Now().UnixNano())) res := make([]cid.Cid, n) for i := 0; i < n; i++ { @@ -200,7 +199,7 @@ func TestNotifyPutAndRemoveCids(t *testing.T) { // Check that the update has been published and can be fetched from subscriber select { - case <-time.After(time.Second * 5): + case <-time.After(time.Second * 10): t.Fatal("timed out waiting for sync to propogate") case downstream := <-watcher: if !downstream.Equals(c) { @@ -214,7 +213,7 @@ func TestNotifyPutAndRemoveCids(t *testing.T) { require.NoError(t, err) // Check that the update has been published and can be fetched from subscriber select { - case <-time.After(time.Second * 5): + case <-time.After(time.Second * 10): t.Fatal("timed out waiting for sync to propogate") case downstream := <-watcher: if !downstream.Equals(c) { @@ -228,7 +227,7 @@ func TestNotifyPutAndRemoveCids(t *testing.T) { require.NoError(t, err) // Check that the update has been published and can be fetched from subscriber select { - case <-time.After(time.Second * 5): + case <-time.After(time.Second * 10): t.Fatal("timed out waiting for sync to propogate") case downstream := <-watcher: if !downstream.Equals(c) { diff --git a/go.mod b/go.mod index 5f918c27..fa771a96 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,13 @@ require ( github.com/ipfs/go-datastore v0.4.6 github.com/ipfs/go-ds-leveldb v0.4.2 github.com/ipfs/go-log/v2 v2.3.0 + github.com/ipld/go-car/v2 v2.0.3-0.20210827150014-1bac13d05359 github.com/ipld/go-ipld-prime v0.12.0 github.com/lib/pq v1.10.2 github.com/libp2p/go-libp2p v0.15.0-rc.1 github.com/libp2p/go-libp2p-core v0.9.0 github.com/mitchellh/go-homedir v1.1.0 + github.com/multiformats/go-multicodec v0.3.0 github.com/multiformats/go-multihash v0.0.16 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 diff --git a/go.sum b/go.sum index c28d0894..3745a713 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= @@ -131,6 +132,7 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -139,6 +141,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -217,6 +220,7 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= @@ -365,6 +369,7 @@ github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= +github.com/ipfs/go-bitswap v0.1.8 h1:38X1mKXkiU6Nzw4TOSWD8eTVY5eX3slQunv3QEWfXKg= github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -382,6 +387,7 @@ github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-cid v0.0.8-0.20210716091050-de6c03deae1c/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -414,6 +420,7 @@ github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE= +github.com/ipfs/go-ipfs-blockstore v1.0.3/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE= github.com/ipfs/go-ipfs-blockstore v1.0.4-0.20210205083733-fb07d7bc5aec/go.mod h1:feuklK+m9POeWJzYQO7l05yNEgUiX5oELBNA8/Be33E= github.com/ipfs/go-ipfs-blockstore v1.0.4 h1:DZdeya9Vu4ttvlGheQPGrj6kWehXnYZRFCp9EsZQ1hI= github.com/ipfs/go-ipfs-blockstore v1.0.4/go.mod h1:uL7/gTJ8QIZ3MtA3dWf+s1a0U3fJy2fcEZAsovpRp+w= @@ -423,6 +430,7 @@ github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcB github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= @@ -441,6 +449,7 @@ github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqt github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRDI/HQQ= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= @@ -487,7 +496,10 @@ github.com/ipfs/go-unixfs v0.2.6 h1:gq3U3T2vh8x6tXhfo3uSO3n+2z4yW0tYtNgVP/3sIyA= github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipld/go-car v0.1.0 h1:AaIEA5ITRnFA68uMyuIPYGM2XXllxsu8sNjFJP797us= github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= +github.com/ipld/go-car/v2 v2.0.3-0.20210827150014-1bac13d05359 h1:Nuj2+LnmCWXrmvlrutu9VljH13yq1umFJylcDcHC8Sc= +github.com/ipld/go-car/v2 v2.0.3-0.20210827150014-1bac13d05359/go.mod h1:wqj2VLWbxMMjKm4rvEKXKXfUuwZpknmrOZN8GjQFHHs= github.com/ipld/go-codec-dagpb v1.2.0 h1:2umV7ud8HBMkRuJgd8gXw95cLhwmcYrihS3cQEy9zpI= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= @@ -544,6 +556,7 @@ github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6 github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.8/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -641,6 +654,7 @@ github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQO github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= github.com/libp2p/go-libp2p-discovery v0.5.1 h1:CJylx+h2+4+s68GvrM4pGNyfNhOYviWBPtVv5PA7sfo= github.com/libp2p/go-libp2p-discovery v0.5.1/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= +github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= @@ -677,6 +691,7 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU github.com/libp2p/go-libp2p-quic-transport v0.11.2 h1:p1YQDZRHH4Cv2LPtHubqlQ9ggz4CKng/REZuXZbZMhM= github.com/libp2p/go-libp2p-quic-transport v0.11.2/go.mod h1:wlanzKtIh6pHrq+0U3p3DY9PJfGqxMgPaGKaK5LifwQ= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= +github.com/libp2p/go-libp2p-record v0.1.1 h1:ZJK2bHXYUBqObHX+rHLSNrM3M8fmJUlUHrodDPPATmY= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= @@ -822,10 +837,12 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -911,6 +928,7 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= +github.com/multiformats/go-multicodec v0.2.1-0.20210713081508-b421db6850ae/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.0 h1:tstDwfIjiHbnIjeM5Lp+pMrSeN+LCMsEwOrkPmWm03A= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= @@ -985,6 +1003,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1042,6 +1062,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= @@ -1078,6 +1100,7 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -1138,6 +1161,8 @@ github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvS github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= @@ -1182,6 +1207,11 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1240,11 +1270,13 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e h1:VvfwVmMH40bpMeizC9/K7ipM5Qjucuu16RWfneFPyhQ= golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -1252,6 +1284,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20210615023648-acb5c1269671 h1:ddvpKwqE7dm58PoWjRCmaCiA3DANEW0zWGfNYQD212Y= +golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1268,10 +1302,12 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -1417,6 +1453,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1429,6 +1466,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1478,6 +1516,7 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1499,6 +1538,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1-0.20210225150353-54dc8c5edb56/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1651,6 +1691,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/suppliers/car_ciditerator.go b/internal/suppliers/car_ciditerator.go new file mode 100644 index 00000000..1b4c2429 --- /dev/null +++ b/internal/suppliers/car_ciditerator.go @@ -0,0 +1,57 @@ +package suppliers + +import ( + "context" + "io" + + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/blockstore" +) + +type carCidIterator struct { + closer io.Closer + cancel context.CancelFunc + cidsChan <-chan cid.Cid + errChan <-chan error +} + +func newCarCidIterator(path string, opts ...car.ReadOption) (*carCidIterator, error) { + robs, err := blockstore.OpenReadOnly(path, opts...) + if err != nil { + return nil, err + } + errChan := make(chan error) + ctx, cancel := context.WithCancel(context.Background()) + ctx = blockstore.WithAsyncErrorHandler(ctx, func(err error) { errChan <- err }) + cidsChan, err := robs.AllKeysChan(ctx) + if err != nil { + cancel() + _ = robs.Close() + return nil, err + } + + return &carCidIterator{ + closer: robs, + cancel: cancel, + cidsChan: cidsChan, + errChan: errChan, + }, nil +} + +func (c *carCidIterator) Next() (cid.Cid, error) { + select { + case err := <-c.errChan: + return cid.Undef, err + case nextCid, ok := <-c.cidsChan: + if ok { + return nextCid, nil + } + return cid.Undef, io.EOF + } +} + +func (c *carCidIterator) Close() error { + c.cancel() + return c.closer.Close() +} diff --git a/internal/suppliers/car_ciditerator_test.go b/internal/suppliers/car_ciditerator_test.go new file mode 100644 index 00000000..52db3db4 --- /dev/null +++ b/internal/suppliers/car_ciditerator_test.go @@ -0,0 +1,62 @@ +package suppliers + +import ( + "context" + "io" + "testing" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/blockstore" + "github.com/stretchr/testify/require" +) + +func TestCarCidIteratorReturnsExpectedCids(t *testing.T) { + tests := []struct { + name string + carPath string + }{ + { + "CARv1ReturnsExpectedCIDs", + "../../testdata/sample-v1.car", + }, + { + "CARv2ReturnsExpectedCIDs", + "../../testdata/sample-wrapped-v2.car", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + subject, err := newCarCidIterator(tt.carPath) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, subject.Close()) }) + + // Open ReadOnly blockstore used to provide wanted case for testing + want, err := blockstore.OpenReadOnly(tt.carPath) + require.NoError(t, err) + + // Wait at most 3 seconds for iteration over wanted CIDs. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + + // Fail immediately if error is encountered while iterating over CIDs. + ctx = blockstore.WithAsyncErrorHandler(ctx, func(err error) { require.Fail(t, "expected no error", "%v", err) }) + t.Cleanup(cancel) + + // Instantiate wanted CIDs channel + keysChan, err := want.AllKeysChan(ctx) + require.NoError(t, err) + + // Assert CIDs are consistent with the iterator + for wantCid := range keysChan { + gotCid, gotErr := subject.Next() + require.NoError(t, gotErr) + require.Equal(t, wantCid, gotCid) + } + + // Assert there are no more CIDs left to consume via the iterator. + gotCid, gotErr := subject.Next() + require.Equal(t, io.EOF, gotErr) + require.Equal(t, cid.Undef, gotCid) + }) + } +} diff --git a/internal/suppliers/car_supplier.go b/internal/suppliers/car_supplier.go new file mode 100644 index 00000000..242d6781 --- /dev/null +++ b/internal/suppliers/car_supplier.go @@ -0,0 +1,187 @@ +package suppliers + +import ( + "crypto/sha256" + "encoding/base64" + "io" + "path/filepath" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipld/go-car/v2" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" +) + +const carPathKeyPrefix = "car://" + +var ( + _ CidIteratorSupplier = (*CarSupplier)(nil) + _ io.Closer = (*CarSupplier)(nil) + _ io.Closer = (*carCidIterator)(nil) + _ CidIterator = (*carCidIterator)(nil) +) + +type CarSupplier struct { + ds datastore.Datastore + opts []car.ReadOption +} + +func NewCarSupplier(ds datastore.Datastore, opts ...car.ReadOption) *CarSupplier { + return &CarSupplier{ + ds: ds, + opts: opts, + } +} + +// Put makes the CAR at given path suppliable by this supplier. +// The return CID can then be used via Supply to get an iterator over CIDs that belong to the CAR. +// The ID is generated based on the content of the CAR. +// When the CAR ID is already known, PutWithID should be used instead. +// This function accepts both CARv1 and CARv2 formats. +func (cs *CarSupplier) Put(path string) (cid.Cid, error) { + // Clean path to CAR. + path = filepath.Clean(path) + + // Generate a CID for the CAR at given path. + id, err := generateID(path) + if err != nil { + return cid.Undef, err + } + + return cs.PutWithID(id, path) +} + +// PutWithID makes the CAR at given path suppliable by this supplier identified by the given ID. +// The return CID can then be used via Supply to get an iterator over CIDs that belong to the CAR. +// When the CAR ID is not known, Put should be used instead. +// This function accepts both CARv1 and CARv2 formats. +func (cs *CarSupplier) PutWithID(id cid.Cid, path string) (cid.Cid, error) { + // Clean path to CAR. + path = filepath.Clean(path) + + // Store mapping of CAR ID to path, used to instantiate CID iterator. + carIdKey := toCarIdKey(id) + if err := cs.ds.Put(carIdKey, []byte(path)); err != nil { + return cid.Undef, err + } + + // Store mapping of path to CAR ID, used to lookup the CAR by path when it is removed. + if err := cs.ds.Put(toPathKey(path), id.Bytes()); err != nil { + return cid.Undef, err + } + return id, nil +} + +func toCarIdKey(id cid.Cid) datastore.Key { + return datastore.NewKey(id.String()) +} + +// Remove removes the CAR at the given path from the list of suppliable CID iterators. +// If the CAR at given path is not known, this function will return an error. +// This function accepts both CARv1 and CARv2 formats. +func (cs *CarSupplier) Remove(path string) (cid.Cid, error) { + // Clean path. + path = filepath.Clean(path) + + // Find the CAR ID that corresponds to the given path + pathKey := toPathKey(path) + id, err := cs.getCarIDFromPathKey(pathKey) + if err != nil { + if err == datastore.ErrNotFound { + err = ErrNotFound + } + return cid.Undef, err + } + + // Delete mapping of CAR ID to path. + carIdKey := toCarIdKey(id) + if err := cs.ds.Delete(carIdKey); err != nil { + // TODO improve error handling logic + // we shouldn't typically get NotFound error here. + // If we do then a put must have failed prematurely + // See what we can do to opportunistically heal the datastore. + return cid.Undef, err + } + + // Delete mapping of path to CAR ID. + if err := cs.ds.Delete(pathKey); err != nil { + // TODO improve error handling logic + // we shouldn't typically get NotFound error here. + // If we do then a put must have failed prematurely + // See what we can do to opportunistically heal the datastore. + return cid.Undef, err + } + return id, nil +} + +// Supply supplies an iterator over CIDs of the CAR file that corresponds to the given key. +// An error is returned if no CAR file is found for the key. +func (cs *CarSupplier) Supply(key cid.Cid) (CidIterator, error) { + b, err := cs.ds.Get(toCarIdKey(key)) + if err != nil { + if err == datastore.ErrNotFound { + return nil, ErrNotFound + } + return nil, err + } + path := string(b) + return newCarCidIterator(path, cs.opts...) +} + +// Close permanently closes this supplier. +// After calling Close this supplier is no longer usable. +func (cs *CarSupplier) Close() error { + return cs.ds.Close() +} + +func (cs *CarSupplier) getCarIDFromPathKey(pathKey datastore.Key) (cid.Cid, error) { + carIdBytes, err := cs.ds.Get(pathKey) + if err != nil { + return cid.Undef, err + } + _, c, err := cid.CidFromBytes(carIdBytes) + return c, err +} + +// generateID generates a unique ID for a CAR at a given path. +// The ID is in form of a CID, generated by hashing the list of all CIDs inside the CAR payload. +// This implies that different CARs that have the same CID list appearing in the same order will have the same ID, regardless of version. +// For example, CARv1 and wrapped CARv2 version of it will have the same CID list. +// This function accepts both CARv1 and CARv2 payloads +func generateID(path string, opts ...car.ReadOption) (cid.Cid, error) { + // TODO investigate if there is a more efficient and version-agnostic way to generate CID for a CAR file. + // HINT it will most likely be more efficient to generate the ID using the index of a CAR if it is an indexed CARv2 + // and fall back on current approach otherwise. Note, the CAR index has the multihashes of CIDs not full CIDs, + // and that should be enough for the purposes of ID generation. + + // Instantiate iterator over CAR CIDs. + cri, err := newCarCidIterator(path, opts...) + if err != nil { + return cid.Undef, err + } + defer cri.Close() + // Instantiate a reader over the CID iterator to turn CIDs into bytes. + // Note we use the multihash of CIDs instead of the entire CID. + // TODO consider implementing an efficient multihash iterator for cars. + reader := NewCidIteratorReadCloser(cri, func(cid cid.Cid) ([]byte, error) { return cid.Hash(), nil }) + + // Generate multihash of CAR's CIDs. + mh, err := multihash.SumStream(reader, multihash.SHA2_256, -1) + if err != nil { + return cid.Undef, err + } + // TODO Figure out what the codec should be. + // HINT we could use the root CID codec or the first CID's codec. + // Construct the ID for the CAR in form of a CID. + return cid.NewCidV1(uint64(multicodec.DagCbor), mh), nil +} + +func toPathKey(path string) datastore.Key { + // Hash the path to get a fixed length string as key, regardless of how long the path is. + pathHash := sha256.New().Sum([]byte(path)) + encPathHash := base64.StdEncoding.EncodeToString(pathHash) + + // Prefix the hash with some constant for namespacing and debug readability. + return datastore.NewKey(carPathKeyPrefix + encPathHash) +} diff --git a/internal/suppliers/car_supplier_test.go b/internal/suppliers/car_supplier_test.go new file mode 100644 index 00000000..48d4387d --- /dev/null +++ b/internal/suppliers/car_supplier_test.go @@ -0,0 +1,81 @@ +package suppliers + +import ( + "io" + "testing" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/stretchr/testify/require" +) + +func TestPutCarReturnsExpectedCidIterator(t *testing.T) { + tests := []struct { + name string + carPath string + }{ + { + "CARv1ReturnsExpectedCIDs", + "../../testdata/sample-v1.car", + }, + { + "CARv2ReturnsExpectedCIDs", + "../../testdata/sample-wrapped-v2.car", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ds := datastore.NewMapDatastore() + subject := NewCarSupplier(ds) + t.Cleanup(func() { require.NoError(t, subject.Close()) }) + + wantIterator, err := newCarCidIterator(tt.carPath) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, wantIterator.Close()) }) + + gotCid, err := subject.Put(tt.carPath) + require.NoError(t, err) + + gotIterator, err := subject.Supply(gotCid) + require.NoError(t, err) + + for { + wantNext, wantErr := wantIterator.Next() + gotNext, gotErr := gotIterator.Next() + require.Equal(t, wantErr, gotErr) + require.Equal(t, wantNext, gotNext) + + if wantErr == io.EOF { + break + } + } + }) + } +} + +func TestRemovedPathIsNoLongerSupplied(t *testing.T) { + path := "../../testdata/sample-wrapped-v2.car" + ds := datastore.NewMapDatastore() + subject := NewCarSupplier(ds) + t.Cleanup(func() { require.NoError(t, subject.Close()) }) + + id, err := subject.Put(path) + require.NoError(t, err) + require.NotEqual(t, cid.Undef, id) + + removedId, err := subject.Remove(path) + require.NoError(t, err) + require.NotEqual(t, cid.Undef, id) + require.Equal(t, id, removedId) + + _, err = subject.Remove(path) + require.EqualError(t, err, "no CID iterator found for given key") +} + +func TestCARsWithSameCidsHaveSameID(t *testing.T) { + oneId, err := generateID("../../testdata/sample-v1.car") + require.NoError(t, err) + anotherId, err := generateID("../../testdata/sample-wrapped-v2.car") + require.NoError(t, err) + require.Equal(t, oneId, anotherId) +} diff --git a/internal/suppliers/ciditerator_reader.go b/internal/suppliers/ciditerator_reader.go new file mode 100644 index 00000000..5e818d65 --- /dev/null +++ b/internal/suppliers/ciditerator_reader.go @@ -0,0 +1,50 @@ +package suppliers + +import ( + "bytes" + "io" + + "github.com/ipfs/go-cid" +) + +var _ io.ReadCloser = (*CidIteratorReadCloser)(nil) + +type CidIteratorReadCloser struct { + toBytes func(cid cid.Cid) ([]byte, error) + iter CidIterator + reachedEnd bool + buf bytes.Buffer +} + +func NewCidIteratorReadCloser(iter CidIterator, marshaller func(cid cid.Cid) ([]byte, error)) *CidIteratorReadCloser { + return &CidIteratorReadCloser{ + toBytes: marshaller, + iter: iter, + } +} + +func (c *CidIteratorReadCloser) Read(p []byte) (n int, err error) { + // While there is not enough buffered bytes attempt to populate the buffer. + for !c.reachedEnd && c.buf.Len() < len(p) { + next, err := c.iter.Next() + if err != nil { + if err != io.EOF { + return 0, err + } + c.reachedEnd = true + break + } + b, err := c.toBytes(next) + if err != nil { + return 0, err + } + if _, err = c.buf.Write(b); err != nil { + return 0, err + } + } + return c.buf.Read(p) +} + +func (c *CidIteratorReadCloser) Close() error { + return c.iter.Close() +} diff --git a/internal/suppliers/ciditerator_reader_test.go b/internal/suppliers/ciditerator_reader_test.go new file mode 100644 index 00000000..3cacc9d8 --- /dev/null +++ b/internal/suppliers/ciditerator_reader_test.go @@ -0,0 +1,56 @@ +package suppliers + +import ( + "io" + "testing" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" +) + +func TestCidIteratorReadCloserReturnsCidsConsistentWithIterator(t *testing.T) { + tests := []struct { + name string + carPath string + }{ + { + "CARv1ReturnsExpectedCIDs", + "../../testdata/sample-v1.car", + }, + { + "CARv2ReturnsExpectedCIDs", + "../../testdata/sample-wrapped-v2.car", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iter, err := newCarCidIterator(tt.carPath) + require.NoError(t, err) + subject := NewCidIteratorReadCloser(iter, func(cid cid.Cid) ([]byte, error) { return cid.Bytes(), nil }) + t.Cleanup(func() { require.NoError(t, subject.Close()) }) + + control, err := newCarCidIterator(tt.carPath) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, control.Close()) }) + + // Assert CIDs are consistent with the iterator + for { + wantNext, wantErr := control.Next() + if wantErr == io.EOF { + gotBytes := make([]byte, 1) + _, err := subject.Read(gotBytes) + require.Equal(t, io.EOF, err) + break + } + require.NoError(t, wantErr) + wantBytes := wantNext.Bytes() + wantBytesLen := len(wantBytes) + gotBytes := make([]byte, wantBytesLen) + read, err := subject.Read(gotBytes) + require.NoError(t, err) + require.Equal(t, wantBytesLen, read) + require.Equal(t, wantBytes, gotBytes) + } + }) + } +} diff --git a/internal/suppliers/interface.go b/internal/suppliers/interface.go new file mode 100644 index 00000000..15e1f549 --- /dev/null +++ b/internal/suppliers/interface.go @@ -0,0 +1,30 @@ +package suppliers + +import ( + "errors" + "io" + + "github.com/ipfs/go-cid" +) + +// ErrNotFound signals that CidIteratorSupplier has no iterator corresponding to the given key. +var ErrNotFound = errors.New("no CID iterator found for given key") + +type ( + // CidIteratorSupplier supplies iterators by key. + CidIteratorSupplier interface { + // Supply supplies a CID iterator for a given key. + // If no such iterator is found, this function returns ErrNotFound error. + Supply(key cid.Cid) (CidIterator, error) + } + + // CidIterator provides an iterator over a list of CIDs. + // Once the iteration over CIDs is complete, the user should close this iterator by calling CidIterator.Close. + // See CidIterator.Next + CidIterator interface { + io.Closer + // Next returns the next CID supplied by this iterator. + // If no more CIDs are left to return, this function returns io.EOF error. + Next() (cid.Cid, error) + } +) diff --git a/main.go b/main.go index cfe9c04a..329a4d87 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ func main() { signal.Stop(interrupt) }() + // TODO parameterize log level if err := logging.SetLogLevel("*", "info"); err != nil { log.Fatal(err) } diff --git a/testdata/sample-v1.car b/testdata/sample-v1.car new file mode 100644 index 00000000..47a61c8c Binary files /dev/null and b/testdata/sample-v1.car differ diff --git a/testdata/sample-wrapped-v2.car b/testdata/sample-wrapped-v2.car new file mode 100644 index 00000000..7307bff7 Binary files /dev/null and b/testdata/sample-wrapped-v2.car differ