copyFrom supports passing additional blocks (#471)

* test copyFrom with additional blocks

* copyFrom supports passing additional blocks

* standard fix

* copyFrom uses additional blocks to fill in tree

* only verify batch when upgraded

* add test for additional blocks filling in tree

* add test for both filling in and upgrade

* copyFrom only flushes tree once done

* only copy until specified length

* partial clone test asserts no over copy

* copy tree data last to avoid side effects

* additional blocks need to be committed atomically with the rest of the update

* remove duplicate tests

lib/core.js

@@ -225,7 +225,7 @@ module.exports = class Core {
     return false
   }
 
-  async copyFrom (src, signature, { length = src.tree.length } = {}) {
+  async copyFrom (src, signature, { length = src.tree.length, additional = [] } = {}) {
     await this._mutex.lock()
 
     try {
@@ -235,16 +235,19 @@ module.exports = class Core {
       throw err
     }
 
+    const initialLength = this.tree.length
+
     try {
       const updates = []
 
       let pos = 0
+      const copyLength = Math.min(src.tree.length, length)
 
-      while (pos < length) {
+      while (pos < copyLength) {
         const segmentStart = maximumSegmentStart(pos, src.bitfield, this.bitfield)
         if (segmentStart >= length || segmentStart < 0) break
 
-        const segmentEnd = Math.min(length, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
+        const segmentEnd = Math.min(src.tree.length, minimumSegmentEnd(segmentStart, src.bitfield, this.bitfield))
 
         const segment = []
 
@@ -268,7 +271,7 @@ module.exports = class Core {
         })
       }
 
-      for (let i = 0; i < length * 2; i++) {
+      for (let i = 0; i < copyLength * 2; i++) {
         const node = await src.tree.get(i, false)
         if (node === null) continue
 
@@ -277,35 +280,67 @@ module.exports = class Core {
 
       await this.tree.flush()
 
-      if (length > this.tree.length) {
-        this.tree.fork = src.tree.fork
-        this.tree.roots = [...src.tree.roots]
-        this.tree.length = src.tree.length
-        this.tree.byteLength = src.tree.byteLength
-
-        if (length < this.tree.length) {
-          const batch = await src.tree.truncate(length)
-          this.tree.roots = [...batch.roots]
-          this.tree.length = batch.length
-          this.tree.byteLength = batch.byteLength
+      let batch = this.tree.batch()
+
+      // add additional blocks
+      if (length > src.tree.length) {
+        const missing = length - src.tree.length
+
+        if (additional.length < missing) {
+          throw INVALID_OPERATION('Insufficient additional nodes were passed')
         }
 
+        const source = src.tree.batch()
+
+        batch.roots = [...source.roots]
+        batch.length = source.length
+        batch.byteLength = source.byteLength
+
+        const blocks = additional.length === missing ? additional : additional.slice(0, missing)
+
+        await this.blocks.putBatch(source.length, blocks, source.byteLength)
+        this.bitfield.setRange(source.length, missing, true)
+
+        updates.push({
+          drop: false,
+          start: source.length,
+          length: missing
+        })
+
+        for (const block of blocks) await batch.append(block)
+      } else {
+        const source = length < src.tree.length ? await src.tree.truncate(length) : src.tree.batch()
+
+        this.tree.roots = [...source.roots]
+        this.tree.length = source.length
+        this.tree.byteLength = source.byteLength
+
+        // update batch
+        batch = this.tree.batch()
+      }
+
+      // verify if upgraded
+      if (batch.length > initialLength) {
         try {
-          const batch = this.tree.batch()
           batch.signature = signature
+
           this._verifyBatchUpgrade(batch, this.header.manifest)
           this.tree.signature = signature
         } catch (err) {
           this.tree.signature = null
           // TODO: how to handle signature failure?
           throw err
         }
-
-        this.header.tree.length = this.tree.length
-        this.header.tree.rootHash = this.tree.hash()
-        this.header.tree.signature = this.tree.signature
       }
 
+      await batch.commit()
+
+      this.tree.fork = src.tree.fork
+
+      this.header.tree.length = this.tree.length
+      this.header.tree.rootHash = this.tree.hash()
+      this.header.tree.signature = this.tree.signature
+
       this.header.userData = src.header.userData.slice(0)
       this.header.hints.contiguousLength = Math.min(src.header.hints.contiguousLength, this.header.tree.length)
 

test/core.js

@@ -403,6 +403,161 @@ test('core - partial clone', async function (t) {
     b4a.from('0'),
     b4a.from('1')
   ])
+
+  await t.exception(copy.blocks.get(2))
+})
+
+test('core - clone with additional', async function (t) {
+  const { core } = await create()
+  const { core: copy } = await create({ keyPair: core.header.keyPair })
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.keyPair.publicKey } })
+
+  await core.append([b4a.from('a'), b4a.from('b')])
+  await copy.copyFrom(core, core.tree.signature)
+
+  t.is(copy.header.keyPair.publicKey, core.header.keyPair.publicKey)
+  t.is(copy.header.keyPair.publicKey, clone.header.keyPair.publicKey)
+
+  // copy should be independent
+  await core.append([b4a.from('c')])
+
+  await clone.copyFrom(copy, core.tree.signature, { length: 3, additional: [b4a.from('c')] })
+
+  t.is(clone.header.tree.length, 3)
+  t.alike(clone.header.tree.signature, core.header.tree.signature)
+
+  t.is(clone.tree.length, core.tree.length)
+  t.is(clone.tree.byteLength, core.tree.byteLength)
+  t.alike(clone.roots, core.roots)
+
+  t.alike(await clone.blocks.get(0), b4a.from('a'))
+  t.alike(await clone.blocks.get(1), b4a.from('b'))
+  t.alike(await clone.blocks.get(2), b4a.from('c'))
+})
+
+test('core - clone with additional, larger tree', async function (t) {
+  const { core } = await create()
+  const { core: copy } = await create({ keyPair: core.header.keyPair })
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.keyPair.publicKey } })
+
+  await core.append([b4a.from('a'), b4a.from('b')])
+  await copy.copyFrom(core, core.tree.signature)
+
+  t.is(copy.header.keyPair.publicKey, core.header.keyPair.publicKey)
+  t.is(copy.header.keyPair.publicKey, clone.header.keyPair.publicKey)
+
+  const additional = [
+    b4a.from('c'),
+    b4a.from('d'),
+    b4a.from('e'),
+    b4a.from('f'),
+    b4a.from('g'),
+    b4a.from('h'),
+    b4a.from('i'),
+    b4a.from('j')
+  ]
+
+  await core.append(additional)
+
+  // copy should be independent
+  await clone.copyFrom(copy, core.tree.signature, { length: core.tree.length, additional })
+
+  t.is(clone.header.tree.length, core.header.tree.length)
+  t.alike(clone.header.tree.signature, core.header.tree.signature)
+
+  t.is(clone.tree.length, core.tree.length)
+  t.is(clone.tree.byteLength, core.tree.byteLength)
+  t.alike(clone.roots, core.roots)
+
+  t.alike(await clone.blocks.get(0), b4a.from('a'))
+  t.alike(await clone.blocks.get(1), b4a.from('b'))
+  t.alike(await clone.blocks.get(2), b4a.from('c'))
+  t.alike(await clone.blocks.get(3), b4a.from('d'))
+  t.alike(await clone.blocks.get(4), b4a.from('e'))
+  t.alike(await clone.blocks.get(5), b4a.from('f'))
+  t.alike(await clone.blocks.get(6), b4a.from('g'))
+  t.alike(await clone.blocks.get(7), b4a.from('h'))
+  t.alike(await clone.blocks.get(8), b4a.from('i'))
+  t.alike(await clone.blocks.get(9), b4a.from('j'))
+})
+
+test('core - clone with too many additional', async function (t) {
+  const { core } = await create()
+  const { core: copy } = await create({ keyPair: core.header.keyPair })
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.keyPair.publicKey } })
+
+  await core.append([b4a.from('a'), b4a.from('b')])
+  await copy.copyFrom(core, core.tree.signature)
+
+  t.is(copy.header.keyPair.publicKey, core.header.keyPair.publicKey)
+  t.is(copy.header.keyPair.publicKey, clone.header.keyPair.publicKey)
+
+  // copy should be independent
+  await core.append([b4a.from('c')])
+
+  await clone.copyFrom(copy, core.tree.signature, {
+    length: 3,
+    additional: [
+      b4a.from('c'),
+      b4a.from('d')
+    ]
+  })
+
+  t.is(clone.header.tree.length, 3)
+  t.alike(clone.header.tree.signature, core.header.tree.signature)
+
+  t.is(clone.tree.length, core.tree.length)
+  t.is(clone.tree.byteLength, core.tree.byteLength)
+  t.alike(clone.roots, core.roots)
+
+  t.alike(await clone.blocks.get(0), b4a.from('a'))
+  t.alike(await clone.blocks.get(1), b4a.from('b'))
+  t.alike(await clone.blocks.get(2), b4a.from('c'))
+
+  await t.exception(clone.blocks.get(3))
+})
+
+test('core - clone fills in with additional', async function (t) {
+  const { core } = await create()
+  const { core: copy } = await create({ keyPair: core.header.keyPair })
+  const { core: clone } = await create({ keyPair: { publicKey: core.header.keyPair.publicKey } })
+
+  t.is(copy.header.keyPair.publicKey, core.header.keyPair.publicKey)
+  t.is(copy.header.keyPair.publicKey, clone.header.keyPair.publicKey)
+
+  await clone.copyFrom(core, core.tree.signature)
+
+  // copy should be independent
+  await core.append([b4a.from('a')])
+  await copy.copyFrom(core, core.tree.signature)
+
+  // upgrade clone
+  {
+    const p = await core.tree.proof({ upgrade: { start: 0, length: 1 } })
+    t.ok(await clone.verify(p))
+  }
+
+  await core.append([b4a.from('b')])
+
+  // verify state
+  t.is(copy.tree.length, 1)
+  t.is(clone.tree.length, 1)
+
+  await t.exception(clone.blocks.get(0))
+  await t.exception(copy.blocks.get(1))
+
+  // copy should both fill in and upgrade
+  await clone.copyFrom(copy, core.tree.signature, { length: 2, additional: [b4a.from('b')] })
+
+  t.is(clone.header.tree.length, 2)
+  t.alike(clone.header.tree.signature, core.header.tree.signature)
+
+  t.is(clone.tree.length, core.tree.length)
+  t.is(clone.tree.byteLength, core.tree.byteLength)
+  t.alike(clone.roots, core.roots)
+
+  t.alike(await clone.blocks.get(0), b4a.from('a'))
+  t.alike(await clone.blocks.get(1), b4a.from('b'))
 })
 
 async function create (opts) {