wdb: fix initial migrations.

lib/migrations/README.md

@@ -64,6 +64,24 @@ migration to run specific to that db.
 incremented from there. (Currently, wallet and chain both have one migration)
 
 ## Writing migrations
+### Migration requirements
+  1. Migration must not depend on the methods, but instead interact
+  with the database directly. This makes sure that future code changes
+  to the chain/wallet db do not affect old migrations. Especially if someone
+  is upgrading multiple versions at once.
+  2. Migration must allow for interruption. Using batch provided to the migration
+  is useful for this purpose. If migration does not utilize the provided batch,
+  it should be possible to stop the migration and resume or restart it later.
+  This makes sure that interruption by user or the system does not corrupt
+  the database.
+  3. Each migration should copy the existing db keys and use those
+  in the migration, instead of depending on layout. Future updates may modify
+  the layout and break old migrations.
+  4. Migration Tests should also follow the same 3 rules.
+
+NOTE: Migration tests can use database dumps from the previous/current versions.
+NOTE: Migration test db generation must be deterministic.
+
 ### Databases and migrations
   HSD has two separate databases with their own migrations: ChainDB and
 WalletDB. Depending which database your migration affects, you will need to

lib/wallet/migrations.js

@@ -8,6 +8,9 @@
 
 const assert = require('bsert');
 const Logger = require('blgr');
+const bdb = require('bdb');
+const bio = require('bufio');
+const {encoding} = bio;
 const Network = require('../protocol/network');
 const AbstractMigration = require('../migrations/migration');
 const {
@@ -16,7 +19,6 @@ const {
   types,
   oldLayout
 } = require('../migrations/migrator');
-const MigrationState = require('../migrations/state');
 const layouts = require('./layout');
 const layout = layouts.wdb;
 
@@ -37,7 +39,7 @@ class MigrateMigrations extends AbstractMigration {
     this.logger = options.logger.context('wallet-migrations-migrate');
     this.db = options.db;
     this.ldb = options.ldb;
-    this.layout = options.layout;
+    this.layout = MigrateMigrations.layout();
   }
 
   async check() {
@@ -52,16 +54,27 @@ class MigrateMigrations extends AbstractMigration {
 
   async migrate(b) {
     this.logger.info('Migrating migrations..');
-    const state = new MigrationState();
-    state.nextMigration = 1;
+    let nextMigration = 1;
 
-    if (await this.ldb.get(oldLayout.M.encode(0))) {
-      b.del(oldLayout.M.encode(0));
-      state.nextMigration = 2;
+    if (await this.ldb.get(this.layout.oldLayout.wdb.M.encode(0))) {
+      b.del(this.layout.oldLayout.wdb.M.encode(0));
+      nextMigration = 2;
     }
 
     this.db.writeVersion(b, 1);
-    b.put(this.layout.M.encode(), state.encode());
+    b.put(
+      this.layout.newLayout.wdb.M.encode(),
+      this.encodeMigrationState(nextMigration)
+    );
+  }
+
+  encodeMigrationState(nextMigration) {
+    const size = 4 + 1 + 1;
+    const encoded = Buffer.alloc(size);
+
+    encoding.writeVarint(encoded, nextMigration, 4);
+
+    return encoded;
   }
 
   static info() {
@@ -70,6 +83,21 @@ class MigrateMigrations extends AbstractMigration {
       description: 'Wallet migration layout has changed.'
     };
   }
+
+  static layout() {
+    return {
+      oldLayout: {
+        wdb: {
+          M: bdb.key('M', ['uint32'])
+        }
+      },
+      newLayout: {
+        wdb: {
+          M: bdb.key('M')
+        }
+      }
+    };
+  }
 }
 
 /**

test/data/migrations/wallet-0-migrate-migrations.js

@@ -0,0 +1,35 @@
+'use strict';
+
+// walletdb version 0 to 1 migration.
+
+const Network = require('../../../lib/protocol/network');
+const WalletDB = require('../../../lib/wallet/walletdb');
+const cutil = require('../../util/common');
+const wutils = require('../../util/wallet');
+
+const NETWORK = Network.get('regtest');
+
+(async () => {
+  const wdb = new WalletDB({
+    network: NETWORK,
+    memory: true
+  });
+
+  await wdb.open();
+  console.log(JSON.stringify({
+    data: await getMigrationDump(wdb)
+  }, null, 2));
+
+  await wdb.close();
+})().catch((err) => {
+  console.error(err.stack);
+  process.exit(1);
+});
+
+async function getMigrationDump(wdb) {
+  const prefixes = [
+    'M'
+  ];
+
+  return wutils.dumpWDB(wdb, prefixes.map(cutil.prefix2hex));
+}

test/data/migrations/wallet-0-migrate-migrations.json

@@ -0,0 +1,21 @@
+{
+  "description": "Migrate migrations. Affects wdb layout M.",
+  "cases": [
+    {
+      "description": "Migration after migration flag was set.",
+      "before": {
+        "4d00000000": "00"
+      },
+      "after": {
+        "4d": "000000000200"
+      }
+    },
+    {
+      "description": "Migration before flag was set.",
+      "before": {},
+      "after": {
+        "4d": "000000000200"
+      }
+    }
+  ]
+}

test/util/common.js

@@ -225,6 +225,10 @@ common.enableLogger = () => {
   Logger.global.closed = false;
 };
 
+common.prefix2hex = function prefix2hex(prefix) {
+  return Buffer.from(prefix, 'ascii').toString('hex');
+};
+
 function parseUndo(data) {
   const br = bio.read(data);
   const items = [];

test/util/wallet.js

@@ -54,6 +54,22 @@ walletUtils.curEntry = (wdb) => {
   return new ChainEntry(walletUtils.curBlock(wdb));
 };
 
+walletUtils.dumpWDB = async (wdb, prefixes) => {
+  const data = await wdb.dump();
+  const filtered = {};
+
+  for (const [key, value] of Object.entries(data)) {
+    for (const prefix of prefixes) {
+      if (key.startsWith(prefix)) {
+        filtered[key] = value;
+        break;
+      }
+    }
+  }
+
+  return filtered;
+};
+
 function fromU32(num) {
   const data = Buffer.allocUnsafe(4);
   data.writeUInt32LE(num, 0, true);

test/wallet-migration-test.js

@@ -473,6 +473,70 @@ describe('Wallet Migrations', function() {
     });
   });
 
+  describe('Migrations #0 & #1 (data)', function() {
+    const location = testdir('migrate-wallet-0-1-int');
+    const migrationBAK = WalletMigrator.migrations;
+    const data = require('./data/migrations/wallet-0-migrate-migrations.json');
+    const Migration = WalletMigrator.MigrateMigrations;
+    const layout = Migration.layout();
+
+    const walletOptions = {
+      prefix: location,
+      memory: false,
+      network
+    };
+
+    let wdb, ldb;
+    beforeEach(async () => {
+      WalletMigrator.migrations = {};
+      await fs.mkdirp(location);
+
+      wdb = new WalletDB(walletOptions);
+      ldb = wdb.db;
+    });
+
+    afterEach(async () => {
+      WalletMigrator.migrations = migrationBAK;
+      await rimraf(location);
+    });
+
+    for (let i = 0; i < data.cases.length; i++) {
+      it(`should migrate ${data.cases[i].description}`, async () => {
+        const before = data.cases[i].before;
+        const after = data.cases[i].after;
+        await ldb.open();
+        const b = ldb.batch();
+
+        for (const [key, value] of Object.entries(before)) {
+          const bkey = Buffer.from(key, 'hex');
+          const bvalue = Buffer.from(value, 'hex');
+
+          b.put(bkey, bvalue);
+        }
+
+        writeVersion(b, 'wallet', 0);
+
+        await b.write();
+        await ldb.close();
+
+        WalletMigrator.migrations = {
+          0: Migration,
+          1: WalletMigrator.MigrateChangeAddress
+        };
+
+        wdb.options.walletMigrate = 1;
+        wdb.version = 1;
+
+        await wdb.open();
+        await checkVersion(ldb, layouts.wdb.V.encode(), 1);
+        await checkEntries(ldb, after);
+        const oldM = await ldb.get(layout.oldLayout.wdb.M.encode(0));
+        assert.strictEqual(oldM, null);
+        await wdb.close();
+      });
+    }
+  });
+
   describe('Migrate change address (integration)', function() {
     const location = testdir('wallet-change');
     const migrationsBAK = WalletMigrator.migrations;
@@ -971,3 +1035,24 @@ function getVersion(data, name) {
 
   return data.readUInt32LE(name.length);
 }
+
+async function checkVersion(db, versionDBKey, expectedVersion) {
+  const data = await db.get(versionDBKey);
+  const version = getVersion(data, 'wallet');
+
+  assert.strictEqual(version, expectedVersion);
+}
+
+async function checkEntries(db, data) {
+  for (const [key, value] of Object.entries(data)) {
+    const bkey = Buffer.from(key, 'hex');
+    const bvalue = Buffer.from(value, 'hex');
+
+    const stored = await db.get(bkey);
+
+    assert(stored,
+      `Value for ${key} not found in db, expected: ${value}`);
+    assert.bufferEqual(stored, bvalue,
+      `Value for ${key}: ${stored.toString('hex')} does not match expected: ${value}`);
+  }
+}