have AccountDB use address hash for uniqueness (openethereum#1533)

* partially done alternate migration scheme

* finish altering migration framework

* migrate tests to new migration framework

* address comments

* remove superfluous newline
[ci skip]

* TempIdx -> TempIndex
[ci skip]

* modify account_db to work on address hash, not address

* add a database migration for new accountdb

* preserve first 96 bits of keys when combining

* handle metadata keys in migration and preserve first 96 bits

* fix comments and hash address instead of hash

* different migrations based on pruning

* migrations mutably borrow self

* batch abstraction for migration

* added missing licence headers

* overlay recent v7 migration

* better error handling, migrate version key as well

* fix migration tests

* commit final batch and migrate journaled insertions

* two passes on journal to migrate all possible deleted keys

ethcore/src/account_db.rs

@@ -1,26 +1,59 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
 //! DB backend wrapper for Account trie
 use util::*;
 
 static NULL_RLP_STATIC: [u8; 1] = [0x80; 1];
 
+// combines a key with an address hash to ensure uniqueness.
+// leaves the first 96 bits untouched in order to support partial key lookup.
+#[inline]
+fn combine_key<'a>(address_hash: &'a H256, key: &'a H256) -> H256 {
+	let mut dst = key.clone();
+	{
+		let last_src: &[u8] = &*address_hash;
+		let last_dst: &mut [u8] = &mut *dst;
+		for (k, a) in last_dst[12..].iter_mut().zip(&last_src[12..]) {
+			*k ^= *a
+		}
+	}
+
+	dst
+}
+
 // TODO: introduce HashDBMut?
 /// DB backend wrapper for Account trie
 /// Transforms trie node keys for the database
 pub struct AccountDB<'db> {
 	db: &'db HashDB,
-	address: H256,
-}
-
-#[inline]
-fn combine_key<'a>(address: &'a H256, key: &'a H256) -> H256 {
-	address ^ key
+	address_hash: H256,
 }
 
 impl<'db> AccountDB<'db> {
-	pub fn new(db: &'db HashDB, address: &Address) -> AccountDB<'db> {
+	/// Create a new AccountDB from an address.
+	pub fn new(db: &'db HashDB, address: &Address) -> Self {
+		Self::from_hash(db, address.sha3())
+	}
+
+	/// Create a new AcountDB from an address' hash.
+	pub fn from_hash(db: &'db HashDB, address_hash: H256) -> Self {
 		AccountDB {
 			db: db,
-			address: address.into(),
+			address_hash: address_hash,
 		}
 	}
 }
@@ -34,14 +67,14 @@ impl<'db> HashDB for AccountDB<'db>{
 		if key == &SHA3_NULL_RLP {
 			return Some(&NULL_RLP_STATIC);
 		}
-		self.db.get(&combine_key(&self.address, key))
+		self.db.get(&combine_key(&self.address_hash, key))
 	}
 
 	fn contains(&self, key: &H256) -> bool {
 		if key == &SHA3_NULL_RLP {
 			return true;
 		}
-		self.db.contains(&combine_key(&self.address, key))
+		self.db.contains(&combine_key(&self.address_hash, key))
 	}
 
 	fn insert(&mut self, _value: &[u8]) -> H256 {
@@ -60,20 +93,26 @@ impl<'db> HashDB for AccountDB<'db>{
 /// DB backend wrapper for Account trie
 pub struct AccountDBMut<'db> {
 	db: &'db mut HashDB,
-	address: H256,
+	address_hash: H256,
 }
 
 impl<'db> AccountDBMut<'db> {
-	pub fn new(db: &'db mut HashDB, address: &Address) -> AccountDBMut<'db> {
+	/// Create a new AccountDB from an address.
+	pub fn new(db: &'db mut HashDB, address: &Address) -> Self {
+		Self::from_hash(db, address.sha3())
+	}
+
+	/// Create a new AcountDB from an address' hash.
+	pub fn from_hash(db: &'db mut HashDB, address_hash: H256) -> Self {
 		AccountDBMut {
 			db: db,
-			address: address.into(),
+			address_hash: address_hash,
 		}
 	}
 
 	#[allow(dead_code)]
 	pub fn immutable(&'db self) -> AccountDB<'db> {
-		AccountDB { db: self.db, address: self.address.clone() }
+		AccountDB { db: self.db, address_hash: self.address_hash.clone() }
 	}
 }
 
@@ -86,22 +125,22 @@ impl<'db> HashDB for AccountDBMut<'db>{
 		if key == &SHA3_NULL_RLP {
 			return Some(&NULL_RLP_STATIC);
 		}
-		self.db.get(&combine_key(&self.address, key))
+		self.db.get(&combine_key(&self.address_hash, key))
 	}
 
 	fn contains(&self, key: &H256) -> bool {
 		if key == &SHA3_NULL_RLP {
 			return true;
 		}
-		self.db.contains(&combine_key(&self.address, key))
+		self.db.contains(&combine_key(&self.address_hash, key))
 	}
 
 	fn insert(&mut self, value: &[u8]) -> H256 {
 		if value == &NULL_RLP {
 			return SHA3_NULL_RLP.clone();
 		}
 		let k = value.sha3();
-		let ak = combine_key(&self.address, &k);
+		let ak = combine_key(&self.address_hash, &k);
 		self.db.emplace(ak, value.to_vec());
 		k
 	}
@@ -110,15 +149,15 @@ impl<'db> HashDB for AccountDBMut<'db>{
 		if key == SHA3_NULL_RLP {
 			return;
 		}
-		let key = combine_key(&self.address, &key);
+		let key = combine_key(&self.address_hash, &key);
 		self.db.emplace(key, value.to_vec())
 	}
 
 	fn remove(&mut self, key: &H256) {
 		if key == &SHA3_NULL_RLP {
 			return;
 		}
-		let key = combine_key(&self.address, key);
+		let key = combine_key(&self.address_hash, key);
 		self.db.remove(&key)
 	}
 }

ethcore/src/migrations/extras/mod.rs

@@ -1,5 +1,21 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
 //! Extras database migrations.
 
 mod v6;
 
-pub use self::v6::ToV6;
+pub use self::v6::ToV6;

ethcore/src/migrations/extras/v6.rs

@@ -1,3 +1,19 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
 use util::migration::SimpleMigration;
 
 /// This migration reduces the sizes of keys and moves `ExtrasIndex` byte from back to the front.
@@ -22,7 +38,7 @@ impl SimpleMigration for ToV6 {
 		6
 	}
 
-	fn simple_migrate(&self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
+	fn simple_migrate(&mut self, key: Vec<u8>, value: Vec<u8>) -> Option<(Vec<u8>, Vec<u8>)> {
 
 		//// at this version all extras keys are 33 bytes long.
 		if key.len() == 33 {

ethcore/src/migrations/mod.rs

@@ -1,3 +1,4 @@
 //! Database migrations.
 
 pub mod extras;
+pub mod state;

ethcore/src/migrations/state/mod.rs

@@ -0,0 +1,21 @@
+// Copyright 2015, 2016 Ethcore (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! State database migrations.
+
+mod v7;
+
+pub use self::v7::{ArchiveV7, OverlayRecentV7};