diff --git a/cmd/pics/state.go b/cmd/pics/state.go
index 9e5564a3bf5db0d196f43bf28e99720a79a3baee..3a96c2ee6e5f6bdef6a684668d0265ad9e4795b9 100644
--- a/cmd/pics/state.go
+++ b/cmd/pics/state.go
@@ -116,17 +116,14 @@ func keyTape(t *trie.Trie, number int) error {
}
var bucketLabels = map[string]string{
- dbutils.BlockReceiptsPrefix: "Receipts",
- dbutils.AccountsHistoryBucket: "History Of Accounts",
- dbutils.HeaderPrefix: "Headers",
- //dbutils.ConfigPrefix: "Config",
+ dbutils.BlockReceiptsPrefix: "Receipts",
+ dbutils.Log: "Event Logs",
+ dbutils.AccountsHistoryBucket: "History Of Accounts",
+ dbutils.StorageHistoryBucket: "History Of Storage",
+ dbutils.HeaderPrefix: "Headers",
dbutils.BlockBodyPrefix: "Block Bodies",
dbutils.HeaderNumberPrefix: "Header Numbers",
- dbutils.AccountChangeSetBucket: "Account Change Sets",
- dbutils.StorageChangeSetBucket: "Storage Change Sets",
- dbutils.CurrentStateBucket: "Current State",
dbutils.TxLookupPrefix: "Transaction Index",
- dbutils.StorageHistoryBucket: "History Of Storage",
dbutils.CodeBucket: "Code Of Contracts",
dbutils.Senders: "Senders",
dbutils.SyncStageProgress: "Sync Progress",
@@ -137,6 +134,7 @@ var bucketLabels = map[string]string{
dbutils.PlainAccountChangeSetBucket: "Account Changes",
dbutils.PlainStorageChangeSetBucket: "Storage Changes",
dbutils.IncarnationMapBucket: "Incarnations",
+ dbutils.Senders: "Transaction Senders",
}
/*dbutils.PlainContractCodeBucket,
diff --git a/docs/programmers_guide/changes_0_AccountChanges.png b/docs/programmers_guide/changes_0_AccountChanges.png
index 0913e16ef541fadbf02bdb54c6e02da099081e98..b93285ef852702ac613c3a2662c5b81447371bc6 100644
Binary files a/docs/programmers_guide/changes_0_AccountChanges.png and b/docs/programmers_guide/changes_0_AccountChanges.png differ
diff --git a/docs/programmers_guide/changes_0_BlockBodies.png b/docs/programmers_guide/changes_0_BlockBodies.png
index 185223d58184f7082d0de8fefc3ad9c6d0161923..906d19129c29602a1308b220a8fae34239b47ea9 100644
Binary files a/docs/programmers_guide/changes_0_BlockBodies.png and b/docs/programmers_guide/changes_0_BlockBodies.png differ
diff --git a/docs/programmers_guide/changes_0_HeaderNumbers.png b/docs/programmers_guide/changes_0_HeaderNumbers.png
index 85a9ee60e1632a0601f720e4eebc441c6f3953d0..5dc556db065b410e1324e57f29bf001bad9718f1 100644
Binary files a/docs/programmers_guide/changes_0_HeaderNumbers.png and b/docs/programmers_guide/changes_0_HeaderNumbers.png differ
diff --git a/docs/programmers_guide/changes_0_Headers.png b/docs/programmers_guide/changes_0_Headers.png
index 8a513612b792a2ed375324b10e0ffc70e2bcb57c..c8a401d8725f534640da8ed4b59fbfbd2cc8b48a 100644
Binary files a/docs/programmers_guide/changes_0_Headers.png and b/docs/programmers_guide/changes_0_Headers.png differ
diff --git a/docs/programmers_guide/changes_0_HistoryOfAccounts.png b/docs/programmers_guide/changes_0_HistoryOfAccounts.png
index 139d25951a0cfc5b8be7e3b74b1db530157b8536..948fec8e40994919600771df678751ec9a360e8c 100644
Binary files a/docs/programmers_guide/changes_0_HistoryOfAccounts.png and b/docs/programmers_guide/changes_0_HistoryOfAccounts.png differ
diff --git a/docs/programmers_guide/changes_0_PlainState.png b/docs/programmers_guide/changes_0_PlainState.png
index 32d6a1af2896961132556c2d62a9d3174dc3486f..89fd7d7f386cf00471adc0f4025b5673309ab4f6 100644
Binary files a/docs/programmers_guide/changes_0_PlainState.png and b/docs/programmers_guide/changes_0_PlainState.png differ
diff --git a/docs/programmers_guide/changes_0_Receipts.png b/docs/programmers_guide/changes_0_Receipts.png
index 9408a15b5d9dbd56a96924151190abcf56bdeda3..160dcf8487e22ace9b494e43d0a1652538920a2c 100644
Binary files a/docs/programmers_guide/changes_0_Receipts.png and b/docs/programmers_guide/changes_0_Receipts.png differ
diff --git a/docs/programmers_guide/changes_1_AccountChanges.png b/docs/programmers_guide/changes_1_AccountChanges.png
index 69c05edce4a273537ba110cf9bf67e0795dea443..d678afe237efba66081e826814c23259537c6996 100644
Binary files a/docs/programmers_guide/changes_1_AccountChanges.png and b/docs/programmers_guide/changes_1_AccountChanges.png differ
diff --git a/docs/programmers_guide/changes_1_BlockBodies.png b/docs/programmers_guide/changes_1_BlockBodies.png
index 94b2ccbfbff5faddbec270a4eca3ada95b78530d..236be2ba046fee1d6bc468c96ecd7f8f64d0372b 100644
Binary files a/docs/programmers_guide/changes_1_BlockBodies.png and b/docs/programmers_guide/changes_1_BlockBodies.png differ
diff --git a/docs/programmers_guide/changes_1_HashedState.png b/docs/programmers_guide/changes_1_HashedState.png
index a4db3f3686b1c365dd765630a3132fb59aadc00b..9e763dc6c7d5562c0d44fd12bd88066ce460f11e 100644
Binary files a/docs/programmers_guide/changes_1_HashedState.png and b/docs/programmers_guide/changes_1_HashedState.png differ
diff --git a/docs/programmers_guide/changes_1_HeaderNumbers.png b/docs/programmers_guide/changes_1_HeaderNumbers.png
index 49d5d04be202d5988f3cbeca517b67effc720831..6620a4d195b4c335c1bb4ffbf8a99423345bde13 100644
Binary files a/docs/programmers_guide/changes_1_HeaderNumbers.png and b/docs/programmers_guide/changes_1_HeaderNumbers.png differ
diff --git a/docs/programmers_guide/changes_1_Headers.png b/docs/programmers_guide/changes_1_Headers.png
index 1fed83c633624653035dc09b7b9bb6d37307ca25..bd44bf1fbb660bd7debcc57c5724a3462adadf1d 100644
Binary files a/docs/programmers_guide/changes_1_Headers.png and b/docs/programmers_guide/changes_1_Headers.png differ
diff --git a/docs/programmers_guide/changes_1_HistoryOfAccounts.png b/docs/programmers_guide/changes_1_HistoryOfAccounts.png
index dfe3a35f8d41c065f4674c3abb92b6eea1c97458..3cc580beabdb9bc4ab7fdbf75e3951a15e0d65fc 100644
Binary files a/docs/programmers_guide/changes_1_HistoryOfAccounts.png and b/docs/programmers_guide/changes_1_HistoryOfAccounts.png differ
diff --git a/docs/programmers_guide/changes_1_PlainState.png b/docs/programmers_guide/changes_1_PlainState.png
index 45991261f5a98225a86ee77b9932ec81c818bfe8..dbf51e0817f9bb09e61ab2b2ca61a6597f93e615 100644
Binary files a/docs/programmers_guide/changes_1_PlainState.png and b/docs/programmers_guide/changes_1_PlainState.png differ
diff --git a/docs/programmers_guide/changes_1_Receipts.png b/docs/programmers_guide/changes_1_Receipts.png
index e3c6f945a1b9aac36060d0d4a36504212262071b..61cbca007f58d58807619a8f7e4a606597e4f04a 100644
Binary files a/docs/programmers_guide/changes_1_Receipts.png and b/docs/programmers_guide/changes_1_Receipts.png differ
diff --git a/docs/programmers_guide/changes_1_Senders.png b/docs/programmers_guide/changes_1_Senders.png
index 8abda6822e37b7d8f341920782b1ce6b291a9bc8..46f4720a7fea29dd0b7129085cba77f902adb886 100644
Binary files a/docs/programmers_guide/changes_1_Senders.png and b/docs/programmers_guide/changes_1_Senders.png differ
diff --git a/docs/programmers_guide/changes_1_SyncProgress.png b/docs/programmers_guide/changes_1_SyncProgress.png
index 9ab201f6cac04950b8b4aaefa15cf0dc13297482..d485a4bcbe57f2ce666c64095313016caf604a67 100644
Binary files a/docs/programmers_guide/changes_1_SyncProgress.png and b/docs/programmers_guide/changes_1_SyncProgress.png differ
diff --git a/docs/programmers_guide/changes_1_TransactionIndex.png b/docs/programmers_guide/changes_1_TransactionIndex.png
index 190a7723d7e0017a53a41e5da4a2d992a2410142..aa6be5a39b6a190d17193d74a4a15091f4844157 100644
Binary files a/docs/programmers_guide/changes_1_TransactionIndex.png and b/docs/programmers_guide/changes_1_TransactionIndex.png differ
diff --git a/docs/programmers_guide/changes_1_TransactionSenders.png b/docs/programmers_guide/changes_1_TransactionSenders.png
new file mode 100644
index 0000000000000000000000000000000000000000..b9bec4da23af242459b61acabb4f80b928cfb9d4
Binary files /dev/null and b/docs/programmers_guide/changes_1_TransactionSenders.png differ
diff --git a/docs/programmers_guide/db_walkthrough.MD b/docs/programmers_guide/db_walkthrough.MD
index 21d8ce5da52e6782100840bf26bce1863bf472e7..ab98ca722829ce3fee98a0bc1f2c5ecea2b58f41 100644
--- a/docs/programmers_guide/db_walkthrough.MD
+++ b/docs/programmers_guide/db_walkthrough.MD
@@ -1,13 +1,10 @@
-Database Walkthrough
-====================
-
This document attempts to explain how Turbo-Geth organises its persistent data in its database and how this organisation is
different from go-ethereum, the project from which it is derived. We start from a very simple genesis block and then
apply one block containing an ETH transfer. For each step, we show visualisations produced by the code available in
Turbo-Geth and code added to a fork of go-ethereum.
Genesis in Turbo-Geth
----------------------
+====================
For the genesis block, we generate 3 different private keys and construct Ethereum addresses from them.
Then, we endow one of the accounts with 9 ETH, and two others with 0.2 and 0.3 ETH, respectively.
This is how the initial state trie looks:
@@ -85,32 +82,28 @@ You can check that for the genesis block
* logs bloom is 256-byte zeros (see the 8 white rows of all 32-byte 0s)
* total difficulty is 0x80, which is RLP encoding of value 0
-Bucket "Block Bodies"
+Table "Block Bodies"
---------------------
-The keys in this bucket are concatenations of 8-byte encoding of the block number and 32-byte block hash.
-The values are RLP-encoded list of 3 structures:
+The keys in this table are concatenations of 8-byte encoding of the block number and 32-byte block hash.
+The values are RLP-encoded list of 2 structures:
1. List of transactions
-2. List transaction sender addresses, one for each transaction in the first list
-3. List of ommers (a.k.a uncles)
+2. List of ommers (a.k.a uncles)
In the case of the genesis block, all of these lists are empty (RLP encodings `0xC0`), and the prefix `0xC3` means
in RLP "some number of sub-structures with the total length of 3 bytes".
-The reason turbo-geth keeps the list of transaction sender addresses for each transaction has to do with the fact that
-transactions themselves do not contain this information directly. Sender's address can be "recovered" from the digital
-signature, but this recovery can be computationally intensive, therefore we "memoise" it.
-Next three buckets, "Last Header", "Last Fast", and "Last Block", always contain just one record each, and their
+Next three tables, "Last Header", "Last Fast", and "Last Block", always contain just one record each, and their
keys are always the same, the ASCII-encodings of the strings `LastHeader`, `LastFast`, and `LastBlock`, respectively.
The values record the block/header hash of the last header, receipt or block chains that the node has managed to sync
from its peers in the network. The value in "Last Fast" bucket is not really used at the moment, because turbo-geth
does not support Fast Sync.
-Bucket "Header Numbers"
+Table "Header Numbers"
-----------------------
Is a mapping of 32-byte header/block hashes to the corresponding block numbers (encoded
@@ -118,18 +111,18 @@ in 8 bytes):
-Bucket "Receipts"
+Table "Receipts"
-----------------
Records the list of transaction receipts for each block:
-The first 8 bytes of the key (or 16 nibbles, equaling to 0s here) encode the block number, which is 0 for the Genesis block.
-The remaining 32 bytes of the key encode the block hash. The value is the RLP-encoded list of receipts. In our case, there were
-no transactions in the Genesis block, therefore, we have RLP encoding of an empty list, `0xC0`.
+The 8 bytes of the key (or 16 nibbles, equaling to 0s here) encode the block number, which is 0 for the Genesis block.
+The value is the CBOR-encoded list of receipts. In our case, there were
+no transactions in the Genesis block, therefore, we have CBOR encoding of an empty list, `0xf6`.
-Bucket "PlainState"
+Table "PlainState"
-----------------
@@ -171,31 +164,31 @@ incarnation 0 (not set), and all contract accounts will start their existence wi
Contract accounts may also contain a contract code hash and storage root. These two pieces of information would make
the record in the "Accounts" bucket contain 5 instead of 3 fields.
-Bucket "History Of Accounts"
+Table "History Of Accounts"
----------------------------
-key="account addresses + inverted incarnation", value="encoded array of block numbers, where account changed/created".
+key="account addresses + incarnation", value="encoded array of block numbers, where account changed/created".
The history of accounts records how the accounts changed at each block. But, instead of recording,
at each change, the value that the accounts had *after* the change, it records what value the accounts
had *before* the change. That explains the empty values here -- it records the fact that these
three accounts in questions did not exist prior to the block 0.
-Bucket "Change Sets"
+Table "Change Sets"
--------------------
Records the history of changes in accounts and contract storage. But, unlike in "History of Accounts"
and "History of Storage", where keys are derived from accounts' addresses
-(key="address hash + block number"), in the "Change Sets" bucket, keys are derived from the block numbers:
+(key="address hash + block number"), in the "Change Sets" table, keys are derived from the block numbers:
In the cases of our Genesis block, the key is composed from the encoding of the block number (`0x20`), and the
ASCII-code of `hAT` (meaning **h**istory of **A**counts **T**rie).
-The "Change Set" bucket records changes that happen to accounts and contract storage slots at every block.
-It is important to note that the values recorded in the "Changes Set" bucket are not the values the accounts
+The "Change Set" table records changes that happen to accounts and contract storage slots at every block.
+It is important to note that the values recorded in the "Changes Set" table are not the values the accounts
(or storage slots) had AFTER the change, it records what value the accounts (or storage slots)
had BEFORE the change. That explains the empty values here - it records the fact that these
three accounts in question did not exist prior to the block 0.
@@ -212,61 +205,14 @@ as well as linear-time merge of multiple changesets.
In our example, all values are empty strings, therefore we see 3 zero offsets (24 white boxes with zeros in them).
5. Values themselves. In our example, they are empty, so this 5th part is not present.
-Buckets "HashedState" and "IntermediateTrieHashes"
+Tables "HashedState" and "IntermediateTrieHashes"
--------------------------------------------------
-These buckets used to calculate Merkle Trie root hash of whole current state.
-Because TurboGeth does it completely different from go-ethereum,
-we created [dedicated article](guide.md#organising-ethereum-state-into-a-merkle-tree) for it.
-
-Genesis in go-ethereum
-------------------------------
-
-Now we will create the same Genesis state and block in go-ethereum (in archive mode to make sure we compare like for like).
-Here is how the database looks like. Since go-ethereum uses LevelDB, and LevelDB does not have a concept of "buckets" (or
-"tables"), go-ethereum emulates them by adding table-specific prefixes to all the keys, with the exception of the keys that
-describe the state trie (bucket "Hashes" in our example). In the illustration, these prefixes are mostly removed for better
-comparison with turbo-geth. They were not removed only for the buckets "LastBlock", "LastHeader" and "LastFast", because
-othewise they key would be empty.
-
-
-
-The buckets "Headers", "Config", "Last Header", "Last Fast", "Last Block", all look identical
-to those in the turbo-geth database. We will walk through the ones that are different.
-
-In the bucket "Block Bodies", the value is slightly different:
-
-
-
-The difference is that the block body has 2 elements instead of 3 in turbo-geth. The missing element is the list
-of the sender addresses that go-ethereum does not store, but recomputes after loading or caches in memory.
-
-The buckets "Accounts", "History Of Accounts", and "Change Sets" are missing, because go-ethereum uses a very
-different mechanism for storing the state and its history:
-
-
-
-In the illustration showing the state trie, one can find 4 parts of the diagram that consist of the coloured boxes
-(that excludes the leaves that contain account balances and nonces). These parts are usually called "trie nodes",
-and in the diagram above we see 2 types of trie nodes:
-1. Branch node. This is the horizontal line of 3 coloured boxes on the top. It branches the traversal of the state
-trie from top to bottom 3-ways.
-2. Leaf node. These are 3 vertical lines of 63 coloured boxes.
-
-Each type of trie nodes can be serialised (using RLP encoding), to convert it to a string of bytes. What we see in
-the values of the records in the "Hashes" bucket just above are the RLP-encodings of these 4 trie nodes.
-What we see in the keys of these records are the results of `Keccak256` function applied to the values. In a way,
-this is similar to the "Preimages" bucket, with the different type of values.
-
-If you look closely, you may notice that the keys of the last 3 records are actually contained inside the value
-of the first record. This is because the first value correponds to that 3-way branch node, and the hashes of the
-leaf nodes are used like "pointers" to thoese nodes. Continuing the "pointer" analogy, you can say that
-"dereferencing" these pointers mean fetching the corresponding records from this "Hashes" bucket. Using such
-"derederencing" process, one can traverse the state trie from the top to any leaf at the bottom. Each step in
-such traversal requires finding the corresponding record in the "Hashes" bucket.
+These tables are used to calculate Merkle Trie root hash of whole current state. For Genesis block, these tables
+are not populated, because the root hash is known in advance.
Block with 1 Ethereum transaction in Turbo-Geth
-------------------------------
+===============================================
The first block contains transaction that transfers 0.001 ETH from the account holding 9 ETH to
a new account with the address `0x0100000000000000000000000000000000000000` (this is what one gets
specifying `common.Address{1}` in the code).
@@ -286,13 +232,18 @@ omit the records that are the same as before the insertion of the block.
-Bucket "Receipts" now has some non-empty information for the block 1, because we had 1 transaction:
+Table "Receipts"
+----------------
+
+Table `Receipts` now has some non-empty information for the block 1, because we had 1 transaction:
The first 8 bytes of the key (or 16 nibbles, 15 zeroes followed by `1`) encode the block number, which is 1.
-The remaining 32 bytes of the key encode the block hash.
-The value is the RLP-encoded list of receipts. In our case, a transaction was sent and as a matter of fact,
+Receipt records are only kept for "canonical" blocks, and not for forked blocks, that is why receipt keys
+do not include 32-byte block hash.
+
+The value is the CBOR-encoded list of receipts. In our case, a transaction was sent and as a matter of fact,
instead of having `0xC0` (that represent an empty list) we have `0xc6c501825208c0` that is the receipt for the 0.001 ETH transaction.
First prefix, `c6` means that the following 6 bytes is a sequence of RLP-encoded sub-structures.
In our case, there is only one sub-structure, and its
@@ -304,7 +255,16 @@ Next, we have a byte array of length 2 (prefix `82`), `5208`, which is
And the last structure, `c0` is RLP encoding of an empty list. Usually, the log (events) emitted by the transaction,
would be here. But since our transaction did not emit anything, the list is empty.
-Next bucket is "History Of Accounts":
+Table "Senders"
+---------------
+
+The reason turbo-geth keeps the list of transaction sender addresses for each transaction has to do with the fact that
+transactions themselves do not contain this information directly. Sender's address can be "recovered" from the digital
+signature, but this recovery can be computationally intensive, therefore we "memoise" it.
+
+
+Table "History Of Accounts"
+---------------------------
@@ -380,6 +340,58 @@ Type "help", "copyright", "credits" or "license" for more information.
````
Which is 0.001 ETH that is exactly how much has been sent to the recipient address, meaning that this account represent the recipient of the transaction.
+
+
+Because TurboGeth does it completely different from go-ethereum,
+we created [dedicated article](guide.md#organising-ethereum-state-into-a-merkle-tree) for it.
+
+Genesis in go-ethereum
+------------------------------
+
+Now we will create the same Genesis state and block in go-ethereum (in archive mode to make sure we compare like for like).
+Here is how the database looks like. Since go-ethereum uses LevelDB, and LevelDB does not have a concept of "buckets" (or
+"tables"), go-ethereum emulates them by adding table-specific prefixes to all the keys, with the exception of the keys that
+describe the state trie (bucket "Hashes" in our example). In the illustration, these prefixes are mostly removed for better
+comparison with turbo-geth. They were not removed only for the buckets "LastBlock", "LastHeader" and "LastFast", because
+othewise they key would be empty.
+
+
+
+The buckets "Headers", "Config", "Last Header", "Last Fast", "Last Block", all look identical
+to those in the turbo-geth database. We will walk through the ones that are different.
+
+In the bucket "Block Bodies", the value is slightly different:
+
+
+
+The difference is that the block body has 2 elements instead of 3 in turbo-geth. The missing element is the list
+of the sender addresses that go-ethereum does not store, but recomputes after loading or caches in memory.
+
+The buckets "Accounts", "History Of Accounts", and "Change Sets" are missing, because go-ethereum uses a very
+different mechanism for storing the state and its history:
+
+
+
+In the illustration showing the state trie, one can find 4 parts of the diagram that consist of the coloured boxes
+(that excludes the leaves that contain account balances and nonces). These parts are usually called "trie nodes",
+and in the diagram above we see 2 types of trie nodes:
+1. Branch node. This is the horizontal line of 3 coloured boxes on the top. It branches the traversal of the state
+trie from top to bottom 3-ways.
+2. Leaf node. These are 3 vertical lines of 63 coloured boxes.
+
+Each type of trie nodes can be serialised (using RLP encoding), to convert it to a string of bytes. What we see in
+the values of the records in the "Hashes" bucket just above are the RLP-encodings of these 4 trie nodes.
+What we see in the keys of these records are the results of `Keccak256` function applied to the values. In a way,
+this is similar to the "Preimages" bucket, with the different type of values.
+
+If you look closely, you may notice that the keys of the last 3 records are actually contained inside the value
+of the first record. This is because the first value correponds to that 3-way branch node, and the hashes of the
+leaf nodes are used like "pointers" to thoese nodes. Continuing the "pointer" analogy, you can say that
+"dereferencing" these pointers mean fetching the corresponding records from this "Hashes" bucket. Using such
+"derederencing" process, one can traverse the state trie from the top to any leaf at the bottom. Each step in
+such traversal requires finding the corresponding record in the "Hashes" bucket.
+
+
Block 1 in go-ethereum
----------------------