diff --git a/common/changeset/storage_changeset.go b/common/changeset/storage_changeset.go
index 512af2ef6a458d85e3435c87ee22d1b83f340a4a..a21eeae9cfdad78f13b83a5dbf6deb41a81b7004 100644
--- a/common/changeset/storage_changeset.go
+++ b/common/changeset/storage_changeset.go
@@ -11,8 +11,9 @@ import (
)
const (
+ DefaultIncarnation = uint64(1)
storageEnodingIndexSize = 4
- storageEnodingStartElem = uint32(4)
+ storageEnodingStartElem = 4
storageEnodingLengthOfNumOfElements = 4
storageEnodingLengthOfDict = 2
storageEnodingLengthOfNumTypeOfElements = 2
@@ -40,6 +41,7 @@ numOfUint32Values uint16
[len(val0), len(val0)+len(val1), ..., len(val0)+len(val1)+...+len(val_{numOfUint8Values-1})] []uint8
[len(valnumOfUint8Values), len(val0)+len(val1), ..., len(val0)+len(val1)+...+len(val_{numOfUint16Values-1})] []uint16
[len(valnumOfUint16Values), len(val0)+len(val1), ..., len(val0)+len(val1)+...+len(val_{numOfUint32Values-1})] []uint32
+[elementNum:incarnation] - optional [uint32:uint64...]
*/
func EncodeStorage(s *ChangeSet) ([]byte, error) {
@@ -54,30 +56,32 @@ func EncodeStorage(s *ChangeSet) ([]byte, error) {
return nil, err
}
- addrHashesMap := make(map[common.Hash]uint32)
- addrHashList := make([]byte, 0)
+ addrHashesMap := make([]uint32, n)
+ var addrHashList []byte
+ notDefaultIncarnationList := make([]byte, 0)
//collect information about unique addHashes and non default incarnations
- nextIDAddrHash := uint32(0)
- var addrHash common.Hash
- for i := 0; i < n; i++ {
- //copy addrHash
- copy(
- addrHash[:],
- s.Changes[i].Key[0:common.HashLength],
- )
-
- //fill addrHashesMap and addrHashList
- if _, ok := addrHashesMap[addrHash]; !ok {
- addrHashesMap[addrHash] = nextIDAddrHash
- nextIDAddrHash++
- addrHashList = append(addrHashList, addrHash[:]...)
+ var addrID uint32
+ var addrIdxToIncarnation [12]byte
+ for i, change := range s.Changes {
+ if i == 0 || !bytes.Equal(change.Key[:common.HashLength], s.Changes[i-1].Key[:common.HashLength]) {
+ if i > 0 {
+ addrID++
+ }
+ addrHashList = append(addrHashList, change.Key[:common.HashLength]...)
+ incarnation := ^binary.BigEndian.Uint64(change.Key[common.HashLength : common.HashLength+common.IncarnationLength])
+ if incarnation != DefaultIncarnation {
+ binary.BigEndian.PutUint32(addrIdxToIncarnation[:4], addrID)
+ binary.BigEndian.PutUint64(addrIdxToIncarnation[4:12], ^incarnation)
+ notDefaultIncarnationList = append(notDefaultIncarnationList, addrIdxToIncarnation[:]...)
+ }
}
+ addrHashesMap[i] = addrID
}
//write numOfUniqAddrHashes
numOfUniqAddrHashes := make([]byte, storageEnodingLengthOfDict)
- binary.BigEndian.PutUint16(numOfUniqAddrHashes, uint16(len(addrHashesMap)))
+ binary.BigEndian.PutUint16(numOfUniqAddrHashes, uint16(addrID+1))
if _, err := buf.Write(numOfUniqAddrHashes); err != nil {
return nil, err
}
@@ -87,60 +91,62 @@ func EncodeStorage(s *ChangeSet) ([]byte, error) {
return nil, err
}
- lenOfAddr := getNumOfBytesByLen(len(addrHashesMap))
- values := new(bytes.Buffer)
- lengthes := make([]byte, storageEnodingLengthOfNumTypeOfElements*3)
+ lenOfAddr := getNumOfBytesByLen(int(addrID + 1))
numOfUint8 := uint16(0)
numOfUint16 := uint16(0)
numOfUint32 := uint16(0)
- keys := new(bytes.Buffer)
lengthOfValues := uint32(0)
row := make([]byte, lenOfAddr+common.HashLength)
- for i := 0; i < len(s.Changes); i++ {
+ for i, change := range s.Changes {
writeKeyRow(
- addrHashesMap[common.BytesToHash(s.Changes[i].Key[0:common.HashLength])],
+ addrHashesMap[i],
row[0:lenOfAddr],
)
- copy(row[lenOfAddr:lenOfAddr+common.HashLength], common.CopyBytes(s.Changes[i].Key[common.IncarnationLength+common.HashLength:common.IncarnationLength+2*common.HashLength]))
- keys.Write(row)
-
- lengthOfValues += uint32(len(s.Changes[i].Value))
+ copy(row[lenOfAddr:lenOfAddr+common.HashLength], change.Key[common.IncarnationLength+common.HashLength:common.IncarnationLength+2*common.HashLength])
+ buf.Write(row)
+ }
+ // Remember position to fix up the lengths later
+ lengthPos := buf.Len()
+ uint16b := make([]byte, 2)
+ // Reserve 3 positions for lengths
+ buf.Write(uint16b)
+ buf.Write(uint16b)
+ buf.Write(uint16b)
+ uint32b := make([]byte, 4)
+ for _, change := range s.Changes {
+ lengthOfValues += uint32(len(change.Value))
switch {
case lengthOfValues <= 255:
numOfUint8++
- lengthes = append(lengthes, uint8(lengthOfValues))
+ buf.WriteByte(uint8(lengthOfValues))
case lengthOfValues <= 65535:
numOfUint16++
- uint16b := make([]byte, 2)
binary.BigEndian.PutUint16(uint16b, uint16(lengthOfValues))
- lengthes = append(lengthes, uint16b...)
+ buf.Write(uint16b)
default:
numOfUint32++
- uint32b := make([]byte, 4)
binary.BigEndian.PutUint32(uint32b, lengthOfValues)
- lengthes = append(lengthes, uint32b...)
+ buf.Write(uint32b)
}
- values.Write(s.Changes[i].Value)
}
- binary.BigEndian.PutUint16(lengthes[0:storageEnodingLengthOfNumTypeOfElements], numOfUint8)
- binary.BigEndian.PutUint16(lengthes[storageEnodingLengthOfNumTypeOfElements:2*storageEnodingLengthOfNumTypeOfElements], numOfUint16)
- binary.BigEndian.PutUint16(lengthes[2*storageEnodingLengthOfNumTypeOfElements:3*storageEnodingLengthOfNumTypeOfElements], numOfUint32)
- if _, err := buf.Write(keys.Bytes()); err != nil {
- return nil, err
+ for _, change := range s.Changes {
+ buf.Write(change.Value)
}
- if _, err := buf.Write(lengthes); err != nil {
- return nil, err
- }
-
- if _, err := buf.Write(values.Bytes()); err != nil {
- return nil, err
+ if len(notDefaultIncarnationList) > 0 {
+ if _, err := buf.Write(notDefaultIncarnationList); err != nil {
+ return nil, err
+ }
}
- byt := buf.Bytes()
- return byt, nil
+ b := buf.Bytes()
+ // Fix up the lengths
+ binary.BigEndian.PutUint16(b[lengthPos:], numOfUint8)
+ binary.BigEndian.PutUint16(b[lengthPos+2:], numOfUint16)
+ binary.BigEndian.PutUint16(b[lengthPos+4:], numOfUint32)
+ return b, nil
}
func DecodeStorage(b []byte) (*ChangeSet, error) {
@@ -153,50 +159,65 @@ func DecodeStorage(b []byte) (*ChangeSet, error) {
return h, fmt.Errorf("decode: input too short (%d bytes)", len(b))
}
- //numOfElements uint32
- numOfElements := binary.BigEndian.Uint32(b[0:storageEnodingLengthOfNumOfElements])
+ numOfElements := int(binary.BigEndian.Uint32(b[0:storageEnodingLengthOfNumOfElements]))
h.Changes = make([]Change, numOfElements)
if numOfElements == 0 {
return h, nil
}
- dictLen := binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict])
- addMap := make(map[uint32]common.Hash)
- for i := 0; i < int(dictLen); i++ {
+ dictLen := int(binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict]))
+ addMap := make(map[uint32][]byte)
+ for i := 0; i < dictLen; i++ {
elemStart := storageEnodingLengthOfNumOfElements + storageEnodingLengthOfDict + i*common.HashLength
- addMap[uint32(i)] = common.BytesToHash(b[elemStart : elemStart+common.HashLength])
+ addMap[uint32(i)] = b[elemStart : elemStart+common.HashLength]
}
lenOfValsPos := storageEnodingStartElem +
- 2 + uint32(dictLen)*common.HashLength +
- numOfElements*uint32(getNumOfBytesByLen(int(dictLen))+common.HashLength)
+ 2 + dictLen*common.HashLength +
+ numOfElements*(getNumOfBytesByLen(dictLen)+common.HashLength)
numOfUint8 := int(binary.BigEndian.Uint16(b[lenOfValsPos : lenOfValsPos+2]))
numOfUint16 := int(binary.BigEndian.Uint16(b[lenOfValsPos+2 : lenOfValsPos+4]))
numOfUint32 := int(binary.BigEndian.Uint16(b[lenOfValsPos+4 : lenOfValsPos+6]))
lenOfValsPos = lenOfValsPos + 3*storageEnodingLengthOfNumTypeOfElements
- valuesPos := lenOfValsPos + uint32(numOfUint8) + uint32(numOfUint16*2) + uint32(numOfUint32*4)
+ valuesPos := lenOfValsPos + numOfUint8 + numOfUint16*2 + numOfUint32*4
- elementStart := storageEnodingStartElem + storageEnodingLengthOfDict + uint32(dictLen)*common.HashLength
- key := make([]byte, common.HashLength*2+common.IncarnationLength)
+ incarnationPosition := lenOfValsPos + calculateIncarnationPos3(b[lenOfValsPos:], numOfUint8, numOfUint16, numOfUint32)
+ notDefaultIncarnation := make(map[uint32]uint64)
+
+ if len(b) > incarnationPosition {
+ if len(b[incarnationPosition:])%12 != 0 {
+ return h, fmt.Errorf("decode: incarnatin part is incorrect(%d bytes)", len(b[incarnationPosition:]))
+ }
+ for i := incarnationPosition; i < len(b); i += 12 {
+ id := binary.BigEndian.Uint32(b[i : i+4])
+ inc := ^binary.BigEndian.Uint64(b[i+4 : i+12])
+ notDefaultIncarnation[id] = inc
+ }
+ }
+
+ elementStart := storageEnodingStartElem + storageEnodingLengthOfDict + dictLen*common.HashLength
- lenOfAddHash := uint32(getNumOfBytesByLen(len(addMap)))
- //lastValLen:=0
- for i := uint32(0); i < numOfElements; i++ {
+ lenOfAddHash := getNumOfBytesByLen(len(addMap))
+ for i := 0; i < numOfElements; i++ {
//copy addrHash
+ key := make([]byte, common.HashLength*2+common.IncarnationLength)
elem := elementStart + i*(lenOfAddHash+common.HashLength)
- copy(
- key[0:common.HashLength],
- readFromMap(addMap, b[elem:elem+lenOfAddHash]).Bytes(),
- )
+ addrIdx := getUint32(b[elem : elem+lenOfAddHash])
+ copy(key[:common.HashLength], addMap[addrIdx])
+ if inc, ok := notDefaultIncarnation[addrIdx]; ok {
+ binary.BigEndian.PutUint64(key[common.HashLength:], ^inc)
+ } else {
+ binary.BigEndian.PutUint64(key[common.HashLength:], ^DefaultIncarnation)
+ }
//copy key hash
copy(
key[common.HashLength+common.IncarnationLength:2*common.HashLength+common.IncarnationLength],
common.CopyBytes(b[elem+lenOfAddHash:elem+lenOfAddHash+common.HashLength]),
)
- binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], ^uint64(1))
- h.Changes[i].Key = common.CopyBytes(key)
+
+ h.Changes[i].Key = key
h.Changes[i].Value = findVal(b[lenOfValsPos:valuesPos], b[valuesPos:], i, numOfUint8, numOfUint16, numOfUint32)
}
return h, nil
@@ -216,41 +237,57 @@ func getNumOfBytesByLen(n int) int {
}
}
-func findVal(lenOfVals []byte, values []byte, i uint32, numOfUint8, numOfUint16, numOfUint32 int) []byte {
- lenOfValStart := uint32(0)
- lenOfValEnd := uint32(0)
+func calculateIncarnationPos3(b []byte, numOfUint8, numOfUint16, numOfUint32 int) int {
+ res := 0
+ end := 0
+ switch {
+ case numOfUint32 > 0:
+ end = numOfUint8 + numOfUint16*2 + numOfUint32*4
+ res = int(binary.BigEndian.Uint32(b[end-4:end])) + end
+ case numOfUint16 > 0:
+ end = numOfUint8 + numOfUint16*2
+ res = int(binary.BigEndian.Uint16(b[end-2:end])) + end
+ case numOfUint8 > 0:
+ end = numOfUint8
+ res = int(b[end-1]) + end
+ default:
+ return 0
+ }
+ return res
+}
+
+func findVal(lenOfVals []byte, values []byte, i int, numOfUint8, numOfUint16, numOfUint32 int) []byte {
+ lenOfValStart := 0
+ lenOfValEnd := 0
switch {
- case i < uint32(numOfUint8):
- lenOfValEnd = uint32(lenOfVals[i])
+ case i < numOfUint8:
+ lenOfValEnd = int(lenOfVals[i])
if i > 0 {
- lenOfValStart = uint32(lenOfVals[i-1])
+ lenOfValStart = int(lenOfVals[i-1])
}
- return common.CopyBytes(values[lenOfValStart:lenOfValEnd])
- case i < uint32(numOfUint8)+uint32(numOfUint16):
- one := uint32(numOfUint8) + (i-uint32(numOfUint8))*2
- lenOfValEnd = uint32(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
- if i-1 < uint32(numOfUint8) {
- lenOfValStart = uint32(lenOfVals[i-1])
+ case i < numOfUint8+numOfUint16:
+ one := (i-numOfUint8)*2 + numOfUint8
+ lenOfValEnd = int(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
+ if i-1 < numOfUint8 {
+ lenOfValStart = int(lenOfVals[i-1])
} else {
- one = uint32(numOfUint8) + (i-1-uint32(numOfUint8))*2
- lenOfValStart = uint32(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
+ one = (i-1)*2 - numOfUint8
+ lenOfValStart = int(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
}
- return common.CopyBytes(values[lenOfValStart:lenOfValEnd])
- case i < uint32(numOfUint8)+uint32(numOfUint16)+uint32(numOfUint32):
- fmt.Println(i, uint32(numOfUint8), numOfUint16, numOfUint32)
- one := uint32(numOfUint8) + uint32(numOfUint16)*2 + (i-uint32(numOfUint8)-uint32(numOfUint16))*4
- lenOfValEnd = binary.BigEndian.Uint32(lenOfVals[one : one+4])
- if i-1 < uint32(numOfUint8)+uint32(numOfUint16) {
- one = uint32(numOfUint8) + (i-1-uint32(numOfUint8))*2
- lenOfValStart = uint32(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
+ case i < numOfUint8+numOfUint16+numOfUint32:
+ one := numOfUint8 + numOfUint16*2 + (i-numOfUint8-numOfUint16)*4
+ lenOfValEnd = int(binary.BigEndian.Uint32(lenOfVals[one : one+4]))
+ if i-1 < numOfUint8+numOfUint16 {
+ one = (i-1)*2 - numOfUint8
+ lenOfValStart = int(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
} else {
- one := uint32(numOfUint8) + uint32(numOfUint16)*2 + (i-1-uint32(numOfUint8)-uint32(numOfUint16))*4
- lenOfValStart = binary.BigEndian.Uint32(lenOfVals[one : one+4])
+ one = numOfUint8 + numOfUint16*2 + (i-1-numOfUint8-numOfUint16)*4
+ lenOfValStart = int(binary.BigEndian.Uint32(lenOfVals[one : one+4]))
}
- return common.CopyBytes(values[lenOfValStart:lenOfValEnd])
default:
panic("findval err")
}
+ return common.CopyBytes(values[lenOfValStart:lenOfValEnd])
}
func writeKeyRow(id uint32, row []byte) {
@@ -267,16 +304,17 @@ func writeKeyRow(id uint32, row []byte) {
panic("wrong size of row")
}
}
-func readFromMap(m map[uint32]common.Hash, row []byte) common.Hash {
+
+func getUint32(row []byte) uint32 {
switch len(row) {
case 1:
- return m[uint32(row[0])]
+ return uint32(row[0])
case 2:
- return m[uint32(binary.BigEndian.Uint16(row))]
+ return uint32(binary.BigEndian.Uint16(row))
case 4:
- return m[binary.BigEndian.Uint32(row)]
+ return binary.BigEndian.Uint32(row)
case 8:
- return m[uint32(binary.BigEndian.Uint64(row))]
+ return uint32(binary.BigEndian.Uint64(row))
default:
panic("wrong")
}
@@ -292,50 +330,65 @@ func (b StorageChangeSetBytes) Walk(f func(k, v []byte) error) error {
return fmt.Errorf("decode: input too short (%d bytes)", len(b))
}
- numOfItems := binary.BigEndian.Uint32(b[0:4])
+ numOfItems := int(binary.BigEndian.Uint32(b[0:4]))
if numOfItems == 0 {
return nil
}
- numOfUniqueItems := binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict])
+ numOfUniqueItems := int(binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict]))
lenOfValsPos := storageEnodingStartElem +
storageEnodingLengthOfDict +
- uint32(numOfUniqueItems)*common.HashLength +
- numOfItems*uint32(getNumOfBytesByLen(int(numOfUniqueItems))+common.HashLength)
+ numOfUniqueItems*common.HashLength +
+ numOfItems*(getNumOfBytesByLen(numOfUniqueItems)+common.HashLength)
numOfUint8 := int(binary.BigEndian.Uint16(b[lenOfValsPos : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements]))
numOfUint16 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2]))
numOfUint32 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2 : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*3]))
lenOfValsPos += storageEnodingLengthOfNumTypeOfElements * 3
- valuesPos := lenOfValsPos +
- uint32(numOfUint8) +
- uint32(numOfUint16*2) +
- uint32(numOfUint32*4)
+ valuesPos := lenOfValsPos + numOfUint8 + numOfUint16*2 + numOfUint32*4
+
+ incarnationPosition := lenOfValsPos + calculateIncarnationPos3(b[lenOfValsPos:], numOfUint8, numOfUint16, numOfUint32)
+ notDefaultIncarnation := make(map[uint32]uint64)
+
+ if len(b) > incarnationPosition {
+ if len(b[incarnationPosition:])%12 != 0 {
+ return fmt.Errorf("decode: incarnatin part is incorrect(%d bytes)", len(b[incarnationPosition:]))
+ }
+ for i := incarnationPosition; i < len(b); i += 12 {
+ id := binary.BigEndian.Uint32(b[i : i+4])
+ inc := ^binary.BigEndian.Uint64(b[i+4 : i+12])
+ notDefaultIncarnation[id] = inc
+ }
+ }
- addrHashMap := make(map[uint32]common.Hash, numOfUniqueItems)
+ addrHashMap := make(map[uint32][]byte, numOfUniqueItems)
for i := uint32(0); i < uint32(numOfUniqueItems); i++ {
elemStart := storageEnodingStartElem + storageEnodingLengthOfDict + i*(common.HashLength)
- addrHashMap[i] = common.BytesToHash(b[elemStart : elemStart+common.HashLength])
+ addrHashMap[i] = b[elemStart : elemStart+common.HashLength]
}
key := make([]byte, common.HashLength*2+common.IncarnationLength)
- elemLength := uint32(getNumOfBytesByLen(int(numOfUniqueItems)))
- for i := uint32(0); i < numOfItems; i++ {
+ elemLength := getNumOfBytesByLen(numOfUniqueItems)
+ for i := 0; i < numOfItems; i++ {
elemStart := storageEnodingStartElem +
storageEnodingLengthOfDict +
- uint32(numOfUniqueItems)*(common.HashLength) +
+ numOfUniqueItems*(common.HashLength) +
i*(elemLength+common.HashLength)
- //copy addrHash
- copy(key[0:common.HashLength], readFromMap(addrHashMap, b[elemStart:elemStart+elemLength]).Bytes())
+ addrIdx := getUint32(b[elemStart : elemStart+elemLength])
+ copy(key[:common.HashLength], addrHashMap[addrIdx])
+ if inc, ok := notDefaultIncarnation[addrIdx]; ok {
+ binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], ^inc)
+ } else {
+ binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], ^DefaultIncarnation)
+ }
//copy key hash
copy(
key[common.HashLength+common.IncarnationLength:2*common.HashLength+common.IncarnationLength],
b[elemStart+elemLength:elemStart+elemLength+common.HashLength],
)
- binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], ^uint64(1))
err := f(common.CopyBytes(key), findVal(b[lenOfValsPos:valuesPos], b[valuesPos:], i, numOfUint8, numOfUint16, numOfUint32))
if err != nil {
return err
@@ -352,52 +405,50 @@ func (b StorageChangeSetBytes) Find(addrHash []byte, keyHash []byte) ([]byte, er
return nil, fmt.Errorf("decode: input too short (%d bytes)", len(b))
}
- numOfItems := binary.BigEndian.Uint32(b[0:storageEnodingLengthOfNumOfElements])
+ numOfItems := int(binary.BigEndian.Uint32(b[:storageEnodingLengthOfNumOfElements]))
if numOfItems == 0 {
return nil, nil
}
- numOfUniqueItems := binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict])
+ numOfUniqueItems := int(binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements:]))
var addHashID uint32
found := false
- var elemStart uint32
+ elemStart := storageEnodingLengthOfNumOfElements + storageEnodingLengthOfDict
//todo[boris] here should be binary search
- for i := uint32(0); i < uint32(numOfUniqueItems); i++ {
- elemStart = storageEnodingLengthOfNumOfElements + storageEnodingLengthOfDict + i*common.HashLength
+ for i := 0; i < numOfUniqueItems; i++ {
if bytes.Equal(addrHash, b[elemStart:elemStart+common.HashLength]) {
found = true
- addHashID = i
+ addHashID = uint32(i)
break
}
+ elemStart += common.HashLength
}
if !found {
+ fmt.Printf("did not find addrHash %x\n", addrHash)
return nil, ErrNotFound
}
+ elemLength := getNumOfBytesByLen(numOfUniqueItems)
lenOfValsPos := storageEnodingStartElem +
storageEnodingLengthOfDict +
- uint32(numOfUniqueItems)*common.HashLength +
- numOfItems*uint32(getNumOfBytesByLen(int(numOfUniqueItems))+common.HashLength)
+ numOfUniqueItems*common.HashLength +
+ numOfItems*(elemLength+common.HashLength)
- numOfUint8 := int(binary.BigEndian.Uint16(b[lenOfValsPos : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements]))
- numOfUint16 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2]))
- numOfUint32 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2 : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*3]))
+ numOfUint8 := int(binary.BigEndian.Uint16(b[lenOfValsPos:]))
+ numOfUint16 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements:]))
+ numOfUint32 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2:]))
lenOfValsPos += storageEnodingLengthOfNumTypeOfElements * 3
- valuesPos := lenOfValsPos +
- uint32(numOfUint8) +
- uint32(numOfUint16*2) +
- uint32(numOfUint32*4)
+ valuesPos := lenOfValsPos + numOfUint8 + numOfUint16*2 + numOfUint32*4
//here should be binary search too
- elemLength := uint32(getNumOfBytesByLen(int(numOfUniqueItems)))
encodedAddHashID := make([]byte, elemLength)
writeKeyRow(addHashID, encodedAddHashID)
- for i := uint32(0); i < numOfItems; i++ {
+ for i := 0; i < numOfItems; i++ {
elemStart := storageEnodingStartElem +
storageEnodingLengthOfDict +
- uint32(numOfUniqueItems)*(common.HashLength) +
+ numOfUniqueItems*(common.HashLength) +
i*(elemLength+common.HashLength)
if !bytes.Equal(encodedAddHashID, b[elemStart:elemStart+elemLength]) {
diff --git a/common/changeset/storage_changeset_test.go b/common/changeset/storage_changeset_test.go
index c7ddea801424710b7b73e802a053d19db736576d..cc9c5ffe154de3b328563a2896560bb5a21dd69a 100644
--- a/common/changeset/storage_changeset_test.go
+++ b/common/changeset/storage_changeset_test.go
@@ -3,6 +3,7 @@ package changeset
import (
"bytes"
"fmt"
+ "math/rand"
"reflect"
"sort"
"strconv"
@@ -17,7 +18,7 @@ const (
defaultIncarnation = 1
)
-func TestEncodingStorageWithoutNotDefaultIncarnation(t *testing.T) {
+func TestEncodingStorageWithoutNotDefaultIncarnation1(t *testing.T) {
f := func(t *testing.T, numOfElements int) {
// empty StorageChangeSset first
ch := NewStorageChangeSet()
@@ -84,7 +85,7 @@ func TestEncodingStorageWithtRandomIncarnation(t *testing.T) {
addrHash, _ := common.HashData([]byte("addrHash" + strconv.Itoa(i)))
key, _ := common.HashData([]byte("key" + strconv.Itoa(i)))
val, _ := common.HashData([]byte("val" + strconv.Itoa(i)))
- err = ch.Add(dbutils.GenerateCompositeStorageKey(addrHash, defaultIncarnation, key), val.Bytes())
+ err = ch.Add(dbutils.GenerateCompositeStorageKey(addrHash, rand.Uint64(), key), val.Bytes())
if err != nil {
t.Error(err)
}
@@ -150,10 +151,10 @@ func TestEncodingStorageWithoutNotDefaultIncarnationWalk(t *testing.T) {
i := 0
err = StorageChangeSetBytes(b).Walk(func(k, v []byte) error {
if !bytes.Equal(k, ch.Changes[i].Key) {
- t.Error(i, "key was incorrect", k, ch.Changes[i].Key)
+ t.Errorf("%d key was incorrect %x %x", i, k, ch.Changes[i].Key)
}
if !bytes.Equal(v, ch.Changes[i].Value) {
- t.Error(i, "val is incorrect", v, ch.Changes[i].Value)
+ t.Errorf("%d val is incorrect %x %x", i, v, ch.Changes[i].Value)
}
i++
return nil
@@ -202,7 +203,7 @@ func TestEncodingStorageWithoutNotDefaultIncarnationFind(t *testing.T) {
t.Error(err, i)
}
if !bytes.Equal(val, v.Value) {
- t.Error("not equal for ", v, val)
+ t.Errorf("not equal for %x %x", v, val)
}
}
}
diff --git a/common/dbutils/composite_keys.go b/common/dbutils/composite_keys.go
index fa06e765f55842164b92c8dba7993b57080f9818..4ad89e9f2a5b58d386b5324b56911360dc72c529 100644
--- a/common/dbutils/composite_keys.go
+++ b/common/dbutils/composite_keys.go
@@ -110,7 +110,7 @@ func GenerateCompositeStorageKey(addressHash common.Hash, incarnation uint64, se
func GenerateStoragePrefix(addressHash common.Hash, incarnation uint64) []byte {
prefix := make([]byte, common.HashLength+8)
copy(prefix, addressHash[:])
- binary.BigEndian.PutUint64(prefix[common.HashLength:], incarnation^^uint64(0))
+ binary.BigEndian.PutUint64(prefix[common.HashLength:], ^incarnation)
return prefix
}
diff --git a/core/state/database.go b/core/state/database.go
index 904db090333a85e4f6a4f3b62328bfb61715ffd3..abae6f48a6dcccf9ac9a37a7c12ec3814b5ea9fe 100644
--- a/core/state/database.go
+++ b/core/state/database.go
@@ -859,6 +859,7 @@ func (tds *TrieDbState) GetBlockNr() uint64 {
}
func (tds *TrieDbState) UnwindTo(blockNr uint64) error {
+ //fmt.Printf("Unwind from block %d to block %d\n", tds.blockNr, blockNr)
tds.StartNewBuffer()
b := tds.currentBuffer
diff --git a/core/state/db_state_writer.go b/core/state/db_state_writer.go
index a575f89cafcb01581f244ba58be95f774b4ee37c..f06dc2e431816e5f4ad65c53835b95624cba4565 100644
--- a/core/state/db_state_writer.go
+++ b/core/state/db_state_writer.go
@@ -187,13 +187,16 @@ func (dsw *DbStateWriter) WriteHistory() error {
}
if debug.IsThinHistory() {
for _, change := range storageChanges.Changes {
- value, err1 := dsw.tds.db.Get(dbutils.StorageHistoryBucket, change.Key)
+ keyNoInc := make([]byte, len(change.Key)-common.IncarnationLength)
+ copy(keyNoInc, change.Key[:common.HashLength])
+ copy(keyNoInc[common.HashLength:], change.Key[common.HashLength+common.IncarnationLength:])
+ value, err1 := dsw.tds.db.Get(dbutils.StorageHistoryBucket, keyNoInc)
if err1 != nil && err1 != ethdb.ErrKeyNotFound {
return fmt.Errorf("db.Get failed: %w", err1)
}
index := dbutils.WrapHistoryIndex(value)
index.Append(dsw.tds.blockNr)
- if err := dsw.tds.db.Put(dbutils.StorageHistoryBucket, change.Key, *index); err != nil {
+ if err := dsw.tds.db.Put(dbutils.StorageHistoryBucket, keyNoInc, *index); err != nil {
return err
}
}
diff --git a/core/state/history_test.go b/core/state/history_test.go
index ffc634bec264e547658438366e74c5637d023947..31ccfdbad451633dc80b4dd37d9fd0cfe7a84493 100644
--- a/core/state/history_test.go
+++ b/core/state/history_test.go
@@ -324,7 +324,7 @@ func TestMutationCommitThinHistory(t *testing.T) {
resultHash := common.BytesToHash(res)
if resultHash != v {
- t.Fatal("incorrect storage history for ", addrHash.String(), v, resultHash)
+ t.Fatalf("incorrect storage history for %x %x %x", addrHash.String(), v, resultHash)
}
}
}
diff --git a/ethdb/bolt_db.go b/ethdb/bolt_db.go
index 2044597d03855648e3873f5e0847a0867db60c96..740730dd2ccbf30d59a2de267ba2dd65874366fb 100644
--- a/ethdb/bolt_db.go
+++ b/ethdb/bolt_db.go
@@ -507,6 +507,9 @@ func (db *BoltDatabase) walkAsOfThinStorage(startkey []byte, fixedbits uint, tim
if csB == nil {
return fmt.Errorf("storageChangeBucket not found")
}
+ startkeyNoInc := make([]byte, len(startkey)-common.IncarnationLength)
+ copy(startkeyNoInc, startkey[:common.HashLength])
+ copy(startkeyNoInc[common.HashLength:], startkey[common.HashLength+common.IncarnationLength:])
//for storage
mainCursor := newSplitCursor(
b,
@@ -518,8 +521,8 @@ func (db *BoltDatabase) walkAsOfThinStorage(startkey []byte, fixedbits uint, tim
//for historic data
historyCursor := newSplitCursor(
hB,
- startkey,
- fixedbits,
+ startkeyNoInc,
+ fixedbits-8*common.IncarnationLength,
common.HashLength, /* part1end */
common.HashLength+common.IncarnationLength, /* part2start */
)
@@ -912,7 +915,15 @@ func BoltDBFindByHistory(tx *bolt.Tx, hBucket []byte, key []byte, timestamp uint
if hB == nil {
return nil, ErrKeyNotFound
}
- v, _ := hB.Get(key)
+ var v []byte
+ if bytes.Equal(dbutils.StorageHistoryBucket, hBucket) {
+ keyNoInc := make([]byte, len(key)-common.IncarnationLength)
+ copy(keyNoInc, key[:common.HashLength])
+ copy(keyNoInc[common.HashLength:], key[common.HashLength+common.IncarnationLength:])
+ v, _ = hB.Get(keyNoInc)
+ } else {
+ v, _ = hB.Get(key)
+ }
index := dbutils.WrapHistoryIndex(v)
changeSetBlock, ok := index.Search(timestamp)
@@ -933,9 +944,9 @@ func BoltDBFindByHistory(tx *bolt.Tx, hBucket []byte, key []byte, timestamp uint
err error
)
switch {
- case debug.IsThinHistory() && bytes.Equal(dbutils.AccountsHistoryBucket, hBucket):
+ case bytes.Equal(dbutils.AccountsHistoryBucket, hBucket):
data, err = changeset.AccountChangeSetBytes(changeSetData).FindLast(key)
- case debug.IsThinHistory() && bytes.Equal(dbutils.StorageHistoryBucket, hBucket):
+ case bytes.Equal(dbutils.StorageHistoryBucket, hBucket):
data, err = changeset.StorageChangeSetBytes(changeSetData).Find(key[:common.HashLength], key[common.HashLength+common.IncarnationLength:])
default:
data, err = changeset.FindLast(changeSetData, key)