diff --git a/common/etl/dataprovider.go b/common/etl/dataprovider.go index c52eb5965ec3af1f739e7fd275e4d6a65923ad73..908ecaf835666f14428c9711ae02478cf5360fb4 100644 --- a/common/etl/dataprovider.go +++ b/common/etl/dataprovider.go @@ -14,7 +14,7 @@ import ( type dataProvider interface { Next(decoder Decoder) ([]byte, []byte, error) - Dispose() error + Dispose() (uint64, error) } type fileDataProvider struct { @@ -74,16 +74,20 @@ func (p *fileDataProvider) Next(decoder Decoder) ([]byte, []byte, error) { return readElementFromDisk(decoder) } -func (p *fileDataProvider) Dispose() error { +func (p *fileDataProvider) Dispose() (uint64, error) { + info, errStat := os.Stat(p.file.Name()) errClose := p.file.Close() errRemove := os.Remove(p.file.Name()) if errClose != nil { - return errClose + return 0, errClose } if errRemove != nil { - return errRemove + return 0, errRemove } - return nil + if errStat != nil { + return 0, errStat + } + return uint64(info.Size()), nil } func (p *fileDataProvider) String() string { @@ -119,8 +123,8 @@ func (p *memoryDataProvider) Next(decoder Decoder) ([]byte, []byte, error) { return entry.key, entry.value, nil } -func (p *memoryDataProvider) Dispose() error { - return nil +func (p *memoryDataProvider) Dispose() (uint64, error) { + return 0 /* doesn't take space on disk */, nil } func (p *memoryDataProvider) String() string { diff --git a/common/etl/etl.go b/common/etl/etl.go index c1b5f98801784cf34c917edf99d9056fb5d6c09d..e7d7a4b7f2d0eba5651ea006410daeea3f504f52 100644 --- a/common/etl/etl.go +++ b/common/etl/etl.go @@ -6,6 +6,7 @@ import ( "io" "time" + "github.com/c2h5oh/datasize" "github.com/ledgerwatch/turbo-geth/common" "github.com/ledgerwatch/turbo-geth/ethdb" "github.com/ledgerwatch/turbo-geth/log" @@ -120,12 +121,15 @@ func extractBucketIntoFiles( return collector.flushBuffer(nil, true) } func disposeProviders(providers []dataProvider) { + totalSize := uint64(0) for _, p := range providers { - err := p.Dispose() + providerSize, err := p.Dispose() if err != nil { log.Warn("promoting hashed state, error while disposing provider", "provier", p, "err", err) } + totalSize += providerSize } + log.Info("etl: temp files removed successfully", "total size", datasize.ByteSize(totalSize).HumanReadable()) } type bucketState struct {