diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go
index 258f24d3204d289c015f4ea7ebf7d4a2b6bf6afd..ac09ae998156682e412697bdc87ca09c3009ed55 100644
--- a/cmd/swarm/main.go
+++ b/cmd/swarm/main.go
@@ -322,23 +322,23 @@ Downloads a swarm bzz uri to the given dir. When no dir is provided, working dir
Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
Subcommands: []cli.Command{
{
- Action: add,
+ Action: manifestAdd,
CustomHelpTemplate: helpTemplate,
Name: "add",
Usage: "add a new path to the manifest",
- ArgsUsage: "<MANIFEST> <path> <hash> [<content-type>]",
+ ArgsUsage: "<MANIFEST> <path> <hash>",
Description: "Adds a new path to the manifest",
},
{
- Action: update,
+ Action: manifestUpdate,
CustomHelpTemplate: helpTemplate,
Name: "update",
Usage: "update the hash for an already existing path in the manifest",
- ArgsUsage: "<MANIFEST> <path> <newhash> [<newcontent-type>]",
+ ArgsUsage: "<MANIFEST> <path> <newhash>",
Description: "Update the hash for an already existing path in the manifest",
},
{
- Action: remove,
+ Action: manifestRemove,
CustomHelpTemplate: helpTemplate,
Name: "remove",
Usage: "removes a path from the manifest",
diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go
index 82166edf6c808b2a04157890dab71871339792e0..0216ffc1dd7039b5a2f83a11e887d978bdf82421 100644
--- a/cmd/swarm/manifest.go
+++ b/cmd/swarm/manifest.go
@@ -18,10 +18,8 @@
package main
import (
- "encoding/json"
"fmt"
- "mime"
- "path/filepath"
+ "os"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
@@ -30,127 +28,118 @@ import (
"gopkg.in/urfave/cli.v1"
)
-const bzzManifestJSON = "application/bzz-manifest+json"
-
-func add(ctx *cli.Context) {
+// manifestAdd adds a new entry to the manifest at the given path.
+// New entry hash, the last argument, must be the hash of a manifest
+// with only one entry, which meta-data will be added to the original manifest.
+// On success, this function will print new (updated) manifest's hash.
+func manifestAdd(ctx *cli.Context) {
args := ctx.Args()
- if len(args) < 3 {
- utils.Fatalf("Need at least three arguments <MHASH> <path> <HASH> [<content-type>]")
+ if len(args) != 3 {
+ utils.Fatalf("Need exactly three arguments <MHASH> <path> <HASH>")
}
var (
mhash = args[0]
path = args[1]
hash = args[2]
-
- ctype string
- wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot api.Manifest
)
- if len(args) > 3 {
- ctype = args[3]
- } else {
- ctype = mime.TypeByExtension(filepath.Ext(path))
+ bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+ client := swarm.NewClient(bzzapi)
+
+ m, _, err := client.DownloadManifest(hash)
+ if err != nil {
+ utils.Fatalf("Error downloading manifest to add: %v", err)
+ }
+ l := len(m.Entries)
+ if l == 0 {
+ utils.Fatalf("No entries in manifest %s", hash)
+ } else if l > 1 {
+ utils.Fatalf("Too many entries in manifest %s", hash)
}
- newManifest := addEntryToManifest(ctx, mhash, path, hash, ctype)
+ newManifest := addEntryToManifest(client, mhash, path, m.Entries[0])
fmt.Println(newManifest)
-
- if !wantManifest {
- // Print the manifest. This is the only output to stdout.
- mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
- fmt.Println(string(mrootJSON))
- return
- }
}
-func update(ctx *cli.Context) {
-
+// manifestUpdate replaces an existing entry of the manifest at the given path.
+// New entry hash, the last argument, must be the hash of a manifest
+// with only one entry, which meta-data will be added to the original manifest.
+// On success, this function will print hash of the updated manifest.
+func manifestUpdate(ctx *cli.Context) {
args := ctx.Args()
- if len(args) < 3 {
- utils.Fatalf("Need at least three arguments <MHASH> <path> <HASH>")
+ if len(args) != 3 {
+ utils.Fatalf("Need exactly three arguments <MHASH> <path> <HASH>")
}
var (
mhash = args[0]
path = args[1]
hash = args[2]
-
- ctype string
- wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot api.Manifest
)
- if len(args) > 3 {
- ctype = args[3]
- } else {
- ctype = mime.TypeByExtension(filepath.Ext(path))
- }
- newManifest := updateEntryInManifest(ctx, mhash, path, hash, ctype)
- fmt.Println(newManifest)
+ bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+ client := swarm.NewClient(bzzapi)
- if !wantManifest {
- // Print the manifest. This is the only output to stdout.
- mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
- fmt.Println(string(mrootJSON))
- return
+ m, _, err := client.DownloadManifest(hash)
+ if err != nil {
+ utils.Fatalf("Error downloading manifest to update: %v", err)
+ }
+ l := len(m.Entries)
+ if l == 0 {
+ utils.Fatalf("No entries in manifest %s", hash)
+ } else if l > 1 {
+ utils.Fatalf("Too many entries in manifest %s", hash)
}
+
+ newManifest, _, defaultEntryUpdated := updateEntryInManifest(client, mhash, path, m.Entries[0], true)
+ if defaultEntryUpdated {
+ // Print informational message to stderr
+ // allowing the user to get the new manifest hash from stdout
+ // without the need to parse the complete output.
+ fmt.Fprintln(os.Stderr, "Manifest default entry is updated, too")
+ }
+ fmt.Println(newManifest)
}
-func remove(ctx *cli.Context) {
+// manifestRemove removes an existing entry of the manifest at the given path.
+// On success, this function will print hash of the manifest which does not
+// contain the path.
+func manifestRemove(ctx *cli.Context) {
args := ctx.Args()
- if len(args) < 2 {
- utils.Fatalf("Need at least two arguments <MHASH> <path>")
+ if len(args) != 2 {
+ utils.Fatalf("Need exactly two arguments <MHASH> <path>")
}
var (
mhash = args[0]
path = args[1]
-
- wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot api.Manifest
)
- newManifest := removeEntryFromManifest(ctx, mhash, path)
- fmt.Println(newManifest)
+ bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+ client := swarm.NewClient(bzzapi)
- if !wantManifest {
- // Print the manifest. This is the only output to stdout.
- mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
- fmt.Println(string(mrootJSON))
- return
- }
+ newManifest := removeEntryFromManifest(client, mhash, path)
+ fmt.Println(newManifest)
}
-func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
-
- var (
- bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
- client = swarm.NewClient(bzzapi)
- longestPathEntry = api.ManifestEntry{}
- )
+func addEntryToManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry) string {
+ var longestPathEntry = api.ManifestEntry{}
mroot, isEncrypted, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
- //TODO: check if the "hash" to add is valid and present in swarm
- _, _, err = client.DownloadManifest(hash)
- if err != nil {
- utils.Fatalf("Hash to add is not present: %v", err)
- }
-
// See if we path is in this Manifest or do we have to dig deeper
- for _, entry := range mroot.Entries {
- if path == entry.Path {
+ for _, e := range mroot.Entries {
+ if path == e.Path {
utils.Fatalf("Path %s already present, not adding anything", path)
} else {
- if entry.ContentType == bzzManifestJSON {
- prfxlen := strings.HasPrefix(path, entry.Path)
+ if e.ContentType == api.ManifestType {
+ prfxlen := strings.HasPrefix(path, e.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
- longestPathEntry = entry
+ longestPathEntry = e
}
}
}
@@ -159,25 +148,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
if longestPathEntry.Path != "" {
// Load the child Manifest add the entry there
newPath := path[len(longestPathEntry.Path):]
- newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
+ newHash := addEntryToManifest(client, longestPathEntry.Hash, newPath, entry)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}
- for _, entry := range mroot.Entries {
- if longestPathEntry.Path == entry.Path {
- entry.Hash = newHash
+ for _, e := range mroot.Entries {
+ if longestPathEntry.Path == e.Path {
+ e.Hash = newHash
}
- newMRoot.Entries = append(newMRoot.Entries, entry)
+ newMRoot.Entries = append(newMRoot.Entries, e)
}
mroot = newMRoot
} else {
// Add the entry in the leaf Manifest
- newEntry := api.ManifestEntry{
- Hash: hash,
- Path: path,
- ContentType: ctype,
- }
- mroot.Entries = append(mroot.Entries, newEntry)
+ entry.Path = path
+ mroot.Entries = append(mroot.Entries, entry)
}
newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
@@ -185,14 +170,16 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
utils.Fatalf("Manifest upload failed: %v", err)
}
return newManifestHash
-
}
-func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
-
+// updateEntryInManifest updates an existing entry o path with a new one in the manifest with provided mhash
+// finding the path recursively through all nested manifests. Argument isRoot is used for default
+// entry update detection. If the updated entry has the same hash as the default entry, then the
+// default entry in root manifest will be updated too.
+// Returned values are the new manifest hash, hash of the entry that was replaced by the new entry and
+// a a bool that is true if default entry is updated.
+func updateEntryInManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry, isRoot bool) (newManifestHash, oldHash string, defaultEntryUpdated bool) {
var (
- bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
- client = swarm.NewClient(bzzapi)
newEntry = api.ManifestEntry{}
longestPathEntry = api.ManifestEntry{}
)
@@ -202,17 +189,18 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
utils.Fatalf("Manifest download failed: %v", err)
}
- //TODO: check if the "hash" with which to update is valid and present in swarm
-
// See if we path is in this Manifest or do we have to dig deeper
- for _, entry := range mroot.Entries {
- if path == entry.Path {
- newEntry = entry
+ for _, e := range mroot.Entries {
+ if path == e.Path {
+ newEntry = e
+ // keep the reference of the hash of the entry that should be replaced
+ // for default entry detection
+ oldHash = e.Hash
} else {
- if entry.ContentType == bzzManifestJSON {
- prfxlen := strings.HasPrefix(path, entry.Path)
+ if e.ContentType == api.ManifestType {
+ prfxlen := strings.HasPrefix(path, e.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
- longestPathEntry = entry
+ longestPathEntry = e
}
}
}
@@ -225,50 +213,50 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
if longestPathEntry.Path != "" {
// Load the child Manifest add the entry there
newPath := path[len(longestPathEntry.Path):]
- newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
+ var newHash string
+ newHash, oldHash, _ = updateEntryInManifest(client, longestPathEntry.Hash, newPath, entry, false)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}
- for _, entry := range mroot.Entries {
- if longestPathEntry.Path == entry.Path {
- entry.Hash = newHash
+ for _, e := range mroot.Entries {
+ if longestPathEntry.Path == e.Path {
+ e.Hash = newHash
}
- newMRoot.Entries = append(newMRoot.Entries, entry)
+ newMRoot.Entries = append(newMRoot.Entries, e)
}
mroot = newMRoot
}
- if newEntry.Path != "" {
+ // update the manifest if the new entry is found and
+ // check if default entry should be updated
+ if newEntry.Path != "" || isRoot {
// Replace the hash for leaf Manifest
newMRoot := &api.Manifest{}
- for _, entry := range mroot.Entries {
- if newEntry.Path == entry.Path {
- myEntry := api.ManifestEntry{
- Hash: hash,
- Path: entry.Path,
- ContentType: ctype,
- }
- newMRoot.Entries = append(newMRoot.Entries, myEntry)
- } else {
+ for _, e := range mroot.Entries {
+ if newEntry.Path == e.Path {
+ entry.Path = e.Path
newMRoot.Entries = append(newMRoot.Entries, entry)
+ } else if isRoot && e.Path == "" && e.Hash == oldHash {
+ entry.Path = e.Path
+ newMRoot.Entries = append(newMRoot.Entries, entry)
+ defaultEntryUpdated = true
+ } else {
+ newMRoot.Entries = append(newMRoot.Entries, e)
}
}
mroot = newMRoot
}
- newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
+ newManifestHash, err = client.UploadManifest(mroot, isEncrypted)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
- return newManifestHash
+ return newManifestHash, oldHash, defaultEntryUpdated
}
-func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
-
+func removeEntryFromManifest(client *swarm.Client, mhash, path string) string {
var (
- bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
- client = swarm.NewClient(bzzapi)
entryToRemove = api.ManifestEntry{}
longestPathEntry = api.ManifestEntry{}
)
@@ -283,7 +271,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if path == entry.Path {
entryToRemove = entry
} else {
- if entry.ContentType == bzzManifestJSON {
+ if entry.ContentType == api.ManifestType {
prfxlen := strings.HasPrefix(path, entry.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
longestPathEntry = entry
@@ -299,7 +287,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if longestPathEntry.Path != "" {
// Load the child Manifest remove the entry there
newPath := path[len(longestPathEntry.Path):]
- newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
+ newHash := removeEntryFromManifest(client, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}
diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..08fe0b2eb7abd51cb1eabf8014f5b3f4c0de9442
--- /dev/null
+++ b/cmd/swarm/manifest_test.go
@@ -0,0 +1,579 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/swarm/api"
+ swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+)
+
+// TestManifestChange tests manifest add, update and remove
+// cli commands without encryption.
+func TestManifestChange(t *testing.T) {
+ testManifestChange(t, false)
+}
+
+// TestManifestChange tests manifest add, update and remove
+// cli commands with encryption enabled.
+func TestManifestChangeEncrypted(t *testing.T) {
+ testManifestChange(t, true)
+}
+
+// testManifestChange performs cli commands:
+// - manifest add
+// - manifest update
+// - manifest remove
+// on a manifest, testing the functionality of this
+// comands on paths that are in root manifest or a nested one.
+// Argument encrypt controls whether to use encryption or not.
+func testManifestChange(t *testing.T, encrypt bool) {
+ t.Parallel()
+ cluster := newTestCluster(t, 1)
+ defer cluster.Shutdown()
+
+ tmp, err := ioutil.TempDir("", "swarm-manifest-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(tmp)
+
+ origDir := filepath.Join(tmp, "orig")
+ if err := os.Mkdir(origDir, 0777); err != nil {
+ t.Fatal(err)
+ }
+
+ indexDataFilename := filepath.Join(origDir, "index.html")
+ err = ioutil.WriteFile(indexDataFilename, []byte("<h1>Test</h1>"), 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Files paths robots.txt and robots.html share the same prefix "robots."
+ // which will result a manifest with a nested manifest under path "robots.".
+ // This will allow testing manifest changes on both root and nested manifest.
+ err = ioutil.WriteFile(filepath.Join(origDir, "robots.txt"), []byte("Disallow: /"), 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = ioutil.WriteFile(filepath.Join(origDir, "robots.html"), []byte("<strong>No Robots Allowed</strong>"), 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = ioutil.WriteFile(filepath.Join(origDir, "mutants.txt"), []byte("Frank\nMarcus"), 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ args := []string{
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "--recursive",
+ "--defaultpath",
+ indexDataFilename,
+ "up",
+ origDir,
+ }
+ if encrypt {
+ args = append(args, "--encrypt")
+ }
+
+ origManifestHash := runSwarmExpectHash(t, args...)
+
+ checkHashLength(t, origManifestHash, encrypt)
+
+ client := swarm.NewClient(cluster.Nodes[0].URL)
+
+ // upload a new file and use its manifest to add it the original manifest.
+ t.Run("add", func(t *testing.T) {
+ humansData := []byte("Ann\nBob")
+ humansDataFilename := filepath.Join(tmp, "humans.txt")
+ err = ioutil.WriteFile(humansDataFilename, humansData, 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ humansManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "up",
+ humansDataFilename,
+ )
+
+ newManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "manifest",
+ "add",
+ origManifestHash,
+ "humans.txt",
+ humansManifestHash,
+ )
+
+ checkHashLength(t, newManifestHash, encrypt)
+
+ newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+ var found bool
+ for _, e := range newManifest.Entries {
+ if e.Path == "humans.txt" {
+ found = true
+ if e.Size != int64(len(humansData)) {
+ t.Errorf("expected humans.txt size %v, got %v", len(humansData), e.Size)
+ }
+ if e.ModTime.IsZero() {
+ t.Errorf("got zero mod time for humans.txt")
+ }
+ ct := "text/plain; charset=utf-8"
+ if e.ContentType != ct {
+ t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+ }
+ break
+ }
+ }
+ if !found {
+ t.Fatal("no humans.txt in new manifest")
+ }
+
+ checkFile(t, client, newManifestHash, "humans.txt", humansData)
+ })
+
+ // upload a new file and use its manifest to add it the original manifest,
+ // but ensure that the file will be in the nested manifest of the original one.
+ t.Run("add nested", func(t *testing.T) {
+ robotsData := []byte(`{"disallow": "/"}`)
+ robotsDataFilename := filepath.Join(tmp, "robots.json")
+ err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ robotsManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "up",
+ robotsDataFilename,
+ )
+
+ newManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "manifest",
+ "add",
+ origManifestHash,
+ "robots.json",
+ robotsManifestHash,
+ )
+
+ checkHashLength(t, newManifestHash, encrypt)
+
+ newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+ var found bool
+ loop:
+ for _, e := range newManifest.Entries {
+ if e.Path == "robots." {
+ nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
+ for _, e := range nestedManifest.Entries {
+ if e.Path == "json" {
+ found = true
+ if e.Size != int64(len(robotsData)) {
+ t.Errorf("expected robots.json size %v, got %v", len(robotsData), e.Size)
+ }
+ if e.ModTime.IsZero() {
+ t.Errorf("got zero mod time for robots.json")
+ }
+ ct := "application/json"
+ if e.ContentType != ct {
+ t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+ }
+ break loop
+ }
+ }
+ }
+ }
+ if !found {
+ t.Fatal("no robots.json in new manifest")
+ }
+
+ checkFile(t, client, newManifestHash, "robots.json", robotsData)
+ })
+
+ // upload a new file and use its manifest to change the file it the original manifest.
+ t.Run("update", func(t *testing.T) {
+ indexData := []byte("<h1>Ethereum Swarm</h1>")
+ indexDataFilename := filepath.Join(tmp, "index.html")
+ err = ioutil.WriteFile(indexDataFilename, indexData, 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ indexManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "up",
+ indexDataFilename,
+ )
+
+ newManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "manifest",
+ "update",
+ origManifestHash,
+ "index.html",
+ indexManifestHash,
+ )
+
+ checkHashLength(t, newManifestHash, encrypt)
+
+ newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+ var found bool
+ for _, e := range newManifest.Entries {
+ if e.Path == "index.html" {
+ found = true
+ if e.Size != int64(len(indexData)) {
+ t.Errorf("expected index.html size %v, got %v", len(indexData), e.Size)
+ }
+ if e.ModTime.IsZero() {
+ t.Errorf("got zero mod time for index.html")
+ }
+ ct := "text/html; charset=utf-8"
+ if e.ContentType != ct {
+ t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+ }
+ break
+ }
+ }
+ if !found {
+ t.Fatal("no index.html in new manifest")
+ }
+
+ checkFile(t, client, newManifestHash, "index.html", indexData)
+
+ // check default entry change
+ checkFile(t, client, newManifestHash, "", indexData)
+ })
+
+ // upload a new file and use its manifest to change the file it the original manifest,
+ // but ensure that the file is in the nested manifest of the original one.
+ t.Run("update nested", func(t *testing.T) {
+ robotsData := []byte(`<string>Only humans allowed!!!</strong>`)
+ robotsDataFilename := filepath.Join(tmp, "robots.html")
+ err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ humansManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "up",
+ robotsDataFilename,
+ )
+
+ newManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "manifest",
+ "update",
+ origManifestHash,
+ "robots.html",
+ humansManifestHash,
+ )
+
+ checkHashLength(t, newManifestHash, encrypt)
+
+ newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+ var found bool
+ loop:
+ for _, e := range newManifest.Entries {
+ if e.Path == "robots." {
+ nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
+ for _, e := range nestedManifest.Entries {
+ if e.Path == "html" {
+ found = true
+ if e.Size != int64(len(robotsData)) {
+ t.Errorf("expected robots.html size %v, got %v", len(robotsData), e.Size)
+ }
+ if e.ModTime.IsZero() {
+ t.Errorf("got zero mod time for robots.html")
+ }
+ ct := "text/html; charset=utf-8"
+ if e.ContentType != ct {
+ t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+ }
+ break loop
+ }
+ }
+ }
+ }
+ if !found {
+ t.Fatal("no robots.html in new manifest")
+ }
+
+ checkFile(t, client, newManifestHash, "robots.html", robotsData)
+ })
+
+ // remove a file from the manifest.
+ t.Run("remove", func(t *testing.T) {
+ newManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "manifest",
+ "remove",
+ origManifestHash,
+ "mutants.txt",
+ )
+
+ checkHashLength(t, newManifestHash, encrypt)
+
+ newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+ var found bool
+ for _, e := range newManifest.Entries {
+ if e.Path == "mutants.txt" {
+ found = true
+ break
+ }
+ }
+ if found {
+ t.Fatal("mutants.txt is not removed")
+ }
+ })
+
+ // remove a file from the manifest, but ensure that the file is in
+ // the nested manifest of the original one.
+ t.Run("remove nested", func(t *testing.T) {
+ newManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "manifest",
+ "remove",
+ origManifestHash,
+ "robots.html",
+ )
+
+ checkHashLength(t, newManifestHash, encrypt)
+
+ newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+ var found bool
+ loop:
+ for _, e := range newManifest.Entries {
+ if e.Path == "robots." {
+ nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
+ for _, e := range nestedManifest.Entries {
+ if e.Path == "html" {
+ found = true
+ break loop
+ }
+ }
+ }
+ }
+ if found {
+ t.Fatal("robots.html in not removed")
+ }
+ })
+}
+
+// TestNestedDefaultEntryUpdate tests if the default entry is updated
+// if the file in nested manifest used for it is also updated.
+func TestNestedDefaultEntryUpdate(t *testing.T) {
+ testNestedDefaultEntryUpdate(t, false)
+}
+
+// TestNestedDefaultEntryUpdateEncrypted tests if the default entry
+// of encrypted upload is updated if the file in nested manifest
+// used for it is also updated.
+func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
+ testNestedDefaultEntryUpdate(t, true)
+}
+
+func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
+ t.Parallel()
+ cluster := newTestCluster(t, 1)
+ defer cluster.Shutdown()
+
+ tmp, err := ioutil.TempDir("", "swarm-manifest-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(tmp)
+
+ origDir := filepath.Join(tmp, "orig")
+ if err := os.Mkdir(origDir, 0777); err != nil {
+ t.Fatal(err)
+ }
+
+ indexData := []byte("<h1>Test</h1>")
+ indexDataFilename := filepath.Join(origDir, "index.html")
+ err = ioutil.WriteFile(indexDataFilename, indexData, 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Add another file with common prefix as the default entry to test updates of
+ // default entry with nested manifests.
+ err = ioutil.WriteFile(filepath.Join(origDir, "index.txt"), []byte("Test"), 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ args := []string{
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "--recursive",
+ "--defaultpath",
+ indexDataFilename,
+ "up",
+ origDir,
+ }
+ if encrypt {
+ args = append(args, "--encrypt")
+ }
+
+ origManifestHash := runSwarmExpectHash(t, args...)
+
+ checkHashLength(t, origManifestHash, encrypt)
+
+ client := swarm.NewClient(cluster.Nodes[0].URL)
+
+ newIndexData := []byte("<h1>Ethereum Swarm</h1>")
+ newIndexDataFilename := filepath.Join(tmp, "index.html")
+ err = ioutil.WriteFile(newIndexDataFilename, newIndexData, 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ newIndexManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "up",
+ newIndexDataFilename,
+ )
+
+ newManifestHash := runSwarmExpectHash(t,
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "manifest",
+ "update",
+ origManifestHash,
+ "index.html",
+ newIndexManifestHash,
+ )
+
+ checkHashLength(t, newManifestHash, encrypt)
+
+ newManifest := downloadManifest(t, client, newManifestHash, encrypt)
+
+ var found bool
+ for _, e := range newManifest.Entries {
+ if e.Path == "index." {
+ found = true
+ newManifest = downloadManifest(t, client, e.Hash, encrypt)
+ break
+ }
+ }
+ if !found {
+ t.Fatal("no index. path in new manifest")
+ }
+
+ found = false
+ for _, e := range newManifest.Entries {
+ if e.Path == "html" {
+ found = true
+ if e.Size != int64(len(newIndexData)) {
+ t.Errorf("expected index.html size %v, got %v", len(newIndexData), e.Size)
+ }
+ if e.ModTime.IsZero() {
+ t.Errorf("got zero mod time for index.html")
+ }
+ ct := "text/html; charset=utf-8"
+ if e.ContentType != ct {
+ t.Errorf("expected content type %q, got %q", ct, e.ContentType)
+ }
+ break
+ }
+ }
+ if !found {
+ t.Fatal("no html in new manifest")
+ }
+
+ checkFile(t, client, newManifestHash, "index.html", newIndexData)
+
+ // check default entry change
+ checkFile(t, client, newManifestHash, "", newIndexData)
+}
+
+func runSwarmExpectHash(t *testing.T, args ...string) (hash string) {
+ t.Helper()
+ hashRegexp := `[a-f\d]{64,128}`
+ up := runSwarm(t, args...)
+ _, matches := up.ExpectRegexp(hashRegexp)
+ up.ExpectExit()
+
+ if len(matches) < 1 {
+ t.Fatal("no matches found")
+ }
+ return matches[0]
+}
+
+func checkHashLength(t *testing.T, hash string, encrypted bool) {
+ t.Helper()
+ l := len(hash)
+ if encrypted && l != 128 {
+ t.Errorf("expected hash length 128, got %v", l)
+ }
+ if !encrypted && l != 64 {
+ t.Errorf("expected hash length 64, got %v", l)
+ }
+}
+
+func downloadManifest(t *testing.T, client *swarm.Client, hash string, encrypted bool) (manifest *api.Manifest) {
+ t.Helper()
+ m, isEncrypted, err := client.DownloadManifest(hash)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if encrypted != isEncrypted {
+ t.Error("new manifest encryption flag is not correct")
+ }
+ return m
+}
+
+func checkFile(t *testing.T, client *swarm.Client, hash, path string, expected []byte) {
+ t.Helper()
+ f, err := client.Download(hash, path)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ got, err := ioutil.ReadAll(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(got, expected) {
+ t.Errorf("expected file content %q, got %q", expected, got)
+ }
+}
diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go
index 8ba0e7c5f0c0289210e0242d9e38c554f047b097..9eae2a3f806bfb8d747a0b3ccc730e5d0e24f84b 100644
--- a/cmd/swarm/upload.go
+++ b/cmd/swarm/upload.go
@@ -98,6 +98,17 @@ func upload(ctx *cli.Context) {
if !recursive {
return "", errors.New("Argument is a directory and recursive upload is disabled")
}
+ if defaultPath != "" {
+ // construct absolute default path
+ absDefaultPath, _ := filepath.Abs(defaultPath)
+ absFile, _ := filepath.Abs(file)
+ // make sure absolute directory ends with only one "/"
+ // to trim it from absolute default path and get relative default path
+ absFile = strings.TrimRight(absFile, "/") + "/"
+ if absDefaultPath != "" && absFile != "" && strings.HasPrefix(absDefaultPath, absFile) {
+ defaultPath = strings.TrimPrefix(absDefaultPath, absFile)
+ }
+ }
return client.UploadDirectory(file, defaultPath, "", toEncrypt)
}
} else {
diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go
index 2afc9b3a1186e6b49a028d0067271ac75a5dd28c..c3199dadc6baf928c187724146d4a5231b3612a8 100644
--- a/cmd/swarm/upload_test.go
+++ b/cmd/swarm/upload_test.go
@@ -273,3 +273,84 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
}
}
}
+
+// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
+// default paths and with encryption.
+func TestCLISwarmUpDefaultPath(t *testing.T) {
+ testCLISwarmUpDefaultPath(false, false, t)
+ testCLISwarmUpDefaultPath(false, true, t)
+ testCLISwarmUpDefaultPath(true, false, t)
+ testCLISwarmUpDefaultPath(true, true, t)
+}
+
+func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
+ cluster := newTestCluster(t, 1)
+ defer cluster.Shutdown()
+
+ tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(tmp)
+
+ err = ioutil.WriteFile(filepath.Join(tmp, "index.html"), []byte("<h1>Test</h1>"), 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = ioutil.WriteFile(filepath.Join(tmp, "robots.txt"), []byte("Disallow: /"), 0666)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ defaultPath := "index.html"
+ if absDefaultPath {
+ defaultPath = filepath.Join(tmp, defaultPath)
+ }
+
+ args := []string{
+ "--bzzapi",
+ cluster.Nodes[0].URL,
+ "--recursive",
+ "--defaultpath",
+ defaultPath,
+ "up",
+ tmp,
+ }
+ if toEncrypt {
+ args = append(args, "--encrypt")
+ }
+
+ up := runSwarm(t, args...)
+ hashRegexp := `[a-f\d]{64,128}`
+ _, matches := up.ExpectRegexp(hashRegexp)
+ up.ExpectExit()
+ hash := matches[0]
+
+ client := swarm.NewClient(cluster.Nodes[0].URL)
+
+ m, isEncrypted, err := client.DownloadManifest(hash)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if toEncrypt != isEncrypted {
+ t.Error("downloaded manifest is not encrypted")
+ }
+
+ var found bool
+ var entriesCount int
+ for _, e := range m.Entries {
+ entriesCount++
+ if e.Path == "" {
+ found = true
+ }
+ }
+
+ if !found {
+ t.Error("manifest default entry was not found")
+ }
+
+ if entriesCount != 3 {
+ t.Errorf("manifest contains %v entries, expected %v", entriesCount, 3)
+ }
+}
diff --git a/swarm/api/api.go b/swarm/api/api.go
index b418c45e1ce625cb77df472ae98cf9e4737ed01f..99d971b105ce5afbb9d286956768ed9b7d4162dc 100644
--- a/swarm/api/api.go
+++ b/swarm/api/api.go
@@ -704,11 +704,12 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content []
return fkey, newMkey.String(), nil
}
-func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath string, mw *ManifestWriter) (storage.Address, error) {
+func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath, defaultPath string, mw *ManifestWriter) (storage.Address, error) {
apiUploadTarCount.Inc(1)
var contentKey storage.Address
tr := tar.NewReader(bodyReader)
defer bodyReader.Close()
+ var defaultPathFound bool
for {
hdr, err := tr.Next()
if err == io.EOF {
@@ -737,6 +738,25 @@ func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestP
apiUploadTarFail.Inc(1)
return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err)
}
+ if hdr.Name == defaultPath {
+ entry := &ManifestEntry{
+ Hash: contentKey.Hex(),
+ Path: "", // default entry
+ ContentType: hdr.Xattrs["user.swarm.content-type"],
+ Mode: hdr.Mode,
+ Size: hdr.Size,
+ ModTime: hdr.ModTime,
+ }
+ contentKey, err = mw.AddEntry(ctx, nil, entry)
+ if err != nil {
+ apiUploadTarFail.Inc(1)
+ return nil, fmt.Errorf("error adding default manifest entry from tar stream: %s", err)
+ }
+ defaultPathFound = true
+ }
+ }
+ if defaultPath != "" && !defaultPathFound {
+ return contentKey, fmt.Errorf("default path %q not found", defaultPath)
}
return contentKey, nil
}
diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go
index b3a5e929d0163f312f3f391bd1b0f32045cbf1de..8a9efe3608c93862a470f720bc0f9cfe783156ff 100644
--- a/swarm/api/client/client.go
+++ b/swarm/api/client/client.go
@@ -138,7 +138,7 @@ func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, er
if file.Size <= 0 {
return "", errors.New("file size must be greater than zero")
}
- return c.TarUpload(manifest, &FileUploader{file}, toEncrypt)
+ return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt)
}
// Download downloads a file with the given path from the swarm manifest with
@@ -175,7 +175,15 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo
} else if !stat.IsDir() {
return "", fmt.Errorf("not a directory: %s", dir)
}
- return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}, toEncrypt)
+ if defaultPath != "" {
+ if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil {
+ if os.IsNotExist(err) {
+ return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir)
+ }
+ return "", fmt.Errorf("default path: %v", err)
+ }
+ }
+ return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt)
}
// DownloadDirectory downloads the files contained in a swarm manifest under
@@ -389,21 +397,11 @@ func (u UploaderFunc) Upload(upload UploadFn) error {
// DirectoryUploader uploads all files in a directory, optionally uploading
// a file to the default path
type DirectoryUploader struct {
- Dir string
- DefaultPath string
+ Dir string
}
// Upload performs the upload of the directory and default path
func (d *DirectoryUploader) Upload(upload UploadFn) error {
- if d.DefaultPath != "" {
- file, err := Open(d.DefaultPath)
- if err != nil {
- return err
- }
- if err := upload(file); err != nil {
- return err
- }
- }
return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
@@ -441,7 +439,7 @@ type UploadFn func(file *File) error
// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
// returning the resulting manifest hash
-func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (string, error) {
+func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) {
reqR, reqW := io.Pipe()
defer reqR.Close()
addr := hash
@@ -458,6 +456,11 @@ func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (stri
return "", err
}
req.Header.Set("Content-Type", "application/x-tar")
+ if defaultPath != "" {
+ q := req.URL.Query()
+ q.Set("defaultpath", defaultPath)
+ req.URL.RawQuery = q.Encode()
+ }
// use 'Expect: 100-continue' so we don't send the request body if
// the server refuses the request
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
index dc608e3f1f869d88d14df0c471c695b62e1c5125..ae82a91d798a198bbb005c3626cd508d7083c49b 100644
--- a/swarm/api/client/client_test.go
+++ b/swarm/api/client/client_test.go
@@ -194,7 +194,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
// upload the directory
client := NewClient(srv.URL)
- defaultPath := filepath.Join(dir, testDirFiles[0])
+ defaultPath := testDirFiles[0]
hash, err := client.UploadDirectory(dir, defaultPath, "", false)
if err != nil {
t.Fatalf("error uploading directory: %s", err)
diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index bd6949de6caf7c5d73e5e75002ad3a5f1a146444..5a5c42adc07383e86a08014d939c0c1c85fb6099 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -336,7 +336,9 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) {
log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context()))
- key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, mw)
+ defaultPath := r.URL.Query().Get("defaultpath")
+
+ key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, defaultPath, mw)
if err != nil {
return nil, err
}
diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go
index fbd143f295a145006426444e86be0c49b44c34f6..2a163dd39c9e40b40e0acfedba0acbaa694110f2 100644
--- a/swarm/api/manifest.go
+++ b/swarm/api/manifest.go
@@ -106,13 +106,18 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC
}
// AddEntry stores the given data and adds the resulting key to the manifest
-func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (storage.Address, error) {
- key, _, err := m.api.Store(ctx, data, e.Size, m.trie.encrypted)
- if err != nil {
- return nil, err
- }
+func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (key storage.Address, err error) {
entry := newManifestTrieEntry(e, nil)
- entry.Hash = key.Hex()
+ if data != nil {
+ key, _, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted)
+ if err != nil {
+ return nil, err
+ }
+ entry.Hash = key.Hex()
+ }
+ if entry.Hash == "" {
+ return key, errors.New("missing entry hash")
+ }
m.trie.addEntry(entry, m.quitC)
return key, nil
}