From b319f027a0be232a9cb307336b0349b36737c7f1 Mon Sep 17 00:00:00 2001
From: Lewis Marshall <lewis@lmars.net>
Date: Tue, 4 Apr 2017 23:20:07 +0100
Subject: [PATCH] cmd/swarm, swarm/api/client: add HTTP API client and 'swarm
ls' command (#3742)
This adds a swarm ls command which lists files and directories stored in a
manifest. Rather than listing all files, it uses "directory prefixes" in case there are a
lot of files in a manifest but you just want to traverse it.
This also includes some refactoring to the tests and the introduction of a
swarm/api/client package to make things easier to test.
---
cmd/swarm/list.go | 58 +++++++
cmd/swarm/main.go | 9 +
cmd/swarm/manifest.go | 51 +++---
cmd/swarm/upload.go | 163 +----------------
swarm/api/client/client.go | 298 ++++++++++++++++++++++++++++++++
swarm/api/client/client_test.go | 105 +++++++++++
swarm/api/http/server.go | 42 +++--
swarm/api/http/server_test.go | 52 +-----
swarm/swarm.go | 5 +-
swarm/testutil/http.go | 56 ++++++
10 files changed, 594 insertions(+), 245 deletions(-)
create mode 100644 cmd/swarm/list.go
create mode 100644 swarm/api/client/client.go
create mode 100644 swarm/api/client/client_test.go
create mode 100644 swarm/testutil/http.go
diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go
new file mode 100644
index 000000000..3a68fef03
--- /dev/null
+++ b/cmd/swarm/list.go
@@ -0,0 +1,58 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "text/tabwriter"
+
+ "github.com/ethereum/go-ethereum/cmd/utils"
+ swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+ "gopkg.in/urfave/cli.v1"
+)
+
+func list(ctx *cli.Context) {
+ args := ctx.Args()
+
+ if len(args) < 1 {
+ utils.Fatalf("Please supply a manifest reference as the first argument")
+ } else if len(args) > 2 {
+ utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
+ }
+ manifest := args[0]
+
+ var prefix string
+ if len(args) == 2 {
+ prefix = args[1]
+ }
+
+ bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+ client := swarm.NewClient(bzzapi)
+ entries, err := client.ManifestFileList(manifest, prefix)
+ if err != nil {
+ utils.Fatalf("Failed to generate file and directory list: %s", err)
+ }
+
+ w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
+ defer w.Flush()
+ fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
+ for _, entry := range entries {
+ fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
+ }
+}
diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go
index 3c82497e5..731c300f8 100644
--- a/cmd/swarm/main.go
+++ b/cmd/swarm/main.go
@@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable.
ArgsUsage: " <file>",
Description: `
"upload a file or directory to swarm using the HTTP API and prints the root hash",
+`,
+ },
+ {
+ Action: list,
+ Name: "ls",
+ Usage: "list files and directories contained in a manifest",
+ ArgsUsage: " <manifest> [<prefix>]",
+ Description: `
+Lists files and directories contained in a manifest.
`,
},
{
diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go
index 2b6b02313..698b8ddb8 100644
--- a/cmd/swarm/manifest.go
+++ b/cmd/swarm/manifest.go
@@ -25,6 +25,7 @@ import (
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
+ swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
@@ -41,7 +42,7 @@ func add(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot manifest
+ mroot swarm.Manifest
)
if len(args) > 3 {
@@ -75,7 +76,7 @@ func update(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot manifest
+ mroot swarm.Manifest
)
if len(args) > 3 {
ctype = args[3]
@@ -105,7 +106,7 @@ func remove(ctx *cli.Context) {
path = args[1]
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot manifest
+ mroot swarm.Manifest
)
newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -123,21 +124,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
- client = &client{api: bzzapi}
- longestPathEntry = manifestEntry{
+ client = swarm.NewClient(bzzapi)
+ longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
- mroot, err := client.downloadManifest(mhash)
+ mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
//TODO: check if the "hash" to add is valid and present in swarm
- _, err = client.downloadManifest(hash)
+ _, err = client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Hash to add is not present: %v", err)
}
@@ -162,7 +163,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
- newMRoot := manifest{}
+ newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@@ -172,7 +173,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot = newMRoot
} else {
// Add the entry in the leaf Manifest
- newEntry := manifestEntry{
+ newEntry := swarm.ManifestEntry{
Path: path,
Hash: hash,
ContentType: ctype,
@@ -180,7 +181,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot.Entries = append(mroot.Entries, newEntry)
}
- newManifestHash, err := client.uploadManifest(mroot)
+ newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@@ -192,20 +193,20 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
- client = &client{api: bzzapi}
- newEntry = manifestEntry{
+ client = swarm.NewClient(bzzapi)
+ newEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
- longestPathEntry = manifestEntry{
+ longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
- mroot, err := client.downloadManifest(mhash)
+ mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
@@ -236,7 +237,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
- newMRoot := manifest{}
+ newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@@ -249,10 +250,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
if newEntry.Path != "" {
// Replace the hash for leaf Manifest
- newMRoot := manifest{}
+ newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if newEntry.Path == entry.Path {
- myEntry := manifestEntry{
+ myEntry := swarm.ManifestEntry{
Path: entry.Path,
Hash: hash,
ContentType: ctype,
@@ -265,7 +266,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
mroot = newMRoot
}
- newManifestHash, err := client.uploadManifest(mroot)
+ newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@@ -276,20 +277,20 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
- client = &client{api: bzzapi}
- entryToRemove = manifestEntry{
+ client = swarm.NewClient(bzzapi)
+ entryToRemove = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
- longestPathEntry = manifestEntry{
+ longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
- mroot, err := client.downloadManifest(mhash)
+ mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
@@ -318,7 +319,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests
- newMRoot := manifest{}
+ newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@@ -330,7 +331,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if entryToRemove.Path != "" {
// remove the entry in this Manifest
- newMRoot := manifest{}
+ newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if entryToRemove.Path != entry.Path {
newMRoot.Entries = append(newMRoot.Entries, entry)
@@ -339,7 +340,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
mroot = newMRoot
}
- newManifestHash, err := client.uploadManifest(mroot)
+ newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go
index 7b4961778..696b907d2 100644
--- a/cmd/swarm/upload.go
+++ b/cmd/swarm/upload.go
@@ -18,21 +18,15 @@
package main
import (
- "bytes"
"encoding/json"
"fmt"
- "io"
- "io/ioutil"
- "mime"
- "net/http"
"os"
"os/user"
"path"
- "path/filepath"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
- "github.com/ethereum/go-ethereum/log"
+ swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
@@ -50,7 +44,7 @@ func upload(ctx *cli.Context) {
var (
file = args[0]
- client = &client{api: bzzapi}
+ client = swarm.NewClient(bzzapi)
)
fi, err := os.Stat(expandPath(file))
if err != nil {
@@ -63,25 +57,25 @@ func upload(ctx *cli.Context) {
if !wantManifest {
utils.Fatalf("Manifest is required for directory uploads")
}
- mhash, err := client.uploadDirectory(file, defaultPath)
+ mhash, err := client.UploadDirectory(file, defaultPath)
if err != nil {
utils.Fatalf("Failed to upload directory: %v", err)
}
fmt.Println(mhash)
return
}
- entry, err := client.uploadFile(file, fi)
+ entry, err := client.UploadFile(file, fi)
if err != nil {
utils.Fatalf("Upload failed: %v", err)
}
- mroot := manifest{[]manifestEntry{entry}}
+ mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
if !wantManifest {
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
return
}
- hash, err := client.uploadManifest(mroot)
+ hash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@@ -111,148 +105,3 @@ func homeDir() string {
}
return ""
}
-
-// client wraps interaction with the swarm HTTP gateway.
-type client struct {
- api string
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifestEntry struct {
- Hash string `json:"hash,omitempty"`
- ContentType string `json:"contentType,omitempty"`
- Path string `json:"path,omitempty"`
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifest struct {
- Entries []manifestEntry `json:"entries,omitempty"`
-}
-
-func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
- mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
- if err != nil {
- return "", fmt.Errorf("failed to upload empty manifest")
- }
- if len(defaultPath) > 0 {
- fi, err := os.Stat(defaultPath)
- if err != nil {
- return "", err
- }
- mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
- if err != nil {
- return "", err
- }
- }
- prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
- err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
- if err != nil || fi.IsDir() {
- return err
- }
- if !strings.HasPrefix(path, dir) {
- return fmt.Errorf("path %s outside directory %s", path, dir)
- }
- uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
- mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
- return err
- })
- return mhash, err
-}
-
-func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
- hash, err := c.uploadFileContent(file, fi)
- m := manifestEntry{
- Hash: hash,
- ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
- }
- return m, err
-}
-
-func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
- fd, err := os.Open(file)
- if err != nil {
- return "", err
- }
- defer fd.Close()
- log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
- return c.postRaw("application/octet-stream", fi.Size(), fd)
-}
-
-func (c *client) uploadManifest(m manifest) (string, error) {
- jsm, err := json.Marshal(m)
- if err != nil {
- panic(err)
- }
- log.Info("Uploading swarm manifest")
- return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
-}
-
-func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
- fd, err := os.Open(fpath)
- if err != nil {
- return "", err
- }
- defer fd.Close()
- log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
- req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
- if err != nil {
- return "", err
- }
- req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
- req.ContentLength = fi.Size()
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- return "", err
- }
- defer resp.Body.Close()
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("bad status: %s", resp.Status)
- }
- content, err := ioutil.ReadAll(resp.Body)
- return string(content), err
-}
-
-func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
- req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
- if err != nil {
- return "", err
- }
- req.Header.Set("content-type", mimetype)
- req.ContentLength = size
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- return "", err
- }
- defer resp.Body.Close()
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("bad status: %s", resp.Status)
- }
- content, err := ioutil.ReadAll(resp.Body)
- return string(content), err
-}
-
-func (c *client) downloadManifest(mhash string) (manifest, error) {
-
- mroot := manifest{}
- req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
- if err != nil {
- return mroot, err
- }
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- return mroot, err
- }
- defer resp.Body.Close()
-
- if resp.StatusCode >= 400 {
- return mroot, fmt.Errorf("bad status: %s", resp.Status)
-
- }
- content, err := ioutil.ReadAll(resp.Body)
-
- err = json.Unmarshal(content, &mroot)
- if err != nil {
- return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
- }
- return mroot, err
-}
diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go
new file mode 100644
index 000000000..15e44f35d
--- /dev/null
+++ b/swarm/api/client/client.go
@@ -0,0 +1,298 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "mime"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/ethereum/go-ethereum/log"
+)
+
+var (
+ DefaultGateway = "http://localhost:8500"
+ DefaultClient = NewClient(DefaultGateway)
+)
+
+// Manifest represents a swarm manifest.
+type Manifest struct {
+ Entries []ManifestEntry `json:"entries,omitempty"`
+}
+
+// ManifestEntry represents an entry in a swarm manifest.
+type ManifestEntry struct {
+ Hash string `json:"hash,omitempty"`
+ ContentType string `json:"contentType,omitempty"`
+ Path string `json:"path,omitempty"`
+}
+
+func NewClient(gateway string) *Client {
+ return &Client{
+ Gateway: gateway,
+ }
+}
+
+// Client wraps interaction with a swarm HTTP gateway.
+type Client struct {
+ Gateway string
+}
+
+func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
+ mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
+ if err != nil {
+ return "", fmt.Errorf("failed to upload empty manifest")
+ }
+ if len(defaultPath) > 0 {
+ fi, err := os.Stat(defaultPath)
+ if err != nil {
+ return "", err
+ }
+ mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
+ if err != nil {
+ return "", err
+ }
+ }
+ prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
+ err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
+ if err != nil || fi.IsDir() {
+ return err
+ }
+ if !strings.HasPrefix(path, dir) {
+ return fmt.Errorf("path %s outside directory %s", path, dir)
+ }
+ uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
+ mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
+ return err
+ })
+ return mhash, err
+}
+
+func (c *Client) UploadFile(file string, fi os.FileInfo) (ManifestEntry, error) {
+ hash, err := c.uploadFileContent(file, fi)
+ m := ManifestEntry{
+ Hash: hash,
+ ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+ }
+ return m, err
+}
+
+func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
+ fd, err := os.Open(file)
+ if err != nil {
+ return "", err
+ }
+ defer fd.Close()
+ log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
+ return c.postRaw("application/octet-stream", fi.Size(), fd)
+}
+
+func (c *Client) UploadManifest(m Manifest) (string, error) {
+ jsm, err := json.Marshal(m)
+ if err != nil {
+ panic(err)
+ }
+ log.Info("Uploading swarm manifest")
+ return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+}
+
+func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
+ fd, err := os.Open(fpath)
+ if err != nil {
+ return "", err
+ }
+ defer fd.Close()
+ log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
+ req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
+ if err != nil {
+ return "", err
+ }
+ req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
+ req.ContentLength = fi.Size()
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode >= 400 {
+ return "", fmt.Errorf("bad status: %s", resp.Status)
+ }
+ content, err := ioutil.ReadAll(resp.Body)
+ return string(content), err
+}
+
+func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
+ req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
+ if err != nil {
+ return "", err
+ }
+ req.Header.Set("content-type", mimetype)
+ req.ContentLength = size
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode >= 400 {
+ return "", fmt.Errorf("bad status: %s", resp.Status)
+ }
+ content, err := ioutil.ReadAll(resp.Body)
+ return string(content), err
+}
+
+func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
+
+ mroot := Manifest{}
+ req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
+ if err != nil {
+ return mroot, err
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return mroot, err
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode >= 400 {
+ return mroot, fmt.Errorf("bad status: %s", resp.Status)
+
+ }
+ content, err := ioutil.ReadAll(resp.Body)
+
+ err = json.Unmarshal(content, &mroot)
+ if err != nil {
+ return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+ }
+ return mroot, err
+}
+
+// ManifestFileList downloads the manifest with the given hash and generates a
+// list of files and directory prefixes which have the specified prefix.
+//
+// For example, if the manifest represents the following directory structure:
+//
+// file1.txt
+// file2.txt
+// dir1/file3.txt
+// dir1/dir2/file4.txt
+//
+// Then:
+//
+// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
+// - a prefix of "file" would return [file1.txt, file2.txt]
+// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
+func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
+ manifest, err := c.DownloadManifest(hash)
+ if err != nil {
+ return nil, err
+ }
+
+ // handleFile handles a manifest entry which is a direct reference to a
+ // file (i.e. it is not a swarm manifest)
+ handleFile := func(entry ManifestEntry) {
+ // ignore the file if it doesn't have the specified prefix
+ if !strings.HasPrefix(entry.Path, prefix) {
+ return
+ }
+ // if the path after the prefix contains a directory separator,
+ // add a directory prefix to the entries, otherwise add the
+ // file
+ suffix := strings.TrimPrefix(entry.Path, prefix)
+ if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
+ entries = append(entries, ManifestEntry{
+ Path: prefix + suffix[:sepIndex+1],
+ ContentType: "DIR",
+ })
+ } else {
+ if entry.Path == "" {
+ entry.Path = "/"
+ }
+ entries = append(entries, entry)
+ }
+ }
+
+ // handleManifest handles a manifest entry which is a reference to
+ // another swarm manifest.
+ handleManifest := func(entry ManifestEntry) error {
+ // if the manifest's path is a prefix of the specified prefix
+ // then just recurse into the manifest by stripping its path
+ // from the prefix
+ if strings.HasPrefix(prefix, entry.Path) {
+ subPrefix := strings.TrimPrefix(prefix, entry.Path)
+ subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
+ if err != nil {
+ return err
+ }
+ // prefix the manifest's path to the sub entries and
+ // add them to the returned entries
+ for i, subEntry := range subEntries {
+ subEntry.Path = entry.Path + subEntry.Path
+ subEntries[i] = subEntry
+ }
+ entries = append(entries, subEntries...)
+ return nil
+ }
+
+ // if the manifest's path has the specified prefix, then if the
+ // path after the prefix contains a directory separator, add a
+ // directory prefix to the entries, otherwise recurse into the
+ // manifest
+ if strings.HasPrefix(entry.Path, prefix) {
+ suffix := strings.TrimPrefix(entry.Path, prefix)
+ sepIndex := strings.Index(suffix, "/")
+ if sepIndex > -1 {
+ entries = append(entries, ManifestEntry{
+ Path: prefix + suffix[:sepIndex+1],
+ ContentType: "DIR",
+ })
+ return nil
+ }
+ subEntries, err := c.ManifestFileList(entry.Hash, "")
+ if err != nil {
+ return err
+ }
+ // prefix the manifest's path to the sub entries and
+ // add them to the returned entries
+ for i, subEntry := range subEntries {
+ subEntry.Path = entry.Path + subEntry.Path
+ subEntries[i] = subEntry
+ }
+ entries = append(entries, subEntries...)
+ return nil
+ }
+ return nil
+ }
+
+ for _, entry := range manifest.Entries {
+ if entry.ContentType == "application/bzz-manifest+json" {
+ if err := handleManifest(entry); err != nil {
+ return nil, err
+ }
+ } else {
+ handleFile(entry)
+ }
+ }
+
+ return
+}
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
new file mode 100644
index 000000000..135474475
--- /dev/null
+++ b/swarm/api/client/client_test.go
@@ -0,0 +1,105 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "reflect"
+ "sort"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/swarm/testutil"
+)
+
+func TestClientManifestFileList(t *testing.T) {
+ srv := testutil.NewTestSwarmServer(t)
+ defer srv.Close()
+
+ dir, err := ioutil.TempDir("", "swarm-client-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ files := []string{
+ "file1.txt",
+ "file2.txt",
+ "dir1/file3.txt",
+ "dir1/file4.txt",
+ "dir2/file5.txt",
+ "dir2/dir3/file6.txt",
+ "dir2/dir4/file7.txt",
+ "dir2/dir4/file8.txt",
+ }
+ for _, file := range files {
+ path := filepath.Join(dir, file)
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ t.Fatalf("error creating dir for %s: %s", path, err)
+ }
+ if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+ t.Fatalf("error writing file %s: %s", path, err)
+ }
+ }
+
+ client := NewClient(srv.URL)
+
+ hash, err := client.UploadDirectory(dir, "")
+ if err != nil {
+ t.Fatalf("error uploading directory: %s", err)
+ }
+
+ ls := func(prefix string) []string {
+ entries, err := client.ManifestFileList(hash, prefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ paths := make([]string, len(entries))
+ for i, entry := range entries {
+ paths[i] = entry.Path
+ }
+ sort.Strings(paths)
+ return paths
+ }
+
+ tests := map[string][]string{
+ "": []string{"dir1/", "dir2/", "file1.txt", "file2.txt"},
+ "file": []string{"file1.txt", "file2.txt"},
+ "file1": []string{"file1.txt"},
+ "file2.txt": []string{"file2.txt"},
+ "file12": []string{},
+ "dir": []string{"dir1/", "dir2/"},
+ "dir1": []string{"dir1/"},
+ "dir1/": []string{"dir1/file3.txt", "dir1/file4.txt"},
+ "dir1/file": []string{"dir1/file3.txt", "dir1/file4.txt"},
+ "dir1/file3.txt": []string{"dir1/file3.txt"},
+ "dir1/file34": []string{},
+ "dir2/": []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"},
+ "dir2/file": []string{"dir2/file5.txt"},
+ "dir2/dir": []string{"dir2/dir3/", "dir2/dir4/"},
+ "dir2/dir3/": []string{"dir2/dir3/file6.txt"},
+ "dir2/dir4/": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
+ "dir2/dir4/file": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
+ "dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"},
+ "dir2/dir4/file78": []string{},
+ }
+ for prefix, expected := range tests {
+ actual := ls(prefix)
+ if !reflect.DeepEqual(actual, expected) {
+ t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+ }
+ }
+}
diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index ae113750f..44e2c203a 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -56,9 +56,9 @@ type sequentialReader struct {
lock sync.Mutex
}
-// Server is the basic configuration needs for the HTTP server and also
+// ServerConfig is the basic configuration needed for the HTTP server and also
// includes CORS settings.
-type Server struct {
+type ServerConfig struct {
Addr string
CorsString string
}
@@ -69,13 +69,9 @@ type Server struct {
// https://github.com/atom/electron/blob/master/docs/api/protocol.md
// starts up http server
-func StartHttpServer(api *api.Api, server *Server) {
- serveMux := http.NewServeMux()
- serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
- handler(w, r, api)
- })
+func StartHttpServer(api *api.Api, config *ServerConfig) {
var allowedOrigins []string
- for _, domain := range strings.Split(server.CorsString, ",") {
+ for _, domain := range strings.Split(config.CorsString, ",") {
allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
}
c := cors.New(cors.Options{
@@ -84,13 +80,21 @@ func StartHttpServer(api *api.Api, server *Server) {
MaxAge: 600,
AllowedHeaders: []string{"*"},
})
- hdlr := c.Handler(serveMux)
+ hdlr := c.Handler(NewServer(api))
+
+ go http.ListenAndServe(config.Addr, hdlr)
+ log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", config.Addr))
+}
- go http.ListenAndServe(server.Addr, hdlr)
- log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", server.Addr))
+func NewServer(api *api.Api) *Server {
+ return &Server{api}
+}
+
+type Server struct {
+ api *api.Api
}
-func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
requestURL := r.URL
// This is wrong
// if requestURL.Host == "" {
@@ -135,7 +139,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
return
}
- key, err := a.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
+ key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
if err == nil {
log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
} else {
@@ -160,7 +164,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
mime := r.Header.Get("Content-Type")
// TODO proper root hash separation
log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
- newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
+ newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
if err == nil {
log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
w.Header().Set("Content-Type", "text/plain")
@@ -178,7 +182,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
} else {
path = api.RegularSlashes(path)
log.Debug(fmt.Sprintf("Delete '%s'.", path))
- newKey, err := a.Modify(path, "", "", nameresolver)
+ newKey, err := s.api.Modify(path, "", "", nameresolver)
if err == nil {
log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
w.Header().Set("Content-Type", "text/plain")
@@ -199,16 +203,16 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
parsedurl, _ := api.Parse(path)
if parsedurl == path {
- key, err := a.Resolve(parsedurl, nameresolver)
+ key, err := s.api.Resolve(parsedurl, nameresolver)
if err != nil {
log.Error(fmt.Sprintf("%v", err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
- reader = a.Retrieve(key)
+ reader = s.api.Retrieve(key)
} else {
var status int
- readertmp, _, status, err := a.Get(path, nameresolver)
+ readertmp, _, status, err := s.api.Get(path, nameresolver)
if err != nil {
http.Error(w, err.Error(), status)
return
@@ -247,7 +251,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
http.Redirect(w, r, path+"/", http.StatusFound)
return
}
- reader, mimeType, status, err := a.Get(path, nameresolver)
+ reader, mimeType, status, err := s.api.Get(path, nameresolver)
if err != nil {
if _, ok := err.(api.ErrResolve); ok {
log.Debug(fmt.Sprintf("%v", err))
diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index 88b49b9a5..45a867f51 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-package http
+package http_test
import (
"bytes"
@@ -22,19 +22,16 @@ import (
"net/http"
"sync"
"testing"
- "time"
"github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage"
+ "github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestBzzrGetPath(t *testing.T) {
var err error
- maxproxyattempts := 3
-
testmanifest := []string{
`{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`,
`{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`,
@@ -54,61 +51,30 @@ func TestBzzrGetPath(t *testing.T) {
key := [3]storage.Key{}
- dir, _ := ioutil.TempDir("", "bzz-storage-test")
-
- storeparams := &storage.StoreParams{
- ChunkDbPath: dir,
- DbCapacity: 5000000,
- CacheCapacity: 5000,
- Radius: 0,
- }
-
- localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
- if err != nil {
- t.Fatal(err)
- }
- chunker := storage.NewTreeChunker(storage.NewChunkerParams())
- dpa := &storage.DPA{
- Chunker: chunker,
- ChunkStore: localStore,
- }
- dpa.Start()
- defer dpa.Stop()
+ srv := testutil.NewTestSwarmServer(t)
+ defer srv.Close()
wg := &sync.WaitGroup{}
for i, mf := range testmanifest {
reader[i] = bytes.NewReader([]byte(mf))
- key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil)
+ key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil)
if err != nil {
t.Fatal(err)
}
wg.Wait()
}
- a := api.NewApi(dpa, nil)
-
- /// \todo iterate port numbers up if fail
- StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""})
- // how to wait for ListenAndServe to have initialized? This is pretty cruuuude
- // if we fix it we don't need maxproxyattempts anymore either
- time.Sleep(1000 * time.Millisecond)
- for i := 0; i <= maxproxyattempts; i++ {
- _, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
- if i == maxproxyattempts {
- t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err)
- } else if err != nil {
- time.Sleep(100 * time.Millisecond)
- continue
- }
- break
+ _, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+ if err != nil {
+ t.Fatalf("Failed to connect to proxy: %v", err)
}
for k, v := range testrequests {
var resp *http.Response
var respbody []byte
- url := "http://127.0.0.1:8504/bzzr:/"
+ url := srv.URL + "/bzzr:/"
if k[:] != "" {
url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
}
diff --git a/swarm/swarm.go b/swarm/swarm.go
index 0ce31bcad..add28d205 100644
--- a/swarm/swarm.go
+++ b/swarm/swarm.go
@@ -195,7 +195,10 @@ func (self *Swarm) Start(net *p2p.Server) error {
// start swarm http proxy server
if self.config.Port != "" {
addr := ":" + self.config.Port
- go httpapi.StartHttpServer(self.api, &httpapi.Server{Addr: addr, CorsString: self.corsString})
+ go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{
+ Addr: addr,
+ CorsString: self.corsString,
+ })
}
log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port))
diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go
new file mode 100644
index 000000000..bf98d16eb
--- /dev/null
+++ b/swarm/testutil/http.go
@@ -0,0 +1,56 @@
+package testutil
+
+import (
+ "io/ioutil"
+ "net/http/httptest"
+ "os"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/swarm/api"
+ httpapi "github.com/ethereum/go-ethereum/swarm/api/http"
+ "github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
+ dir, err := ioutil.TempDir("", "swarm-storage-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ storeparams := &storage.StoreParams{
+ ChunkDbPath: dir,
+ DbCapacity: 5000000,
+ CacheCapacity: 5000,
+ Radius: 0,
+ }
+ localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
+ if err != nil {
+ os.RemoveAll(dir)
+ t.Fatal(err)
+ }
+ chunker := storage.NewTreeChunker(storage.NewChunkerParams())
+ dpa := &storage.DPA{
+ Chunker: chunker,
+ ChunkStore: localStore,
+ }
+ dpa.Start()
+ a := api.NewApi(dpa, nil)
+ srv := httptest.NewServer(httpapi.NewServer(a))
+ return &TestSwarmServer{
+ Server: srv,
+ Dpa: dpa,
+ dir: dir,
+ }
+}
+
+type TestSwarmServer struct {
+ *httptest.Server
+
+ Dpa *storage.DPA
+ dir string
+}
+
+func (t *TestSwarmServer) Close() {
+ t.Server.Close()
+ t.Dpa.Stop()
+ os.RemoveAll(t.dir)
+}
--
GitLab