diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go
index 3a68fef030b065a0ef957f65c187b2ea768e4ed9..06d3883cfe1d554ea1b4cfd75bc83bd5d6c9743d 100644
--- a/cmd/swarm/list.go
+++ b/cmd/swarm/list.go
@@ -44,7 +44,7 @@ func list(ctx *cli.Context) {
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
- entries, err := client.ManifestFileList(manifest, prefix)
+ list, err := client.List(manifest, prefix)
if err != nil {
utils.Fatalf("Failed to generate file and directory list: %s", err)
}
@@ -52,7 +52,10 @@ func list(ctx *cli.Context) {
w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
- for _, entry := range entries {
+ for _, prefix := range list.CommonPrefixes {
+ fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
+ }
+ for _, entry := range list.Entries {
fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
}
}
diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go
index 698b8ddb8860326a7efcfb57574fc34cdaeaa8ca..9729022c05cca31c925658fa5943524fea044559 100644
--- a/cmd/swarm/manifest.go
+++ b/cmd/swarm/manifest.go
@@ -25,6 +25,7 @@ import (
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
+ "github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
@@ -42,7 +43,7 @@ func add(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot swarm.Manifest
+ mroot api.Manifest
)
if len(args) > 3 {
@@ -76,7 +77,7 @@ func update(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot swarm.Manifest
+ mroot api.Manifest
)
if len(args) > 3 {
ctype = args[3]
@@ -106,7 +107,7 @@ func remove(ctx *cli.Context) {
path = args[1]
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
- mroot swarm.Manifest
+ mroot api.Manifest
)
newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -125,11 +126,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
- longestPathEntry = swarm.ManifestEntry{
- Path: "",
- Hash: "",
- ContentType: "",
- }
+ longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
@@ -163,7 +160,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
- newMRoot := swarm.Manifest{}
+ newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@@ -173,9 +170,9 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot = newMRoot
} else {
// Add the entry in the leaf Manifest
- newEntry := swarm.ManifestEntry{
- Path: path,
+ newEntry := api.ManifestEntry{
Hash: hash,
+ Path: path,
ContentType: ctype,
}
mroot.Entries = append(mroot.Entries, newEntry)
@@ -192,18 +189,10 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
var (
- bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
- client = swarm.NewClient(bzzapi)
- newEntry = swarm.ManifestEntry{
- Path: "",
- Hash: "",
- ContentType: "",
- }
- longestPathEntry = swarm.ManifestEntry{
- Path: "",
- Hash: "",
- ContentType: "",
- }
+ bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+ client = swarm.NewClient(bzzapi)
+ newEntry = api.ManifestEntry{}
+ longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
@@ -237,7 +226,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
- newMRoot := swarm.Manifest{}
+ newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@@ -250,12 +239,12 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
if newEntry.Path != "" {
// Replace the hash for leaf Manifest
- newMRoot := swarm.Manifest{}
+ newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if newEntry.Path == entry.Path {
- myEntry := swarm.ManifestEntry{
- Path: entry.Path,
+ myEntry := api.ManifestEntry{
Hash: hash,
+ Path: entry.Path,
ContentType: ctype,
}
newMRoot.Entries = append(newMRoot.Entries, myEntry)
@@ -276,18 +265,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
var (
- bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
- client = swarm.NewClient(bzzapi)
- entryToRemove = swarm.ManifestEntry{
- Path: "",
- Hash: "",
- ContentType: "",
- }
- longestPathEntry = swarm.ManifestEntry{
- Path: "",
- Hash: "",
- ContentType: "",
- }
+ bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+ client = swarm.NewClient(bzzapi)
+ entryToRemove = api.ManifestEntry{}
+ longestPathEntry = api.ManifestEntry{}
)
mroot, err := client.DownloadManifest(mhash)
@@ -319,7 +300,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests
- newMRoot := swarm.Manifest{}
+ newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@@ -331,7 +312,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if entryToRemove.Path != "" {
// remove the entry in this Manifest
- newMRoot := swarm.Manifest{}
+ newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if entryToRemove.Path != entry.Path {
newMRoot.Entries = append(newMRoot.Entries, entry)
diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go
index 46f10c4bec34afd7eaa56751ddb3479145d3d0a6..42673ae214a1b1b0e3847aef9358703cf7d75986 100644
--- a/cmd/swarm/upload.go
+++ b/cmd/swarm/upload.go
@@ -18,13 +18,15 @@
package main
import (
- "encoding/json"
"fmt"
"io"
"io/ioutil"
+ "mime"
+ "net/http"
"os"
"os/user"
"path"
+ "path/filepath"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
@@ -42,12 +44,10 @@ func upload(ctx *cli.Context) {
defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name)
fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
mimeType = ctx.GlobalString(SwarmUploadMimeType.Name)
+ client = swarm.NewClient(bzzapi)
+ file string
)
- var client = swarm.NewClient(bzzapi)
- var entry swarm.ManifestEntry
- var file string
-
if len(args) != 1 {
if fromStdin {
tmp, err := ioutil.TempFile("", "swarm-stdin")
@@ -66,41 +66,47 @@ func upload(ctx *cli.Context) {
utils.Fatalf("Need filename as the first and only argument")
}
} else {
- file = args[0]
+ file = expandPath(args[0])
+ }
+
+ if !wantManifest {
+ f, err := swarm.Open(file)
+ if err != nil {
+ utils.Fatalf("Error opening file: %s", err)
+ }
+ defer f.Close()
+ hash, err := client.UploadRaw(f, f.Size)
+ if err != nil {
+ utils.Fatalf("Upload failed: %s", err)
+ }
+ fmt.Println(hash)
+ return
}
- fi, err := os.Stat(expandPath(file))
+ stat, err := os.Stat(file)
if err != nil {
- utils.Fatalf("Failed to stat file: %v", err)
+ utils.Fatalf("Error opening file: %s", err)
}
- if fi.IsDir() {
+ var hash string
+ if stat.IsDir() {
if !recursive {
utils.Fatalf("Argument is a directory and recursive upload is disabled")
}
- if !wantManifest {
- utils.Fatalf("Manifest is required for directory uploads")
+ hash, err = client.UploadDirectory(file, defaultPath, "")
+ } else {
+ if mimeType == "" {
+ mimeType = detectMimeType(file)
}
- mhash, err := client.UploadDirectory(file, defaultPath)
+ f, err := swarm.Open(file)
if err != nil {
- utils.Fatalf("Failed to upload directory: %v", err)
+ utils.Fatalf("Error opening file: %s", err)
}
- fmt.Println(mhash)
- return
+ defer f.Close()
+ f.ContentType = mimeType
+ hash, err = client.Upload(f, "")
}
- entry, err = client.UploadFile(file, fi, mimeType)
if err != nil {
- utils.Fatalf("Upload failed: %v", err)
- }
- mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
- if !wantManifest {
- // Print the manifest. This is the only output to stdout.
- mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
- fmt.Println(string(mrootJSON))
- return
- }
- hash, err := client.UploadManifest(mroot)
- if err != nil {
- utils.Fatalf("Manifest upload failed: %v", err)
+ utils.Fatalf("Upload failed: %s", err)
}
fmt.Println(hash)
}
@@ -128,3 +134,19 @@ func homeDir() string {
}
return ""
}
+
+func detectMimeType(file string) string {
+ if ext := filepath.Ext(file); ext != "" {
+ return mime.TypeByExtension(ext)
+ }
+ f, err := os.Open(file)
+ if err != nil {
+ return ""
+ }
+ defer f.Close()
+ buf := make([]byte, 512)
+ if n, _ := f.Read(buf); n > 0 {
+ return http.DetectContentType(buf)
+ }
+ return ""
+}
diff --git a/swarm/api/api.go b/swarm/api/api.go
index 7af27208df2ddd54f8c160a4893cdab43b932fd4..ba1156f7ea1b7293a4a72f9f6d35d0fe227b8dbe 100644
--- a/swarm/api/api.go
+++ b/swarm/api/api.go
@@ -17,6 +17,7 @@
package api
import (
+ "errors"
"fmt"
"io"
"net/http"
@@ -70,86 +71,50 @@ func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key stor
type ErrResolve error
// DNS Resolver
-func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) {
- log.Trace(fmt.Sprintf("Resolving : %v", hostPort))
- if hashMatcher.MatchString(hostPort) || self.dns == nil {
- log.Trace(fmt.Sprintf("host is a contentHash: '%v'", hostPort))
- return storage.Key(common.Hex2Bytes(hostPort)), nil
+func (self *Api) Resolve(uri *URI) (storage.Key, error) {
+ log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr))
+ if hashMatcher.MatchString(uri.Addr) {
+ log.Trace(fmt.Sprintf("addr is a hash: %q", uri.Addr))
+ return storage.Key(common.Hex2Bytes(uri.Addr)), nil
}
- if !nameresolver {
- return nil, fmt.Errorf("'%s' is not a content hash value.", hostPort)
+ if uri.Immutable() {
+ return nil, errors.New("refusing to resolve immutable address")
}
- contentHash, err := self.dns.Resolve(hostPort)
- if err != nil {
- err = ErrResolve(err)
- log.Warn(fmt.Sprintf("DNS error : %v", err))
- }
- log.Trace(fmt.Sprintf("host lookup: %v -> %v", hostPort, contentHash))
- return contentHash[:], err
-}
-func Parse(uri string) (hostPort, path string) {
- if uri == "" {
- return
- }
- parts := slashes.Split(uri, 3)
- var i int
- if len(parts) == 0 {
- return
+ if self.dns == nil {
+ return nil, fmt.Errorf("unable to resolve addr %q, resolver not configured", uri.Addr)
}
- // beginning with slash is now optional
- for len(parts[i]) == 0 {
- i++
- }
- hostPort = parts[i]
- for i < len(parts)-1 {
- i++
- if len(path) > 0 {
- path = path + "/" + parts[i]
- } else {
- path = parts[i]
- }
+ hash, err := self.dns.Resolve(uri.Addr)
+ if err != nil {
+ log.Warn(fmt.Sprintf("DNS error resolving addr %q: %s", uri.Addr, err))
+ return nil, ErrResolve(err)
}
- log.Debug(fmt.Sprintf("host: '%s', path '%s' requested.", hostPort, path))
- return
-}
-
-func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) {
- hostPort, path = Parse(uri)
- //resolving host and port
- contentHash, err := self.Resolve(hostPort, nameresolver)
- log.Debug(fmt.Sprintf("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path))
- return contentHash[:], hostPort, path, err
+ log.Trace(fmt.Sprintf("addr lookup: %v -> %v", uri.Addr, hash))
+ return hash[:], nil
}
// Put provides singleton manifest creation on top of dpa store
-func (self *Api) Put(content, contentType string) (string, error) {
+func (self *Api) Put(content, contentType string) (storage.Key, error) {
r := strings.NewReader(content)
wg := &sync.WaitGroup{}
key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
if err != nil {
- return "", err
+ return nil, err
}
manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
r = strings.NewReader(manifest)
key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
if err != nil {
- return "", err
+ return nil, err
}
wg.Wait()
- return key.String(), nil
+ return key, nil
}
// Get uses iterative manifest retrieval and prefix matching
// to resolve path to content using dpa retrieve
// it returns a section reader, mimeType, status and an error
-func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) {
- key, _, path, err := self.parseAndResolve(uri, nameresolver)
- if err != nil {
- return nil, "", 500, fmt.Errorf("can't resolve: %v", err)
- }
-
- quitC := make(chan bool)
- trie, err := loadManifest(self.dpa, key, quitC)
+func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
+ trie, err := loadManifest(self.dpa, key, nil)
if err != nil {
log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
return
@@ -173,32 +138,25 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR
return
}
-func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) {
- root, _, path, err := self.parseAndResolve(uri, nameresolver)
- if err != nil {
- return "", fmt.Errorf("can't resolve: %v", err)
- }
-
+func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
quitC := make(chan bool)
- trie, err := loadManifest(self.dpa, root, quitC)
+ trie, err := loadManifest(self.dpa, key, quitC)
if err != nil {
- return
+ return nil, err
}
-
if contentHash != "" {
- entry := &manifestTrieEntry{
+ entry := newManifestTrieEntry(&ManifestEntry{
Path: path,
- Hash: contentHash,
ContentType: contentType,
- }
+ }, nil)
+ entry.Hash = contentHash
trie.addEntry(entry, quitC)
} else {
trie.deleteEntry(path, quitC)
}
- err = trie.recalcAndStore()
- if err != nil {
- return
+ if err := trie.recalcAndStore(); err != nil {
+ return nil, err
}
- return trie.hash.String(), nil
+ return trie.hash, nil
}
diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go
index 16e90dd329cf4ec8128be0c7d31bc41ec078ca25..c2d78c2dcf0d7928ffc3028ae3cf25b1eb348a56 100644
--- a/swarm/api/api_test.go
+++ b/swarm/api/api_test.go
@@ -23,6 +23,7 @@ import (
"os"
"testing"
+ "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage"
)
@@ -81,8 +82,9 @@ func expResponse(content string, mimeType string, status int) *Response {
}
// func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
-func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
- reader, mimeType, status, err := api.Get(bzzhash, true)
+func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
+ key := storage.Key(common.Hex2Bytes(bzzhash))
+ reader, mimeType, status, err := api.Get(key, path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -107,11 +109,11 @@ func TestApiPut(t *testing.T) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
- bzzhash, err := api.Put(content, exp.MimeType)
+ key, err := api.Put(content, exp.MimeType)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
- resp := testGet(t, api, bzzhash)
+ resp := testGet(t, api, key.String(), "")
checkResponse(t, resp, exp)
})
}
diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go
index ef5335be3696d0fae8e7db3ee66fb4015e7b02f4..f9c3e51e8f8a429e0e8125f7c1f01766f93b5bd7 100644
--- a/swarm/api/client/client.go
+++ b/swarm/api/client/client.go
@@ -17,18 +17,23 @@
package client
import (
+ "archive/tar"
"bytes"
"encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
"mime"
+ "mime/multipart"
"net/http"
+ "net/textproto"
"os"
"path/filepath"
+ "strconv"
"strings"
- "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/swarm/api"
)
var (
@@ -36,18 +41,6 @@ var (
DefaultClient = NewClient(DefaultGateway)
)
-// Manifest represents a swarm manifest.
-type Manifest struct {
- Entries []ManifestEntry `json:"entries,omitempty"`
-}
-
-// ManifestEntry represents an entry in a swarm manifest.
-type ManifestEntry struct {
- Hash string `json:"hash,omitempty"`
- ContentType string `json:"contentType,omitempty"`
- Path string `json:"path,omitempty"`
-}
-
func NewClient(gateway string) *Client {
return &Client{
Gateway: gateway,
@@ -59,160 +52,207 @@ type Client struct {
Gateway string
}
-func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
- mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
- if err != nil {
- return "", fmt.Errorf("failed to upload empty manifest")
+// UploadRaw uploads raw data to swarm and returns the resulting hash
+func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
+ if size <= 0 {
+ return "", errors.New("data size must be greater than zero")
}
- if len(defaultPath) > 0 {
- fi, err := os.Stat(defaultPath)
- if err != nil {
- return "", err
- }
- mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
- if err != nil {
- return "", err
- }
+ req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
+ if err != nil {
+ return "", err
}
- prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
- err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
- if err != nil || fi.IsDir() {
- return err
- }
- if !strings.HasPrefix(path, dir) {
- return fmt.Errorf("path %s outside directory %s", path, dir)
- }
- uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
- mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
- return err
- })
- return mhash, err
-}
-
-func (c *Client) UploadFile(file string, fi os.FileInfo, mimetype_hint string) (ManifestEntry, error) {
- var mimetype string
- hash, err := c.uploadFileContent(file, fi)
- if mimetype_hint != "" {
- mimetype = mimetype_hint
- log.Info("Mime type set by override", "mime", mimetype)
- } else {
- ext := filepath.Ext(file)
- log.Info("Ext", "ext", ext, "file", file)
- if ext != "" {
- mimetype = mime.TypeByExtension(filepath.Ext(fi.Name()))
- log.Info("Mime type set by fileextension", "mime", mimetype, "ext", filepath.Ext(file))
- } else {
- f, err := os.Open(file)
- if err == nil {
- first512 := make([]byte, 512)
- fread, _ := f.ReadAt(first512, 0)
- if fread > 0 {
- mimetype = http.DetectContentType(first512[:fread])
- log.Info("Mime type set by autodetection", "mime", mimetype)
- }
- }
- f.Close()
- }
-
+ req.ContentLength = size
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
}
- m := ManifestEntry{
- Hash: hash,
- ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
- return m, err
-}
-
-func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
- fd, err := os.Open(file)
+ data, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
- defer fd.Close()
- log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
- return c.postRaw("application/octet-stream", fi.Size(), fd)
+ return string(data), nil
}
-func (c *Client) UploadManifest(m Manifest) (string, error) {
- jsm, err := json.Marshal(m)
+// DownloadRaw downloads raw data from swarm
+func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
+ uri := c.Gateway + "/bzzr:/" + hash
+ res, err := http.DefaultClient.Get(uri)
if err != nil {
- panic(err)
+ return nil, err
+ }
+ if res.StatusCode != http.StatusOK {
+ res.Body.Close()
+ return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
- log.Info("Uploading swarm manifest")
- return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+ return res.Body, nil
}
-func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
- fd, err := os.Open(fpath)
- if err != nil {
- return "", err
- }
- defer fd.Close()
- log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
- req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
+// File represents a file in a swarm manifest and is used for uploading and
+// downloading content to and from swarm
+type File struct {
+ io.ReadCloser
+ api.ManifestEntry
+}
+
+// Open opens a local file which can then be passed to client.Upload to upload
+// it to swarm
+func Open(path string) (*File, error) {
+ f, err := os.Open(path)
if err != nil {
- return "", err
+ return nil, err
}
- req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
- req.ContentLength = fi.Size()
- resp, err := http.DefaultClient.Do(req)
+ stat, err := f.Stat()
if err != nil {
- return "", err
+ f.Close()
+ return nil, err
}
- defer resp.Body.Close()
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("bad status: %s", resp.Status)
+ return &File{
+ ReadCloser: f,
+ ManifestEntry: api.ManifestEntry{
+ ContentType: mime.TypeByExtension(filepath.Ext(path)),
+ Mode: int64(stat.Mode()),
+ Size: stat.Size(),
+ ModTime: stat.ModTime(),
+ },
+ }, nil
+}
+
+// Upload uploads a file to swarm and either adds it to an existing manifest
+// (if the manifest argument is non-empty) or creates a new manifest containing
+// the file, returning the resulting manifest hash (the file will then be
+// available at bzz:/<hash>/<path>)
+func (c *Client) Upload(file *File, manifest string) (string, error) {
+ if file.Size <= 0 {
+ return "", errors.New("file size must be greater than zero")
}
- content, err := ioutil.ReadAll(resp.Body)
- return string(content), err
+ return c.TarUpload(manifest, &FileUploader{file})
}
-func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
- req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
+// Download downloads a file with the given path from the swarm manifest with
+// the given hash (i.e. it gets bzz:/<hash>/<path>)
+func (c *Client) Download(hash, path string) (*File, error) {
+ uri := c.Gateway + "/bzz:/" + hash + "/" + path
+ res, err := http.DefaultClient.Get(uri)
if err != nil {
- return "", err
+ return nil, err
}
- req.Header.Set("content-type", mimetype)
- req.ContentLength = size
- resp, err := http.DefaultClient.Do(req)
+ if res.StatusCode != http.StatusOK {
+ res.Body.Close()
+ return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ return &File{
+ ReadCloser: res.Body,
+ ManifestEntry: api.ManifestEntry{
+ ContentType: res.Header.Get("Content-Type"),
+ Size: res.ContentLength,
+ },
+ }, nil
+}
+
+// UploadDirectory uploads a directory tree to swarm and either adds the files
+// to an existing manifest (if the manifest argument is non-empty) or creates a
+// new manifest, returning the resulting manifest hash (files from the
+// directory will then be available at bzz:/<hash>/path/to/file), with
+// the file specified in defaultPath being uploaded to the root of the manifest
+// (i.e. bzz:/<hash>/)
+func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
+ stat, err := os.Stat(dir)
if err != nil {
return "", err
+ } else if !stat.IsDir() {
+ return "", fmt.Errorf("not a directory: %s", dir)
}
- defer resp.Body.Close()
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("bad status: %s", resp.Status)
- }
- content, err := ioutil.ReadAll(resp.Body)
- return string(content), err
+ return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
}
-func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
+// DownloadDirectory downloads the files contained in a swarm manifest under
+// the given path into a local directory (existing files will be overwritten)
+func (c *Client) DownloadDirectory(hash, path, destDir string) error {
+ stat, err := os.Stat(destDir)
+ if err != nil {
+ return err
+ } else if !stat.IsDir() {
+ return fmt.Errorf("not a directory: %s", destDir)
+ }
- mroot := Manifest{}
- req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
+ uri := c.Gateway + "/bzz:/" + hash + "/" + path
+ req, err := http.NewRequest("GET", uri, nil)
if err != nil {
- return mroot, err
+ return err
}
- resp, err := http.DefaultClient.Do(req)
+ req.Header.Set("Accept", "application/x-tar")
+ res, err := http.DefaultClient.Do(req)
if err != nil {
- return mroot, err
+ return err
}
- defer resp.Body.Close()
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ tr := tar.NewReader(res.Body)
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ return nil
+ } else if err != nil {
+ return err
+ }
+ // ignore the default path file
+ if hdr.Name == "" {
+ continue
+ }
- if resp.StatusCode >= 400 {
- return mroot, fmt.Errorf("bad status: %s", resp.Status)
+ dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
+ if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+ return err
+ }
+ var mode os.FileMode = 0644
+ if hdr.Mode > 0 {
+ mode = os.FileMode(hdr.Mode)
+ }
+ dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+ if err != nil {
+ return err
+ }
+ n, err := io.Copy(dst, tr)
+ dst.Close()
+ if err != nil {
+ return err
+ } else if n != hdr.Size {
+ return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
+ }
+ }
+}
+// UploadManifest uploads the given manifest to swarm
+func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
+ data, err := json.Marshal(m)
+ if err != nil {
+ return "", err
}
- content, err := ioutil.ReadAll(resp.Body)
+ return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
+}
- err = json.Unmarshal(content, &mroot)
+// DownloadManifest downloads a swarm manifest
+func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
+ res, err := c.DownloadRaw(hash)
if err != nil {
- return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+ return nil, err
+ }
+ defer res.Close()
+ var manifest api.Manifest
+ if err := json.NewDecoder(res).Decode(&manifest); err != nil {
+ return nil, err
}
- return mroot, err
+ return &manifest, nil
}
-// ManifestFileList downloads the manifest with the given hash and generates a
-// list of files and directory prefixes which have the specified prefix.
+// List list files in a swarm manifest which have the given prefix, grouping
+// common prefixes using "/" as a delimiter.
//
// For example, if the manifest represents the following directory structure:
//
@@ -226,97 +266,200 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
// - a prefix of "file" would return [file1.txt, file2.txt]
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
-func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
- manifest, err := c.DownloadManifest(hash)
+//
+// where entries ending with "/" are common prefixes.
+func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
+ res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
if err != nil {
return nil, err
}
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ var list api.ManifestList
+ if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
+ return nil, err
+ }
+ return &list, nil
+}
+
+// Uploader uploads files to swarm using a provided UploadFn
+type Uploader interface {
+ Upload(UploadFn) error
+}
+
+type UploaderFunc func(UploadFn) error
+
+func (u UploaderFunc) Upload(upload UploadFn) error {
+ return u(upload)
+}
+
+// DirectoryUploader uploads all files in a directory, optionally uploading
+// a file to the default path
+type DirectoryUploader struct {
+ Dir string
+ DefaultPath string
+}
- // handleFile handles a manifest entry which is a direct reference to a
- // file (i.e. it is not a swarm manifest)
- handleFile := func(entry ManifestEntry) {
- // ignore the file if it doesn't have the specified prefix
- if !strings.HasPrefix(entry.Path, prefix) {
- return
+// Upload performs the upload of the directory and default path
+func (d *DirectoryUploader) Upload(upload UploadFn) error {
+ if d.DefaultPath != "" {
+ file, err := Open(d.DefaultPath)
+ if err != nil {
+ return err
}
- // if the path after the prefix contains a directory separator,
- // add a directory prefix to the entries, otherwise add the
- // file
- suffix := strings.TrimPrefix(entry.Path, prefix)
- if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
- entries = append(entries, ManifestEntry{
- Path: prefix + suffix[:sepIndex+1],
- ContentType: "DIR",
- })
- } else {
- if entry.Path == "" {
- entry.Path = "/"
- }
- entries = append(entries, entry)
+ if err := upload(file); err != nil {
+ return err
}
}
-
- // handleManifest handles a manifest entry which is a reference to
- // another swarm manifest.
- handleManifest := func(entry ManifestEntry) error {
- // if the manifest's path is a prefix of the specified prefix
- // then just recurse into the manifest by stripping its path
- // from the prefix
- if strings.HasPrefix(prefix, entry.Path) {
- subPrefix := strings.TrimPrefix(prefix, entry.Path)
- subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
- if err != nil {
- return err
- }
- // prefix the manifest's path to the sub entries and
- // add them to the returned entries
- for i, subEntry := range subEntries {
- subEntry.Path = entry.Path + subEntry.Path
- subEntries[i] = subEntry
- }
- entries = append(entries, subEntries...)
+ return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if f.IsDir() {
return nil
}
+ file, err := Open(path)
+ if err != nil {
+ return err
+ }
+ relPath, err := filepath.Rel(d.Dir, path)
+ if err != nil {
+ return err
+ }
+ file.Path = filepath.ToSlash(relPath)
+ return upload(file)
+ })
+}
- // if the manifest's path has the specified prefix, then if the
- // path after the prefix contains a directory separator, add a
- // directory prefix to the entries, otherwise recurse into the
- // manifest
- if strings.HasPrefix(entry.Path, prefix) {
- suffix := strings.TrimPrefix(entry.Path, prefix)
- sepIndex := strings.Index(suffix, "/")
- if sepIndex > -1 {
- entries = append(entries, ManifestEntry{
- Path: prefix + suffix[:sepIndex+1],
- ContentType: "DIR",
- })
- return nil
- }
- subEntries, err := c.ManifestFileList(entry.Hash, "")
- if err != nil {
- return err
- }
- // prefix the manifest's path to the sub entries and
- // add them to the returned entries
- for i, subEntry := range subEntries {
- subEntry.Path = entry.Path + subEntry.Path
- subEntries[i] = subEntry
- }
- entries = append(entries, subEntries...)
- return nil
+// FileUploader uploads a single file
+type FileUploader struct {
+ File *File
+}
+
+// Upload performs the upload of the file
+func (f *FileUploader) Upload(upload UploadFn) error {
+ return upload(f.File)
+}
+
+// UploadFn is the type of function passed to an Uploader to perform the upload
+// of a single file (for example, a directory uploader would call a provided
+// UploadFn for each file in the directory tree)
+type UploadFn func(file *File) error
+
+// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
+// returning the resulting manifest hash
+func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
+ reqR, reqW := io.Pipe()
+ defer reqR.Close()
+ req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+ if err != nil {
+ return "", err
+ }
+ req.Header.Set("Content-Type", "application/x-tar")
+
+ // use 'Expect: 100-continue' so we don't send the request body if
+ // the server refuses the request
+ req.Header.Set("Expect", "100-continue")
+
+ tw := tar.NewWriter(reqW)
+
+ // define an UploadFn which adds files to the tar stream
+ uploadFn := func(file *File) error {
+ hdr := &tar.Header{
+ Name: file.Path,
+ Mode: file.Mode,
+ Size: file.Size,
+ ModTime: file.ModTime,
+ Xattrs: map[string]string{
+ "user.swarm.content-type": file.ContentType,
+ },
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ return err
}
- return nil
+ _, err = io.Copy(tw, file)
+ return err
}
- for _, entry := range manifest.Entries {
- if entry.ContentType == "application/bzz-manifest+json" {
- if err := handleManifest(entry); err != nil {
- return nil, err
- }
- } else {
- handleFile(entry)
+ // run the upload in a goroutine so we can send the request headers and
+ // wait for a '100 Continue' response before sending the tar stream
+ go func() {
+ err := uploader.Upload(uploadFn)
+ if err == nil {
+ err = tw.Close()
}
+ reqW.CloseWithError(err)
+ }()
+
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ data, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return "", err
+ }
+ return string(data), nil
+}
+
+// MultipartUpload uses the given Uploader to upload files to swarm as a
+// multipart form, returning the resulting manifest hash
+func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
+ reqR, reqW := io.Pipe()
+ defer reqR.Close()
+ req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+ if err != nil {
+ return "", err
}
- return
+ // use 'Expect: 100-continue' so we don't send the request body if
+ // the server refuses the request
+ req.Header.Set("Expect", "100-continue")
+
+ mw := multipart.NewWriter(reqW)
+ req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
+
+ // define an UploadFn which adds files to the multipart form
+ uploadFn := func(file *File) error {
+ hdr := make(textproto.MIMEHeader)
+ hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
+ hdr.Set("Content-Type", file.ContentType)
+ hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
+ w, err := mw.CreatePart(hdr)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(w, file)
+ return err
+ }
+
+ // run the upload in a goroutine so we can send the request headers and
+ // wait for a '100 Continue' response before sending the multipart form
+ go func() {
+ err := uploader.Upload(uploadFn)
+ if err == nil {
+ err = mw.Close()
+ }
+ reqW.CloseWithError(err)
+ }()
+
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ data, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return "", err
+ }
+ return string(data), nil
}
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
index 135474475081aaa25b6e4fce6d408241cfcc420a..4d02ceaf4ea93a98d87e632ad6a87207ef34fd33 100644
--- a/swarm/api/client/client_test.go
+++ b/swarm/api/client/client_test.go
@@ -17,6 +17,7 @@
package client
import (
+ "bytes"
"io/ioutil"
"os"
"path/filepath"
@@ -24,52 +25,221 @@ import (
"sort"
"testing"
+ "github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
-func TestClientManifestFileList(t *testing.T) {
+// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
+func TestClientUploadDownloadRaw(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
+ client := NewClient(srv.URL)
+
+ // upload some raw data
+ data := []byte("foo123")
+ hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // check we can download the same data
+ res, err := client.DownloadRaw(hash)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Close()
+ gotData, err := ioutil.ReadAll(res)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(gotData, data) {
+ t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
+ }
+}
+
+// TestClientUploadDownloadFiles test uploading and downloading files to swarm
+// manifests
+func TestClientUploadDownloadFiles(t *testing.T) {
+ srv := testutil.NewTestSwarmServer(t)
+ defer srv.Close()
+
+ client := NewClient(srv.URL)
+ upload := func(manifest, path string, data []byte) string {
+ file := &File{
+ ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+ ManifestEntry: api.ManifestEntry{
+ Path: path,
+ ContentType: "text/plain",
+ Size: int64(len(data)),
+ },
+ }
+ hash, err := client.Upload(file, manifest)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return hash
+ }
+ checkDownload := func(manifest, path string, expected []byte) {
+ file, err := client.Download(manifest, path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer file.Close()
+ if file.Size != int64(len(expected)) {
+ t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size)
+ }
+ if file.ContentType != file.ContentType {
+ t.Fatalf("expected downloaded file to have type %q, got %q", file.ContentType, file.ContentType)
+ }
+ data, err := ioutil.ReadAll(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(data, expected) {
+ t.Fatalf("expected downloaded data to be %q, got %q", expected, data)
+ }
+ }
+
+ // upload a file to the root of a manifest
+ rootData := []byte("some-data")
+ rootHash := upload("", "", rootData)
+
+ // check we can download the root file
+ checkDownload(rootHash, "", rootData)
+
+ // upload another file to the same manifest
+ otherData := []byte("some-other-data")
+ newHash := upload(rootHash, "some/other/path", otherData)
+
+ // check we can download both files from the new manifest
+ checkDownload(newHash, "", rootData)
+ checkDownload(newHash, "some/other/path", otherData)
+
+ // replace the root file with different data
+ newHash = upload(newHash, "", otherData)
+
+ // check both files have the other data
+ checkDownload(newHash, "", otherData)
+ checkDownload(newHash, "some/other/path", otherData)
+}
+
+var testDirFiles = []string{
+ "file1.txt",
+ "file2.txt",
+ "dir1/file3.txt",
+ "dir1/file4.txt",
+ "dir2/file5.txt",
+ "dir2/dir3/file6.txt",
+ "dir2/dir4/file7.txt",
+ "dir2/dir4/file8.txt",
+}
+
+func newTestDirectory(t *testing.T) string {
dir, err := ioutil.TempDir("", "swarm-client-test")
if err != nil {
t.Fatal(err)
}
- files := []string{
- "file1.txt",
- "file2.txt",
- "dir1/file3.txt",
- "dir1/file4.txt",
- "dir2/file5.txt",
- "dir2/dir3/file6.txt",
- "dir2/dir4/file7.txt",
- "dir2/dir4/file8.txt",
- }
- for _, file := range files {
+
+ for _, file := range testDirFiles {
path := filepath.Join(dir, file)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ os.RemoveAll(dir)
t.Fatalf("error creating dir for %s: %s", path, err)
}
- if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+ if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil {
+ os.RemoveAll(dir)
t.Fatalf("error writing file %s: %s", path, err)
}
}
+ return dir
+}
+
+// TestClientUploadDownloadDirectory tests uploading and downloading a
+// directory of files to a swarm manifest
+func TestClientUploadDownloadDirectory(t *testing.T) {
+ srv := testutil.NewTestSwarmServer(t)
+ defer srv.Close()
+
+ dir := newTestDirectory(t)
+ defer os.RemoveAll(dir)
+
+ // upload the directory
client := NewClient(srv.URL)
+ defaultPath := filepath.Join(dir, testDirFiles[0])
+ hash, err := client.UploadDirectory(dir, defaultPath, "")
+ if err != nil {
+ t.Fatalf("error uploading directory: %s", err)
+ }
+
+ // check we can download the individual files
+ checkDownloadFile := func(path string, expected []byte) {
+ file, err := client.Download(hash, path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer file.Close()
+ data, err := ioutil.ReadAll(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(data, expected) {
+ t.Fatalf("expected data to be %q, got %q", expected, data)
+ }
+ }
+ for _, file := range testDirFiles {
+ checkDownloadFile(file, []byte(file))
+ }
+
+ // check we can download the default path
+ checkDownloadFile("", []byte(testDirFiles[0]))
+
+ // check we can download the directory
+ tmp, err := ioutil.TempDir("", "swarm-client-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(tmp)
+ if err := client.DownloadDirectory(hash, "", tmp); err != nil {
+ t.Fatal(err)
+ }
+ for _, file := range testDirFiles {
+ data, err := ioutil.ReadFile(filepath.Join(tmp, file))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(data, []byte(file)) {
+ t.Fatalf("expected data to be %q, got %q", file, data)
+ }
+ }
+}
+
+// TestClientFileList tests listing files in a swarm manifest
+func TestClientFileList(t *testing.T) {
+ srv := testutil.NewTestSwarmServer(t)
+ defer srv.Close()
- hash, err := client.UploadDirectory(dir, "")
+ dir := newTestDirectory(t)
+ defer os.RemoveAll(dir)
+
+ client := NewClient(srv.URL)
+ hash, err := client.UploadDirectory(dir, "", "")
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
ls := func(prefix string) []string {
- entries, err := client.ManifestFileList(hash, prefix)
+ list, err := client.List(hash, prefix)
if err != nil {
t.Fatal(err)
}
- paths := make([]string, len(entries))
- for i, entry := range entries {
- paths[i] = entry.Path
+ paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries))
+ for _, prefix := range list.CommonPrefixes {
+ paths = append(paths, prefix)
+ }
+ for _, entry := range list.Entries {
+ paths = append(paths, entry.Path)
}
sort.Strings(paths)
return paths
@@ -99,7 +269,59 @@ func TestClientManifestFileList(t *testing.T) {
for prefix, expected := range tests {
actual := ls(prefix)
if !reflect.DeepEqual(actual, expected) {
- t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+ t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual)
+ }
+ }
+}
+
+// TestClientMultipartUpload tests uploading files to swarm using a multipart
+// upload
+func TestClientMultipartUpload(t *testing.T) {
+ srv := testutil.NewTestSwarmServer(t)
+ defer srv.Close()
+
+ // define an uploader which uploads testDirFiles with some data
+ data := []byte("some-data")
+ uploader := UploaderFunc(func(upload UploadFn) error {
+ for _, name := range testDirFiles {
+ file := &File{
+ ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+ ManifestEntry: api.ManifestEntry{
+ Path: name,
+ ContentType: "text/plain",
+ Size: int64(len(data)),
+ },
+ }
+ if err := upload(file); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+
+ // upload the files as a multipart upload
+ client := NewClient(srv.URL)
+ hash, err := client.MultipartUpload("", uploader)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // check we can download the individual files
+ checkDownloadFile := func(path string) {
+ file, err := client.Download(hash, path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer file.Close()
+ gotData, err := ioutil.ReadAll(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(gotData, data) {
+ t.Fatalf("expected data to be %q, got %q", data, gotData)
}
}
+ for _, file := range testDirFiles {
+ checkDownloadFile(file)
+ }
}
diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go
index c2583e265f6d81a8d93d841829b13dc7b6354c2d..e7deaa32ffd6b024798e6484866be4e560cb59d3 100644
--- a/swarm/api/filesystem.go
+++ b/swarm/api/filesystem.go
@@ -22,6 +22,7 @@ import (
"io"
"net/http"
"os"
+ "path"
"path/filepath"
"sync"
@@ -43,6 +44,8 @@ func NewFileSystem(api *Api) *FileSystem {
// Upload replicates a local directory as a manifest file and uploads it
// using dpa store
// TODO: localpath should point to a manifest
+//
+// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Upload(lpath, index string) (string, error) {
var list []*manifestTrieEntry
localpath, err := filepath.Abs(filepath.Clean(lpath))
@@ -72,9 +75,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
if path[:start] != localpath {
return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
}
- entry := &manifestTrieEntry{
- Path: filepath.ToSlash(path),
- }
+ entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil)
list = append(list, entry)
}
return err
@@ -91,9 +92,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
if localpath[:start] != dir {
return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir)
}
- entry := &manifestTrieEntry{
- Path: filepath.ToSlash(localpath),
- }
+ entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil)
list = append(list, entry)
}
@@ -153,11 +152,10 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
}
entry.Path = RegularSlashes(entry.Path[start:])
if entry.Path == index {
- ientry := &manifestTrieEntry{
- Path: "",
- Hash: entry.Hash,
+ ientry := newManifestTrieEntry(&ManifestEntry{
ContentType: entry.ContentType,
- }
+ }, nil)
+ ientry.Hash = entry.Hash
trie.addEntry(ientry, quitC)
}
trie.addEntry(entry, quitC)
@@ -174,6 +172,8 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
// Download replicates the manifest path structure on the local filesystem
// under localpath
+//
+// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Download(bzzpath, localpath string) error {
lpath, err := filepath.Abs(filepath.Clean(localpath))
if err != nil {
@@ -185,10 +185,15 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
}
//resolving host and port
- key, _, path, err := self.api.parseAndResolve(bzzpath, true)
+ uri, err := Parse(path.Join("bzz:/", bzzpath))
+ if err != nil {
+ return err
+ }
+ key, err := self.api.Resolve(uri)
if err != nil {
return err
}
+ path := uri.Path
if len(path) > 0 {
path += "/"
diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go
index 4a27cb1da6a49aa27fb861dbf74bd9deaad7c851..8a15e735dcb0086f79ddde5d0996437214fe3931 100644
--- a/swarm/api/filesystem_test.go
+++ b/swarm/api/filesystem_test.go
@@ -23,6 +23,9 @@ import (
"path/filepath"
"sync"
"testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/swarm/storage"
)
var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test")
@@ -51,16 +54,17 @@ func TestApiDirUpload0(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
content := readPath(t, "testdata", "test0", "index.html")
- resp := testGet(t, api, bzzhash+"/index.html")
+ resp := testGet(t, api, bzzhash, "index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
content = readPath(t, "testdata", "test0", "index.css")
- resp = testGet(t, api, bzzhash+"/index.css")
+ resp = testGet(t, api, bzzhash, "index.css")
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
- _, _, _, err = api.Get(bzzhash, true)
+ key := storage.Key(common.Hex2Bytes(bzzhash))
+ _, _, _, err = api.Get(key, "")
if err == nil {
t.Fatalf("expected error: %v", err)
}
@@ -90,7 +94,8 @@ func TestApiDirUploadModify(t *testing.T) {
return
}
- bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true)
+ key := storage.Key(common.Hex2Bytes(bzzhash))
+ key, err = api.Modify(key, "index.html", "", "")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
@@ -107,32 +112,33 @@ func TestApiDirUploadModify(t *testing.T) {
t.Errorf("unexpected error: %v", err)
return
}
- bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true)
+ key, err = api.Modify(key, "index2.html", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
- bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true)
+ key, err = api.Modify(key, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
+ bzzhash = key.String()
content := readPath(t, "testdata", "test0", "index.html")
- resp := testGet(t, api, bzzhash+"/index2.html")
+ resp := testGet(t, api, bzzhash, "index2.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
- resp = testGet(t, api, bzzhash+"/img/logo.png")
+ resp = testGet(t, api, bzzhash, "img/logo.png")
exp = expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
content = readPath(t, "testdata", "test0", "index.css")
- resp = testGet(t, api, bzzhash+"/index.css")
+ resp = testGet(t, api, bzzhash, "index.css")
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
- _, _, _, err = api.Get(bzzhash, true)
+ _, _, _, err = api.Get(key, "")
if err == nil {
t.Errorf("expected error: %v", err)
}
@@ -149,7 +155,7 @@ func TestApiDirUploadWithRootFile(t *testing.T) {
}
content := readPath(t, "testdata", "test0", "index.html")
- resp := testGet(t, api, bzzhash)
+ resp := testGet(t, api, bzzhash, "")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
@@ -165,7 +171,7 @@ func TestApiFileUpload(t *testing.T) {
}
content := readPath(t, "testdata", "test0", "index.html")
- resp := testGet(t, api, bzzhash+"/index.html")
+ resp := testGet(t, api, bzzhash, "index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
@@ -181,7 +187,7 @@ func TestApiFileUploadWithRootFile(t *testing.T) {
}
content := readPath(t, "testdata", "test0", "index.html")
- resp := testGet(t, api, bzzhash)
+ resp := testGet(t, api, bzzhash, "")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
diff --git a/swarm/api/http/roundtripper_test.go b/swarm/api/http/roundtripper_test.go
index fc74f5d3a711813888b72ad7309a052dc97e1461..f99c4f35e058cd05536d24a29fc9fe7e6c3d4b34 100644
--- a/swarm/api/http/roundtripper_test.go
+++ b/swarm/api/http/roundtripper_test.go
@@ -18,14 +18,14 @@ package http
import (
"io/ioutil"
+ "net"
"net/http"
+ "net/http/httptest"
"strings"
"testing"
"time"
)
-const port = "3222"
-
func TestRoundTripper(t *testing.T) {
serveMux := http.NewServeMux()
serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
@@ -36,9 +36,12 @@ func TestRoundTripper(t *testing.T) {
http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
}
})
- go http.ListenAndServe(":"+port, serveMux)
- rt := &RoundTripper{Port: port}
+ srv := httptest.NewServer(serveMux)
+ defer srv.Close()
+
+ host, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
+ rt := &RoundTripper{Host: host, Port: port}
trans := &http.Transport{}
trans.RegisterProtocol("bzz", rt)
client := &http.Client{Transport: trans}
diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index 44e2c203a41b6c6398618ef604c16f0cb8d392cc..849b9e10f497d503c4df861d0f599b1d1fc22616 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -20,13 +20,19 @@ A simple http server interface to Swarm
package http
import (
- "bytes"
+ "archive/tar"
+ "encoding/json"
+ "errors"
"fmt"
"io"
+ "io/ioutil"
+ "mime"
+ "mime/multipart"
"net/http"
- "regexp"
+ "os"
+ "path"
+ "strconv"
"strings"
- "sync"
"time"
"github.com/ethereum/go-ethereum/common"
@@ -36,26 +42,6 @@ import (
"github.com/rs/cors"
)
-const (
- rawType = "application/octet-stream"
-)
-
-var (
- // accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw)
- bzzPrefix = regexp.MustCompile("^/+bzz[ir]?:/+")
- trailingSlashes = regexp.MustCompile("/+$")
- rootDocumentUri = regexp.MustCompile("^/+bzz[i]?:/+[^/]+$")
- // forever = func() time.Time { return time.Unix(0, 0) }
- forever = time.Now
-)
-
-type sequentialReader struct {
- reader io.Reader
- pos int64
- ahead map[int64](chan bool)
- lock sync.Mutex
-}
-
// ServerConfig is the basic configuration needed for the HTTP server and also
// includes CORS settings.
type ServerConfig struct {
@@ -94,242 +80,569 @@ type Server struct {
api *api.Api
}
-func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- requestURL := r.URL
- // This is wrong
- // if requestURL.Host == "" {
- // var err error
- // requestURL, err = url.Parse(r.Referer() + requestURL.String())
- // if err != nil {
- // http.Error(w, err.Error(), http.StatusBadRequest)
- // return
- // }
- // }
- log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept")))
- uri := requestURL.Path
- var raw, nameresolver bool
- var proto string
-
- // HTTP-based URL protocol handler
- log.Debug(fmt.Sprintf("BZZ request URI: '%s'", uri))
-
- path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string {
- proto = p
- return ""
- })
+// Request wraps http.Request and also includes the parsed bzz URI
+type Request struct {
+ http.Request
+
+ uri *api.URI
+}
- // protocol identification (ugly)
- if proto == "" {
- log.Error(fmt.Sprintf("[BZZ] Swarm: Protocol error in request `%s`.", uri))
- http.Error(w, "Invalid request URL: need access protocol (bzz:/, bzzr:/, bzzi:/) as first element in path.", http.StatusBadRequest)
+// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
+// body in swarm and returns the resulting storage key as a text/plain response
+func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
+ if r.uri.Path != "" {
+ s.BadRequest(w, r, "raw POST request cannot contain a path")
return
}
- if len(proto) > 4 {
- raw = proto[1:5] == "bzzr"
- nameresolver = proto[1:5] != "bzzi"
+
+ if r.Header.Get("Content-Length") == "" {
+ s.BadRequest(w, r, "missing Content-Length header in request")
+ return
}
- log.Debug("", "msg", log.Lazy{Fn: func() string {
- return fmt.Sprintf("[BZZ] Swarm: %s request over protocol %s '%s' received.", r.Method, proto, path)
- }})
+ key, err := s.api.Store(r.Body, r.ContentLength, nil)
+ if err != nil {
+ s.Error(w, r, err)
+ return
+ }
+ s.logDebug("content for %s stored", key.Log())
+
+ w.Header().Set("Content-Type", "text/plain")
+ w.WriteHeader(http.StatusOK)
+ fmt.Fprint(w, key)
+}
+
+// HandlePostFiles handles a POST request (or deprecated PUT request) to
+// bzz:/<hash>/<path> which contains either a single file or multiple files
+// (either a tar archive or multipart form), adds those files either to an
+// existing manifest or to a new manifest under <path> and returns the
+// resulting manifest hash as a text/plain response
+func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
+ contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+ if err != nil {
+ s.BadRequest(w, r, err.Error())
+ return
+ }
- switch {
- case r.Method == "POST" || r.Method == "PUT":
- if r.Header.Get("content-length") == "" {
- http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
+ var key storage.Key
+ if r.uri.Addr != "" {
+ key, err = s.api.Resolve(r.uri)
+ if err != nil {
+ s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
- key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
- if err == nil {
- log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
- } else {
- http.Error(w, err.Error(), http.StatusBadRequest)
+ } else {
+ key, err = s.api.NewManifest()
+ if err != nil {
+ s.Error(w, r, err)
return
}
- if r.Method == "POST" {
- if raw {
- w.Header().Set("Content-Type", "text/plain")
- http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(common.Bytes2Hex(key))))
- } else {
- http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest)
- return
+ }
+
+ newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
+ switch contentType {
+
+ case "application/x-tar":
+ return s.handleTarUpload(r, mw)
+
+ case "multipart/form-data":
+ return s.handleMultipartUpload(r, params["boundary"], mw)
+
+ default:
+ return s.handleDirectUpload(r, mw)
+ }
+ })
+ if err != nil {
+ s.Error(w, r, fmt.Errorf("error creating manifest: %s", err))
+ return
+ }
+
+ w.Header().Set("Content-Type", "text/plain")
+ w.WriteHeader(http.StatusOK)
+ fmt.Fprint(w, newKey)
+}
+
+func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error {
+ tr := tar.NewReader(req.Body)
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ return nil
+ } else if err != nil {
+ return fmt.Errorf("error reading tar stream: %s", err)
+ }
+
+ // only store regular files
+ if !hdr.FileInfo().Mode().IsRegular() {
+ continue
+ }
+
+ // add the entry under the path from the request
+ path := path.Join(req.uri.Path, hdr.Name)
+ entry := &api.ManifestEntry{
+ Path: path,
+ ContentType: hdr.Xattrs["user.swarm.content-type"],
+ Mode: hdr.Mode,
+ Size: hdr.Size,
+ ModTime: hdr.ModTime,
+ }
+ s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
+ contentKey, err := mw.AddEntry(tr, entry)
+ if err != nil {
+ return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
+ }
+ s.logDebug("content for %s stored", contentKey.Log())
+ }
+}
+
+func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
+ mr := multipart.NewReader(req.Body, boundary)
+ for {
+ part, err := mr.NextPart()
+ if err == io.EOF {
+ return nil
+ } else if err != nil {
+ return fmt.Errorf("error reading multipart form: %s", err)
+ }
+
+ var size int64
+ var reader io.Reader = part
+ if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
+ size, err = strconv.ParseInt(contentLength, 10, 64)
+ if err != nil {
+ return fmt.Errorf("error parsing multipart content length: %s", err)
}
+ reader = part
} else {
- // PUT
- if raw {
- http.Error(w, "No PUT to /raw allowed.", http.StatusBadRequest)
- return
- } else {
- path = api.RegularSlashes(path)
- mime := r.Header.Get("Content-Type")
- // TODO proper root hash separation
- log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
- newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
- if err == nil {
- log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
- w.Header().Set("Content-Type", "text/plain")
- http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
- } else {
- http.Error(w, "PUT to "+path+"failed.", http.StatusBadRequest)
- return
- }
+ // copy the part to a tmp file to get its size
+ tmp, err := ioutil.TempFile("", "swarm-multipart")
+ if err != nil {
+ return err
}
- }
- case r.Method == "DELETE":
- if raw {
- http.Error(w, "No DELETE to /raw allowed.", http.StatusBadRequest)
- return
- } else {
- path = api.RegularSlashes(path)
- log.Debug(fmt.Sprintf("Delete '%s'.", path))
- newKey, err := s.api.Modify(path, "", "", nameresolver)
- if err == nil {
- log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
- w.Header().Set("Content-Type", "text/plain")
- http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
- } else {
- http.Error(w, "DELETE to "+path+"failed.", http.StatusBadRequest)
- return
+ defer os.Remove(tmp.Name())
+ defer tmp.Close()
+ size, err = io.Copy(tmp, part)
+ if err != nil {
+ return fmt.Errorf("error copying multipart content: %s", err)
+ }
+ if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
+ return fmt.Errorf("error copying multipart content: %s", err)
}
+ reader = tmp
+ }
+
+ // add the entry under the path from the request
+ name := part.FileName()
+ if name == "" {
+ name = part.FormName()
+ }
+ path := path.Join(req.uri.Path, name)
+ entry := &api.ManifestEntry{
+ Path: path,
+ ContentType: part.Header.Get("Content-Type"),
+ Size: size,
+ ModTime: time.Now(),
}
- case r.Method == "GET" || r.Method == "HEAD":
- path = trailingSlashes.ReplaceAllString(path, "")
- if path == "" {
- http.Error(w, "Empty path not allowed", http.StatusBadRequest)
+ s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
+ contentKey, err := mw.AddEntry(reader, entry)
+ if err != nil {
+ return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
+ }
+ s.logDebug("content for %s stored", contentKey.Log())
+ }
+}
+
+func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
+ key, err := mw.AddEntry(req.Body, &api.ManifestEntry{
+ Path: req.uri.Path,
+ ContentType: req.Header.Get("Content-Type"),
+ Mode: 0644,
+ Size: req.ContentLength,
+ ModTime: time.Now(),
+ })
+ if err != nil {
+ return err
+ }
+ s.logDebug("content for %s stored", key.Log())
+ return nil
+}
+
+// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
+// <path> from <manifest> and returns the resulting manifest hash as a
+// text/plain response
+func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
+ key, err := s.api.Resolve(r.uri)
+ if err != nil {
+ s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+ return
+ }
+
+ newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
+ s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log())
+ return mw.RemoveEntry(r.uri.Path)
+ })
+ if err != nil {
+ s.Error(w, r, fmt.Errorf("error updating manifest: %s", err))
+ return
+ }
+
+ w.Header().Set("Content-Type", "text/plain")
+ w.WriteHeader(http.StatusOK)
+ fmt.Fprint(w, newKey)
+}
+
+// HandleGetRaw handles a GET request to bzzr://<key> and responds with
+// the raw content stored at the given storage key
+func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
+ key, err := s.api.Resolve(r.uri)
+ if err != nil {
+ s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+ return
+ }
+
+ // if path is set, interpret <key> as a manifest and return the
+ // raw entry at the given path
+ if r.uri.Path != "" {
+ walker, err := s.api.NewManifestWalker(key, nil)
+ if err != nil {
+ s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key))
return
}
- if raw {
- var reader storage.LazySectionReader
- parsedurl, _ := api.Parse(path)
-
- if parsedurl == path {
- key, err := s.api.Resolve(parsedurl, nameresolver)
- if err != nil {
- log.Error(fmt.Sprintf("%v", err))
- http.Error(w, err.Error(), http.StatusBadRequest)
- return
- }
- reader = s.api.Retrieve(key)
- } else {
- var status int
- readertmp, _, status, err := s.api.Get(path, nameresolver)
- if err != nil {
- http.Error(w, err.Error(), status)
- return
- }
- reader = readertmp
+ var entry *api.ManifestEntry
+ walker.Walk(func(e *api.ManifestEntry) error {
+ // if the entry matches the path, set entry and stop
+ // the walk
+ if e.Path == r.uri.Path {
+ entry = e
+ // return an error to cancel the walk
+ return errors.New("found")
}
- // retrieving content
-
- quitC := make(chan bool)
- size, err := reader.Size(quitC)
- if err != nil {
- log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error()))
- //An error on call to Size means we don't have the root chunk
- http.Error(w, err.Error(), http.StatusNotFound)
- return
+ // ignore non-manifest files
+ if e.ContentType != api.ManifestType {
+ return nil
}
- log.Debug(fmt.Sprintf("Reading %d bytes.", size))
- // setting mime type
- qv := requestURL.Query()
- mimeType := qv.Get("content_type")
- if mimeType == "" {
- mimeType = rawType
+ // if the manifest's path is a prefix of the
+ // requested path, recurse into it by returning
+ // nil and continuing the walk
+ if strings.HasPrefix(r.uri.Path, e.Path) {
+ return nil
}
- w.Header().Set("Content-Type", mimeType)
- http.ServeContent(w, r, uri, forever(), reader)
- log.Debug(fmt.Sprintf("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType))
+ return api.SkipManifest
+ })
+ if entry == nil {
+ http.NotFound(w, &r.Request)
+ return
+ }
+ key = storage.Key(common.Hex2Bytes(entry.Hash))
+ }
- // retrieve path via manifest
- } else {
- log.Debug(fmt.Sprintf("Structured GET request '%s' received.", uri))
- // add trailing slash, if missing
- if rootDocumentUri.MatchString(uri) {
- http.Redirect(w, r, path+"/", http.StatusFound)
- return
+ // check the root chunk exists by retrieving the file's size
+ reader := s.api.Retrieve(key)
+ if _, err := reader.Size(nil); err != nil {
+ s.logDebug("key not found %s: %s", key, err)
+ http.NotFound(w, &r.Request)
+ return
+ }
+
+ // allow the request to overwrite the content type using a query
+ // parameter
+ contentType := "application/octet-stream"
+ if typ := r.URL.Query().Get("content_type"); typ != "" {
+ contentType = typ
+ }
+ w.Header().Set("Content-Type", contentType)
+
+ http.ServeContent(w, &r.Request, "", time.Now(), reader)
+}
+
+// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept
+// header of "application/x-tar" and returns a tar stream of all files
+// contained in the manifest
+func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
+ if r.uri.Path != "" {
+ s.BadRequest(w, r, "files request cannot contain a path")
+ return
+ }
+
+ key, err := s.api.Resolve(r.uri)
+ if err != nil {
+ s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+ return
+ }
+
+ walker, err := s.api.NewManifestWalker(key, nil)
+ if err != nil {
+ s.Error(w, r, err)
+ return
+ }
+
+ tw := tar.NewWriter(w)
+ defer tw.Close()
+ w.Header().Set("Content-Type", "application/x-tar")
+ w.WriteHeader(http.StatusOK)
+
+ err = walker.Walk(func(entry *api.ManifestEntry) error {
+ // ignore manifests (walk will recurse into them)
+ if entry.ContentType == api.ManifestType {
+ return nil
+ }
+
+ // retrieve the entry's key and size
+ reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash)))
+ size, err := reader.Size(nil)
+ if err != nil {
+ return err
+ }
+
+ // write a tar header for the entry
+ hdr := &tar.Header{
+ Name: entry.Path,
+ Mode: entry.Mode,
+ Size: size,
+ ModTime: entry.ModTime,
+ Xattrs: map[string]string{
+ "user.swarm.content-type": entry.ContentType,
+ },
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ return err
+ }
+
+ // copy the file into the tar stream
+ n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
+ if err != nil {
+ return err
+ } else if n != size {
+ return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
+ }
+
+ return nil
+ })
+ if err != nil {
+ s.logError("error generating tar stream: %s", err)
+ }
+}
+
+// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
+// the "list" query parameter set to "true" and returns a list of all files
+// contained in <manifest> under <path> grouped into common prefixes using
+// "/" as a delimiter
+func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
+ // ensure the root path has a trailing slash so that relative URLs work
+ if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+ http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
+ return
+ }
+
+ key, err := s.api.Resolve(r.uri)
+ if err != nil {
+ s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+ return
+ }
+
+ walker, err := s.api.NewManifestWalker(key, nil)
+ if err != nil {
+ s.Error(w, r, err)
+ return
+ }
+
+ var list api.ManifestList
+ prefix := r.uri.Path
+ err = walker.Walk(func(entry *api.ManifestEntry) error {
+ // handle non-manifest files
+ if entry.ContentType != api.ManifestType {
+ // ignore the file if it doesn't have the specified prefix
+ if !strings.HasPrefix(entry.Path, prefix) {
+ return nil
}
- reader, mimeType, status, err := s.api.Get(path, nameresolver)
- if err != nil {
- if _, ok := err.(api.ErrResolve); ok {
- log.Debug(fmt.Sprintf("%v", err))
- status = http.StatusBadRequest
- } else {
- log.Debug(fmt.Sprintf("error retrieving '%s': %v", uri, err))
- status = http.StatusNotFound
- }
- http.Error(w, err.Error(), status)
- return
+
+ // if the path after the prefix contains a slash, add a
+ // common prefix to the list, otherwise add the entry
+ suffix := strings.TrimPrefix(entry.Path, prefix)
+ if index := strings.Index(suffix, "/"); index > -1 {
+ list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+ return nil
}
- // set mime type and status headers
- w.Header().Set("Content-Type", mimeType)
- if status > 0 {
- w.WriteHeader(status)
- } else {
- status = 200
+ if entry.Path == "" {
+ entry.Path = "/"
}
- quitC := make(chan bool)
- size, err := reader.Size(quitC)
- if err != nil {
- log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error()))
- //An error on call to Size means we don't have the root chunk
- http.Error(w, err.Error(), http.StatusNotFound)
- return
+ list.Entries = append(list.Entries, entry)
+ return nil
+ }
+
+ // if the manifest's path is a prefix of the specified prefix
+ // then just recurse into the manifest by returning nil and
+ // continuing the walk
+ if strings.HasPrefix(prefix, entry.Path) {
+ return nil
+ }
+
+ // if the manifest's path has the specified prefix, then if the
+ // path after the prefix contains a slash, add a common prefix
+ // to the list and skip the manifest, otherwise recurse into
+ // the manifest by returning nil and continuing the walk
+ if strings.HasPrefix(entry.Path, prefix) {
+ suffix := strings.TrimPrefix(entry.Path, prefix)
+ if index := strings.Index(suffix, "/"); index > -1 {
+ list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+ return api.SkipManifest
}
- log.Debug(fmt.Sprintf("Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status))
+ return nil
+ }
- http.ServeContent(w, r, path, forever(), reader)
+ // the manifest neither has the prefix or needs recursing in to
+ // so just skip it
+ return api.SkipManifest
+ })
+ if err != nil {
+ s.Error(w, r, err)
+ return
+ }
+ // if the client wants HTML (e.g. a browser) then render the list as a
+ // HTML index with relative URLs
+ if strings.Contains(r.Header.Get("Accept"), "text/html") {
+ w.Header().Set("Content-Type", "text/html")
+ err := htmlListTemplate.Execute(w, &htmlListData{
+ URI: r.uri,
+ List: &list,
+ })
+ if err != nil {
+ s.logError("error rendering list HTML: %s", err)
}
- default:
- http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+ return
}
+
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(&list)
}
-func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) {
- self.lock.Lock()
- // assert self.pos <= off
- if self.pos > off {
- log.Error(fmt.Sprintf("non-sequential read attempted from sequentialReader; %d > %d", self.pos, off))
- panic("Non-sequential read attempt")
- }
- if self.pos != off {
- log.Debug(fmt.Sprintf("deferred read in POST at position %d, offset %d.", self.pos, off))
- wait := make(chan bool)
- self.ahead[off] = wait
- self.lock.Unlock()
- if <-wait {
- // failed read behind
- n = 0
- err = io.ErrUnexpectedEOF
+// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
+// with the content of the file at <path> from the given <manifest>
+func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
+ key, err := s.api.Resolve(r.uri)
+ if err != nil {
+ s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+ return
+ }
+
+ reader, contentType, _, err := s.api.Get(key, r.uri.Path)
+ if err != nil {
+ s.Error(w, r, err)
+ return
+ }
+
+ // check the root chunk exists by retrieving the file's size
+ if _, err := reader.Size(nil); err != nil {
+ s.logDebug("file not found %s: %s", r.uri, err)
+ http.NotFound(w, &r.Request)
+ return
+ }
+
+ w.Header().Set("Content-Type", contentType)
+
+ http.ServeContent(w, &r.Request, "", time.Now(), reader)
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))
+
+ uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+ if err != nil {
+ s.logError("Invalid URI %q: %s", r.URL.Path, err)
+ http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest)
+ return
+ }
+ s.logDebug("%s request received for %s", r.Method, uri)
+
+ req := &Request{Request: *r, uri: uri}
+ switch r.Method {
+ case "POST":
+ if uri.Raw() {
+ s.HandlePostRaw(w, req)
+ } else {
+ s.HandlePostFiles(w, req)
+ }
+
+ case "PUT":
+ // DEPRECATED:
+ // clients should send a POST request (the request creates a
+ // new manifest leaving the existing one intact, so it isn't
+ // strictly a traditional PUT request which replaces content
+ // at a URI, and POST is more ubiquitous)
+ if uri.Raw() {
+ http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
return
+ } else {
+ s.HandlePostFiles(w, req)
}
- self.lock.Lock()
- }
- localPos := 0
- for localPos < len(target) {
- n, err = self.reader.Read(target[localPos:])
- localPos += n
- log.Debug(fmt.Sprintf("Read %d bytes into buffer size %d from POST, error %v.", n, len(target), err))
- if err != nil {
- log.Debug(fmt.Sprintf("POST stream's reading terminated with %v.", err))
- for i := range self.ahead {
- self.ahead[i] <- true
- delete(self.ahead, i)
- }
- self.lock.Unlock()
- return localPos, err
+
+ case "DELETE":
+ if uri.Raw() {
+ http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
+ return
+ }
+ s.HandleDelete(w, req)
+
+ case "GET":
+ if uri.Raw() {
+ s.HandleGetRaw(w, req)
+ return
+ }
+
+ if r.Header.Get("Accept") == "application/x-tar" {
+ s.HandleGetFiles(w, req)
+ return
+ }
+
+ if r.URL.Query().Get("list") == "true" {
+ s.HandleGetList(w, req)
+ return
}
- self.pos += int64(n)
+
+ s.HandleGetFile(w, req)
+
+ default:
+ http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+
+ }
+}
+
+func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWriter) error) (storage.Key, error) {
+ mw, err := s.api.NewManifestWriter(key, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ if err := update(mw); err != nil {
+ return nil, err
}
- wait := self.ahead[self.pos]
- if wait != nil {
- log.Debug(fmt.Sprintf("deferred read in POST at position %d triggered.", self.pos))
- delete(self.ahead, self.pos)
- close(wait)
+
+ key, err = mw.Store()
+ if err != nil {
+ return nil, err
}
- self.lock.Unlock()
- return localPos, err
+ s.logDebug("generated manifest %s", key)
+ return key, nil
+}
+
+func (s *Server) logDebug(format string, v ...interface{}) {
+ log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
+}
+
+func (s *Server) logError(format string, v ...interface{}) {
+ log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
+}
+
+func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) {
+ s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason)
+ http.Error(w, reason, http.StatusBadRequest)
+}
+
+func (s *Server) Error(w http.ResponseWriter, r *Request, err error) {
+ s.logError("error serving %s %s: %s", r.Method, r.uri, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
}
diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index 45a867f516748cce44dab5bdf51d5d376eb0e049..942f3ba0b29cafa68182ad88728407abafbb96d2 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -40,8 +40,8 @@ func TestBzzrGetPath(t *testing.T) {
testrequests := make(map[string]int)
testrequests["/"] = 0
- testrequests["/a"] = 1
- testrequests["/a/b"] = 2
+ testrequests["/a/"] = 1
+ testrequests["/a/b/"] = 2
testrequests["/x"] = 0
testrequests[""] = 0
diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go
new file mode 100644
index 0000000000000000000000000000000000000000..c3ef8c0f4e8837df3874b60f71efd5acf0c48199
--- /dev/null
+++ b/swarm/api/http/templates.go
@@ -0,0 +1,71 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+ "html/template"
+ "path"
+
+ "github.com/ethereum/go-ethereum/swarm/api"
+)
+
+type htmlListData struct {
+ URI *api.URI
+ List *api.ManifestList
+}
+
+var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.FuncMap{"basename": path.Base}).Parse(`
+<!DOCTYPE html>
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <title>Swarm index of {{ .URI }}</title>
+</head>
+
+<body>
+ <h1>Swarm index of {{ .URI }}</h1>
+ <hr>
+ <table>
+ <thead>
+ <tr>
+ <th>Path</th>
+ <th>Type</th>
+ <th>Size</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {{ range .List.CommonPrefixes }}
+ <tr>
+ <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
+ <td>DIR</td>
+ <td>-</td>
+ </tr>
+ {{ end }}
+
+ {{ range .List.Entries }}
+ <tr>
+ <td><a href="{{ basename .Path }}">{{ basename .Path }}</a></td>
+ <td>{{ .ContentType }}</td>
+ <td>{{ .Size }}</td>
+ </tr>
+ {{ end }}
+ </table>
+ <hr>
+</body>
+`[1:]))
diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go
index 199f259e1f5d681b6d0fd4301959db89a6199d17..6b3630fd0bcc12921f55df87bbbd347ea93523bb 100644
--- a/swarm/api/manifest.go
+++ b/swarm/api/manifest.go
@@ -19,8 +19,11 @@ package api
import (
"bytes"
"encoding/json"
+ "errors"
"fmt"
+ "io"
"sync"
+ "time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
@@ -28,25 +31,152 @@ import (
)
const (
- manifestType = "application/bzz-manifest+json"
+ ManifestType = "application/bzz-manifest+json"
)
+// Manifest represents a swarm manifest
+type Manifest struct {
+ Entries []ManifestEntry `json:"entries,omitempty"`
+}
+
+// ManifestEntry represents an entry in a swarm manifest
+type ManifestEntry struct {
+ Hash string `json:"hash,omitempty"`
+ Path string `json:"path,omitempty"`
+ ContentType string `json:"contentType,omitempty"`
+ Mode int64 `json:"mode,omitempty"`
+ Size int64 `json:"size,omitempty"`
+ ModTime time.Time `json:"mod_time,omitempty"`
+ Status int `json:"status,omitempty"`
+}
+
+// ManifestList represents the result of listing files in a manifest
+type ManifestList struct {
+ CommonPrefixes []string `json:"common_prefixes,omitempty"`
+ Entries []*ManifestEntry `json:"entries,omitempty"`
+}
+
+// NewManifest creates and stores a new, empty manifest
+func (a *Api) NewManifest() (storage.Key, error) {
+ var manifest Manifest
+ data, err := json.Marshal(&manifest)
+ if err != nil {
+ return nil, err
+ }
+ return a.Store(bytes.NewReader(data), int64(len(data)), nil)
+}
+
+// ManifestWriter is used to add and remove entries from an underlying manifest
+type ManifestWriter struct {
+ api *Api
+ trie *manifestTrie
+ quitC chan bool
+}
+
+func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWriter, error) {
+ trie, err := loadManifest(a.dpa, key, quitC)
+ if err != nil {
+ return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
+ }
+ return &ManifestWriter{a, trie, quitC}, nil
+}
+
+// AddEntry stores the given data and adds the resulting key to the manifest
+func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) {
+ key, err := m.api.Store(data, e.Size, nil)
+ if err != nil {
+ return nil, err
+ }
+ entry := newManifestTrieEntry(e, nil)
+ entry.Hash = key.String()
+ m.trie.addEntry(entry, m.quitC)
+ return key, nil
+}
+
+// RemoveEntry removes the given path from the manifest
+func (m *ManifestWriter) RemoveEntry(path string) error {
+ m.trie.deleteEntry(path, m.quitC)
+ return nil
+}
+
+// Store stores the manifest, returning the resulting storage key
+func (m *ManifestWriter) Store() (storage.Key, error) {
+ return m.trie.hash, m.trie.recalcAndStore()
+}
+
+// ManifestWalker is used to recursively walk the entries in the manifest and
+// all of its submanifests
+type ManifestWalker struct {
+ api *Api
+ trie *manifestTrie
+ quitC chan bool
+}
+
+func (a *Api) NewManifestWalker(key storage.Key, quitC chan bool) (*ManifestWalker, error) {
+ trie, err := loadManifest(a.dpa, key, quitC)
+ if err != nil {
+ return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
+ }
+ return &ManifestWalker{a, trie, quitC}, nil
+}
+
+// SkipManifest is used as a return value from WalkFn to indicate that the
+// manifest should be skipped
+var SkipManifest = errors.New("skip this manifest")
+
+// WalkFn is the type of function called for each entry visited by a recursive
+// manifest walk
+type WalkFn func(entry *ManifestEntry) error
+
+// Walk recursively walks the manifest calling walkFn for each entry in the
+// manifest, including submanifests
+func (m *ManifestWalker) Walk(walkFn WalkFn) error {
+ return m.walk(m.trie, "", walkFn)
+}
+
+func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
+ for _, entry := range trie.entries {
+ if entry == nil {
+ continue
+ }
+ entry.Path = prefix + entry.Path
+ err := walkFn(&entry.ManifestEntry)
+ if err != nil {
+ if entry.ContentType == ManifestType && err == SkipManifest {
+ continue
+ }
+ return err
+ }
+ if entry.ContentType != ManifestType {
+ continue
+ }
+ if err := trie.loadSubTrie(entry, nil); err != nil {
+ return err
+ }
+ if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
type manifestTrie struct {
dpa *storage.DPA
entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry
hash storage.Key // if hash != nil, it is stored
}
-type manifestJSON struct {
- Entries []*manifestTrieEntry `json:"entries"`
+func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
+ return &manifestTrieEntry{
+ ManifestEntry: *entry,
+ subtrie: subtrie,
+ }
}
type manifestTrieEntry struct {
- Path string `json:"path"`
- Hash string `json:"hash"` // for manifest content type, empty until subtrie is evaluated
- ContentType string `json:"contentType"`
- Status int `json:"status"`
- subtrie *manifestTrie
+ ManifestEntry
+
+ subtrie *manifestTrie
}
func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
@@ -77,7 +207,9 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
}
log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log()))
- man := manifestJSON{}
+ var man struct {
+ Entries []*manifestTrieEntry `json:"entries"`
+ }
err = json.Unmarshal(manifestData, &man)
if err != nil {
err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err)
@@ -116,7 +248,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
cpl++
}
- if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
+ if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
if self.loadSubTrie(oldentry, quitC) != nil {
return
}
@@ -136,12 +268,10 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
subtrie.addEntry(entry, quitC)
subtrie.addEntry(oldentry, quitC)
- self.entries[b] = &manifestTrieEntry{
+ self.entries[b] = newManifestTrieEntry(&ManifestEntry{
Path: commonPrefix,
- Hash: "",
- ContentType: manifestType,
- subtrie: subtrie,
- }
+ ContentType: ManifestType,
+ }, subtrie)
}
func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
@@ -173,7 +303,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
}
epl := len(entry.Path)
- if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
+ if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if self.loadSubTrie(entry, quitC) != nil {
return
}
@@ -198,7 +328,7 @@ func (self *manifestTrie) recalcAndStore() error {
var buffer bytes.Buffer
buffer.WriteString(`{"entries":[`)
- list := &manifestJSON{}
+ list := &Manifest{}
for _, entry := range self.entries {
if entry != nil {
if entry.Hash == "" { // TODO: paralellize
@@ -208,7 +338,7 @@ func (self *manifestTrie) recalcAndStore() error {
}
entry.Hash = entry.subtrie.hash.String()
}
- list.Entries = append(list.Entries, entry)
+ list.Entries = append(list.Entries, entry.ManifestEntry)
}
}
@@ -254,7 +384,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool,
entry := self.entries[i]
if entry != nil {
epl := len(entry.Path)
- if entry.ContentType == manifestType {
+ if entry.ContentType == ManifestType {
l := plen
if epl < l {
l = epl
@@ -300,7 +430,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl))
if (len(path) >= epl) && (path[:epl] == entry.Path) {
log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
- if entry.ContentType == manifestType {
+ if entry.ContentType == ManifestType {
err := self.loadSubTrie(entry, quitC)
if err != nil {
return nil, 0
diff --git a/swarm/api/storage.go b/swarm/api/storage.go
index 31b484675e89785d91c982a06d54eebe5392f371..7e94a9653e0a0a1cd4b8bd9ea6eed39289214ccb 100644
--- a/swarm/api/storage.go
+++ b/swarm/api/storage.go
@@ -16,6 +16,8 @@
package api
+import "path"
+
type Response struct {
MimeType string
Status int
@@ -25,6 +27,8 @@ type Response struct {
}
// implements a service
+//
+// DEPRECATED: Use the HTTP API instead
type Storage struct {
api *Api
}
@@ -35,8 +39,14 @@ func NewStorage(api *Api) *Storage {
// Put uploads the content to the swarm with a simple manifest speficying
// its content type
+//
+// DEPRECATED: Use the HTTP API instead
func (self *Storage) Put(content, contentType string) (string, error) {
- return self.api.Put(content, contentType)
+ key, err := self.api.Put(content, contentType)
+ if err != nil {
+ return "", err
+ }
+ return key.String(), err
}
// Get retrieves the content from bzzpath and reads the response in full
@@ -45,8 +55,18 @@ func (self *Storage) Put(content, contentType string) (string, error) {
// NOTE: if error is non-nil, sResponse may still have partial content
// the actual size of which is given in len(resp.Content), while the expected
// size is resp.Size
+//
+// DEPRECATED: Use the HTTP API instead
func (self *Storage) Get(bzzpath string) (*Response, error) {
- reader, mimeType, status, err := self.api.Get(bzzpath, true)
+ uri, err := Parse(path.Join("bzz:/", bzzpath))
+ if err != nil {
+ return nil, err
+ }
+ key, err := self.api.Resolve(uri)
+ if err != nil {
+ return nil, err
+ }
+ reader, mimeType, status, err := self.api.Get(key, uri.Path)
if err != nil {
return nil, err
}
@@ -65,6 +85,20 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash,
// and merge on to it. creating an entry w conentType (mime)
+//
+// DEPRECATED: Use the HTTP API instead
func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
- return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true)
+ uri, err := Parse("bzz:/" + rootHash)
+ if err != nil {
+ return "", err
+ }
+ key, err := self.api.Resolve(uri)
+ if err != nil {
+ return "", err
+ }
+ key, err = self.api.Modify(key, path, contentHash, contentType)
+ if err != nil {
+ return "", err
+ }
+ return key.String(), nil
}
diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go
index 72caf52df9d3234ffa5575e3a92e91f4d9cbb6fa..d260dd61d8118fbeab668e9a6bfd8bdfc9ed2a33 100644
--- a/swarm/api/storage_test.go
+++ b/swarm/api/storage_test.go
@@ -36,7 +36,7 @@ func TestStoragePutGet(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
// to check put against the Api#Get
- resp0 := testGet(t, api.api, bzzhash)
+ resp0 := testGet(t, api.api, bzzhash, "")
checkResponse(t, resp0, exp)
// check storage#Get
diff --git a/swarm/api/swarmfs_unix.go b/swarm/api/swarmfs_unix.go
index e696c6b9a6ceed507b15ba2822dea6c224b3d7d5..a704c1ec2b084de09fe1bf3d7226865a4e1e3028 100644
--- a/swarm/api/swarmfs_unix.go
+++ b/swarm/api/swarmfs_unix.go
@@ -91,11 +91,16 @@ func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
return nil, fmt.Errorf("%s is already mounted", cleanedMountPoint)
}
- key, _, path, err := self.swarmApi.parseAndResolve(mhash, true)
+ uri, err := Parse("bzz:/" + mhash)
if err != nil {
- return nil, fmt.Errorf("can't resolve %q: %v", mhash, err)
+ return nil, err
+ }
+ key, err := self.swarmApi.Resolve(uri)
+ if err != nil {
+ return nil, err
}
+ path := uri.Path
if len(path) > 0 {
path += "/"
}
diff --git a/swarm/api/uri.go b/swarm/api/uri.go
new file mode 100644
index 0000000000000000000000000000000000000000..68ce04835bebd69bd4a13c192c02db2d4d276127
--- /dev/null
+++ b/swarm/api/uri.go
@@ -0,0 +1,96 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+ "fmt"
+ "net/url"
+ "strings"
+)
+
+// URI is a reference to content stored in swarm.
+type URI struct {
+ // Scheme has one of the following values:
+ //
+ // * bzz - an entry in a swarm manifest
+ // * bzzr - raw swarm content
+ // * bzzi - immutable URI of an entry in a swarm manifest
+ // (address is not resolved)
+ Scheme string
+
+ // Addr is either a hexadecimal storage key or it an address which
+ // resolves to a storage key
+ Addr string
+
+ // Path is the path to the content within a swarm manifest
+ Path string
+}
+
+// Parse parses rawuri into a URI struct, where rawuri is expected to have one
+// of the following formats:
+//
+// * <scheme>:/
+// * <scheme>:/<addr>
+// * <scheme>:/<addr>/<path>
+// * <scheme>://
+// * <scheme>://<addr>
+// * <scheme>://<addr>/<path>
+//
+// with scheme one of bzz, bzzr or bzzi
+func Parse(rawuri string) (*URI, error) {
+ u, err := url.Parse(rawuri)
+ if err != nil {
+ return nil, err
+ }
+ uri := &URI{Scheme: u.Scheme}
+
+ // check the scheme is valid
+ switch uri.Scheme {
+ case "bzz", "bzzi", "bzzr":
+ default:
+ return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
+ }
+
+ // handle URIs like bzz://<addr>/<path> where the addr and path
+ // have already been split by url.Parse
+ if u.Host != "" {
+ uri.Addr = u.Host
+ uri.Path = strings.TrimLeft(u.Path, "/")
+ return uri, nil
+ }
+
+ // URI is like bzz:/<addr>/<path> so split the addr and path from
+ // the raw path (which will be /<addr>/<path>)
+ parts := strings.SplitN(strings.TrimLeft(u.Path, "/"), "/", 2)
+ uri.Addr = parts[0]
+ if len(parts) == 2 {
+ uri.Path = parts[1]
+ }
+ return uri, nil
+}
+
+func (u *URI) Raw() bool {
+ return u.Scheme == "bzzr"
+}
+
+func (u *URI) Immutable() bool {
+ return u.Scheme == "bzzi"
+}
+
+func (u *URI) String() string {
+ return u.Scheme + ":/" + u.Addr + "/" + u.Path
+}
diff --git a/swarm/api/uri_test.go b/swarm/api/uri_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..dcb5fbbfff6fc0ac8bea752f06e15e6539f82c6b
--- /dev/null
+++ b/swarm/api/uri_test.go
@@ -0,0 +1,120 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestParseURI(t *testing.T) {
+ type test struct {
+ uri string
+ expectURI *URI
+ expectErr bool
+ expectRaw bool
+ expectImmutable bool
+ }
+ tests := []test{
+ {
+ uri: "",
+ expectErr: true,
+ },
+ {
+ uri: "foo",
+ expectErr: true,
+ },
+ {
+ uri: "bzz",
+ expectErr: true,
+ },
+ {
+ uri: "bzz:",
+ expectURI: &URI{Scheme: "bzz"},
+ },
+ {
+ uri: "bzzi:",
+ expectURI: &URI{Scheme: "bzzi"},
+ expectImmutable: true,
+ },
+ {
+ uri: "bzzr:",
+ expectURI: &URI{Scheme: "bzzr"},
+ expectRaw: true,
+ },
+ {
+ uri: "bzz:/",
+ expectURI: &URI{Scheme: "bzz"},
+ },
+ {
+ uri: "bzz:/abc123",
+ expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
+ },
+ {
+ uri: "bzz:/abc123/path/to/entry",
+ expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
+ },
+ {
+ uri: "bzzr:/",
+ expectURI: &URI{Scheme: "bzzr"},
+ expectRaw: true,
+ },
+ {
+ uri: "bzzr:/abc123",
+ expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
+ expectRaw: true,
+ },
+ {
+ uri: "bzzr:/abc123/path/to/entry",
+ expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
+ expectRaw: true,
+ },
+ {
+ uri: "bzz://",
+ expectURI: &URI{Scheme: "bzz"},
+ },
+ {
+ uri: "bzz://abc123",
+ expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
+ },
+ {
+ uri: "bzz://abc123/path/to/entry",
+ expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
+ },
+ }
+ for _, x := range tests {
+ actual, err := Parse(x.uri)
+ if x.expectErr {
+ if err == nil {
+ t.Fatalf("expected %s to error", x.uri)
+ }
+ continue
+ }
+ if err != nil {
+ t.Fatalf("error parsing %s: %s", x.uri, err)
+ }
+ if !reflect.DeepEqual(actual, x.expectURI) {
+ t.Fatalf("expected %s to return %#v, got %#v", x.uri, x.expectURI, actual)
+ }
+ if actual.Raw() != x.expectRaw {
+ t.Fatalf("expected %s raw to be %t, got %t", x.uri, x.expectRaw, actual.Raw())
+ }
+ if actual.Immutable() != x.expectImmutable {
+ t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
+ }
+ }
+}
diff --git a/swarm/swarm.go b/swarm/swarm.go
index add28d205de92590c8bf781cc406fb3eab00a283..5a7f43f8be739ef015f93e60f3dd3a3d9e8a016e 100644
--- a/swarm/swarm.go
+++ b/swarm/swarm.go
@@ -53,8 +53,8 @@ type Swarm struct {
privateKey *ecdsa.PrivateKey
corsString string
swapEnabled bool
- lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped
- sfs *api.SwarmFS // need this to cleanup all the active mounts on node exit
+ lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped
+ sfs *api.SwarmFS // need this to cleanup all the active mounts on node exit
}
type SwarmAPI struct {
@@ -241,13 +241,6 @@ func (self *Swarm) Protocols() []p2p.Protocol {
func (self *Swarm) APIs() []rpc.API {
return []rpc.API{
// public APIs
- {
- Namespace: "bzz",
- Version: "0.1",
- Service: api.NewStorage(self.api),
- Public: true,
- },
-
{
Namespace: "bzz",
Version: "0.1",
@@ -255,11 +248,6 @@ func (self *Swarm) APIs() []rpc.API {
Public: true,
},
// admin APIs
- {
- Namespace: "bzz",
- Version: "0.1",
- Service: api.NewFileSystem(self.api),
- Public: false},
{
Namespace: "bzz",
Version: "0.1",
@@ -278,6 +266,20 @@ func (self *Swarm) APIs() []rpc.API {
Service: self.sfs,
Public: false,
},
+ // storage APIs
+ // DEPRECATED: Use the HTTP API instead
+ {
+ Namespace: "bzz",
+ Version: "0.1",
+ Service: api.NewStorage(self.api),
+ Public: true,
+ },
+ {
+ Namespace: "bzz",
+ Version: "0.1",
+ Service: api.NewFileSystem(self.api),
+ Public: false,
+ },
// {Namespace, Version, api.NewAdmin(self), false},
}
}