cmd: support v1.1 Twitter API in faucet, fix puppeth

cmd/faucet/README.md

+# Faucet
+
+The `faucet` is a simplistic web application with the goal of distributing small amounts of Ether in private and test networks.
+
+Users need to post their Ethereum addresses to fund in a Twitter status update or public Facebook post and share the link to the faucet. The faucet will in turn deduplicate user requests and send the Ether. After a funding round, the faucet prevents the same user requesting again for a pre-configured amount of time, proportional to the amount of Ether requested.
+
+## Operation
+
+The `faucet` is a single binary app (everything included) with all configurations set via command line flags and a few files.
+
+First thing's first, the `faucet` needs to connect to an Ethereum network, for which it needs the necessary genesis and network infos. Each of the following flags must be set:
+
+- `--genesis` is a path to a file containin the network `genesis.json`
+- `--network` is the devp2p network id used during connection
+- `--bootnodes` is a list of `enode://` ids to join the network through
+
+The `faucet` will use the `les` protocol to join the configured Ethereum network and will store its data in `$HOME/.faucet` (currently not configurable).
+
+## Funding
+
+To be able to distribute funds, the `faucet` needs access to an already funded Ethereum account. This can be configured via:
+
+- `--account.json` is a path to the Ethereum account's JSON key file
+- `--account.pass` is a path to a text file with the decryption passphrase
+
+The faucet is able to distribute various amounts of Ether in exchange for various timeouts. These can be configured via:
+
+- `--faucet.amount` is the number of Ethers to send by default
+- `--faucet.minutes` is the time to wait before allowing a rerequest
+- `--faucet.tiers` is the funding tiers to support  (x3 time, x2.5 funds)
+
+## Sybil protection
+
+To prevent the same user from exhausting funds in a loop, the `faucet` ties requests to social networks and captcha resolvers.
+
+Captcha protection uses Google's invisible ReCaptcha, thus the `faucet` needs to run on a live domain. The domain needs to be registered in Google's systems to retrieve the captcha API token and secrets. After doing so, captcha protection may be enabled via:
+
+- `--captcha.token` is the API token for ReCaptcha
+- `--captcha.secret` is the API secret for ReCaptcha
+
+Sybil protection via Twitter requires an API key as of 15th December, 2020. To obtain it, a Twitter user must be upgraded to developer status and a new Twitter App deployed with it. The app's `Bearer` token is required by the faucet to retrieve tweet data:
+
+- `--twitter.token` is the Bearer token for `v2` API access
+- `--twitter.token.v1` is the Bearer token for `v1` API access
+
+Sybil protection via Facebook uses the website to directly download post data thus does not currently require an API configuration. 
+
+## Miscellaneous
+
+Beside the above - mostly essential - CLI flags, there are a number that can be used to fine tune the `faucet`'s operation. Please see `faucet --help` for a full list.
\ No newline at end of file

cmd/faucet/faucet.go

@@ -84,7 +84,8 @@ var (
 	noauthFlag = flag.Bool("noauth", false, "Enables funding requests without authentication")
 	logFlag    = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
 
-	twitterBearerToken = flag.String("twitter.token", "", "Twitter bearer token to authenticate with the twitter API")
+	twitterTokenFlag   = flag.String("twitter.token", "", "Bearer token to authenticate with the v2 Twitter API")
+	twitterTokenV1Flag = flag.String("twitter.token.v1", "", "Bearer token to authenticate with the v1.1 Twitter API")
 )
 
 var (
@@ -245,6 +246,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 	cfg.NetworkId = network
 	cfg.Genesis = genesis
 	utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock(nil).Hash())
+
 	lesBackend, err := les.New(stack, &cfg)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to register the Ethereum service: %w", err)
@@ -388,8 +390,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 		if err = conn.ReadJSON(&msg); err != nil {
 			return
 		}
-		if !*noauthFlag && !strings.HasPrefix(msg.URL, "https://gist.github.com/") && !strings.HasPrefix(msg.URL, "https://twitter.com/") &&
-			!strings.HasPrefix(msg.URL, "https://plus.google.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") {
+		if !*noauthFlag && !strings.HasPrefix(msg.URL, "https://twitter.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") {
 			if err = sendError(conn, errors.New("URL doesn't link to supported services")); err != nil {
 				log.Warn("Failed to send URL error to client", "err", err)
 				return
@@ -451,21 +452,8 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			address  common.Address
 		)
 		switch {
-		case strings.HasPrefix(msg.URL, "https://gist.github.com/"):
-			if err = sendError(conn, errors.New("GitHub authentication discontinued at the official request of GitHub")); err != nil {
-				log.Warn("Failed to send GitHub deprecation to client", "err", err)
-				return
-			}
-			continue
-		case strings.HasPrefix(msg.URL, "https://plus.google.com/"):
-			//lint:ignore ST1005 Google is a company name and should be capitalized.
-			if err = sendError(conn, errors.New("Google+ authentication discontinued as the service was sunset")); err != nil {
-				log.Warn("Failed to send Google+ deprecation to client", "err", err)
-				return
-			}
-			continue
 		case strings.HasPrefix(msg.URL, "https://twitter.com/"):
-			id, username, avatar, address, err = authTwitter(msg.URL, *twitterBearerToken)
+			id, username, avatar, address, err = authTwitter(msg.URL, *twitterTokenV1Flag, *twitterTokenFlag)
 		case strings.HasPrefix(msg.URL, "https://www.facebook.com/"):
 			username, avatar, address, err = authFacebook(msg.URL)
 			id = username
@@ -690,23 +678,31 @@ func sendSuccess(conn *websocket.Conn, msg string) error {
 
 // authTwitter tries to authenticate a faucet request using Twitter posts, returning
 // the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success.
-func authTwitter(url string, token string) (string, string, string, common.Address, error) {
+func authTwitter(url string, tokenV1, tokenV2 string) (string, string, string, common.Address, error) {
 	// Ensure the user specified a meaningful URL, no fancy nonsense
 	parts := strings.Split(url, "/")
 	if len(parts) < 4 || parts[len(parts)-2] != "status" {
 		//lint:ignore ST1005 This error is to be displayed in the browser
 		return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL")
 	}
-
+	// Strip any query parameters from the tweet id and ensure it's numeric
+	tweetID := strings.Split(parts[len(parts)-1], "?")[0]
+	if !regexp.MustCompile("^[0-9]+$").MatchString(tweetID) {
+		return "", "", "", common.Address{}, errors.New("Invalid Tweet URL")
+	}
 	// Twitter's API isn't really friendly with direct links.
 	// It is restricted to 300 queries / 15 minute with an app api key.
 	// Anything more will require read only authorization from the users and that we want to avoid.
 
-	// If twitter bearer token is provided, use the twitter api
-	if token != "" {
-		return authTwitterWithToken(parts[len(parts)-1], token)
+	// If Twitter bearer token is provided, use the API, selecting the version
+	// the user would prefer (currently there's a limit of 1 v2 app / developer
+	// but unlimited v1.1 apps).
+	switch {
+	case tokenV1 != "":
+		return authTwitterWithTokenV1(tweetID, tokenV1)
+	case tokenV2 != "":
+		return authTwitterWithTokenV2(tweetID, tokenV2)
 	}
-
 	// Twiter API token isn't provided so we just load the public posts
 	// and scrape it for the Ethereum address and profile URL. We need to load
 	// the mobile page though since the main page loads tweet contents via JS.
@@ -742,19 +738,49 @@ func authTwitter(url string, token string) (string, string, string, common.Addre
 	return username + "@twitter", username, avatar, address, nil
 }
 
-// authTwitterWithToken tries to authenticate a faucet request using Twitter's API, returning
-// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success.
-func authTwitterWithToken(tweetID string, token string) (string, string, string, common.Address, error) {
-	// Strip any query parameters from the tweet id
-	sanitizedTweetID := strings.Split(tweetID, "?")[0]
+// authTwitterWithTokenV1 tries to authenticate a faucet request using Twitter's v1
+// API, returning the user id, username, avatar URL and Ethereum address to fund on
+// success.
+func authTwitterWithTokenV1(tweetID string, token string) (string, string, string, common.Address, error) {
+	// Query the tweet details from Twitter
+	url := fmt.Sprintf("https://api.twitter.com/1.1/statuses/show.json?id=%s", tweetID)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
 
-	// Ensure numeric tweetID
-	if !regexp.MustCompile("^[0-9]+$").MatchString(sanitizedTweetID) {
-		return "", "", "", common.Address{}, errors.New("Invalid Tweet URL")
+	var result struct {
+		Text string `json:"text"`
+		User struct {
+			ID       string `json:"id_str"`
+			Username string `json:"screen_name"`
+			Avatar   string `json:"profile_image_url"`
+		} `json:"user"`
 	}
+	err = json.NewDecoder(res.Body).Decode(&result)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Text))
+	if address == (common.Address{}) {
+		//lint:ignore ST1005 This error is to be displayed in the browser
+		return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	return result.User.ID + "@twitter", result.User.Username, result.User.Avatar, address, nil
+}
 
+// authTwitterWithTokenV2 tries to authenticate a faucet request using Twitter's v2
+// API, returning the user id, username, avatar URL and Ethereum address to fund on
+// success.
+func authTwitterWithTokenV2(tweetID string, token string) (string, string, string, common.Address, error) {
 	// Query the tweet details from Twitter
-	url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", sanitizedTweetID)
+	url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", tweetID)
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return "", "", "", common.Address{}, err
@@ -769,15 +795,13 @@ func authTwitterWithToken(tweetID string, token string) (string, string, string,
 	var result struct {
 		Data struct {
 			AuthorID string `json:"author_id"`
-			ID       string `json:"id"`
 			Text     string `json:"text"`
 		} `json:"data"`
 		Includes struct {
 			Users []struct {
-				ProfileImageURL string `json:"profile_image_url"`
-				Username        string `json:"username"`
-				ID              string `json:"id"`
-				Name            string `json:"name"`
+				ID       string `json:"id"`
+				Username string `json:"username"`
+				Avatar   string `json:"profile_image_url"`
 			} `json:"users"`
 		} `json:"includes"`
 	}
@@ -792,7 +816,7 @@ func authTwitterWithToken(tweetID string, token string) (string, string, string,
 		//lint:ignore ST1005 This error is to be displayed in the browser
 		return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund")
 	}
-	return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].ProfileImageURL, address, nil
+	return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].Avatar, address, nil
 }
 
 // authFacebook tries to authenticate a faucet request using Facebook posts,

cmd/puppeth/module_faucet.go

@@ -46,7 +46,7 @@ ENTRYPOINT [ \
 	"--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}",             \
 	"--account.json", "/account.json", "--account.pass", "/account.pass"                                                                                                    \
 	{{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}{{if .NoAuth}}, "--noauth"{{end}}                          \
-	{{if .TwitterToken}}, "--twitter.token", "{{.TwitterToken}}",
+	{{if .TwitterToken}}, "--twitter.token.v1", "{{.TwitterToken}}"{{end}}                                                                                                  \
 ]`
 
 // faucetComposefile is the docker-compose.yml file required to deploy and maintain

cmd/puppeth/wizard_faucet.go

@@ -102,11 +102,10 @@ func (w *wizard) deployFaucet() {
 			infos.captchaSecret = w.readPassword()
 		}
 	}
-
-	// Accessing the twitter api requires a bearer token, request it
+	// Accessing the Twitter API requires a bearer token, request it
 	if infos.twitterToken != "" {
 		fmt.Println()
-		fmt.Println("Reuse previous twitter API Bearer token (y/n)? (default = yes)")
+		fmt.Println("Reuse previous Twitter API token (y/n)? (default = yes)")
 		if !w.readDefaultYesNo(true) {
 			infos.twitterToken = ""
 		}
@@ -114,17 +113,10 @@ func (w *wizard) deployFaucet() {
 	if infos.twitterToken == "" {
 		// No previous twitter token (or old one discarded)
 		fmt.Println()
-		fmt.Println("Enable twitter API (y/n)? (default = no)")
-		if !w.readDefaultYesNo(false) {
-			log.Warn("The faucet will fallback to using direct calls")
-		} else {
-			// Twitter api explicitly requested, read the bearer token
-			fmt.Println()
-			fmt.Printf("What is the twitter API Bearer token?\n")
-			infos.twitterToken = w.readString()
-		}
+		fmt.Println()
+		fmt.Printf("What is the Twitter API app Bearer token?\n")
+		infos.twitterToken = w.readString()
 	}
-
 	// Figure out where the user wants to store the persistent data
 	fmt.Println()
 	if infos.node.datadir == "" {