From 3de19c8b31ab975eed1f7f276d31761f7f8b9af9 Mon Sep 17 00:00:00 2001
From: Felix Lange <fjl@users.noreply.github.com>
Date: Tue, 12 Feb 2019 10:55:25 +0100
Subject: [PATCH] build: use SFTP for launchpad uploads (#19037)
* build: use sftp for launchpad uploads
* .travis.yml: configure sftp export
* build: update CI docs
---
.travis.yml | 4 ++-
build/ci-notes.md | 13 +++++++---
build/ci.go | 56 ++++++++++++++++++++++++++++-------------
build/dput-launchpad.cf | 8 ++++++
4 files changed, 60 insertions(+), 21 deletions(-)
create mode 100644 build/dput-launchpad.cf
diff --git a/.travis.yml b/.travis.yml
index f08083d3c..c34e3ea8a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -68,8 +68,10 @@ matrix:
- debhelper
- dput
- fakeroot
+ - python-bzrlib
+ - python-paramiko
script:
- - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+ - go run build/ci.go debsrc -upload ppa:ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Linux Azure uploads
- if: type = push
diff --git a/build/ci-notes.md b/build/ci-notes.md
index f5b0e869d..ba27cdb1d 100644
--- a/build/ci-notes.md
+++ b/build/ci-notes.md
@@ -7,11 +7,18 @@ Canonical.
Packages of develop branch commits have suffix -unstable and cannot be installed alongside
the stable version. Switching between release streams requires user intervention.
+## Launchpad
+
The packages are built and served by launchpad.net. We generate a Debian source package
for each distribution and upload it. Their builder picks up the source package, builds it
and installs the new version into the PPA repository. Launchpad requires a valid signature
-by a team member for source package uploads. The signing key is stored in an environment
-variable which Travis CI makes available to certain builds.
+by a team member for source package uploads.
+
+The signing key is stored in an environment variable which Travis CI makes available to
+certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the
+packages. To set this up yourself, you need to create a Launchpad user and add a GPG key
+and SSH key to it. Then encode both keys as base64 and configure 'secret' environment
+variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
We want to build go-ethereum with the most recent version of Go, irrespective of the Go
version that is available in the main Ubuntu repository. In order to make this possible,
@@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools:
$ sudo apt-add-repository ppa:gophers/ubuntu/archive
$ sudo apt-get update
- $ sudo apt-get install build-essential golang-1.10 devscripts debhelper
+ $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko
Create the source packages:
diff --git a/build/ci.go b/build/ci.go
index d2f4ce4fe..14d97135b 100644
--- a/build/ci.go
+++ b/build/ci.go
@@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string {
func archiveUpload(archive string, blobstore string, signer string) error {
// If signing was requested, generate the signature files
if signer != "" {
- pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
- if err != nil {
- return fmt.Errorf("invalid base64 %s", signer)
- }
- if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
+ key := getenvBase64(signer)
+ if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
return err
}
}
@@ -489,6 +486,7 @@ func doDebianSource(cmdline []string) {
var (
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
+ sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
now = time.Now()
)
@@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) {
maybeSkipArchive(env)
// Import the signing key.
- if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
- key, err := base64.StdEncoding.DecodeString(b64key)
- if err != nil {
- log.Fatal("invalid base64 PPA_SIGNING_KEY")
- }
+ if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
gpg := exec.Command("gpg", "--import")
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
@@ -523,12 +517,45 @@ func doDebianSource(cmdline []string) {
build.MustRunCommand("debsign", changes)
}
if *upload != "" {
- build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes)
+ uploadDebianSource(*workdir, *upload, *sshUser, changes)
}
}
}
}
+func uploadDebianSource(workdir, ppa, sshUser, changes string) {
+ // Create the dput config file.
+ dputConfig := filepath.Join(workdir, "dput.cf")
+ p := strings.Split(ppa, "/")
+ if len(p) != 2 {
+ log.Fatal("-upload PPA name must contain single /")
+ }
+ templateData := map[string]string{
+ "LaunchpadUser": p[0],
+ "LaunchpadPPA": p[1],
+ "LaunchpadSSH": sshUser,
+ }
+ if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
+ idfile := filepath.Join(workdir, "sshkey")
+ ioutil.WriteFile(idfile, sshkey, 0600)
+ templateData["IdentityFile"] = idfile
+ }
+ build.Render("build/dput-launchpad.cf", dputConfig, 0644, templateData)
+
+ // Run dput to do the upload.
+ dput := exec.Command("dput", "-c", dputConfig, "--no-upload-log", ppa, changes)
+ dput.Stdin = strings.NewReader("Yes\n") // accept SSH host key
+ build.MustRun(dput)
+}
+
+func getenvBase64(variable string) []byte {
+ dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
+ if err != nil {
+ log.Fatal("invalid base64 " + variable)
+ }
+ return []byte(dec)
+}
+
func makeWorkdir(wdflag string) string {
var err error
if wdflag != "" {
@@ -800,15 +827,10 @@ func doAndroidArchive(cmdline []string) {
os.Rename(archive, meta.Package+".aar")
if *signer != "" && *deploy != "" {
// Import the signing key into the local GPG instance
- b64key := os.Getenv(*signer)
- key, err := base64.StdEncoding.DecodeString(b64key)
- if err != nil {
- log.Fatalf("invalid base64 %s", *signer)
- }
+ key := getenvBase64(*signer)
gpg := exec.Command("gpg", "--import")
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
-
keyID, err := build.PGPKeyID(string(key))
if err != nil {
log.Fatal(err)
diff --git a/build/dput-launchpad.cf b/build/dput-launchpad.cf
new file mode 100644
index 000000000..3063c3c07
--- /dev/null
+++ b/build/dput-launchpad.cf
@@ -0,0 +1,8 @@
+[{{.LaunchpadUser}}/{{.LaunchpadPPA}}]
+fqdn = ppa.launchpad.net
+method = sftp
+incoming = ~{{.LaunchpadUser}}/ubuntu/{{.LaunchpadPPA}}/
+login = {{.LaunchpadSSH}}
+{{ if .IdentityFile }}
+ ssh_options = IdentityFile {{.IdentityFile}}
+{{ end }}
--
GitLab