From 734e00af9e9a5487f57b00ae33cbb52dbe0c9565 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= <peterke@gmail.com>
Date: Tue, 5 Nov 2019 15:32:42 +0200
Subject: [PATCH] travis, build, internal: use own Go bundle for PPA builds
(#20240)
* build: bump PPAs to Go 1.13 (via longsleep), keep Trusty on 1.11
* travis, build, vendor: use own Go bundle for PPA builds
* travis, build, internal, vendor: smarter Go bundler, own untar
* build: updated ci-notes with new Go bundling, only make, don't test
---
.travis.yml | 5 ++-
build/ci-notes.md | 17 ++++---
build/ci.go | 70 ++++++++++++++++++++---------
build/deb/ethereum/deb.control | 2 +-
build/deb/ethereum/deb.rules | 4 +-
internal/build/archive.go | 46 +++++++++++++++++++
internal/build/gosrc.go | 81 ++++++++++++++++++++++++++++++++++
7 files changed, 193 insertions(+), 32 deletions(-)
create mode 100644 internal/build/gosrc.go
diff --git a/.travis.yml b/.travis.yml
index 996ec4fdc..a09450500 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -75,9 +75,12 @@ jobs:
- fakeroot
- python-bzrlib
- python-paramiko
+ cache:
+ directories:
+ - $HOME/.gobundle
script:
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
+ - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -goversion 1.13.4 -gohash 95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 -gobundle $HOME/.gobundle/go.tar.gz
# This builder does the Linux Azure uploads
- stage: build
diff --git a/build/ci-notes.md b/build/ci-notes.md
index 13e1fd230..edd9adc1c 100644
--- a/build/ci-notes.md
+++ b/build/ci-notes.md
@@ -22,19 +22,18 @@ variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
We want to build go-ethereum with the most recent version of Go, irrespective of the Go
version that is available in the main Ubuntu repository. In order to make this possible,
-our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.11, which is co-installable alongside the regular golang package. PPA dependencies
-can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
+we bundle the entire Go sources into our own source archive and start the built job by
+compiling Go and then using that to build go-ethereum. On Trusty we have a special case
+requiring the `~gophers/ubuntu/archive` PPA since Trusty can't even build Go itself. PPA
+deps are set at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
## Building Packages Locally (for testing)
You need to run Ubuntu to do test packaging.
-Add the gophers PPA and install Go 1.11 and Debian packaging tools:
+Install any version of Go and Debian packaging tools:
- $ sudo apt-add-repository ppa:gophers/ubuntu/archive
- $ sudo apt-get update
- $ sudo apt-get install build-essential golang-1.11 devscripts debhelper python-bzrlib python-paramiko
+ $ sudo apt-get install build-essential golang-go devscripts debhelper python-bzrlib python-paramiko
Create the source packages:
@@ -42,10 +41,10 @@ Create the source packages:
Then go into the source package directory for your running distribution and build the package:
- $ cd dist/ethereum-unstable-1.6.0+xenial
+ $ cd dist/ethereum-unstable-1.9.6+bionic
$ dpkg-buildpackage
Built packages are placed in the dist/ directory.
$ cd ..
- $ dpkg-deb -c geth-unstable_1.6.0+xenial_amd64.deb
+ $ dpkg-deb -c geth-unstable_1.9.6+bionic_amd64.deb
diff --git a/build/ci.go b/build/ci.go
index 347cf95e6..ac5c72b6b 100644
--- a/build/ci.go
+++ b/build/ci.go
@@ -58,6 +58,7 @@ import (
"strings"
"time"
+ "github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/internal/build"
"github.com/ethereum/go-ethereum/params"
)
@@ -138,7 +139,18 @@ var (
// Note: zesty is unsupported because it was officially deprecated on Launchpad.
// Note: artful is unsupported because it was officially deprecated on Launchpad.
// Note: cosmic is unsupported because it was officially deprecated on Launchpad.
- debDistros = []string{"trusty", "xenial", "bionic", "disco", "eoan"}
+ debDistroGoBoots = map[string]string{
+ "trusty": "golang-1.11",
+ "xenial": "golang-go",
+ "bionic": "golang-go",
+ "disco": "golang-go",
+ "eoan": "golang-go",
+ }
+
+ debGoBootPaths = map[string]string{
+ "golang-1.11": "/usr/lib/go-1.11",
+ "golang-go": "/usr/lib/go",
+ }
)
var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -459,11 +471,14 @@ func maybeSkipArchive(env build.Environment) {
// Debian Packaging
func doDebianSource(cmdline []string) {
var (
- signer = flag.String("signer", "", `Signing key name, also used as package author`)
- upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
- sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
- workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
- now = time.Now()
+ goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`)
+ gobundle = flag.String("gobundle", "/tmp/go.tar.gz", `Filesystem path to cache the downloaded Go bundles at`)
+ gohash = flag.String("gohash", "", `SHA256 checksum of the Go sources requested to build with`)
+ signer = flag.String("signer", "", `Signing key name, also used as package author`)
+ upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
+ sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
+ workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
+ now = time.Now()
)
flag.CommandLine.Parse(cmdline)
*workdir = makeWorkdir(*workdir)
@@ -476,12 +491,25 @@ func doDebianSource(cmdline []string) {
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
}
-
+ // Download and verify the Go source package
+ if err := build.EnsureGoSources(*goversion, hexutil.MustDecode("0x"+*gohash), *gobundle); err != nil {
+ log.Fatalf("Failed to ensure Go source package: %v", err)
+ }
// Create Debian packages and upload them
for _, pkg := range debPackages {
- for _, distro := range debDistros {
- meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
+ for distro, goboot := range debDistroGoBoots {
+ // Prepare the debian package with the go-ethereum sources
+ meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
pkgdir := stageDebianSource(*workdir, meta)
+
+ // Ship the Go sources along so we have a proper thing to build with
+ if err := build.ExtractTarballArchive(*gobundle, pkgdir); err != nil {
+ log.Fatalf("Failed to extract Go sources: %v", err)
+ }
+ if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
+ log.Fatalf("Failed to rename Go source folder: %v", err)
+ }
+ // Run the packaging and upload to the PPA
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
debuild.Dir = pkgdir
build.MustRun(debuild)
@@ -561,7 +589,9 @@ type debPackage struct {
}
type debMetadata struct {
- Env build.Environment
+ Env build.Environment
+ GoBootPackage string
+ GoBootPath string
PackageName string
@@ -590,19 +620,21 @@ func (d debExecutable) Package() string {
return d.BinaryName
}
-func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
+func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
if author == "" {
// No signing key, use default author.
author = "Ethereum Builds <fjl@ethereum.org>"
}
return debMetadata{
- PackageName: name,
- Env: env,
- Author: author,
- Distro: distro,
- Version: version,
- Time: t.Format(time.RFC1123Z),
- Executables: exes,
+ GoBootPackage: goboot,
+ GoBootPath: debGoBootPaths[goboot],
+ PackageName: name,
+ Env: env,
+ Author: author,
+ Distro: distro,
+ Version: version,
+ Time: t.Format(time.RFC1123Z),
+ Executables: exes,
}
}
@@ -667,7 +699,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
if err := os.Mkdir(pkgdir, 0755); err != nil {
log.Fatal(err)
}
-
// Copy the source code.
build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))
@@ -685,7 +716,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe)
build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe)
}
-
return pkgdir
}
diff --git a/build/deb/ethereum/deb.control b/build/deb/ethereum/deb.control
index 5b3ff9354..501a32cb4 100644
--- a/build/deb/ethereum/deb.control
+++ b/build/deb/ethereum/deb.control
@@ -2,7 +2,7 @@ Source: {{.Name}}
Section: science
Priority: extra
Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.11
+Build-Depends: debhelper (>= 8.0.0), {{.GoBootPackage}}
Standards-Version: 3.9.5
Homepage: https://ethereum.org
Vcs-Git: git://github.com/ethereum/go-ethereum.git
diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules
index 5280e0e55..1370a52f1 100644
--- a/build/deb/ethereum/deb.rules
+++ b/build/deb/ethereum/deb.rules
@@ -6,9 +6,11 @@
# Launchpad rejects Go's access to $HOME/.cache, use custom folder
export GOCACHE=/tmp/go-build
+export GOROOT_BOOTSTRAP={{.GoBootPath}}
override_dh_auto_build:
- build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+ (cd .go/src && ./make.bash)
+ build/env.sh .go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
override_dh_auto_test:
diff --git a/internal/build/archive.go b/internal/build/archive.go
index ac680ba63..8571edd5a 100644
--- a/internal/build/archive.go
+++ b/internal/build/archive.go
@@ -183,3 +183,49 @@ func (a *TarballArchive) Close() error {
}
return a.file.Close()
}
+
+func ExtractTarballArchive(archive string, dest string) error {
+ // We're only interested in gzipped archives, wrap the reader now
+ ar, err := os.Open(archive)
+ if err != nil {
+ return err
+ }
+ defer ar.Close()
+
+ gzr, err := gzip.NewReader(ar)
+ if err != nil {
+ return err
+ }
+ defer gzr.Close()
+
+ // Iterate over all the files in the tarball
+ tr := tar.NewReader(gzr)
+ for {
+ // Fetch the next tarball header and abort if needed
+ header, err := tr.Next()
+ if err != nil {
+ if err == io.EOF {
+ return nil
+ }
+ return err
+ }
+ // Figure out the target and create it
+ target := filepath.Join(dest, header.Name)
+
+ switch header.Typeflag {
+ case tar.TypeDir:
+ if err := os.MkdirAll(target, 0755); err != nil {
+ return err
+ }
+ case tar.TypeReg:
+ file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+ if err != nil {
+ return err
+ }
+ if _, err := io.Copy(file, tr); err != nil {
+ return err
+ }
+ file.Close()
+ }
+ }
+}
diff --git a/internal/build/gosrc.go b/internal/build/gosrc.go
new file mode 100644
index 000000000..c85e46968
--- /dev/null
+++ b/internal/build/gosrc.go
@@ -0,0 +1,81 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// EnsureGoSources ensures that path contains a file with the given SHA256 hash,
+// and if not, it downloads a fresh Go source package from upstream and replaces
+// path with it (if the hash matches).
+func EnsureGoSources(version string, hash []byte, path string) error {
+ // Sanity check the destination path to ensure we don't do weird things
+ if !strings.HasSuffix(path, ".tar.gz") {
+ return fmt.Errorf("destination path (%s) must end with .tar.gz", path)
+ }
+ // If the file exists, validate it's hash
+ if archive, err := ioutil.ReadFile(path); err == nil { // Go sources are ~20MB, it's fine to read all
+ hasher := sha256.New()
+ hasher.Write(archive)
+ have := hasher.Sum(nil)
+
+ if bytes.Equal(have, hash) {
+ fmt.Printf("Go %s [%x] available at %s\n", version, hash, path)
+ return nil
+ }
+ fmt.Printf("Go %s hash mismatch (have %x, want %x) at %s, deleting old archive\n", version, have, hash, path)
+ if err := os.Remove(path); err != nil {
+ return err
+ }
+ }
+ // Archive missing or bad hash, download a new one
+ fmt.Printf("Downloading Go %s [want %x] into %s\n", version, hash, path)
+
+ res, err := http.Get(fmt.Sprintf("https://dl.google.com/go/go%s.src.tar.gz", version))
+ if err != nil || res.StatusCode != http.StatusOK {
+ return fmt.Errorf("failed to access Go sources: code %d, err %v", res.StatusCode, err)
+ }
+ defer res.Body.Close()
+
+ archive, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return err
+ }
+ // Sanity check the downloaded archive, save if checks out
+ hasher := sha256.New()
+ hasher.Write(archive)
+
+ if have := hasher.Sum(nil); !bytes.Equal(have, hash) {
+ return fmt.Errorf("downloaded Go %s hash mismatch (have %x, want %x)", version, have, hash)
+ }
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(path, archive, 0644); err != nil {
+ return err
+ }
+ fmt.Printf("Downloaded Go %s [%x] into %s\n", version, hash, path)
+ return nil
+}
--
GitLab