servo/python/servo/bootstrap_commands.py

# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

from __future__ import print_function, unicode_literals

import os
import os.path as path
import shutil
import subprocess
import sys
import tarfile
import urllib2
from distutils.version import LooseVersion

from mach.decorators import (
    CommandArgument,
    CommandProvider,
    Command,
)

from servo.command_base import CommandBase, cd, host_triple


def download(desc, src, dst):
    print("Downloading %s..." % desc)
    dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())

    try:
        resp = urllib2.urlopen(src)
        fsize = int(resp.info().getheader('Content-Length').strip())
        recved = 0
        chunk_size = 8192

        with open(dst, 'wb') as fd:
            while True:
                chunk = resp.read(chunk_size)
                if not chunk:
                    break
                recved += len(chunk)
                if not dumb:
                    pct = recved * 100.0 / fsize
                    print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
                    sys.stdout.flush()
                fd.write(chunk)

        if not dumb:
            print()
    except urllib2.HTTPError, e:
        print("Download failed (%d): %s - %s" % (e.code, e.reason, src))

        cpu_type = subprocess.check_output(["uname", "-m"]).strip().lower()
        if e.code == 404 and cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
            # i686
            print("Note: Servo does not currently bootstrap 32bit snapshots of Rust")
            print("See https://github.com/servo/servo/issues/3899")

        sys.exit(1)


def extract(src, dst, movedir=None):
    tarfile.open(src).extractall(dst)

    if movedir:
        for f in os.listdir(movedir):
            frm = path.join(movedir, f)
            to = path.join(dst, f)
            os.rename(frm, to)
        os.rmdir(movedir)

    os.remove(src)


@CommandProvider
class MachCommands(CommandBase):
    @Command('env',
             description='Print environment setup commands',
             category='bootstrap')
    def env(self):
        env = self.build_env()
        print("export PATH=%s" % env["PATH"])
        if sys.platform == "darwin":
            print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
        else:
            print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])

    @Command('bootstrap-rust',
             description='Download the Rust compiler snapshot',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if a snapshot already exists')
    def bootstrap_rustc(self, force=False):
        rust_dir = path.join(
            self.context.sharedir, "rust", *self.rust_snapshot_path().split("/"))
        if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc")):
            print("Snapshot Rust compiler already downloaded.", end=" ")
            print("Use |bootstrap-rust --force| to download again.")
            return

        if path.isdir(rust_dir):
            shutil.rmtree(rust_dir)
        os.makedirs(rust_dir)

        snapshot_url = ("https://servo-rust.s3.amazonaws.com/%s.tar.gz"
                        % self.rust_snapshot_path())
        tgz_file = rust_dir + '.tar.gz'

        download("Rust snapshot", snapshot_url, tgz_file)

        print("Extracting Rust snapshot...")
        snap_dir = path.join(rust_dir,
                             path.basename(tgz_file).replace(".tar.gz", ""))
        extract(tgz_file, rust_dir, movedir=snap_dir)
        print("Snapshot Rust ready.")

    @Command('bootstrap-rust-docs',
             description='Download the Rust documentation',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if docs already exist')
    def bootstrap_rustc_docs(self, force=False):
        self.ensure_bootstrapped()
        hash_dir = path.join(self.context.sharedir, "rust",
                             self.rust_snapshot_path().split("/")[0])
        docs_dir = path.join(hash_dir, self.rust_snapshot_path().split("/")[1], "doc")
        if not force and path.exists(docs_dir):
            print("Snapshot Rust docs already downloaded.", end=" ")
            print("Use |bootstrap-rust-docs --force| to download again.")
            return

        if path.isdir(docs_dir):
            shutil.rmtree(docs_dir)
        docs_name = self.rust_snapshot_path().replace("rustc-", "rust-docs-")
        snapshot_url = ("https://servo-rust.s3.amazonaws.com/%s.tar.gz"
                        % docs_name)
        tgz_file = path.join(hash_dir, 'doc.tar.gz')

        download("Rust docs", snapshot_url, tgz_file)

        print("Extracting Rust docs...")
        temp_dir = path.join(hash_dir, "temp_docs")
        if path.isdir(temp_dir):
            shutil.rmtree(temp_dir)
        extract(tgz_file, temp_dir)
        shutil.move(path.join(temp_dir, docs_name.split("/")[1],
                              "rust-docs", "share", "doc", "rust", "html"),
                    docs_dir)
        shutil.rmtree(temp_dir)
        print("Rust docs ready.")

    @Command('bootstrap-cargo',
             description='Download the Cargo build tool',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Force download even if cargo already exists')
    def bootstrap_cargo(self, force=False):
        cargo_dir = path.join(self.context.sharedir, "cargo",
                              self.cargo_build_id())
        if not force and path.exists(path.join(cargo_dir, "bin", "cargo")):
            print("Cargo already downloaded.", end=" ")
            print("Use |bootstrap_cargo --force| to download again.")
            return

        if path.isdir(cargo_dir):
            shutil.rmtree(cargo_dir)
        os.makedirs(cargo_dir)

        tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
        nightly_url = "https://static-rust-lang-org.s3.amazonaws.com/cargo-dist/%s/%s" % \
            (self.cargo_build_id(), tgz_file)

        download("Cargo nightly", nightly_url, tgz_file)

        print("Extracting Cargo nightly...")
        nightly_dir = path.join(cargo_dir,
                                path.basename(tgz_file).replace(".tar.gz", ""))
        extract(tgz_file, cargo_dir, movedir=nightly_dir)
        print("Cargo ready.")

    @Command('update-submodules',
             description='Update submodules',
             category='bootstrap')
    def update_submodules(self):
        # Ensure that the installed git version is >= 1.8.1
        gitversion_output = subprocess.check_output(["git", "--version"])
        gitversion = LooseVersion(gitversion_output.split(" ")[-1])
        if gitversion < LooseVersion("1.8.1"):
            print("Git version 1.8.1 or above required. Current version is {}"
                  .format(gitversion))
            sys.exit(1)
        submodules = subprocess.check_output(["git", "submodule", "status"])
        for line in submodules.split('\n'):
            components = line.strip().split(' ')
            if len(components) > 1:
                module_path = components[1]
                if path.exists(module_path):
                    with cd(module_path):
                        output = subprocess.check_output(
                            ["git", "status", "--porcelain"])
                        if len(output) != 0:
                            print("error: submodule %s is not clean"
                                  % module_path)
                            print("\nClean the submodule and try again.")
                            return 1
        subprocess.check_call(
            ["git", "submodule", "--quiet", "sync", "--recursive"])
        subprocess.check_call(
            ["git", "submodule", "update", "--init", "--recursive"])

    @Command('clean-snapshots',
             description='Clean unused snapshots of Rust and Cargo',
             category='bootstrap')
    @CommandArgument('--force', '-f',
                     action='store_true',
                     help='Actually remove stuff')
    def clean_snapshots(self, force=False):
        rust_current = self.rust_snapshot_path().split('/')[0]
        cargo_current = self.cargo_build_id()
        print("Current Rust version: " + rust_current)
        print("Current Cargo version: " + cargo_current)
        removing_anything = False
        for current, base in [(rust_current, "rust"), (cargo_current, "cargo")]:
            base = path.join(self.context.sharedir, base)
            for name in os.listdir(base):
                if name != current:
                    removing_anything = True
                    name = path.join(base, name)
                    if force:
                        print("Removing " + name)
                        shutil.rmtree(name)
                    else:
                        print("Would remove " + name)
        if not removing_anything:
            print("Nothing to remove.")
        elif not force:
            print("Nothing done. "
                  "Run `./mach clean-snapshots -f` to actually remove.")