#!/usr/bin/python3
# Run the commands of a Debusine task in a throwaway VM (docs/ARCHITECTURE.md,
# "Isolation pro Auftrag").  Installed as /usr/local/bin/{unshare,sbuild,
# autopkgtest} (Debusine runs unshare with its own PATH, where /usr/local/bin
# comes first): Debusine and these programs stay unpatched and get their
# updates from apt.
#
# Per work request one VM from the node's image (build-isolation-image.sh),
# with a temporary overlay; the work request's directory and the image cache
# are shared over 9p at the same paths.  Commands run over ssh as
# debusine-worker, so their input and output never pass through the shared
# directory.  Between commands the VM is paused, and links it left pointing
# outside the work request's directory are removed: what Debusine reads and
# uploads cannot change any more.  The VM ends when Debusine removes the
# work request's directory.  Its network reaches only the node's apt proxy
# (10.0.2.100:3142).  Commands that do not belong to a work request run
# unchanged on the node.

import fcntl
import json
import os
import re
import shlex
import shutil
import socket
import subprocess
import sys
import time
from pathlib import Path

HOME = Path(os.environ.get("DUR_ISOLATE_HOME", "/var/lib/debusine/worker"))
WORKDIR = HOME / "workdir"
SHARED_RO = [HOME / "system-images"]
STATE = HOME / "isolation"
IMAGE = STATE / "image.qcow2"
KEY = STATE / "id_ed25519"
USER = "debusine-worker"
PROXY = "guestfwd=tcp:10.0.2.100:3142-cmd:nc 127.0.0.1 3142"
MACHINE = {  # qemu binary, machine, console
    "x86_64": ("qemu-system-x86_64", "q35", "ttyS0"),
    "aarch64": ("qemu-system-aarch64", "virt", "ttyAMA0"),
}
# limits: a VM ends after this long; it runs below the node's own work
MAX_LIFETIME = float(os.environ.get("DUR_ISOLATE_MAX_HOURS", 24)) * 3600
NICE = int(os.environ.get("DUR_ISOLATE_NICE", 10))
# the VM ends before a build fills the node's disk (shared directory and
# overlay are both on it)
MIN_FREE = int(os.environ.get("DUR_ISOLATE_MIN_FREE_MB", 1024)) << 20


def free(path):
    st = os.statvfs(path)
    return st.f_bavail * st.f_frsize


def real(name):
    """The program this wrapper stands in for: the next one on PATH, or the
    one dpkg-divert moved aside (setup-worker.sh)."""
    here = Path(sys.argv[0]).resolve().parent
    for directory in os.environ.get("PATH", "/usr/bin:/bin").split(os.pathsep):
        for candidate in (Path(directory, name), Path(directory, f"{name}.distrib")):
            if candidate.resolve().parent != here and os.access(candidate, os.X_OK):
                return str(candidate)
    sys.exit(f"dur-isolate: {name} not found")


def work_request(args):
    """The work request directory the command belongs to, if any."""
    pattern = re.compile(re.escape(str(WORKDIR)) + r"/[^/\s]+")
    for text in [os.getcwd(), *args]:
        if m := pattern.search(text):
            return Path(m.group(0))
    return None


def ram_size():
    with open("/proc/meminfo") as meminfo:
        for line in meminfo:
            if line.startswith("MemAvailable:"):
                return max(512, int(line.split()[1]) // 1024 - 256)
    return 1024


def qmp(state, command):
    with socket.socket(socket.AF_UNIX) as s:
        s.connect(str(state / "qmp"))
        f = s.makefile("rw")
        f.readline()
        for execute in ("qmp_capabilities", command):
            f.write(json.dumps({"execute": execute}) + "\n")
            f.flush()
            while "return" not in (reply := json.loads(f.readline())):
                if "error" in reply:
                    raise RuntimeError(reply["error"])


def ssh(state, script, **kwargs):
    port = (state / "port").read_text().strip()
    return subprocess.run(
        ["ssh", "-T", "-q", "-p", port, "-i", str(KEY),
         "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null",
         "-o", "BatchMode=yes", "root@127.0.0.1", script],
        **kwargs,
    )


def daemon(wr, state):
    """Run the VM of work request wr until its directory is gone."""
    qemu, machine, console = MACHINE[os.uname().machine]
    with socket.socket() as s:
        s.bind(("127.0.0.1", 0))
        port = s.getsockname()[1]
    shares = [(wr, "work", False)] + [
        (path, f"ro{i}", True) for i, path in enumerate(SHARED_RO)]
    ram, cpus = ram_size(), len(os.sched_getaffinity(0))
    # for the builds' parallel= (run)
    (state / "size").write_text(f"{ram} {cpus}\n")
    cmd = [
        qemu, "-machine", f"{machine},accel=kvm", "-cpu", "host",
        "-m", str(ram), "-smp", str(cpus),
        "-display", "none", "-monitor", "none",
        # kept with the state, for errors; gone with the work request
        "-serial", f"file:{state / 'console.log'}",
        # the kernel directly, without firmware
        "-kernel", str(STATE / "vmlinuz"), "-initrd", str(STATE / "initrd.img"),
        "-append", f"root=LABEL=debvm rw console={console} quiet",
        # the overlay is thrown away: flushes to the node's disk would only
        # slow dpkg down
        "-drive", f"file={IMAGE},if=virtio,snapshot=on,cache=unsafe",
        "-netdev", f"user,id=net0,restrict=on,"
                   f"hostfwd=tcp:127.0.0.1:{port}-:22,{PROXY}",
        "-device", "virtio-net-pci,netdev=net0",
        "-device", "virtio-rng-pci",
        "-qmp", f"unix:{state / 'qmp'},server=on,wait=off",
    ]
    for path, tag, readonly in shares:
        cmd += ["-virtfs", f"local,path={path},mount_tag={tag},"
                f"security_model=none,id={tag},readonly={'on' if readonly else 'off'}"]
    # snapshot=on keeps the overlay in $TMPDIR: the state directory; it can
    # grow up to the image's size
    log = STATE / f"{wr.name}.log"  # kept if the VM ends unexpectedly
    vm = subprocess.Popen(cmd, env={**os.environ, "TMPDIR": str(state)},
                          stdin=subprocess.DEVNULL, stderr=log.open("wb"),
                          preexec_fn=lambda: os.nice(NICE))
    (state / "daemon").write_text(f"{os.getpid()}\n")
    (state / "port").write_text(f"{port}\n")
    started = time.monotonic()
    reason = None
    try:
        while wr.exists() and time.monotonic() - started < MAX_LIFETIME:
            try:
                vm.wait(timeout=5)
                break
            except subprocess.TimeoutExpired:
                pass
            try:
                low = min(free(wr), free(state)) < MIN_FREE
            except OSError:  # the work request just ended
                continue
            if low:
                reason = f"less than {MIN_FREE >> 20} MB free on the node"
                break
    finally:
        if vm.poll() is None:
            vm.kill()
            vm.wait()
        if wr.exists():  # ended before its work request: note why
            tail = []
            for path in (state / "console.log", log):
                try:
                    tail += path.read_text(errors="replace").splitlines()[-15:]
                except OSError:
                    pass
            if time.monotonic() - started >= MAX_LIFETIME:
                reason = f"time limit of {MAX_LIFETIME / 3600:g} h reached"
            if reason:
                tail.append(reason)
            (STATE / f"{wr.name}.lost").write_text("\n".join(tail) + "\n")
        # a new VM of the same work request may own the state by now
        try:
            mine = (state / "daemon").read_text().strip() == str(os.getpid())
        except OSError:
            mine = False
        if mine:
            shutil.rmtree(state, ignore_errors=True)
            (STATE / f"{wr.name}.lock").unlink(missing_ok=True)
        if not wr.exists():
            log.unlink(missing_ok=True)


def clean_up():
    """Remove what VMs of finished work requests left (e.g. after a crash)."""
    for entry in STATE.glob("WR-*"):
        name = entry.name.split(".")[0]
        if not (WORKDIR / name).exists():
            if entry.is_dir():
                shutil.rmtree(entry, ignore_errors=True)
            else:
                entry.unlink(missing_ok=True)


def start(wr, state):
    """Boot the VM of wr and mount the shared directories."""
    clean_up()
    for _ in range(100):  # a VM that just ended may still be cleaning up
        if not state.exists():
            break
        time.sleep(0.1)
    state.mkdir(mode=0o700)
    pid = os.fork()
    if pid == 0:
        os.setsid()
        devnull = os.open(os.devnull, os.O_RDWR)
        for fd in (0, 1, 2):
            os.dup2(devnull, fd)
        os.closerange(3, os.sysconf("SC_OPEN_MAX"))  # e.g. the lock
        daemon(wr, state)
        os._exit(0)
    mounts = [f"mkdir -p {shlex.quote(str(wr))} && "
              f"mount -t 9p -o trans=virtio,version=9p2000.L,msize=1048576 "
              f"work {shlex.quote(str(wr))}"]
    for i, path in enumerate(SHARED_RO):
        mounts.append(f"mkdir -p {shlex.quote(str(path))} && "
                      f"mount -t 9p -o trans=virtio,version=9p2000.L,ro "
                      f"ro{i} {shlex.quote(str(path))}")
    deadline = time.monotonic() + 300
    while time.monotonic() < deadline:
        if os.waitpid(pid, os.WNOHANG)[0]:  # qemu did not start, or ended
            vm_gone(wr)
        if (state / "port").exists() and ssh(
                state, " && ".join(mounts), stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL).returncode == 0:
            return
        time.sleep(0.3)
    sys.exit("dur-isolate: the VM did not come up")


def drop_escaping_symlinks(directory):
    root = directory.resolve()
    for dirpath, dirnames, filenames in os.walk(directory):
        for name in dirnames + filenames:
            path = Path(dirpath, name)
            if path.is_symlink() and not path.resolve().is_relative_to(root):
                path.unlink()


def why_gone(wr):
    """The end of the console and qemu log of wr's VM, once it has ended."""
    lost = STATE / f"{wr.name}.lost"
    for _ in range(100):  # the daemon notes it once qemu has ended
        if lost.exists():
            break
        time.sleep(0.1)
    try:
        return lost.read_text()
    except OSError:
        return ""


def vm_gone(wr):
    """Fail: the VM, and with it any unshare instance, is gone.  Why goes to
    the task's log (Debusine keeps it); requeue-lost-work.py retries the
    work request on finding this text."""
    print(f"dur-isolate: the VM of {wr.name} ended unexpectedly:\n{why_gone(wr)}",
          file=sys.stderr, end="")
    sys.exit(125)


def drop_new_entries(before):
    """Remove what a command added to the current directory (its output,
    e.g. sbuild's build log), so that a second run starts as the first."""
    for entry in set(os.listdir()) - before:
        path = Path(entry)
        if path.is_dir() and not path.is_symlink():
            shutil.rmtree(path, ignore_errors=True)
        else:
            path.unlink(missing_ok=True)


# Self-contained commands: if the VM ends while one runs (e.g. out of
# memory), it runs once more in a new VM.  Commands in an unshare instance
# cannot, the instance went with the VM.
RETRY = ("sbuild", "autopkgtest")


def run(name, args, wr, retried=False):
    state = STATE / wr.name
    before = set(os.listdir()) if name in RETRY and not retried and \
        Path.cwd().resolve().is_relative_to(wr.resolve()) else None
    with open(STATE / f"{wr.name}.lock", "a+") as lock:
        fcntl.flock(lock, fcntl.LOCK_EX)
        lost = STATE / f"{wr.name}.lost"
        if lost.exists() and name not in RETRY:
            vm_gone(wr)  # not in a new, empty VM
        if not (state / "port").exists():
            start(wr, state)
            lost.unlink(missing_ok=True)
        lock.seek(0)
        active = int(lock.read() or 0)
        if active == 0:
            try:
                qmp(state, "cont")
            except (OSError, RuntimeError):
                if name not in RETRY:
                    vm_gone(wr)
                shutil.rmtree(state, ignore_errors=True)  # a new VM
                start(wr, state)
                lost.unlink(missing_ok=True)
        lock.seek(0), lock.truncate(), lock.write(f"{active + 1}")
        lock.flush()

    # unshare extracts an environment below wr: in the VM it goes to the
    # VM's disk (9p cannot change ownership for user namespaces)
    roots = sorted({a for a in args if re.fullmatch(
        re.escape(str(wr)) + r"/(.+/)?unshare-[^/]+/root", a)})
    setup = [f"mountpoint -q {shlex.quote(r)} || {{ d=$(mktemp -d /var/tmp/root.XXXXXX)"
             f" && chown {USER}: \"$d\" && mount --bind \"$d\" {shlex.quote(r)}; }}"
             for r in roots]
    # the caller's environment, but with the VM's paths
    env = {k: v for k, v in os.environ.items()
           if k not in ("PATH", "HOME", "TMPDIR", "USER", "LOGNAME", "SHELL")}
    env["TMPDIR"] = "/var/tmp"
    if "HOME" in os.environ:
        env["HOME"] = f"/home/{USER}"
    # Cross builds run their tests too, like native builds: qemu-user in
    # the VM runs the host's programs.  sbuild would set the profile and
    # option "nocheck" unless they are given; debhelper's option
    # "crossbuildcanrunhostbinaries" says that host programs run here.
    archs = dict(a[2:].split("=", 1) for a in args
                 if a.startswith(("--build=", "--host=")))
    if name == "sbuild" and archs.get("host", archs.get("build")) != \
            archs.get("build") and not any(a.startswith("--profiles")
                                           for a in args):
        env["DEB_BUILD_PROFILES"] = "cross"
        env["DEB_BUILD_OPTIONS"] = " ".join([
            o for o in env.get("DEB_BUILD_OPTIONS", "").split()
            if o != "nocheck"] + ["crossbuildcanrunhostbinaries"])
    # builds use the VM's processors as far as its memory goes, about
    # 1 GiB a job (sbuild passes DEB_BUILD_OPTIONS on to the build)
    options = env.get("DEB_BUILD_OPTIONS", "").split()
    if name == "sbuild" and not any(o.startswith("parallel=") for o in options):
        try:
            ram, cpus = map(int, (state / "size").read_text().split())
        except (OSError, ValueError):
            ram, cpus = 0, 1
        if (jobs := min(cpus, ram // 1024)) > 1:
            env["DEB_BUILD_OPTIONS"] = " ".join(options + [f"parallel={jobs}"])
    command = shlex.join([
        "runuser", "-u", USER, "--", "sh", "-c", 'cd "$0" && exec env "$@"',
        os.getcwd(), *(f"{k}={v}" for k, v in env.items()), name, *args])
    # input from a file the VM shares (the environment's tarball): read it
    # there instead of sending it through ssh
    stdin = None
    try:
        source = Path(os.readlink("/proc/self/fd/0"))
        if any(source.is_relative_to(path) for path in SHARED_RO) and \
                os.path.isfile(source) and os.lseek(0, 0, os.SEEK_CUR) == 0:
            command += f" < {shlex.quote(str(source))}"
            stdin = subprocess.DEVNULL
    except OSError:
        pass
    try:
        rc = ssh(state, " && ".join(setup + [f"exec {command}"]),
                 stdin=stdin).returncode
    finally:
        gone = False
        with open(STATE / f"{wr.name}.lock", "a+") as lock:
            fcntl.flock(lock, fcntl.LOCK_EX)
            lock.seek(0)
            active = int(lock.read() or 1) - 1
            if active == 0:
                try:
                    qmp(state, "stop")
                except (OSError, RuntimeError):
                    gone = True  # nothing can change the directory any more
                # always, also when the VM ended itself
                drop_escaping_symlinks(wr)
            lock.seek(0), lock.truncate(), lock.write(f"{active}")
    if gone:  # the VM ended during the command
        if name in RETRY and not retried:
            print(f"dur-isolate: the VM of {wr.name} ended, running {name}"
                  f" again in a new VM:\n{why_gone(wr)}", file=sys.stderr, end="")
            if before is not None:
                drop_new_entries(before)
            return run(name, args, wr, retried=True)
        vm_gone(wr)
    return 125 if rc == 255 else rc


def main():
    name = Path(sys.argv[0]).name
    args = sys.argv[1:]
    wr = work_request(args)
    if wr is None:
        os.execv(real(name), [name, *args])
    if not IMAGE.exists():  # never run a task on the node itself
        sys.exit(f"dur-isolate: no VM image {IMAGE}")
    sys.exit(run(name, args, wr))


if __name__ == "__main__":
    main()
