#!/usr/bin/env python3 # Copyright 2021 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # Usage: # # To get an interactive shell for development: # ./tools/dev_container # # To run a command in the container, e.g. to run presubmits: # ./tools/dev_container ./tools/presubmit # # The state of the container (including build artifacts) are preserved between # calls. To stop the container call: # ./tools/dev_container --stop # # The dev container can also be called with a fresh container for each call that # is cleaned up afterwards (e.g. when run by Kokoro): # # ./tools/dev_container --hermetic CMD # # There's an alternative container which can be used to test crosvm in crOS tree. # It can be launched with: # ./tools/dev_container --cros import argparse from pathlib import Path import shutil import stat from impl.util import ( add_common_args, confirm, cros_repo_root, CROSVM_ROOT, is_cros_repo, is_kiwi_repo, kiwi_repo_root, is_aosp_repo, aosp_repo_root, ) from impl.command import ( chdir, cmd, quoted, ) from typing import Optional, List import getpass import sys import unittest import os import zlib DEV_CONTAINER_NAME = ( f"crosvm_dev_{getpass.getuser()}_{zlib.crc32(os.path.realpath(__file__).encode('utf-8')):x}" ) CROS_CONTAINER_NAME = ( f"crosvm_cros_{getpass.getuser()}_{zlib.crc32(os.path.realpath(__file__).encode('utf-8')):x}" ) DEV_IMAGE_NAME = "gcr.io/crosvm-infra/crosvm_dev" CROS_IMAGE_NAME = "gcr.io/crosvm-infra/crosvm_cros_cloudbuild" DEV_IMAGE_VERSION = (CROSVM_ROOT / "tools/impl/dev_container/version").read_text().strip() CACHE_DIR = os.environ.get("CROSVM_CONTAINER_CACHE", None) COMMON_ARGS = [ # Share cache dir f"--volume {CACHE_DIR}:/cache:rw" if CACHE_DIR else None, # Use tmpfs in the container for faster performance. "--mount type=tmpfs,destination=/tmp", # KVM is required to run a VM for testing. "--device /dev/kvm" if Path("/dev/kvm").is_char_device() else None, # Enable terminal colors f"--env TERM={os.environ.get('TERM', 'xterm-256color')}", ] DOCKER_ARGS = [ *COMMON_ARGS, ] PODMAN_ARGS = [ *COMMON_ARGS, # Allow access to group permissions of the user (e.g. for kvm access). "--group-add keep-groups" if os.name == "posix" else None, # Increase number of PIDs the container can spawn (we run a lot of test processes in parallel) "--pids-limit=4096" if os.name == "posix" else None, ] # Environment variables to pass through to the container if they are specified. ENV_PASSTHROUGH = [ "NEXTEST_PROFILE", "http_proxy", "https_proxy", ] def machine_is_running(docker: cmd): machine_state = docker("machine info").stdout() return "MachineState: Running" in machine_state def container_name(cros: bool): if cros: return CROS_CONTAINER_NAME else: return DEV_CONTAINER_NAME def container_revision(docker: cmd, container_id: str): image = docker("container inspect -f {{.Config.Image}}", container_id).stdout() parts = image.split(":") assert len(parts) == 2, f"Invalid image name {image}" return parts[1] def container_id(docker: cmd, cros: bool): return docker(f"ps -a -q -f name={container_name(cros)}").stdout() def container_is_running(docker: cmd, cros: bool): return bool(docker(f"ps -q -f name={container_name(cros)}").stdout()) def delete_container(docker: cmd, cros: bool): cid = container_id(docker, cros) if cid: print(f"Deleting dev-container {cid}.") docker("rm -f", cid).fg(quiet=True) return True return False def workspace_mount_args(cros: bool): """ Returns arguments for mounting the crosvm sources to /workspace. In ChromeOS checkouts the crosvm repo uses a symlink or worktree checkout, which links to a different folder in the ChromeOS checkout. So we need to mount the whole CrOS checkout. """ if cros: return ["--workdir /home/crosvmdev/chromiumos/src/platform/crosvm"] elif is_cros_repo(): return [ f"--volume {quoted(cros_repo_root())}:/workspace:rw", "--workdir /workspace/src/platform/crosvm", ] elif is_kiwi_repo(): return [ f"--volume {quoted(kiwi_repo_root())}:/workspace:rw", # We override /scratch because we run out of memory if we use memory to back the # `/scratch` mount point. f"--volume {quoted(kiwi_repo_root())}/scratch:/scratch/cargo_target:rw", "--workdir /workspace/platform/crosvm", ] elif is_aosp_repo(): return [ f"--volume {quoted(aosp_repo_root())}:/workspace:rw", "--workdir /workspace/external/crosvm", ] else: return [ f"--volume {quoted(CROSVM_ROOT)}:/workspace:rw", ] def ensure_container_is_alive(docker: cmd, docker_args: List[Optional[str]], cros: bool): cid = container_id(docker, cros) if cid and not container_is_running(docker, cros): print("Existing container is not running.") delete_container(docker, cros) elif cid and not cros and container_revision(docker, cid) != DEV_IMAGE_VERSION: print(f"New image is available.") delete_container(docker, cros) if not container_is_running(docker, cros): # Run neverending sleep to keep container alive while we 'docker exec' commands. print(f"Starting container...") docker( f"run --detach --name {container_name(cros)}", *docker_args, "sleep infinity", ).fg(quiet=False) cid = container_id(docker, cros) else: cid = container_id(docker, cros) print(f"Using existing container ({cid}).") return cid def access_needs_supplemental_group(file_path: str, write=True): """ Checks if access to a given resource/file is granted through user's supplemental group as opposed to ownership or world access. Returns True if and only if access is permitted and is granted through membership in a supplemental group. If access is permitted by other means or no access granted returns False. This function does not consider superuser case. """ if not os.access(file_path, os.W_OK if write else os.R_OK, effective_ids=True): return False euid = os.geteuid() egid = os.getegid() stat_info = os.stat(file_path) if euid == stat_info.st_uid: return False # Access granted to owner desired_mode = stat.S_IWGRP if write else stat.S_IRGRP if not (stat_info.st_mode & desired_mode): return False # Access is not through group ownership if egid == stat_info.st_gid: return False # Access granted through the primary group # Assume access is granted through supplemental group. return True def validate_podman(podman: cmd): graph_driver_name = podman("info --format={{.Store.GraphDriverName}}").stdout() config_file_name = podman("info --format={{.Store.ConfigFile}}").stdout() if graph_driver_name == "vfs": print("You are using vfs as a storage driver. This will be extremely slow.") print("Using the overlay driver is strongly recommended.") print("Note: This will delete all existing podman images and containers.") if confirm(f"Do you want me to update your config in {config_file_name}?"): podman("system reset -f").fg() with open(config_file_name, "a") as config_file: print("[storage]", file=config_file) print('driver = "overlay"', file=config_file) if os.name == "posix": username = os.environ["USER"] subuids = Path("/etc/subuid").read_text() if not username in subuids: print("Rootless podman requires subuid's to be set up for your user.") usermod = cmd( "sudo usermod --add-subuids 900000-965535 --add-subgids 900000-965535", username ) print("I can fix that by running:", usermod) if confirm("Ok?"): usermod.fg() podman("system migrate").fg() if access_needs_supplemental_group("/dev/kvm"): runtime = podman("info --format={{.Host.OCIRuntime.Name}}").stdout() if runtime != "crun": print("On this system access to /dev/kvm relies on supplemental groups.") print("This functionality ('keep-groups') is only available with 'crun'.") print("Please switch podman to use 'crun' runtime.") def main(argv: List[str]): parser = argparse.ArgumentParser() add_common_args(parser) parser.add_argument("--stop", action="store_true") parser.add_argument("--clean", action="store_true") parser.add_argument("--hermetic", action="store_true") parser.add_argument("--no-interactive", action="store_true") parser.add_argument("--use-docker", action="store_true") parser.add_argument("--self-test", action="store_true") parser.add_argument("--pull", action="store_true") parser.add_argument("--cros", action="store_true") parser.add_argument("command", nargs=argparse.REMAINDER) args = parser.parse_args(argv) chdir(CROSVM_ROOT) if CACHE_DIR: Path(CACHE_DIR).mkdir(exist_ok=True) has_docker = shutil.which("docker") != None has_podman = shutil.which("podman") != None if not has_podman and not has_docker: raise Exception("Please install podman (or docker) to use the dev container.") use_docker = args.use_docker if has_docker and not has_podman: use_docker = True # cros container only works in docker if args.cros: use_docker = True if use_docker: print( "WARNING: Running dev_container with docker may cause root-owned files to be created." ) print("Use podman to prevent this.") print() docker = cmd("docker") docker_args = [ *DOCKER_ARGS, *workspace_mount_args(args.cros), ] else: docker = cmd("podman") # On windows, podman uses wsl vm. start the default podman vm for the rest of the script # to work properly. if os.name == "nt" and not machine_is_running(docker): print("Starting podman default machine.") docker("machine start").fg(quiet=True) docker_args = [ *PODMAN_ARGS, *workspace_mount_args(args.cros), ] validate_podman(docker) if args.cros: docker_args.append("--privileged") # cros container requires privileged container docker_args.append(CROS_IMAGE_NAME) else: docker_args.append(DEV_IMAGE_NAME + ":" + DEV_IMAGE_VERSION) # Add environment variables to command line exec_args: List[str] = [] for key in ENV_PASSTHROUGH: value = os.environ.get(key) if value is not None: exec_args.append("--env") exec_args.append(f"{key}={quoted(value)}") if args.self_test: TestDevContainer.docker = docker suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestDevContainer) unittest.TextTestRunner().run(suite) return if args.stop: if not delete_container(docker, args.cros): print(f"container is not running.") return if args.clean: delete_container(docker, args.cros) if args.pull: if args.cros: docker("pull", CROS_IMAGE_NAME).fg() else: docker("pull", f"gcr.io/crosvm-infra/crosvm_dev:{DEV_IMAGE_VERSION}").fg() return command = args.command # Default to interactive mode if a tty is present. tty_args: List[str] = [] if sys.stdin.isatty(): tty_args += ["--tty"] if not args.no_interactive: tty_args += ["--interactive"] # Start an interactive shell by default if args.hermetic: # cmd is passed to entrypoint quoted_cmd = list(map(quoted, command)) docker(f"run --rm", *tty_args, *docker_args, *exec_args, *quoted_cmd).fg() else: # cmd is executed directly cid = ensure_container_is_alive(docker, docker_args, args.cros) if not command: command = ("/bin/bash",) quoted_cmd = list(map(quoted, command)) docker("exec", *tty_args, *exec_args, cid, *quoted_cmd).fg() class TestDevContainer(unittest.TestCase): """ Runs live tests using the docker service. Note: This test is not run by health-check since it cannot be run inside the container. It is run by infra/recipes/health_check.py before running health checks. """ docker: cmd docker_args = [ *workspace_mount_args(cros=False), *DOCKER_ARGS, ] def setUp(self): # Start with a stopped container for each test. delete_container(self.docker, cros=False) def test_stopped_container(self): # Create but do not run a new container. self.docker( f"create --name {DEV_CONTAINER_NAME}", *self.docker_args, "sleep infinity" ).stdout() self.assertTrue(container_id(self.docker, cros=False)) self.assertFalse(container_is_running(self.docker, cros=False)) def test_container_reuse(self): cid = ensure_container_is_alive(self.docker, self.docker_args, cros=False) cid2 = ensure_container_is_alive(self.docker, self.docker_args, cros=False) self.assertEqual(cid, cid2) def test_handling_of_stopped_container(self): cid = ensure_container_is_alive(self.docker, self.docker_args, cros=False) self.docker("kill", cid).fg() # Make sure we can get back into a good state and execute commands. ensure_container_is_alive(self.docker, self.docker_args, cros=False) self.assertTrue(container_is_running(self.docker, cros=False)) main(["true"]) if __name__ == "__main__": main(sys.argv[1:])