=== modified file 'curtin/__init__.py' --- curtin/__init__.py 2015-10-26 17:35:29 +0000 +++ curtin/__init__.py 2016-07-26 19:53:43 +0000 @@ -33,6 +33,8 @@ 'SUBCOMMAND_SYSTEM_INSTALL', # subcommand 'system-upgrade' is present 'SUBCOMMAND_SYSTEM_UPGRADE', + # supports new format of apt configuration + 'APT_CONFIG_V1', ] # vi: ts=4 expandtab syntax=python === added file 'curtin/commands/apt_config.py' --- curtin/commands/apt_config.py 1970-01-01 00:00:00 +0000 +++ curtin/commands/apt_config.py 2016-07-26 19:53:43 +0000 @@ -0,0 +1,691 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Christian Ehrhardt +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . +""" +apt.py +Handling the setup of apt related tasks like proxies, PGP keys, repositories. +""" + +import argparse +import glob +import os +import re +import sys +import yaml + +from curtin.log import LOG +from curtin import (config, util, gpg) + +from . import populate_one_subcmd + +# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + +# place where apt stores cached repository data +APT_LISTS = "/var/lib/apt/lists" + +# Files to store proxy information +APT_CONFIG_FN = "/etc/apt/apt.conf.d/94curtin-config" +APT_PROXY_FN = "/etc/apt/apt.conf.d/90curtin-aptproxy" + +# Default keyserver to use +DEFAULT_KEYSERVER = "keyserver.ubuntu.com" + +# Default archive mirrors +PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/", + "SECURITY": "http://security.ubuntu.com/ubuntu/"} +PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", + "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"} +PRIMARY_ARCHES = ['amd64', 'i386'] +PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el'] + + +def get_default_mirrors(target=None): + """returns the default mirrors for the target. These depend on the + architecture, for more see: + https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" + arch = util.get_architecture(target) + if arch in PRIMARY_ARCHES: + return PRIMARY_ARCH_MIRRORS + if arch in PORTS_ARCHES: + return PORTS_MIRRORS + raise ValueError("No default mirror known for arch %s" % arch) + + +def handle_apt(cfg, target): + """ handle_apt + process the config for apt_config. This can be called from + curthooks if a global apt config was provided or via the "apt" + standalone command. + """ + release = util.lsb_release(target=target)['codename'] + mirrors = find_apt_mirror_info(cfg, target) + LOG.debug("Apt Mirror info: %s", mirrors) + + apply_debconf_selections(cfg, target) + + if not config.value_as_boolean(cfg.get('preserve_sources_list', + True)): + generate_sources_list(cfg, release, mirrors, target) + rename_apt_lists(mirrors, target) + + try: + apply_apt_proxy_config(cfg, target + APT_PROXY_FN, + target + APT_CONFIG_FN) + except (IOError, OSError): + LOG.exception("Failed to apply proxy or apt config info:") + + # Process 'apt_source -> sources {dict}' + if 'sources' in cfg: + params = mirrors + params['RELEASE'] = release + params['MIRROR'] = mirrors["MIRROR"] + + matcher = None + matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH) + if matchcfg: + matcher = re.compile(matchcfg).search + + add_apt_sources(cfg['sources'], target, + template_params=params, aa_repo_match=matcher) + + +def apply_debconf_selections(cfg, target): + """apply_debconf_selections - push content to debconf""" + # debconf_selections: + # set1: | + # cloud-init cloud-init/datasources multiselect MAAS + # set2: pkg pkg/value string bar + selsets = cfg.get('debconf_selections') + if not selsets: + LOG.debug("debconf_selections was not set in config") + return + + # for each entry in selections, chroot and apply them. + # keep a running total of packages we've seen. + pkgs_cfgd = set() + for key, content in selsets.items(): + LOG.debug("setting for %s, %s", key, content) + util.subp(['chroot', target, 'debconf-set-selections'], + data=content.encode()) + for line in content.splitlines(): + if line.startswith("#"): + continue + pkg = re.sub(r"[:\s].*", "", line) + pkgs_cfgd.add(pkg) + + pkgs_installed = util.get_installed_packages(target) + + LOG.debug("pkgs_cfgd: %s", pkgs_cfgd) + LOG.debug("pkgs_installed: %s", pkgs_installed) + need_reconfig = pkgs_cfgd.intersection(pkgs_installed) + + if len(need_reconfig) == 0: + LOG.debug("no need for reconfig") + return + + # For any packages that are already installed, but have preseed data + # we populate the debconf database, but the filesystem configuration + # would be preferred on a subsequent dpkg-reconfigure. + # so, what we have to do is "know" information about certain packages + # to unconfigure them. + unhandled = [] + to_config = [] + for pkg in need_reconfig: + if pkg in CONFIG_CLEANERS: + LOG.debug("unconfiguring %s", pkg) + CONFIG_CLEANERS[pkg](target) + to_config.append(pkg) + else: + unhandled.append(pkg) + + if len(unhandled): + LOG.warn("The following packages were installed and preseeded, " + "but cannot be unconfigured: %s", unhandled) + + if len(to_config): + util.subp(['chroot', target, 'dpkg-reconfigure', + '--frontend=noninteractive'] + + list(to_config), data=None) + + +def clean_cloud_init(target): + """clean out any local cloud-init config""" + flist = glob.glob( + os.path.sep.join([target, "/etc/cloud/cloud.cfg.d/*dpkg*"])) + + LOG.debug("cleaning cloud-init config from: %s", flist) + for dpkg_cfg in flist: + os.unlink(dpkg_cfg) + + +def mirrorurl_to_apt_fileprefix(mirror): + """ mirrorurl_to_apt_fileprefix + Convert a mirror url to the file prefix used by apt on disk to + store cache information for that mirror. + To do so do: + - take off ???:// + - drop tailing / + - convert in string / to _ + """ + string = mirror + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") + if pos >= 0: + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(new_mirrors, target): + """rename_apt_lists - rename apt lists to preserve old cache data""" + default_mirrors = get_default_mirrors(target) + for (name, omirror) in default_mirrors.items(): + nmirror = new_mirrors.get(name) + if not nmirror: + continue + oprefix = os.path.join(target, APT_LISTS, + mirrorurl_to_apt_fileprefix(omirror)) + nprefix = os.path.join(target, APT_LISTS, + mirrorurl_to_apt_fileprefix(nmirror)) + if oprefix == nprefix: + continue + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + newname = "%s%s" % (nprefix, filename[olen:]) + LOG.debug("Renaming apt list %s to %s", filename, newname) + try: + os.rename(filename, newname) + except OSError: + # since this is a best effort task, warn with but don't fail + LOG.warn("Failed to rename apt list:", exc_info=True) + + +def mirror_to_placeholder(tmpl, mirror, placeholder): + """ mirror_to_placeholder + replace the specified mirror in a template with a placeholder string + Checks for existance of the expected mirror and warns if not found + """ + if mirror not in tmpl: + LOG.warn("Expected mirror '%s' not found in: %s", mirror, tmpl) + return tmpl.replace(mirror, placeholder) + + +def map_known_suites(suite): + """there are a few default names which will be auto-extended. + This comes at the inability to use those names literally as suites, + but on the other hand increases readability of the cfg quite a lot""" + mapping = {'updates': '$RELEASE-updates', + 'backports': '$RELEASE-backports', + 'security': '$RELEASE-security', + 'proposed': '$RELEASE-proposed', + 'release': '$RELEASE'} + try: + retsuite = mapping[suite] + except KeyError: + retsuite = suite + return retsuite + + +def disable_suites(cfg, src, release): + """reads the config for suites to be disabled and removes those + from the template""" + retsrc = src + suites_to_disable = cfg.get('disable_suites', None) + if suites_to_disable is not None: + for suite in suites_to_disable: + suite = map_known_suites(suite) + releasesuite = util.render_string(suite, {'RELEASE': release}) + LOG.debug("Disabling suite %s as %s", suite, releasesuite) + + newsrc = "" + for line in retsrc.splitlines(True): + if line.startswith("#"): + newsrc += line + continue + + # sources.list allow options in cols[1] which can have spaces + # so the actual suite can be [2] or later + cols = line.split() + pcol = 2 + if cols[1].startswith("["): + for col in cols[1:]: + pcol += 1 + if col.endswith("]"): + break + + if cols[pcol] == releasesuite: + line = '# suite disabled by curtin: %s' % line + newsrc += line + retsrc = newsrc + + return retsrc + + +def generate_sources_list(cfg, release, mirrors, target): + """ generate_sources_list + create a source.list file based on a custom or default template + by replacing mirrors and release in the template + """ + default_mirrors = get_default_mirrors(target) + aptsrc = "/etc/apt/sources.list" + params = {'RELEASE': release} + for k in mirrors: + params[k] = mirrors[k] + + tmpl = cfg.get('sources_list', None) + if tmpl is None: + LOG.info("No custom template provided, fall back to modify" + "mirrors in %s on the target system", aptsrc) + tmpl = util.load_file(target + aptsrc) + # Strategy if no custom template was provided: + # - Only replacing mirrors + # - no reason to replace "release" as it is from target anyway + # - The less we depend upon, the more stable this is against changes + # - warn if expected original content wasn't found + tmpl = mirror_to_placeholder(tmpl, default_mirrors['PRIMARY'], + "$MIRROR") + tmpl = mirror_to_placeholder(tmpl, default_mirrors['SECURITY'], + "$SECURITY") + try: + os.rename(target + aptsrc, + target + aptsrc + ".curtin") + except OSError: + LOG.exception("failed to backup %s/%s", target, aptsrc) + + rendered = util.render_string(tmpl, params) + disabled = disable_suites(cfg, rendered, release) + util.write_file(target + aptsrc, disabled, mode=0o644) + + # protect the just generated sources.list from cloud-init + clouddir = "/etc/cloud/cloud.cfg.d" + cloudfile = clouddir + "/" + "curtin-preserve-sources.cfg" + # this has to work with older cloud-init as well, so use old key + cloudconf = yaml.dump({'apt_preserve_sources_list': True}, indent=1) + util.subp(['mkdir', '-p', target + clouddir], rcs=[0, 1]) + try: + util.write_file(target + cloudfile, cloudconf, mode=0o644) + except IOError: + LOG.exception("Failed to protect source.list from cloud-init in (%s)", + target + cloudfile) + raise + + +def add_apt_key_raw(key, target): + """ + actual adding of a key as defined in key argument + to the system + """ + LOG.debug("Adding key:\n'%s'", key) + try: + with util.RunInChroot(target, allow_daemons=True) as in_chroot: + in_chroot(['apt-key', 'add', '-'], data=key.encode()) + except util.ProcessExecutionError: + LOG.exception("failed to add apt GPG Key to apt keyring") + raise + + +def add_apt_key(ent, target): + """ + Add key to the system as defined in ent (if any). + Supports raw keys or keyid's + The latter will as a first step fetched to get the raw key + """ + if 'keyid' in ent and 'key' not in ent: + keyserver = DEFAULT_KEYSERVER + if 'keyserver' in ent: + keyserver = ent['keyserver'] + + ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver) + + if 'key' in ent: + add_apt_key_raw(ent['key'], target) + + +def add_apt_sources(srcdict, target, template_params=None, aa_repo_match=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srcdict'. When rendering template, also + include the values in dictionary searchList + """ + if template_params is None: + template_params = {} + + if aa_repo_match is None: + raise ValueError('did not get a valid repo matcher') + + if not isinstance(srcdict, dict): + raise TypeError('unknown apt format: %s' % (srcdict)) + + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent['filename'] = filename + + add_apt_key(ent, target) + + if 'source' not in ent: + continue + source = ent['source'] + source = util.render_string(source, template_params) + + if not ent['filename'].startswith("/"): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if not ent['filename'].endswith(".list"): + ent['filename'] += ".list" + + if aa_repo_match(source): + try: + with util.RunInChroot(target, allow_daemons=True) as in_chroot: + in_chroot(["add-apt-repository", source]) + except util.ProcessExecutionError: + LOG.exception("add-apt-repository failed.") + raise + continue + + sourcefn = target + ent['filename'] + try: + contents = "%s\n" % (source) + util.write_file(sourcefn, contents, omode="a") + except IOError as detail: + LOG.exception("failed write to file %s: %s", sourcefn, detail) + raise + + util.apt_update(target=target, force=True, + comment="apt-source changed config") + + return + + +def search_for_mirror(candidates): + """ + Search through a list of mirror urls for one that works + This needs to return quickly. + """ + if candidates is None: + return None + + for cand in candidates: + try: + if util.is_resolvable_url(cand): + return cand + except Exception: + pass + return None + + +def search_for_mirror_dns(enabled, mirrortext): + "builds a list of potential mirror to check" + if enabled is None or not enabled: + return None + + mydom = "" + doms = [] + + # curtin has no fqdn/hostname in config as cloud-init + # but if we got a hostname by dhcp, then search its domain portion first + try: + (fqdn, _) = util.subp(["hostname", "--fqdn"], rcs=[0], capture=True) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + except util.ProcessExecutionError: + # this can happen if /etc/hostname isn't set up properly yet + # so log, but don't fail + LOG.exception("failed to get fqdn") + + doms.extend((".localdomain", "",)) + + potential_mirror_list = [] + # for curtin just ubuntu instead of fetching from datasource + distro = "ubuntu" + mirrorfmt = "http://%s-%s%s/%s" % (distro, mirrortext, "%s", distro) + for post in doms: + potential_mirror_list.append(mirrorfmt % (post)) + + return search_for_mirror(potential_mirror_list) + + +def update_mirror_info(pmirror, smirror, target=None): + """sets security mirror to primary if not defined. + returns defaults if no mirrors are defined""" + if pmirror is not None: + if smirror is None: + smirror = pmirror + return {'PRIMARY': pmirror, + 'SECURITY': smirror} + return get_default_mirrors(target) + + +def get_arch_mirrorconfig(cfg, mirrortype, arch): + """out of a list of potential mirror configurations select + and return the one matching the architecture (or default)""" + # select the mirror specification (if-any) + mirror_cfg_list = cfg.get(mirrortype, None) + if mirror_cfg_list is None: + return None + + # select the specification matching the target arch + default = None + for mirror_cfg_elem in mirror_cfg_list: + arches = mirror_cfg_elem.get("arches") + if arch in arches: + return mirror_cfg_elem + if "default" in arches: + default = mirror_cfg_elem + return default + + +def get_mirror(cfg, mirrortype, arch): + """pass the three potential stages of mirror specification + returns None is neither of them found anything otherwise the first + hit is returned""" + mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch) + if mcfg is None: + return None + + # directly specified + mirror = mcfg.get("uri", None) + if mirror is None: + # list of mirrors to try to resolve + mirror = search_for_mirror(mcfg.get("search", None)) + + if mirror is None: + # search for predfined dns patterns + if mirrortype == "primary": + pattern = "mirror" + else: + pattern = "%s-mirror" % mirrortype + mirror = search_for_mirror_dns(mcfg.get("search_dns", None), pattern) + + return mirror + + +def find_apt_mirror_info(cfg, target=None): + """find_apt_mirror_info + find an apt_mirror given the cfg provided. + It can check for separate config of primary and security mirrors + If only primary is given security is assumed to be equal to primary + If the generic apt_mirror is given that is defining for both + """ + + arch = util.get_architecture(target) + LOG.debug("got arch for mirror selection: %s", arch) + pmirror = get_mirror(cfg, "primary", arch) + LOG.debug("got primary mirror: %s", pmirror) + smirror = get_mirror(cfg, "security", arch) + LOG.debug("got security mirror: %s", smirror) + + # Note: curtin has no cloud-datasource fallback + + mirror_info = update_mirror_info(pmirror, smirror, target) + + # less complex replacements use only MIRROR, derive from primary + mirror_info["MIRROR"] = mirror_info["PRIMARY"] + + return mirror_info + + +def apply_apt_proxy_config(cfg, proxy_fname, config_fname): + """apply_apt_proxy_config + Applies any apt*proxy config from if specified + """ + # Set up any apt proxy + cfgs = (('proxy', 'Acquire::http::Proxy "%s";'), + ('http_proxy', 'Acquire::http::Proxy "%s";'), + ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'), + ('https_proxy', 'Acquire::https::Proxy "%s";')) + + proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] + if len(proxies): + LOG.debug("write apt proxy info to %s", proxy_fname) + util.write_file(proxy_fname, '\n'.join(proxies) + '\n') + elif os.path.isfile(proxy_fname): + util.del_file(proxy_fname) + LOG.debug("no apt proxy configured, removed %s", proxy_fname) + + if cfg.get('conf', None): + LOG.debug("write apt config info to %s", config_fname) + util.write_file(config_fname, cfg.get('conf')) + elif os.path.isfile(config_fname): + util.del_file(config_fname) + LOG.debug("no apt config configured, removed %s", config_fname) + + +def apt_command(args): + """ Main entry point for curtin apt-config standalone command + This does not read the global config as handled by curthooks, but + instead one can specify a different "target" and a new cfg via --config + """ + cfg = config.load_command_config(args, {}) + + if args.target is not None: + target = args.target + else: + state = util.load_command_environment() + target = state['target'] + + if target is None: + sys.stderr.write("Unable to find target. " + "Use --target or set TARGET_MOUNT_POINT\n") + sys.exit(2) + + apt_cfg = cfg.get("apt") + # if no apt config section is available, do nothing + if apt_cfg is not None: + LOG.debug("Handling apt to target %s with config %s", + target, apt_cfg) + try: + with util.ChrootableTarget(target, allow_daemons=False, + sys_resolvconf=True): + handle_apt(apt_cfg, target) + except (RuntimeError, TypeError, ValueError, IOError): + LOG.exception("Failed to configure apt features '%s'", apt_cfg) + sys.exit(1) + else: + LOG.info("No apt config provided, skipping") + + sys.exit(0) + + +def translate_old_apt_features(cfg): + """translate the few old apt related features into the new config format""" + predef_apt_cfg = cfg.get("apt") + if predef_apt_cfg is None: + cfg['apt'] = {} + predef_apt_cfg = cfg.get("apt") + + if cfg.get('apt_proxy') is not None: + if predef_apt_cfg.get('proxy') is not None: + msg = ("Error in apt_proxy configuration: " + "old and new format of apt features " + "are mutually exclusive") + LOG.error(msg) + raise ValueError(msg) + + cfg['apt']['proxy'] = cfg.get('apt_proxy') + LOG.debug("Transferred %s into new format: %s", cfg.get('apt_proxy'), + cfg.get('apte')) + del cfg['apt_proxy'] + + if cfg.get('apt_mirrors') is not None: + if predef_apt_cfg.get('mirrors') is not None: + msg = ("Error in apt_mirror configuration: " + "old and new format of apt features " + "are mutually exclusive") + LOG.error(msg) + raise ValueError(msg) + + old = cfg.get('apt_mirrors') + cfg['apt']['primary'] = [{"arches": ["default"], + "uri": old.get('ubuntu_archive')}] + cfg['apt']['security'] = [{"arches": ["default"], + "uri": old.get('ubuntu_security')}] + LOG.debug("Transferred %s into new format: %s", cfg.get('apt_mirror'), + cfg.get('apt')) + del cfg['apt_mirrors'] + # to work this also needs to disable the default protection + psl = predef_apt_cfg.get('preserve_sources_list') + if psl is not None: + if config.value_as_boolean(psl) is True: + msg = ("Error in apt_mirror configuration: " + "apt_mirrors and preserve_sources_list: True " + "are mutually exclusive") + LOG.error(msg) + raise ValueError(msg) + cfg['apt']['preserve_sources_list'] = False + + if cfg.get('debconf_selections') is not None: + if predef_apt_cfg.get('debconf_selections') is not None: + msg = ("Error in debconf_selections configuration: " + "old and new format of apt features " + "are mutually exclusive") + LOG.error(msg) + raise ValueError(msg) + + selsets = cfg.get('debconf_selections') + cfg['apt']['debconf_selections'] = selsets + LOG.info("Transferred %s into new format: %s", + cfg.get('debconf_selections'), + cfg.get('apt')) + del cfg['debconf_selections'] + + return cfg + + +CMD_ARGUMENTS = ( + ((('-c', '--config'), + {'help': 'read configuration from cfg', 'action': util.MergedCmdAppend, + 'metavar': 'FILE', 'type': argparse.FileType("rb"), + 'dest': 'cfgopts', 'default': []}), + (('-t', '--target'), + {'help': 'chroot to target. default is env[TARGET_MOUNT_POINT]', + 'action': 'store', 'metavar': 'TARGET', + 'default': os.environ.get('TARGET_MOUNT_POINT')}),) +) + + +def POPULATE_SUBCMD(parser): + """Populate subcommand option parsing for apt-config""" + populate_one_subcmd(parser, CMD_ARGUMENTS, apt_command) + +CONFIG_CLEANERS = { + 'cloud-init': clean_cloud_init, +} + +# vi: ts=4 expandtab syntax=python === modified file 'curtin/commands/curthooks.py' --- curtin/commands/curthooks.py 2016-06-24 19:27:52 +0000 +++ curtin/commands/curthooks.py 2016-07-26 19:53:43 +0000 @@ -16,10 +16,8 @@ # along with Curtin. If not, see . import copy -import glob import os import platform -import re import sys import shutil import textwrap @@ -32,6 +30,7 @@ from curtin import util from curtin import net from curtin.reporter import events +from curtin.commands import apt_config from . import populate_one_subcmd @@ -90,45 +89,15 @@ info.get('perms', "0644"))) -def apt_config(cfg, target): - # cfg['apt_proxy'] - - proxy_cfg_path = os.path.sep.join( - [target, '/etc/apt/apt.conf.d/90curtin-aptproxy']) - if cfg.get('apt_proxy'): - util.write_file( - proxy_cfg_path, - content='Acquire::HTTP::Proxy "%s";\n' % cfg['apt_proxy']) +def do_apt_config(cfg, target): + cfg = apt_config.translate_old_apt_features(cfg) + apt_cfg = cfg.get("apt") + if apt_cfg is not None: + LOG.info("curthooks handling apt to target %s with config %s", + target, apt_cfg) + apt_config.handle_apt(apt_cfg, target) else: - if os.path.isfile(proxy_cfg_path): - os.unlink(proxy_cfg_path) - - # cfg['apt_mirrors'] - # apt_mirrors: - # ubuntu_archive: http://local.archive/ubuntu - # ubuntu_security: http://local.archive/ubuntu - sources_list = os.path.sep.join([target, '/etc/apt/sources.list']) - if (isinstance(cfg.get('apt_mirrors'), dict) and - os.path.isfile(sources_list)): - repls = [ - ('ubuntu_archive', r'http://\S*[.]*archive.ubuntu.com/\S*'), - ('ubuntu_security', r'http://security.ubuntu.com/\S*'), - ] - content = None - for name, regex in repls: - mirror = cfg['apt_mirrors'].get(name) - if not mirror: - continue - - if content is None: - with open(sources_list) as fp: - content = fp.read() - util.write_file(sources_list + ".dist", content) - - content = re.sub(regex, mirror + " ", content) - - if content is not None: - util.write_file(sources_list, content) + LOG.info("No apt config provided, skipping") def disable_overlayroot(cfg, target): @@ -140,15 +109,6 @@ shutil.move(local_conf, local_conf + ".old") -def clean_cloud_init(target): - flist = glob.glob( - os.path.sep.join([target, "/etc/cloud/cloud.cfg.d/*dpkg*"])) - - LOG.debug("cleaning cloud-init config from: %s" % flist) - for dpkg_cfg in flist: - os.unlink(dpkg_cfg) - - def _maybe_remove_legacy_eth0(target, path="/etc/network/interfaces.d/eth0.cfg"): """Ubuntu cloud images previously included a 'eth0.cfg' that had @@ -293,85 +253,6 @@ " System may not boot.", package) -def apply_debconf_selections(cfg, target): - # debconf_selections: - # set1: | - # cloud-init cloud-init/datasources multiselect MAAS - # set2: pkg pkg/value string bar - selsets = cfg.get('debconf_selections') - if not selsets: - LOG.debug("debconf_selections was not set in config") - return - - # for each entry in selections, chroot and apply them. - # keep a running total of packages we've seen. - pkgs_cfgd = set() - for key, content in selsets.items(): - LOG.debug("setting for %s, %s" % (key, content)) - util.subp(['chroot', target, 'debconf-set-selections'], - data=content.encode()) - for line in content.splitlines(): - if line.startswith("#"): - continue - pkg = re.sub(r"[:\s].*", "", line) - pkgs_cfgd.add(pkg) - - pkgs_installed = get_installed_packages(target) - - LOG.debug("pkgs_cfgd: %s" % pkgs_cfgd) - LOG.debug("pkgs_installed: %s" % pkgs_installed) - need_reconfig = pkgs_cfgd.intersection(pkgs_installed) - - if len(need_reconfig) == 0: - LOG.debug("no need for reconfig") - return - - # For any packages that are already installed, but have preseed data - # we populate the debconf database, but the filesystem configuration - # would be preferred on a subsequent dpkg-reconfigure. - # so, what we have to do is "know" information about certain packages - # to unconfigure them. - unhandled = [] - to_config = [] - for pkg in need_reconfig: - if pkg in CONFIG_CLEANERS: - LOG.debug("unconfiguring %s" % pkg) - CONFIG_CLEANERS[pkg](target) - to_config.append(pkg) - else: - unhandled.append(pkg) - - if len(unhandled): - LOG.warn("The following packages were installed and preseeded, " - "but cannot be unconfigured: %s", unhandled) - - util.subp(['chroot', target, 'dpkg-reconfigure', - '--frontend=noninteractive'] + - list(to_config), data=None) - - -def get_installed_packages(target=None): - cmd = [] - if target is not None: - cmd = ['chroot', target] - cmd.extend(['dpkg-query', '--list']) - - (out, _err) = util.subp(cmd, capture=True) - if isinstance(out, bytes): - out = out.decode() - - pkgs_inst = set() - for line in out.splitlines(): - try: - (state, pkg, other) = line.split(None, 2) - except ValueError: - continue - if state.startswith("hi") or state.startswith("ii"): - pkgs_inst.add(re.sub(":.*", "", pkg)) - - return pkgs_inst - - def setup_grub(cfg, target): # target is the path to the mounted filesystem @@ -740,7 +621,7 @@ } needed_packages = [] - installed_packages = get_installed_packages(target) + installed_packages = util.get_installed_packages(target) for cust_cfg, pkg_reqs in custom_configs.items(): if cust_cfg not in cfg: continue @@ -820,7 +701,7 @@ name=stack_prefix, reporting_enabled=True, level="INFO", description="writing config files and configuring apt"): write_files(cfg, target) - apt_config(cfg, target) + do_apt_config(cfg, target) disable_overlayroot(cfg, target) # packages may be needed prior to installing kernel @@ -843,7 +724,6 @@ setup_zipl(cfg, target) install_kernel(cfg, target) run_zipl(cfg, target) - apply_debconf_selections(cfg, target) restore_dist_interfaces(cfg, target) @@ -906,8 +786,4 @@ populate_one_subcmd(parser, CMD_ARGUMENTS, curthooks) -CONFIG_CLEANERS = { - 'cloud-init': clean_cloud_init, -} - # vi: ts=4 expandtab syntax=python === modified file 'curtin/commands/main.py' --- curtin/commands/main.py 2016-04-04 20:12:01 +0000 +++ curtin/commands/main.py 2016-07-26 19:53:43 +0000 @@ -27,7 +27,7 @@ SUB_COMMAND_MODULES = [ 'apply_net', 'block-meta', 'block-wipe', 'curthooks', 'extract', - 'hook', 'in-target', 'install', 'mkfs', 'net-meta', + 'hook', 'in-target', 'install', 'mkfs', 'net-meta', 'apt-config', 'pack', 'swap', 'system-install', 'system-upgrade'] === added file 'curtin/gpg.py' --- curtin/gpg.py 1970-01-01 00:00:00 +0000 +++ curtin/gpg.py 2016-07-26 19:53:43 +0000 @@ -0,0 +1,74 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Scott Moser +# Christian Ehrhardt +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . +""" gpg.py +gpg related utilities to get raw keys data by their id +""" + +from curtin import util + +from .log import LOG + + +def export_armour(key): + """Export gpg key, armoured key gets returned""" + try: + (armour, _) = util.subp(["gpg", "--export", "--armour", key], + capture=True) + except util.ProcessExecutionError as error: + # debug, since it happens for any key not on the system initially + LOG.debug('Failed to export armoured key "%s": %s', key, error) + armour = None + return armour + + +def recv_key(key, keyserver): + """Receive gpg key from the specified keyserver""" + LOG.debug('Receive gpg key "%s"', key) + try: + util.subp(["gpg", "--keyserver", keyserver, "--recv", key], + capture=True) + except util.ProcessExecutionError as error: + raise ValueError(('Failed to import key "%s" ' + 'from server "%s" - error %s') % + (key, keyserver, error)) + + +def delete_key(key): + """Delete the specified key from the local gpg ring""" + try: + util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], + capture=True) + except util.ProcessExecutionError as error: + LOG.warn('Failed delete key "%s": %s', key, error) + + +def getkeybyid(keyid, keyserver='keyserver.ubuntu.com'): + """get gpg keyid from keyserver""" + armour = export_armour(keyid) + if not armour: + try: + recv_key(keyid, keyserver=keyserver) + armour = export_armour(keyid) + except ValueError: + LOG.exception('Failed to obtain gpg key %s', keyid) + raise + finally: + # delete just imported key to leave environment as it was before + delete_key(keyid) + + return armour === modified file 'curtin/util.py' --- curtin/util.py 2016-06-24 19:27:52 +0000 +++ curtin/util.py 2016-07-26 19:53:43 +0000 @@ -16,18 +16,30 @@ # along with Curtin. If not, see . import argparse +import collections import errno import glob import json import os import platform +import re import shutil +import socket import subprocess import stat import sys import tempfile import time +# avoid the dependency to python3-six as used in cloud-init +try: + from urlparse import urlparse +except ImportError: + # python3 + # avoid triggering pylint, https://github.com/PyCQA/pylint/issues/769 + # pylint:disable=import-error,no-name-in-module + from urllib.parse import urlparse + from .log import LOG _INSTALLED_HELPERS_PATH = '/usr/lib/curtin/helpers' @@ -35,6 +47,11 @@ _LSB_RELEASE = {} +_DNS_REDIRECT_IP = None + +# matcher used in template rendering functions +BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)') + def _subp(args, data=None, rcs=None, env=None, capture=False, shell=False, logstring=False, decode="replace"): @@ -486,6 +503,28 @@ return False +def get_installed_packages(target=None): + cmd = [] + if target is not None: + cmd = ['chroot', target] + cmd.extend(['dpkg-query', '--list']) + + (out, _) = subp(cmd, capture=True) + if isinstance(out, bytes): + out = out.decode() + + pkgs_inst = set() + for line in out.splitlines(): + try: + (state, pkg, _) = line.split(None, 2) + except ValueError: + continue + if state.startswith("hi") or state.startswith("ii"): + pkgs_inst.add(re.sub(":.*", "", pkg)) + + return pkgs_inst + + def has_pkg_installed(pkg, target=None): chroot = [] if target is not None: @@ -846,14 +885,17 @@ return (isinstance(exc, IOError) and exc.errno == errno.ENOENT) -def lsb_release(): +def lsb_release(target=None): fmap = {'Codename': 'codename', 'Description': 'description', 'Distributor ID': 'id', 'Release': 'release'} + chroot = [] + if target is not None: + chroot = ['chroot', target] global _LSB_RELEASE if not _LSB_RELEASE: data = {} try: - out, err = subp(['lsb_release', '--all'], capture=True) + out, err = subp(chroot + ['lsb_release', '--all'], capture=True) for line in out.splitlines(): fname, tok, val = line.partition(":") if fname in fmap: @@ -895,4 +937,104 @@ } return platform2arch.get(platform.machine(), platform.machine()) + +def basic_template_render(content, params): + """This does simple replacement of bash variable like templates. + + It identifies patterns like ${a} or $a and can also identify patterns like + ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted + by key 'a'. + """ + + def replacer(match): + """ replacer + replacer used in regex match to replace content + """ + # Only 1 of the 2 groups will actually have a valid entry. + name = match.group(1) + if name is None: + name = match.group(2) + if name is None: + raise RuntimeError("Match encountered but no valid group present") + path = collections.deque(name.split(".")) + selected_params = params + while len(path) > 1: + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not traverse into" + " non-dictionary '%s' of type %s while" + " looking for subkey '%s'" + % (selected_params, + selected_params.__class__.__name__, + key)) + selected_params = selected_params[key] + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not extract key '%s' from non-dictionary" + " '%s' of type %s" + % (key, selected_params, + selected_params.__class__.__name__)) + return str(selected_params[key]) + + return BASIC_MATCHER.sub(replacer, content) + + +def render_string(content, params): + """ render_string + render a string following replacement rules as defined in + basic_template_render returning the string + """ + if not params: + params = {} + return basic_template_render(content, params) + + +def is_resolvable(name): + """determine if a url is resolvable, return a boolean + This also attempts to be resilent against dns redirection. + + Note, that normal nsswitch resolution is used here. So in order + to avoid any utilization of 'search' entries in /etc/resolv.conf + we have to append '.'. + + The top level 'invalid' domain is invalid per RFC. And example.com + should also not exist. The random entry will be resolved inside + the search list. + """ + global _DNS_REDIRECT_IP + if _DNS_REDIRECT_IP is None: + badips = set() + badnames = ("does-not-exist.example.com.", "example.invalid.") + badresults = {} + for iname in badnames: + try: + result = socket.getaddrinfo(iname, None, 0, 0, + socket.SOCK_STREAM, + socket.AI_CANONNAME) + badresults[iname] = [] + for (_, _, _, cname, sockaddr) in result: + badresults[iname].append("%s: %s" % (cname, sockaddr[0])) + badips.add(sockaddr[0]) + except (socket.gaierror, socket.error): + pass + _DNS_REDIRECT_IP = badips + if badresults: + LOG.debug("detected dns redirection: %s", badresults) + + try: + result = socket.getaddrinfo(name, None) + # check first result's sockaddr field + addr = result[0][4][0] + if addr in _DNS_REDIRECT_IP: + return False + return True + except (socket.gaierror, socket.error): + return False + + +def is_resolvable_url(url): + """determine if this url is resolvable (existing or ip).""" + return is_resolvable(urlparse(url).hostname) + + # vi: ts=4 expandtab syntax=python === modified file 'doc/devel/README-vmtest.txt' --- doc/devel/README-vmtest.txt 2016-01-08 17:37:01 +0000 +++ doc/devel/README-vmtest.txt 2016-07-26 19:53:43 +0000 @@ -90,13 +90,13 @@ The tests themselves don't actually have to run as root, but the test setup does. * the 'tools' directory must be in your path. - * test will set apt_proxy in the guests to the value of - 'apt_proxy' environment variable. If that is not set it will + * test will set apt: { proxy } in the guests to the value of + 'apt_proxy' environment variable. If that is not set it will look at the host's apt config and read 'Acquire::HTTP::Proxy' == Environment Variables == Some environment variables affect the running of vmtest - * apt_proxy: + * apt_proxy: test will set apt_proxy in the guests to the value of 'apt_proxy'. If that is not set it will look at the host's apt config and read 'Acquire::HTTP::Proxy' === added file 'doc/topics/apt_source.rst' --- doc/topics/apt_source.rst 1970-01-01 00:00:00 +0000 +++ doc/topics/apt_source.rst 2016-07-26 19:53:43 +0000 @@ -0,0 +1,152 @@ +========== +APT Source +========== + +This part of curtin is meant to allow influencing the apt behaviour and configuration. + +By default - if no apt config is provided - it does nothing. That keeps behavior compatible on upgrades. + +The feature has a target argument which - by default - is used to modify the environment that curtin currently installs (@TARGET_MOUNT_POINT). + +Features +-------- + +* Add PGP keys to the APT trusted keyring + + - add via short keyid + + - add via long key fingerprint + + - specify a custom keyserver to pull from + + - add raw keys (which makes you independent of keyservers) + +* Influence global apt configuration + + - adding ppa's + + - replacing mirror, security mirror and release in sources.list + + - able to provide a fully custom template for sources.list + + - add arbitrary apt.conf settings + + +Configuration +------------- + +The general configuration of the apt feature is under an element called ``apt``. + +This can have various "global" subelements as listed in the examples below. +These global configurations are valid throughput all of the apt feature. +So for exmaple a global specification of ``primary`` for a mirror will apply to all rendered sources entries. + +Then there is a section ``sources`` which can hold a number of subelements itself. +The key is the filename and will be prepended by /etc/apt/sources.list.d/ if it doesn't start with a ``/``. +There are certain cases - where no content is written into a source.list file where the filename will be ignored - yet it can still be used as index for merging. + +The values inside the entries consist of the following optional entries:: +* ``source``: a sources.list entry (some variable replacements apply) + +* ``keyid``: providing a key to import via shortid or fingerprint + +* ``key``: providing a raw PGP key + +* ``keyserver``: specify an alternate keyserver to pull keys from that were specified by keyid + +* ``filename``: for compatibility with the older format (now the key to this dictionary is the filename). If specified this overwrites the filename given as key. + +The section "sources" is is a dictionary (unlike most block/net configs which are lists). This format allows merging between multiple input files than a list like:: + sources: + s1: {'key': 'key1', 'source': 'source1'} + + sources: + s2: {'key': 'key2'} + s1: {'filename': 'foo'} + + This would be merged into + s1: {'key': 'key1', 'source': 'source1', filename: 'foo'} + s2: {'key': 'key2'} + +Here is just one of the most common examples that could be used to install with curtin in a closed environment (derived repository): + +What do we need for that: +* insert the PGP key of the local repository to be trusted + + - since you are locked down you can't pull from keyserver.ubuntu.com + + - if you have an internal keyserver you could pull from there, but let us assume you don't even have that; so you have to provide the raw key + + - in the example I'll use the key of the "Ubuntu CD Image Automatic Signing Key" which makes no sense as it is in the trusted keyring anyway, but it is a good example. (Also the key is shortened to stay readable) + +:: + + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + mQGiBEFEnz8RBAC7LstGsKD7McXZgd58oN68KquARLBl6rjA2vdhwl77KkPPOr3O + RwIbDAAKCRBAl26vQ30FtdxYAJsFjU+xbex7gevyGQ2/mhqidES4MwCggqQyo+w1 + Twx6DKLF+3rF5nf1F3Q= + =PBAe + -----END PGP PUBLIC KEY BLOCK----- + +* replace the mirror from apt pulls repository data + + - lets consider we have a local mirror at ``mymirror.local`` but otherwise following the usual paths + + - make an example with a partial mirror that doesn't mirror the backports suite, so backports have to be disabled + +That would be specified as +:: + + apt: + primary: + - arches [default] + uri: http://mymirror.local/ubuntu/ + disable_suites: [backports] + sources: + localrepokey: + key: | # full key as block + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQGiBEFEnz8RBAC7LstGsKD7McXZgd58oN68KquARLBl6rjA2vdhwl77KkPPOr3O + RwIbDAAKCRBAl26vQ30FtdxYAJsFjU+xbex7gevyGQ2/mhqidES4MwCggqQyo+w1 + Twx6DKLF+3rF5nf1F3Q= + =PBAe + -----END PGP PUBLIC KEY BLOCK----- + +Please also read the section ``Dependencies`` below to avoid loosing some of the configuration content on first boot. + +The file examples/apt-source.yaml holds various further examples that can be configured with this feature. + +Common snippets +--------------- +This is a collection of additional ideas people can use the feature for customizing their to-be-installed system. + +* enable proposed on installing + apt: + sources: + proposed.list: deb $MIRROR $RELEASE-proposed main restricted universe multiverse + +* Make debug symbols available + apt: +  sources: +   ddebs.list: | +   deb http://ddebs.ubuntu.com $RELEASE main restricted universe multiverse +   deb http://ddebs.ubuntu.com $RELEASE-updates main restricted universe multiverse +  deb http://ddebs.ubuntu.com $RELEASE-security main restricted universe multiverse + deb http://ddebs.ubuntu.com $RELEASE-proposed main restricted universe multiverse + +Timing +------ +The feature is implemented at the stage of curthooks_commands, after which runs just after curtin has extracted the image to the target. +It can be ran as standalong command "curtin -v --config apt-config". + +This will pick up the target from the environment variable that is set by curtin, if you want to use it to a different target or outside of usual curtin handling you can add ``--target `` to it to overwrite the target path. +This target should have at least a minimal system with apt and dpkg installed for the functionality to work. + + +Dependencies +------------ +Cloud-init might need to resolve dependencies and install packages in the ephemeral environment to run curtin. +Therefore it is recommended to not only configure curtin for the target, but also the install environment with proper apt configuration via cloud-init. === added file 'examples/apt-source.yaml' --- examples/apt-source.yaml 1970-01-01 00:00:00 +0000 +++ examples/apt-source.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,239 @@ +# YAML example of an apt config. +apt: + sources: + # This is a dictionary (unlike most block/net which are lists) + # The key is the filename and will be prepended by /etc/apt/sources.list.d/ + # if it doesn't start with a '/'. + # There are certain cases - where no content is written into a source.list + # file where the filename will be ignored - yet it can still be used as + # index for merging. + # The values inside the entries consost of the following optional entries: + # 'source': a sources.list entry (some variable replacements apply) + # 'keyid': providing a key to import via shortid or fingerprint + # 'key': providing a raw PGP key + # 'keyserver': specify an alternate keyserver to pull keys from that + # were specified by keyid + # 'filename': for compatibility with the older format (now the key to + # this dictionary is the filename). If specified this + # overwrites the filename given as key. + + # The new format allows merging between multiple input files than a list + # like: + # cloud-config1 + # sources: + # s1: {'key': 'key1', 'source': 'source1'} + # cloud-config2 + # sources: + # s2: {'key': 'key2'} + # s1: {'filename': 'foo'} + # This would be merged to + # sources: + # s1: + # filename: foo + # key: key1 + # source: source1 + # s2: + # key: key2 + + curtin-dev-ppa.list: + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 # GPG key ID published on a key server + # adding a source.list line, importing a gpg key for a given key id and + # storing the deb entry in /etc/apt/sources.list.d/curtin-dev-ppa.list + + # PPA shortcut: + # * Setup correct apt sources.list line + # * Import the signing key from LP + # + # See https://help.launchpad.net/Packaging/PPA for more information + # this requires 'add-apt-repository' + # due to that the filename key is ignored in this case + ignored1: + source: "ppa:curtin-dev/test-archive" # Quote the string + + # additional custom deb source config: + # * Creates a file in /etc/apt/sources.list.d/ for the sources list entry + # * [optional] Import the apt signing key from the keyserver + # * Defaults: + # + keyserver: keyserver.ubuntu.com + # + # See sources.list man page for more information about the format + my-repo1.list: + source: deb http://archive.ubuntu.com/ubuntu xenial-backports main universe multiverse restricted + + # sources can use $MIRROR and $RELEASE and they will be replaced + # with the default or specified mirror and the running release. + # The entry below would be possibly turned into: + # source: deb http://archive.ubuntu.com/ubuntu xenial multiverse + # see below at apt*mirror for more + my-repo2.list: + source: deb $MIRROR $RELEASE multiverse + + # this would have the same end effect as 'ppa:curtin-dev/test-archive' + my-repo3.list: + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 # GPG key ID published on a key server + filename: curtin-dev-ppa.list + + # this would only import the key without adding a ppa or other source spec + # since this doesn't generate a source.list file the filename key is ignored + ignored2: + keyid: F430BBA5 # GPG key ID published on a key server + + # In general keyid's can also be specified via their long fingerprints + # since this doesn't generate a source.list file the filename key is ignored + ignored3: + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + + # Custom apt repository: + # * The apt signing key can also be specified + # by providing a pgp public key block + # * Providing the PGP key here is the most robust method for + # specifying a key, as it removes dependency on a remote key server + my-repo4.list: + source: deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main + key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.0.10 + + mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 + qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj + 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y + IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H + 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP + t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o + uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey + =Y2oI + -----END PGP PUBLIC KEY BLOCK----- + + # Custom gpg key: + # * As with keyid, a key may also be specified without a related source. + # * all other facts mentioned above still apply + # since this doesn't generate a source.list file the filename key is ignored + ignored4: + key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.0.10 + + mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 + qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj + 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y + IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H + 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP + t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o + uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey + =Y2oI + -----END PGP PUBLIC KEY BLOCK----- + # end of apt dictionary + + + # + # All of the following are auxillary configurations to the apt handling + # + + # Preserve existing /etc/apt/sources.list + # Default: True - do not overwrite sources_list. If staying at true + # then any "mirrors" configuration above will have no effect + # Set to False to affect sources.list with the configuration. Without only + # extra source specifications will be written into /etc/apt/sources.list.d/. + preserve_sources_list: false + + # disable_suites by default it is an empty list, so nothing is removed. + # If given, those suites are removed from source.list after all other + # modifications have been made. + # suites are even disabled if no other modification was made, + # but not if is preserve_sources_list is active. + # There is a special alias “$RELEASE” as in the sources that will be replace + # by the matching release. + # To ease configuration and improve readability the following common ubuntu + # suites will be automatically mapped to their full definition. + # updates => $RELEASE-updates + # backports => $RELEASE-backports + # security => $RELEASE-security + # proposed => $RELEASE-proposed + # release => $RELEASE + # Note: Lines don’t get deleted, but disabled by being converted to a comment. + # The following example disables all defaults except $RELEASE-security. On top + # it disables a custom suite called "mysuite" + disable_suites: [$RELEASE-updates, backports, $RELEASE, mysuite] + + # a custom (e.g. localized) mirror that will be set in sources.list and + # deb / deb-src lines + # one can set primary and security mirror to different uri's + # the child elements to the keys primary and secondary are equivalent + primary: + # arches is list of architectures the following config applies to + # the special keyword "default" applies to any architecture not explicitly + # listed. + - arches: [amd64, i386, default] + # uri is just defining the target as-is + uri: http://us.archive.ubuntu.com/ubuntu + # + # via search one can define lists that are + # tried one by one. The first with a working DNS resolution (or if it is an + # IP) will be picked. That way one can keep one configuration for multiple + # subenvironments that select the working one. + search: + - http://cool.but-sometimes-unreachable.com/ubuntu + - http://us.archive.ubuntu.com/ubuntu + # + # This will search for -mirror locally and at the fqdn of the system. + # If resolving that will be used as archive. That allows configuring local + # mirrors via providing certin DNS names via a local nameserver. + # These can even be set to resolve to the public names of defaults like + # archive.ubuntu.com as long as they are reachable from the target. + # For security it will search -security-mirror + search_dns: True + # + # If multiple of a category are given + # 1. uri + # 2. search + # 3. search_dns + # are given the first defining a valid mirror wins (in the + # order as defined here, not the one it is listed in the config). + - arches: [s390x, arm64] + # as above, allowing to have one config for different per arch mirrors + # security is optional, if not defined it is set to the value of primary + security: + uri: http://security.ubuntu.com/ubuntu + [...] + # + # if no mirrors are specified at all, or all lookups fail it will use: + # primary: http://archive.ubuntu.com/ubuntu + # security: http://security.ubuntu.com/ubuntu + + # Provide a custom template for rendering sources.list + # without one provided curtin will try to modify the sources.list it find + # in the target at /etc/apt/. + # Within these source.list templates you can use the following replacement + # variables (all have sane ubuntu defaults, see above for details): + # $RELEASE, $MIRROR, $PRIMARY, $SECURITY + sources_list: | # written by curtin custom template + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + + # any apt config string that will be made available to apt + # see the APT.CONF(5) man page for details what can be specified + conf: | # APT config + APT { + Get { + Assume-Yes "true"; + Fix-Broken "true"; + }; + }; + + # proxies are the most common conf option that is set there is a + # shortcut via apt*proxy. Those get automatically translated into the + # correct Acquire::*::Proxy statements. + proxy: http://[[user][:pass]@]host[:port]/ + http_proxy: http://[[user][:pass]@]host[:port]/ + ftp_proxy: ftp://[[user][:pass]@]host[:port]/ + https_proxy: https://[[user][:pass]@]host[:port]/ + # note: proxy actually being a short synonym to http_proxy + + # 'source' entries in apt-sources that match this python regex + # expression will be passed to add-apt-repository + # The following example is also the builtin default if nothing is specified + add_apt_repo_match: '^[\w-]+:\w' === added file 'examples/tests/apt_config_command.yaml' --- examples/tests/apt_config_command.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/apt_config_command.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,85 @@ +# This pushes curtin through a automatic installation +# where no storage configuration is necessary. +# exercising the standalong curtin apt-config command +-placeholder_simple_install: unused +bucket: + - &run_with_stdin | + #!/bin/sh + input="$1" + shift + printf "%s\n" "$input" | "$@" + + - &run_apt_config_file | + #!/bin/sh + # take the first argument, write it to a tmp file and execute + # curtin apt --config= "$@" + set -e + config="$1" + shift + TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") + trap cleanup EXIT + cleanup() { [ -z "${TEMP_D}" || rm -Rf "${TEMP_D}"; } + cfg_file="${TEMP_D}/curtin-apt.conf" + printf "%s\n" "$config" > "${cfg_file}" + curtin apt-config "--config=${cfg_file}" "$@" + + - &apt_config_ppa | + # this is just a large string + apt: + sources: + ignored: + source: "ppa:curtin-dev/test-archive" + curtin-test1.list: + source: "deb $MIRROR $RELEASE-proposed main" + + - &apt_config_source | + apt: + preserve_sources_list: false + sources: + ignored: + source: "ppa:curtin-dev/test-archive" + # apt-add-repositroy adds the key anyway, but lets pass that code + # path of adding once more in this scope + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBFazYtEBEADXrW53tDOvwcnHwchLapTKK89+wBWR2qQKXx5Mymtjkrb688Fs + ciXcCsvClnNGJ9bEhrJTucyb7WF0KcDVQcvOd0C4HOSEAc0DANBu1Mdp/tmCWuiW + 1TbbhomyHAcHNdbuSZeMDh5xi9M3DYPVq72PwYwjrE4lotVxHeX5nYEH304U+5nJ + tBNpVon91k3ItymQ6Jii+9gVoQ7ujiH1/Gw4/J/1/5zQ3C1mOjq68vLunz5iw1Kn + 7TMVyID6qwq2UFEgudpseLfFZcb/p7KgI0m3S/OViwzSc44m63ggTPMmbeHW51xA + 1rpUChSU+cm0cJ4tNtAcYHRYRltWAo/3J1OzB6Ut5P7vIC5r+QcCyyMbku9NjYaw + dWX4DDKqW3is3qJ/7EeOKPL4N8wuKwuWUC7s2wqsIZL8EmsvR+ZOnTJ3bHZFvsLg + p/OKqmhxMGYXiXOWDOEJ+vwboPxrvhD90JZl8weNGPnpla+EkxRDBSpEb31Vgt5X + AIoxE7XxwfuXS3MGMA7fSqkGPGHfSLYQFFk+CAIeTUV+ypKW94hIxXKgqRxa7dxz + Ymqs+wgIGaWJCnx7z1Kpd3HD9iTAYjyWyhlQ/Tjt43kwUBdALhTL0vYUTGQyTgKt + tAriVf5bqHb6Hj5PS5YZQ/+YoCUI2OTrAWWNyH9rIEZGsFc30oJFPHj3fQARAQAB + tCNMYXVuY2hwYWQgUFBBIGZvciBjdXJ0aW4gZGV2ZWxvcGVyc4kCOAQTAQIAIgUC + VrNi0QIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQVf58jAFlAT4TGg// + SV7vWmkJqr5TSlT9JqCBfmtFjgudxTGG8XM2zwnta+m/3YVOMo0ZjyGL4fUKjCmN + eh6eYihwpRtfdawziaEOydDxNfdjwscV4Qcy7FjHX+DQnNzQyzK+WgWRJwNWloCw + skg2tF+EDRajalTRjHJAn+5zAilXVn71T/hhOCxkF0PBiH9s/e7pW/KcgBEC1MYV + Fs0fLST8SYhsIxttVRWuRkJDrtEY1zeVhkvk+PN6UuCY6/gyRSQ1rhhBF3ePqiba + CmLiUjnJMEm1OJOkuD33IMNPKQi99TZhr8y3AGCcrmAQtJsYLvVDPcsOsjGQHXP4 + 2qQXK+jE/AAUycCQ6tgrAqCcUNQiClP8xUPkZOiDNvVMiPvIj/s79ShkoRaWLMb7 + n9jyDOhs3L7dtmKQwHWq9qJ56fzx1L0/jxSanzm+ZJ/Q7t6E/GFxY1RsAk7xtI1C + SzSmrGKmtlbWlOyqqQb6zhULIJpaXvh/GaYyo0xI3rA+QvPDt/fgUJEBiSidwabW + Q8JU9iI5HXQxbVq1gSdy/z31fue5JuZSqjnjCjgho/UrXa4i1RPtqsY3FoTk7Hmo + C1z2cJc8HQI8JnEX/4qJXvPMRM2JsMD9DqvgsUJG5M9Qchy8cymYY+xeiBVYzJI+ + WHCq6LHqnVxYZ+RM858lSsD6wetN44vguIjL3qJJ+wU= + curtin-test1.list: + source: "deb $MIRROR $RELEASE-proposed main" + +# into ephemeral environment +early_commands: + 00_add_archive: [sh, -c, *run_with_stdin, "curtin-apt", + *apt_config_ppa, curtin, apt-config, --config=-, --target=/] + # tests itself by installing a packet only available in that ppa + 00_install_package: [apt-get, install, --assume-yes, smello] + +# into target environment +late_commands: + 00_add_archive: [sh, -c, *run_apt_config_file, "curtin-apt-file", + *apt_config_source] + 00_install_package: [curtin, in-target, --, apt-get, install, --assume-yes, smello] === added file 'examples/tests/apt_source_custom.yaml' --- examples/tests/apt_source_custom.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/apt_source_custom.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,97 @@ +showtrace: true +apt: + preserve_sources_list: false + primary: + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu + security: + - arches: [default] + uri: http://security.ubuntu.com/ubuntu + sources_list: | # written by curtin custom template + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + # nice line to check in test + conf: | # APT config + ACQUIRE { + Retries "3"; + }; + sources: + curtin-dev-ppa.list: + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 + ignored1: + source: "ppa:curtin-dev/test-archive" + my-repo2.list: + source: deb $MIRROR $RELEASE multiverse + ignored3: + keyid: 0E72 9061 0D2F 6DC4 D65E A921 9A31 4EC5 F470 A0AC + my-repo4.list: + source: deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBFXJ3NcBEAC85PMdaKdItkdjCT1vRJrdwNqj4lN5mu6z4dDVfeZlmozRDBGb + ENSOWCiYz3meANO7bKthQQCqAETSBV72rrDCqFZUpXeyG3zCN98Z/UdJ8zpQD9uw + mq2CaAqWMk6ty+PkHQ4gtIc390lGfRbHNoZ5HaWJNVOK7FCB2hBmnTZW7AViYiYa + YswOjYxaCkwQ/DsMOPD7S5OjwbLucs2YGjkBm7YF1nnXNzyt+BwieKQW/sQ2+ga1 + mkgLW1BTQN3+JreBpeHy/yrRdK4dOZZUar4WPZitZzOW2eNpaaf6hKNA14LB/96a + tEguK8VazoqSQGvNV/R3PjIYmurVP3/Z9bEVgOKhMCflgwKCYgx+tBUypN3zFWv9 + pgVq3iHx1MFCvoP9FsNB7I6jzOxlQP4z25BzR3ympx/QexkFw5CBFXhdrU+qNVBl + SSnz69aLEjCRXqBOnQEr0irs/e/35+yLJdEuw89vSwWwrzbV5r1Y7uxinEGWSydT + qddj97uKOWeMmnp20Be4+nhDDW/BMiTFI4Y3bYeDTrftbWMaSEmtSTw5HHxtAFtg + X9Hyx0Q3eN1w3gRZgIdm0xYTe7bNTofFRdfXzB/9wtNIcaW10+IlODShFHPCnh+d + i56a8LCdZcXiiLfCIhEcnqmM37BVvhjIQKSyOU1eMEgX148aVEz36OVuMwARAQAB + tCdDaHJpc3RpYW4gRWhyaGFyZHQgPGNwYWVsemVyQGdtYWlsLmNvbT6JAjgEEwEC + ACIFAlXJ3NcCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELo+KTOCgLJC + BugP/ir0ES3wCzvHMnkz2UlXt9FR4KqY0L9uFmwu9VYpmfAploEVIOi2HcuxpcRp + hgoQlUtkz3lRhUeZzCxuB1ljM2JKTJiezP1tFTTGCbVYhPyA0LmUiHDWylG7FzPb + TX96HY/G0jf+m4CfR8q3HNHjeDi4VeA2ppBxdHcVE5I7HihDgRPJd+CvCa3nYdAb + nXDKlQZz5aZc7AgrRVamr4mshkzWuwNNCwOt3AIgHDkU/HzA5xlXfwHxOoP6scWH + /T7vFsd/vOikBphGseWPgKm6w1zyQ5Dk/wjRL8UeSJZW+Rh4PuBMbxg01lAZpPTq + tu/bePeNty3g5bhwO6oHMpWhprn3dO37R680qo6UnBPzICeuBUnSYgpPnsQC9maz + FEjiBtMsXSanU5vww7TpxY1JHjk5KFcmKx4sBeablznsm+GuVaDFN8R4eDjrM14r + SOzA9cV0bSQr4dMqA9fZFSx6qLTacIeMfptybW3zaDX/pJOeBBWRAtoAfZIFbBnu + /ZxDDgiQtZzpVK4UkYk5rjjtV/CPVXx64AnTHi35YfUn14KkE+k3odHdvPfBiv9+ + NxfkTuV/koOgpD3+lTIYXyVHS9gwvhfRD/YfdrnVGl7bRZe68j7bfWDuQuSqIhSA + jpeJslJCawnqv6fVB6buj6jjcgHIxqCVn99chaPFSblEIPfXtDVDaHJpc3RpYW4g + RWhyaGFyZHQgPGNocmlzdGlhbi5laHJoYXJkdEBjYW5vbmljYWwuY29tPokCOAQT + AQIAIgUCVsbUOgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQuj4pM4KA + skJNPg/7BF/iXHHdSBigWKXCCvQz58uInoc/R4beIegxRCMq7wkYEey4B7Fd35zY + zv9CBOTV3hZePMCg9jxl4ki2kSsrZSCIEJw4L/aXDtJtx3HT18uTW0QKoU3nK/ro + OtthVqBqmiSEi40UUU+5MGrUjwLSm+PjaaSapjK/lddf0KbXBB78/BtR/XT0gxWM + +o68Oei9Nj1S3h6UndJwNAQ1xaDWmU2T7CRJet3F+cXZd3aDuS2axOTSTZbraSq7 + zdl1xUiKtzXZIp8X1ewne+dzkewZuWj7DOwOBEFK26UhxCjKd5mUr7jpWQ4ampFX + 6xfd/MK8SJFY+iHOBKyzq9po40tE23dqWuaHB+T3MxOgQ9JHCo9x22XNvEuKZW/V + 4WaoGHVkR+jtWNC8Qv/xCMHL3CEvAklKJR68WDhozwUYTgNt5vCoJOviMlbhDSwf + 0zVXpQwMR//4c0QSA0+BPpIEPDnx5vTIHBVXHy4bBBHU2Vi87QIDS0AtiBpNcspN + 6AG0ktuldkE/pqfSTJ2A9HpHZyU+8boagRS5/z102Pjtmf/mzUkcHmfRb9o0DE15 + X5fqpA3lYyx9eHIAgH4eaB1+G20Ez/EY5hr8IMS2nNBSem491UW6DXDYRu6eBLrR + sRmtrJ6DlTZFRFlqVZ47bce/SbeM/xljvRkBxWG6RtDRsTyNVI65Ag0EVcnc1wEQ + ANzk9W058tSHqf05UEtJGrN0K8DLriCvPd7QdFA8yVIZM3WD+m0AMBGXjd8BT5c2 + lt0GmhB8klonHZvPiVLTRTLcSsc3NBopr1HL1bWsgOczwWiXSrc62oGAHUOQT/bv + vS6KIkZgez+qtCo/DCOGJrADaoJBiBCLSsZgowpzazZZDPUF7rAsfcryVCFvftK0 + wAe1OdvUG77NHrMrE1oX3zh82hTqR5azBre6Y81lNwxxug/Xl/RHjNhEOYohcsLS + /xl0m2X831fHzcGGpoISRgrfel+M4RoC7KsLrwVhrF8koCD/ZQlevfLpuRl5LNpO + s1ZtEi8ZvLliih+H+BOlBD0zUc3zZrrks/NCpm1eZba0Z6L48r4TIHW08SGlHx7o + SrXgkq3mtoM8C4uDiLwjav5KxiF7n68s/9LF82aAr7YjNXd+xYZNjsmmFlYj9CGI + lL4jVt4v4EtTONa6pbtCNv5ezOLDZ6BBcQ36xdkrWzdpjQjL2mnh3sqIAGIPu7tH + N8euQ5L1zIvIjVqYlR1eJssp96QYPWYxF7TosfML4BUhCP631IWfuD9X/K2LzDmv + B2gVZo9fbhSC+P7GYVG+tV4VLAMbspAxRXXL69+j98aeV5g59f8OFQPbGpKE/SAY + eIXtq8DD+PYUXXq3VUI2brVLv42LBVdSJpKNKG3decIBABEBAAGJAh8EGAECAAkF + AlXJ3NcCGwwACgkQuj4pM4KAskKzeg/9FxXJLV3eWwY4nn2VhwYTHnHtSUpi8usk + RzIa3Mcj6OEVjU2LZaT3UQF8h6dLM9y+CemcwyjMqm1RQ5+ogfrItby1AaBXwCvm + XCUGw2zFOAnyzSHHoDFj27sllFxDmfSiBY5KP8M+/ywHKZDkRb6EjzMPx5oKFeGW + Hmqaj5FDmTeWChSIHd1ZxobashFauOZDbS/ijRRMsVGFulU2Nb/4QJK73g3orfhY + 5mq1TMkQ5Kcbqh4OmYYYayLtJQcpa6ZVopaRhAJFe30P83zW9pM5LQDpP9JIyY+S + DjasEY4ekYtw6oCKAjpqlwaaNDjl27OkJ7R7laFKy4grZ2TSB/2KTjn/Ea3CH/pA + SrpVis1LvC90XytbBnsEKYXU55H943wmBc6oj+itQhx4WyIiv+UgtHI/DbnYbUru + 71wpfapqGBXYfu/zAra8PITngOFuizeYu+idemu55ANO3keJPKr3ZBUSBBpNFauT + VUUCSnrLt+kpSLopYESiNdsPW/aQTFgFvA4BkBJTIMQsQZXicuXUePYlg5xFzXOv + XgiqkjRA9xBI5JAIUgLRk3ulVFt2bIsTG9XgtGyphEs86Q0MOIMo0WbZGtAYDrZO + DITbm2KzVLGVLn/ZJiW11RSHPNiwgg66/puKdFWrSogYYDJdDEUJtLIhypZ+ORxe + 7oh88hTkC1w= + =UNSw + -----END PGP PUBLIC KEY BLOCK----- === added file 'examples/tests/apt_source_modify.yaml' --- examples/tests/apt_source_modify.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/apt_source_modify.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,92 @@ +showtrace: true +apt: + preserve_sources_list: false + primary: + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu + security: + - arches: [default] + uri: http://security.ubuntu.com/ubuntu + conf: | # APT config + ACQUIRE { + Retries "3"; + }; + sources: + curtin-dev-ppa.list: + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 + ignored1: + source: "ppa:curtin-dev/test-archive" + # intentionally dropped the .list here, has to be added by the code + my-repo2: + source: deb $MIRROR $RELEASE multiverse + ignored3: + keyid: 0E72 9061 0D2F 6DC4 D65E A921 9A31 4EC5 F470 A0AC + my-repo4.list: + source: deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBFXJ3NcBEAC85PMdaKdItkdjCT1vRJrdwNqj4lN5mu6z4dDVfeZlmozRDBGb + ENSOWCiYz3meANO7bKthQQCqAETSBV72rrDCqFZUpXeyG3zCN98Z/UdJ8zpQD9uw + mq2CaAqWMk6ty+PkHQ4gtIc390lGfRbHNoZ5HaWJNVOK7FCB2hBmnTZW7AViYiYa + YswOjYxaCkwQ/DsMOPD7S5OjwbLucs2YGjkBm7YF1nnXNzyt+BwieKQW/sQ2+ga1 + mkgLW1BTQN3+JreBpeHy/yrRdK4dOZZUar4WPZitZzOW2eNpaaf6hKNA14LB/96a + tEguK8VazoqSQGvNV/R3PjIYmurVP3/Z9bEVgOKhMCflgwKCYgx+tBUypN3zFWv9 + pgVq3iHx1MFCvoP9FsNB7I6jzOxlQP4z25BzR3ympx/QexkFw5CBFXhdrU+qNVBl + SSnz69aLEjCRXqBOnQEr0irs/e/35+yLJdEuw89vSwWwrzbV5r1Y7uxinEGWSydT + qddj97uKOWeMmnp20Be4+nhDDW/BMiTFI4Y3bYeDTrftbWMaSEmtSTw5HHxtAFtg + X9Hyx0Q3eN1w3gRZgIdm0xYTe7bNTofFRdfXzB/9wtNIcaW10+IlODShFHPCnh+d + i56a8LCdZcXiiLfCIhEcnqmM37BVvhjIQKSyOU1eMEgX148aVEz36OVuMwARAQAB + tCdDaHJpc3RpYW4gRWhyaGFyZHQgPGNwYWVsemVyQGdtYWlsLmNvbT6JAjgEEwEC + ACIFAlXJ3NcCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELo+KTOCgLJC + BugP/ir0ES3wCzvHMnkz2UlXt9FR4KqY0L9uFmwu9VYpmfAploEVIOi2HcuxpcRp + hgoQlUtkz3lRhUeZzCxuB1ljM2JKTJiezP1tFTTGCbVYhPyA0LmUiHDWylG7FzPb + TX96HY/G0jf+m4CfR8q3HNHjeDi4VeA2ppBxdHcVE5I7HihDgRPJd+CvCa3nYdAb + nXDKlQZz5aZc7AgrRVamr4mshkzWuwNNCwOt3AIgHDkU/HzA5xlXfwHxOoP6scWH + /T7vFsd/vOikBphGseWPgKm6w1zyQ5Dk/wjRL8UeSJZW+Rh4PuBMbxg01lAZpPTq + tu/bePeNty3g5bhwO6oHMpWhprn3dO37R680qo6UnBPzICeuBUnSYgpPnsQC9maz + FEjiBtMsXSanU5vww7TpxY1JHjk5KFcmKx4sBeablznsm+GuVaDFN8R4eDjrM14r + SOzA9cV0bSQr4dMqA9fZFSx6qLTacIeMfptybW3zaDX/pJOeBBWRAtoAfZIFbBnu + /ZxDDgiQtZzpVK4UkYk5rjjtV/CPVXx64AnTHi35YfUn14KkE+k3odHdvPfBiv9+ + NxfkTuV/koOgpD3+lTIYXyVHS9gwvhfRD/YfdrnVGl7bRZe68j7bfWDuQuSqIhSA + jpeJslJCawnqv6fVB6buj6jjcgHIxqCVn99chaPFSblEIPfXtDVDaHJpc3RpYW4g + RWhyaGFyZHQgPGNocmlzdGlhbi5laHJoYXJkdEBjYW5vbmljYWwuY29tPokCOAQT + AQIAIgUCVsbUOgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQuj4pM4KA + skJNPg/7BF/iXHHdSBigWKXCCvQz58uInoc/R4beIegxRCMq7wkYEey4B7Fd35zY + zv9CBOTV3hZePMCg9jxl4ki2kSsrZSCIEJw4L/aXDtJtx3HT18uTW0QKoU3nK/ro + OtthVqBqmiSEi40UUU+5MGrUjwLSm+PjaaSapjK/lddf0KbXBB78/BtR/XT0gxWM + +o68Oei9Nj1S3h6UndJwNAQ1xaDWmU2T7CRJet3F+cXZd3aDuS2axOTSTZbraSq7 + zdl1xUiKtzXZIp8X1ewne+dzkewZuWj7DOwOBEFK26UhxCjKd5mUr7jpWQ4ampFX + 6xfd/MK8SJFY+iHOBKyzq9po40tE23dqWuaHB+T3MxOgQ9JHCo9x22XNvEuKZW/V + 4WaoGHVkR+jtWNC8Qv/xCMHL3CEvAklKJR68WDhozwUYTgNt5vCoJOviMlbhDSwf + 0zVXpQwMR//4c0QSA0+BPpIEPDnx5vTIHBVXHy4bBBHU2Vi87QIDS0AtiBpNcspN + 6AG0ktuldkE/pqfSTJ2A9HpHZyU+8boagRS5/z102Pjtmf/mzUkcHmfRb9o0DE15 + X5fqpA3lYyx9eHIAgH4eaB1+G20Ez/EY5hr8IMS2nNBSem491UW6DXDYRu6eBLrR + sRmtrJ6DlTZFRFlqVZ47bce/SbeM/xljvRkBxWG6RtDRsTyNVI65Ag0EVcnc1wEQ + ANzk9W058tSHqf05UEtJGrN0K8DLriCvPd7QdFA8yVIZM3WD+m0AMBGXjd8BT5c2 + lt0GmhB8klonHZvPiVLTRTLcSsc3NBopr1HL1bWsgOczwWiXSrc62oGAHUOQT/bv + vS6KIkZgez+qtCo/DCOGJrADaoJBiBCLSsZgowpzazZZDPUF7rAsfcryVCFvftK0 + wAe1OdvUG77NHrMrE1oX3zh82hTqR5azBre6Y81lNwxxug/Xl/RHjNhEOYohcsLS + /xl0m2X831fHzcGGpoISRgrfel+M4RoC7KsLrwVhrF8koCD/ZQlevfLpuRl5LNpO + s1ZtEi8ZvLliih+H+BOlBD0zUc3zZrrks/NCpm1eZba0Z6L48r4TIHW08SGlHx7o + SrXgkq3mtoM8C4uDiLwjav5KxiF7n68s/9LF82aAr7YjNXd+xYZNjsmmFlYj9CGI + lL4jVt4v4EtTONa6pbtCNv5ezOLDZ6BBcQ36xdkrWzdpjQjL2mnh3sqIAGIPu7tH + N8euQ5L1zIvIjVqYlR1eJssp96QYPWYxF7TosfML4BUhCP631IWfuD9X/K2LzDmv + B2gVZo9fbhSC+P7GYVG+tV4VLAMbspAxRXXL69+j98aeV5g59f8OFQPbGpKE/SAY + eIXtq8DD+PYUXXq3VUI2brVLv42LBVdSJpKNKG3decIBABEBAAGJAh8EGAECAAkF + AlXJ3NcCGwwACgkQuj4pM4KAskKzeg/9FxXJLV3eWwY4nn2VhwYTHnHtSUpi8usk + RzIa3Mcj6OEVjU2LZaT3UQF8h6dLM9y+CemcwyjMqm1RQ5+ogfrItby1AaBXwCvm + XCUGw2zFOAnyzSHHoDFj27sllFxDmfSiBY5KP8M+/ywHKZDkRb6EjzMPx5oKFeGW + Hmqaj5FDmTeWChSIHd1ZxobashFauOZDbS/ijRRMsVGFulU2Nb/4QJK73g3orfhY + 5mq1TMkQ5Kcbqh4OmYYYayLtJQcpa6ZVopaRhAJFe30P83zW9pM5LQDpP9JIyY+S + DjasEY4ekYtw6oCKAjpqlwaaNDjl27OkJ7R7laFKy4grZ2TSB/2KTjn/Ea3CH/pA + SrpVis1LvC90XytbBnsEKYXU55H943wmBc6oj+itQhx4WyIiv+UgtHI/DbnYbUru + 71wpfapqGBXYfu/zAra8PITngOFuizeYu+idemu55ANO3keJPKr3ZBUSBBpNFauT + VUUCSnrLt+kpSLopYESiNdsPW/aQTFgFvA4BkBJTIMQsQZXicuXUePYlg5xFzXOv + XgiqkjRA9xBI5JAIUgLRk3ulVFt2bIsTG9XgtGyphEs86Q0MOIMo0WbZGtAYDrZO + DITbm2KzVLGVLn/ZJiW11RSHPNiwgg66/puKdFWrSogYYDJdDEUJtLIhypZ+ORxe + 7oh88hTkC1w= + =UNSw + -----END PGP PUBLIC KEY BLOCK----- === added file 'examples/tests/apt_source_modify_arches.yaml' --- examples/tests/apt_source_modify_arches.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/apt_source_modify_arches.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,102 @@ +showtrace: true +apt: + preserve_sources_list: false + primary: + # we don't know on which arch this will run, so we can't put the "right" + # config in an arch, but we can provide various confusing alternatives + # and orders and it has to pick default out of them + - arches: [x86_2048, x86_4096, x86_8192, amd18.5, "foobar"] + uri: http://notthis.com/ubuntu + - arches: ["*"] + uri: http://notthis.com/ubuntu + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu + - arches: [] + uri: http://notthis.com/ubuntu + security: + - arches: [default] + uri: http://security.ubuntu.com/ubuntu + - arches: ["supersecurearchthatdoesnexist"] + uri: http://notthat.com/ubuntu + conf: | # APT config + ACQUIRE { + Retries "3"; + }; + sources: + curtin-dev-ppa.list: + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 + ignored1: + source: "ppa:curtin-dev/test-archive" + my-repo2.list: + source: deb $MIRROR $RELEASE multiverse + ignored3: + keyid: 0E72 9061 0D2F 6DC4 D65E A921 9A31 4EC5 F470 A0AC + my-repo4.list: + source: deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBFXJ3NcBEAC85PMdaKdItkdjCT1vRJrdwNqj4lN5mu6z4dDVfeZlmozRDBGb + ENSOWCiYz3meANO7bKthQQCqAETSBV72rrDCqFZUpXeyG3zCN98Z/UdJ8zpQD9uw + mq2CaAqWMk6ty+PkHQ4gtIc390lGfRbHNoZ5HaWJNVOK7FCB2hBmnTZW7AViYiYa + YswOjYxaCkwQ/DsMOPD7S5OjwbLucs2YGjkBm7YF1nnXNzyt+BwieKQW/sQ2+ga1 + mkgLW1BTQN3+JreBpeHy/yrRdK4dOZZUar4WPZitZzOW2eNpaaf6hKNA14LB/96a + tEguK8VazoqSQGvNV/R3PjIYmurVP3/Z9bEVgOKhMCflgwKCYgx+tBUypN3zFWv9 + pgVq3iHx1MFCvoP9FsNB7I6jzOxlQP4z25BzR3ympx/QexkFw5CBFXhdrU+qNVBl + SSnz69aLEjCRXqBOnQEr0irs/e/35+yLJdEuw89vSwWwrzbV5r1Y7uxinEGWSydT + qddj97uKOWeMmnp20Be4+nhDDW/BMiTFI4Y3bYeDTrftbWMaSEmtSTw5HHxtAFtg + X9Hyx0Q3eN1w3gRZgIdm0xYTe7bNTofFRdfXzB/9wtNIcaW10+IlODShFHPCnh+d + i56a8LCdZcXiiLfCIhEcnqmM37BVvhjIQKSyOU1eMEgX148aVEz36OVuMwARAQAB + tCdDaHJpc3RpYW4gRWhyaGFyZHQgPGNwYWVsemVyQGdtYWlsLmNvbT6JAjgEEwEC + ACIFAlXJ3NcCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELo+KTOCgLJC + BugP/ir0ES3wCzvHMnkz2UlXt9FR4KqY0L9uFmwu9VYpmfAploEVIOi2HcuxpcRp + hgoQlUtkz3lRhUeZzCxuB1ljM2JKTJiezP1tFTTGCbVYhPyA0LmUiHDWylG7FzPb + TX96HY/G0jf+m4CfR8q3HNHjeDi4VeA2ppBxdHcVE5I7HihDgRPJd+CvCa3nYdAb + nXDKlQZz5aZc7AgrRVamr4mshkzWuwNNCwOt3AIgHDkU/HzA5xlXfwHxOoP6scWH + /T7vFsd/vOikBphGseWPgKm6w1zyQ5Dk/wjRL8UeSJZW+Rh4PuBMbxg01lAZpPTq + tu/bePeNty3g5bhwO6oHMpWhprn3dO37R680qo6UnBPzICeuBUnSYgpPnsQC9maz + FEjiBtMsXSanU5vww7TpxY1JHjk5KFcmKx4sBeablznsm+GuVaDFN8R4eDjrM14r + SOzA9cV0bSQr4dMqA9fZFSx6qLTacIeMfptybW3zaDX/pJOeBBWRAtoAfZIFbBnu + /ZxDDgiQtZzpVK4UkYk5rjjtV/CPVXx64AnTHi35YfUn14KkE+k3odHdvPfBiv9+ + NxfkTuV/koOgpD3+lTIYXyVHS9gwvhfRD/YfdrnVGl7bRZe68j7bfWDuQuSqIhSA + jpeJslJCawnqv6fVB6buj6jjcgHIxqCVn99chaPFSblEIPfXtDVDaHJpc3RpYW4g + RWhyaGFyZHQgPGNocmlzdGlhbi5laHJoYXJkdEBjYW5vbmljYWwuY29tPokCOAQT + AQIAIgUCVsbUOgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQuj4pM4KA + skJNPg/7BF/iXHHdSBigWKXCCvQz58uInoc/R4beIegxRCMq7wkYEey4B7Fd35zY + zv9CBOTV3hZePMCg9jxl4ki2kSsrZSCIEJw4L/aXDtJtx3HT18uTW0QKoU3nK/ro + OtthVqBqmiSEi40UUU+5MGrUjwLSm+PjaaSapjK/lddf0KbXBB78/BtR/XT0gxWM + +o68Oei9Nj1S3h6UndJwNAQ1xaDWmU2T7CRJet3F+cXZd3aDuS2axOTSTZbraSq7 + zdl1xUiKtzXZIp8X1ewne+dzkewZuWj7DOwOBEFK26UhxCjKd5mUr7jpWQ4ampFX + 6xfd/MK8SJFY+iHOBKyzq9po40tE23dqWuaHB+T3MxOgQ9JHCo9x22XNvEuKZW/V + 4WaoGHVkR+jtWNC8Qv/xCMHL3CEvAklKJR68WDhozwUYTgNt5vCoJOviMlbhDSwf + 0zVXpQwMR//4c0QSA0+BPpIEPDnx5vTIHBVXHy4bBBHU2Vi87QIDS0AtiBpNcspN + 6AG0ktuldkE/pqfSTJ2A9HpHZyU+8boagRS5/z102Pjtmf/mzUkcHmfRb9o0DE15 + X5fqpA3lYyx9eHIAgH4eaB1+G20Ez/EY5hr8IMS2nNBSem491UW6DXDYRu6eBLrR + sRmtrJ6DlTZFRFlqVZ47bce/SbeM/xljvRkBxWG6RtDRsTyNVI65Ag0EVcnc1wEQ + ANzk9W058tSHqf05UEtJGrN0K8DLriCvPd7QdFA8yVIZM3WD+m0AMBGXjd8BT5c2 + lt0GmhB8klonHZvPiVLTRTLcSsc3NBopr1HL1bWsgOczwWiXSrc62oGAHUOQT/bv + vS6KIkZgez+qtCo/DCOGJrADaoJBiBCLSsZgowpzazZZDPUF7rAsfcryVCFvftK0 + wAe1OdvUG77NHrMrE1oX3zh82hTqR5azBre6Y81lNwxxug/Xl/RHjNhEOYohcsLS + /xl0m2X831fHzcGGpoISRgrfel+M4RoC7KsLrwVhrF8koCD/ZQlevfLpuRl5LNpO + s1ZtEi8ZvLliih+H+BOlBD0zUc3zZrrks/NCpm1eZba0Z6L48r4TIHW08SGlHx7o + SrXgkq3mtoM8C4uDiLwjav5KxiF7n68s/9LF82aAr7YjNXd+xYZNjsmmFlYj9CGI + lL4jVt4v4EtTONa6pbtCNv5ezOLDZ6BBcQ36xdkrWzdpjQjL2mnh3sqIAGIPu7tH + N8euQ5L1zIvIjVqYlR1eJssp96QYPWYxF7TosfML4BUhCP631IWfuD9X/K2LzDmv + B2gVZo9fbhSC+P7GYVG+tV4VLAMbspAxRXXL69+j98aeV5g59f8OFQPbGpKE/SAY + eIXtq8DD+PYUXXq3VUI2brVLv42LBVdSJpKNKG3decIBABEBAAGJAh8EGAECAAkF + AlXJ3NcCGwwACgkQuj4pM4KAskKzeg/9FxXJLV3eWwY4nn2VhwYTHnHtSUpi8usk + RzIa3Mcj6OEVjU2LZaT3UQF8h6dLM9y+CemcwyjMqm1RQ5+ogfrItby1AaBXwCvm + XCUGw2zFOAnyzSHHoDFj27sllFxDmfSiBY5KP8M+/ywHKZDkRb6EjzMPx5oKFeGW + Hmqaj5FDmTeWChSIHd1ZxobashFauOZDbS/ijRRMsVGFulU2Nb/4QJK73g3orfhY + 5mq1TMkQ5Kcbqh4OmYYYayLtJQcpa6ZVopaRhAJFe30P83zW9pM5LQDpP9JIyY+S + DjasEY4ekYtw6oCKAjpqlwaaNDjl27OkJ7R7laFKy4grZ2TSB/2KTjn/Ea3CH/pA + SrpVis1LvC90XytbBnsEKYXU55H943wmBc6oj+itQhx4WyIiv+UgtHI/DbnYbUru + 71wpfapqGBXYfu/zAra8PITngOFuizeYu+idemu55ANO3keJPKr3ZBUSBBpNFauT + VUUCSnrLt+kpSLopYESiNdsPW/aQTFgFvA4BkBJTIMQsQZXicuXUePYlg5xFzXOv + XgiqkjRA9xBI5JAIUgLRk3ulVFt2bIsTG9XgtGyphEs86Q0MOIMo0WbZGtAYDrZO + DITbm2KzVLGVLn/ZJiW11RSHPNiwgg66/puKdFWrSogYYDJdDEUJtLIhypZ+ORxe + 7oh88hTkC1w= + =UNSw + -----END PGP PUBLIC KEY BLOCK----- === added file 'examples/tests/apt_source_modify_disable_suite.yaml' --- examples/tests/apt_source_modify_disable_suite.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/apt_source_modify_disable_suite.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,92 @@ +showtrace: true +apt: + preserve_sources_list: false + primary: + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu + security: + - arches: [default] + uri: http://security.ubuntu.com/ubuntu + disable_suites: [$RELEASE-updates] + conf: | # APT config + ACQUIRE { + Retries "3"; + }; + sources: + curtin-dev-ppa.list: + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 + ignored1: + source: "ppa:curtin-dev/test-archive" + my-repo2.list: + source: deb $MIRROR $RELEASE multiverse + ignored3: + keyid: 0E72 9061 0D2F 6DC4 D65E A921 9A31 4EC5 F470 A0AC + my-repo4.list: + source: deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBFXJ3NcBEAC85PMdaKdItkdjCT1vRJrdwNqj4lN5mu6z4dDVfeZlmozRDBGb + ENSOWCiYz3meANO7bKthQQCqAETSBV72rrDCqFZUpXeyG3zCN98Z/UdJ8zpQD9uw + mq2CaAqWMk6ty+PkHQ4gtIc390lGfRbHNoZ5HaWJNVOK7FCB2hBmnTZW7AViYiYa + YswOjYxaCkwQ/DsMOPD7S5OjwbLucs2YGjkBm7YF1nnXNzyt+BwieKQW/sQ2+ga1 + mkgLW1BTQN3+JreBpeHy/yrRdK4dOZZUar4WPZitZzOW2eNpaaf6hKNA14LB/96a + tEguK8VazoqSQGvNV/R3PjIYmurVP3/Z9bEVgOKhMCflgwKCYgx+tBUypN3zFWv9 + pgVq3iHx1MFCvoP9FsNB7I6jzOxlQP4z25BzR3ympx/QexkFw5CBFXhdrU+qNVBl + SSnz69aLEjCRXqBOnQEr0irs/e/35+yLJdEuw89vSwWwrzbV5r1Y7uxinEGWSydT + qddj97uKOWeMmnp20Be4+nhDDW/BMiTFI4Y3bYeDTrftbWMaSEmtSTw5HHxtAFtg + X9Hyx0Q3eN1w3gRZgIdm0xYTe7bNTofFRdfXzB/9wtNIcaW10+IlODShFHPCnh+d + i56a8LCdZcXiiLfCIhEcnqmM37BVvhjIQKSyOU1eMEgX148aVEz36OVuMwARAQAB + tCdDaHJpc3RpYW4gRWhyaGFyZHQgPGNwYWVsemVyQGdtYWlsLmNvbT6JAjgEEwEC + ACIFAlXJ3NcCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELo+KTOCgLJC + BugP/ir0ES3wCzvHMnkz2UlXt9FR4KqY0L9uFmwu9VYpmfAploEVIOi2HcuxpcRp + hgoQlUtkz3lRhUeZzCxuB1ljM2JKTJiezP1tFTTGCbVYhPyA0LmUiHDWylG7FzPb + TX96HY/G0jf+m4CfR8q3HNHjeDi4VeA2ppBxdHcVE5I7HihDgRPJd+CvCa3nYdAb + nXDKlQZz5aZc7AgrRVamr4mshkzWuwNNCwOt3AIgHDkU/HzA5xlXfwHxOoP6scWH + /T7vFsd/vOikBphGseWPgKm6w1zyQ5Dk/wjRL8UeSJZW+Rh4PuBMbxg01lAZpPTq + tu/bePeNty3g5bhwO6oHMpWhprn3dO37R680qo6UnBPzICeuBUnSYgpPnsQC9maz + FEjiBtMsXSanU5vww7TpxY1JHjk5KFcmKx4sBeablznsm+GuVaDFN8R4eDjrM14r + SOzA9cV0bSQr4dMqA9fZFSx6qLTacIeMfptybW3zaDX/pJOeBBWRAtoAfZIFbBnu + /ZxDDgiQtZzpVK4UkYk5rjjtV/CPVXx64AnTHi35YfUn14KkE+k3odHdvPfBiv9+ + NxfkTuV/koOgpD3+lTIYXyVHS9gwvhfRD/YfdrnVGl7bRZe68j7bfWDuQuSqIhSA + jpeJslJCawnqv6fVB6buj6jjcgHIxqCVn99chaPFSblEIPfXtDVDaHJpc3RpYW4g + RWhyaGFyZHQgPGNocmlzdGlhbi5laHJoYXJkdEBjYW5vbmljYWwuY29tPokCOAQT + AQIAIgUCVsbUOgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQuj4pM4KA + skJNPg/7BF/iXHHdSBigWKXCCvQz58uInoc/R4beIegxRCMq7wkYEey4B7Fd35zY + zv9CBOTV3hZePMCg9jxl4ki2kSsrZSCIEJw4L/aXDtJtx3HT18uTW0QKoU3nK/ro + OtthVqBqmiSEi40UUU+5MGrUjwLSm+PjaaSapjK/lddf0KbXBB78/BtR/XT0gxWM + +o68Oei9Nj1S3h6UndJwNAQ1xaDWmU2T7CRJet3F+cXZd3aDuS2axOTSTZbraSq7 + zdl1xUiKtzXZIp8X1ewne+dzkewZuWj7DOwOBEFK26UhxCjKd5mUr7jpWQ4ampFX + 6xfd/MK8SJFY+iHOBKyzq9po40tE23dqWuaHB+T3MxOgQ9JHCo9x22XNvEuKZW/V + 4WaoGHVkR+jtWNC8Qv/xCMHL3CEvAklKJR68WDhozwUYTgNt5vCoJOviMlbhDSwf + 0zVXpQwMR//4c0QSA0+BPpIEPDnx5vTIHBVXHy4bBBHU2Vi87QIDS0AtiBpNcspN + 6AG0ktuldkE/pqfSTJ2A9HpHZyU+8boagRS5/z102Pjtmf/mzUkcHmfRb9o0DE15 + X5fqpA3lYyx9eHIAgH4eaB1+G20Ez/EY5hr8IMS2nNBSem491UW6DXDYRu6eBLrR + sRmtrJ6DlTZFRFlqVZ47bce/SbeM/xljvRkBxWG6RtDRsTyNVI65Ag0EVcnc1wEQ + ANzk9W058tSHqf05UEtJGrN0K8DLriCvPd7QdFA8yVIZM3WD+m0AMBGXjd8BT5c2 + lt0GmhB8klonHZvPiVLTRTLcSsc3NBopr1HL1bWsgOczwWiXSrc62oGAHUOQT/bv + vS6KIkZgez+qtCo/DCOGJrADaoJBiBCLSsZgowpzazZZDPUF7rAsfcryVCFvftK0 + wAe1OdvUG77NHrMrE1oX3zh82hTqR5azBre6Y81lNwxxug/Xl/RHjNhEOYohcsLS + /xl0m2X831fHzcGGpoISRgrfel+M4RoC7KsLrwVhrF8koCD/ZQlevfLpuRl5LNpO + s1ZtEi8ZvLliih+H+BOlBD0zUc3zZrrks/NCpm1eZba0Z6L48r4TIHW08SGlHx7o + SrXgkq3mtoM8C4uDiLwjav5KxiF7n68s/9LF82aAr7YjNXd+xYZNjsmmFlYj9CGI + lL4jVt4v4EtTONa6pbtCNv5ezOLDZ6BBcQ36xdkrWzdpjQjL2mnh3sqIAGIPu7tH + N8euQ5L1zIvIjVqYlR1eJssp96QYPWYxF7TosfML4BUhCP631IWfuD9X/K2LzDmv + B2gVZo9fbhSC+P7GYVG+tV4VLAMbspAxRXXL69+j98aeV5g59f8OFQPbGpKE/SAY + eIXtq8DD+PYUXXq3VUI2brVLv42LBVdSJpKNKG3decIBABEBAAGJAh8EGAECAAkF + AlXJ3NcCGwwACgkQuj4pM4KAskKzeg/9FxXJLV3eWwY4nn2VhwYTHnHtSUpi8usk + RzIa3Mcj6OEVjU2LZaT3UQF8h6dLM9y+CemcwyjMqm1RQ5+ogfrItby1AaBXwCvm + XCUGw2zFOAnyzSHHoDFj27sllFxDmfSiBY5KP8M+/ywHKZDkRb6EjzMPx5oKFeGW + Hmqaj5FDmTeWChSIHd1ZxobashFauOZDbS/ijRRMsVGFulU2Nb/4QJK73g3orfhY + 5mq1TMkQ5Kcbqh4OmYYYayLtJQcpa6ZVopaRhAJFe30P83zW9pM5LQDpP9JIyY+S + DjasEY4ekYtw6oCKAjpqlwaaNDjl27OkJ7R7laFKy4grZ2TSB/2KTjn/Ea3CH/pA + SrpVis1LvC90XytbBnsEKYXU55H943wmBc6oj+itQhx4WyIiv+UgtHI/DbnYbUru + 71wpfapqGBXYfu/zAra8PITngOFuizeYu+idemu55ANO3keJPKr3ZBUSBBpNFauT + VUUCSnrLt+kpSLopYESiNdsPW/aQTFgFvA4BkBJTIMQsQZXicuXUePYlg5xFzXOv + XgiqkjRA9xBI5JAIUgLRk3ulVFt2bIsTG9XgtGyphEs86Q0MOIMo0WbZGtAYDrZO + DITbm2KzVLGVLn/ZJiW11RSHPNiwgg66/puKdFWrSogYYDJdDEUJtLIhypZ+ORxe + 7oh88hTkC1w= + =UNSw + -----END PGP PUBLIC KEY BLOCK----- === added file 'examples/tests/apt_source_preserve.yaml' --- examples/tests/apt_source_preserve.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/apt_source_preserve.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,98 @@ +showtrace: true +apt: + # this is like the other apt_source test but with preserve true + # this is the default now preserve_sources_list: true + primary: + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu + security: + - arches: [default] + uri: http://security.ubuntu.com/ubuntu + sources_list: | # written by curtin custom template + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + # nice line to check in test + conf: | # APT config + ACQUIRE { + Retries "3"; + }; + sources: + curtin-dev-ppa.list: + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 + ignored1: + source: "ppa:curtin-dev/test-archive" + my-repo2.list: + source: deb $MIRROR $RELEASE multiverse + ignored3: + keyid: 0E72 9061 0D2F 6DC4 D65E A921 9A31 4EC5 F470 A0AC + my-repo4.list: + source: deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBFXJ3NcBEAC85PMdaKdItkdjCT1vRJrdwNqj4lN5mu6z4dDVfeZlmozRDBGb + ENSOWCiYz3meANO7bKthQQCqAETSBV72rrDCqFZUpXeyG3zCN98Z/UdJ8zpQD9uw + mq2CaAqWMk6ty+PkHQ4gtIc390lGfRbHNoZ5HaWJNVOK7FCB2hBmnTZW7AViYiYa + YswOjYxaCkwQ/DsMOPD7S5OjwbLucs2YGjkBm7YF1nnXNzyt+BwieKQW/sQ2+ga1 + mkgLW1BTQN3+JreBpeHy/yrRdK4dOZZUar4WPZitZzOW2eNpaaf6hKNA14LB/96a + tEguK8VazoqSQGvNV/R3PjIYmurVP3/Z9bEVgOKhMCflgwKCYgx+tBUypN3zFWv9 + pgVq3iHx1MFCvoP9FsNB7I6jzOxlQP4z25BzR3ympx/QexkFw5CBFXhdrU+qNVBl + SSnz69aLEjCRXqBOnQEr0irs/e/35+yLJdEuw89vSwWwrzbV5r1Y7uxinEGWSydT + qddj97uKOWeMmnp20Be4+nhDDW/BMiTFI4Y3bYeDTrftbWMaSEmtSTw5HHxtAFtg + X9Hyx0Q3eN1w3gRZgIdm0xYTe7bNTofFRdfXzB/9wtNIcaW10+IlODShFHPCnh+d + i56a8LCdZcXiiLfCIhEcnqmM37BVvhjIQKSyOU1eMEgX148aVEz36OVuMwARAQAB + tCdDaHJpc3RpYW4gRWhyaGFyZHQgPGNwYWVsemVyQGdtYWlsLmNvbT6JAjgEEwEC + ACIFAlXJ3NcCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELo+KTOCgLJC + BugP/ir0ES3wCzvHMnkz2UlXt9FR4KqY0L9uFmwu9VYpmfAploEVIOi2HcuxpcRp + hgoQlUtkz3lRhUeZzCxuB1ljM2JKTJiezP1tFTTGCbVYhPyA0LmUiHDWylG7FzPb + TX96HY/G0jf+m4CfR8q3HNHjeDi4VeA2ppBxdHcVE5I7HihDgRPJd+CvCa3nYdAb + nXDKlQZz5aZc7AgrRVamr4mshkzWuwNNCwOt3AIgHDkU/HzA5xlXfwHxOoP6scWH + /T7vFsd/vOikBphGseWPgKm6w1zyQ5Dk/wjRL8UeSJZW+Rh4PuBMbxg01lAZpPTq + tu/bePeNty3g5bhwO6oHMpWhprn3dO37R680qo6UnBPzICeuBUnSYgpPnsQC9maz + FEjiBtMsXSanU5vww7TpxY1JHjk5KFcmKx4sBeablznsm+GuVaDFN8R4eDjrM14r + SOzA9cV0bSQr4dMqA9fZFSx6qLTacIeMfptybW3zaDX/pJOeBBWRAtoAfZIFbBnu + /ZxDDgiQtZzpVK4UkYk5rjjtV/CPVXx64AnTHi35YfUn14KkE+k3odHdvPfBiv9+ + NxfkTuV/koOgpD3+lTIYXyVHS9gwvhfRD/YfdrnVGl7bRZe68j7bfWDuQuSqIhSA + jpeJslJCawnqv6fVB6buj6jjcgHIxqCVn99chaPFSblEIPfXtDVDaHJpc3RpYW4g + RWhyaGFyZHQgPGNocmlzdGlhbi5laHJoYXJkdEBjYW5vbmljYWwuY29tPokCOAQT + AQIAIgUCVsbUOgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQuj4pM4KA + skJNPg/7BF/iXHHdSBigWKXCCvQz58uInoc/R4beIegxRCMq7wkYEey4B7Fd35zY + zv9CBOTV3hZePMCg9jxl4ki2kSsrZSCIEJw4L/aXDtJtx3HT18uTW0QKoU3nK/ro + OtthVqBqmiSEi40UUU+5MGrUjwLSm+PjaaSapjK/lddf0KbXBB78/BtR/XT0gxWM + +o68Oei9Nj1S3h6UndJwNAQ1xaDWmU2T7CRJet3F+cXZd3aDuS2axOTSTZbraSq7 + zdl1xUiKtzXZIp8X1ewne+dzkewZuWj7DOwOBEFK26UhxCjKd5mUr7jpWQ4ampFX + 6xfd/MK8SJFY+iHOBKyzq9po40tE23dqWuaHB+T3MxOgQ9JHCo9x22XNvEuKZW/V + 4WaoGHVkR+jtWNC8Qv/xCMHL3CEvAklKJR68WDhozwUYTgNt5vCoJOviMlbhDSwf + 0zVXpQwMR//4c0QSA0+BPpIEPDnx5vTIHBVXHy4bBBHU2Vi87QIDS0AtiBpNcspN + 6AG0ktuldkE/pqfSTJ2A9HpHZyU+8boagRS5/z102Pjtmf/mzUkcHmfRb9o0DE15 + X5fqpA3lYyx9eHIAgH4eaB1+G20Ez/EY5hr8IMS2nNBSem491UW6DXDYRu6eBLrR + sRmtrJ6DlTZFRFlqVZ47bce/SbeM/xljvRkBxWG6RtDRsTyNVI65Ag0EVcnc1wEQ + ANzk9W058tSHqf05UEtJGrN0K8DLriCvPd7QdFA8yVIZM3WD+m0AMBGXjd8BT5c2 + lt0GmhB8klonHZvPiVLTRTLcSsc3NBopr1HL1bWsgOczwWiXSrc62oGAHUOQT/bv + vS6KIkZgez+qtCo/DCOGJrADaoJBiBCLSsZgowpzazZZDPUF7rAsfcryVCFvftK0 + wAe1OdvUG77NHrMrE1oX3zh82hTqR5azBre6Y81lNwxxug/Xl/RHjNhEOYohcsLS + /xl0m2X831fHzcGGpoISRgrfel+M4RoC7KsLrwVhrF8koCD/ZQlevfLpuRl5LNpO + s1ZtEi8ZvLliih+H+BOlBD0zUc3zZrrks/NCpm1eZba0Z6L48r4TIHW08SGlHx7o + SrXgkq3mtoM8C4uDiLwjav5KxiF7n68s/9LF82aAr7YjNXd+xYZNjsmmFlYj9CGI + lL4jVt4v4EtTONa6pbtCNv5ezOLDZ6BBcQ36xdkrWzdpjQjL2mnh3sqIAGIPu7tH + N8euQ5L1zIvIjVqYlR1eJssp96QYPWYxF7TosfML4BUhCP631IWfuD9X/K2LzDmv + B2gVZo9fbhSC+P7GYVG+tV4VLAMbspAxRXXL69+j98aeV5g59f8OFQPbGpKE/SAY + eIXtq8DD+PYUXXq3VUI2brVLv42LBVdSJpKNKG3decIBABEBAAGJAh8EGAECAAkF + AlXJ3NcCGwwACgkQuj4pM4KAskKzeg/9FxXJLV3eWwY4nn2VhwYTHnHtSUpi8usk + RzIa3Mcj6OEVjU2LZaT3UQF8h6dLM9y+CemcwyjMqm1RQ5+ogfrItby1AaBXwCvm + XCUGw2zFOAnyzSHHoDFj27sllFxDmfSiBY5KP8M+/ywHKZDkRb6EjzMPx5oKFeGW + Hmqaj5FDmTeWChSIHd1ZxobashFauOZDbS/ijRRMsVGFulU2Nb/4QJK73g3orfhY + 5mq1TMkQ5Kcbqh4OmYYYayLtJQcpa6ZVopaRhAJFe30P83zW9pM5LQDpP9JIyY+S + DjasEY4ekYtw6oCKAjpqlwaaNDjl27OkJ7R7laFKy4grZ2TSB/2KTjn/Ea3CH/pA + SrpVis1LvC90XytbBnsEKYXU55H943wmBc6oj+itQhx4WyIiv+UgtHI/DbnYbUru + 71wpfapqGBXYfu/zAra8PITngOFuizeYu+idemu55ANO3keJPKr3ZBUSBBpNFauT + VUUCSnrLt+kpSLopYESiNdsPW/aQTFgFvA4BkBJTIMQsQZXicuXUePYlg5xFzXOv + XgiqkjRA9xBI5JAIUgLRk3ulVFt2bIsTG9XgtGyphEs86Q0MOIMo0WbZGtAYDrZO + DITbm2KzVLGVLn/ZJiW11RSHPNiwgg66/puKdFWrSogYYDJdDEUJtLIhypZ+ORxe + 7oh88hTkC1w= + =UNSw + -----END PGP PUBLIC KEY BLOCK----- === added file 'examples/tests/apt_source_search.yaml' --- examples/tests/apt_source_search.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/apt_source_search.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,97 @@ +showtrace: true +apt: + preserve_sources_list: false + primary: + - arches: [default] + search: + - http://does.not.exist/ubuntu + - http://does.also.not.exist/ubuntu + - http://us.archive.ubuntu.com/ubuntu + security: + - arches: [default] + search: + - http://does.not.exist/ubuntu + - http://does.also.not.exist/ubuntu + - http://security.ubuntu.com/ubuntu + conf: | # APT config + ACQUIRE { + Retries "3"; + }; + sources: + curtin-dev-ppa.list: + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 + ignored1: + source: "ppa:curtin-dev/test-archive" + my-repo2.list: + source: deb $MIRROR $RELEASE multiverse + ignored3: + keyid: 0E72 9061 0D2F 6DC4 D65E A921 9A31 4EC5 F470 A0AC + my-repo4.list: + source: deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBFXJ3NcBEAC85PMdaKdItkdjCT1vRJrdwNqj4lN5mu6z4dDVfeZlmozRDBGb + ENSOWCiYz3meANO7bKthQQCqAETSBV72rrDCqFZUpXeyG3zCN98Z/UdJ8zpQD9uw + mq2CaAqWMk6ty+PkHQ4gtIc390lGfRbHNoZ5HaWJNVOK7FCB2hBmnTZW7AViYiYa + YswOjYxaCkwQ/DsMOPD7S5OjwbLucs2YGjkBm7YF1nnXNzyt+BwieKQW/sQ2+ga1 + mkgLW1BTQN3+JreBpeHy/yrRdK4dOZZUar4WPZitZzOW2eNpaaf6hKNA14LB/96a + tEguK8VazoqSQGvNV/R3PjIYmurVP3/Z9bEVgOKhMCflgwKCYgx+tBUypN3zFWv9 + pgVq3iHx1MFCvoP9FsNB7I6jzOxlQP4z25BzR3ympx/QexkFw5CBFXhdrU+qNVBl + SSnz69aLEjCRXqBOnQEr0irs/e/35+yLJdEuw89vSwWwrzbV5r1Y7uxinEGWSydT + qddj97uKOWeMmnp20Be4+nhDDW/BMiTFI4Y3bYeDTrftbWMaSEmtSTw5HHxtAFtg + X9Hyx0Q3eN1w3gRZgIdm0xYTe7bNTofFRdfXzB/9wtNIcaW10+IlODShFHPCnh+d + i56a8LCdZcXiiLfCIhEcnqmM37BVvhjIQKSyOU1eMEgX148aVEz36OVuMwARAQAB + tCdDaHJpc3RpYW4gRWhyaGFyZHQgPGNwYWVsemVyQGdtYWlsLmNvbT6JAjgEEwEC + ACIFAlXJ3NcCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELo+KTOCgLJC + BugP/ir0ES3wCzvHMnkz2UlXt9FR4KqY0L9uFmwu9VYpmfAploEVIOi2HcuxpcRp + hgoQlUtkz3lRhUeZzCxuB1ljM2JKTJiezP1tFTTGCbVYhPyA0LmUiHDWylG7FzPb + TX96HY/G0jf+m4CfR8q3HNHjeDi4VeA2ppBxdHcVE5I7HihDgRPJd+CvCa3nYdAb + nXDKlQZz5aZc7AgrRVamr4mshkzWuwNNCwOt3AIgHDkU/HzA5xlXfwHxOoP6scWH + /T7vFsd/vOikBphGseWPgKm6w1zyQ5Dk/wjRL8UeSJZW+Rh4PuBMbxg01lAZpPTq + tu/bePeNty3g5bhwO6oHMpWhprn3dO37R680qo6UnBPzICeuBUnSYgpPnsQC9maz + FEjiBtMsXSanU5vww7TpxY1JHjk5KFcmKx4sBeablznsm+GuVaDFN8R4eDjrM14r + SOzA9cV0bSQr4dMqA9fZFSx6qLTacIeMfptybW3zaDX/pJOeBBWRAtoAfZIFbBnu + /ZxDDgiQtZzpVK4UkYk5rjjtV/CPVXx64AnTHi35YfUn14KkE+k3odHdvPfBiv9+ + NxfkTuV/koOgpD3+lTIYXyVHS9gwvhfRD/YfdrnVGl7bRZe68j7bfWDuQuSqIhSA + jpeJslJCawnqv6fVB6buj6jjcgHIxqCVn99chaPFSblEIPfXtDVDaHJpc3RpYW4g + RWhyaGFyZHQgPGNocmlzdGlhbi5laHJoYXJkdEBjYW5vbmljYWwuY29tPokCOAQT + AQIAIgUCVsbUOgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQuj4pM4KA + skJNPg/7BF/iXHHdSBigWKXCCvQz58uInoc/R4beIegxRCMq7wkYEey4B7Fd35zY + zv9CBOTV3hZePMCg9jxl4ki2kSsrZSCIEJw4L/aXDtJtx3HT18uTW0QKoU3nK/ro + OtthVqBqmiSEi40UUU+5MGrUjwLSm+PjaaSapjK/lddf0KbXBB78/BtR/XT0gxWM + +o68Oei9Nj1S3h6UndJwNAQ1xaDWmU2T7CRJet3F+cXZd3aDuS2axOTSTZbraSq7 + zdl1xUiKtzXZIp8X1ewne+dzkewZuWj7DOwOBEFK26UhxCjKd5mUr7jpWQ4ampFX + 6xfd/MK8SJFY+iHOBKyzq9po40tE23dqWuaHB+T3MxOgQ9JHCo9x22XNvEuKZW/V + 4WaoGHVkR+jtWNC8Qv/xCMHL3CEvAklKJR68WDhozwUYTgNt5vCoJOviMlbhDSwf + 0zVXpQwMR//4c0QSA0+BPpIEPDnx5vTIHBVXHy4bBBHU2Vi87QIDS0AtiBpNcspN + 6AG0ktuldkE/pqfSTJ2A9HpHZyU+8boagRS5/z102Pjtmf/mzUkcHmfRb9o0DE15 + X5fqpA3lYyx9eHIAgH4eaB1+G20Ez/EY5hr8IMS2nNBSem491UW6DXDYRu6eBLrR + sRmtrJ6DlTZFRFlqVZ47bce/SbeM/xljvRkBxWG6RtDRsTyNVI65Ag0EVcnc1wEQ + ANzk9W058tSHqf05UEtJGrN0K8DLriCvPd7QdFA8yVIZM3WD+m0AMBGXjd8BT5c2 + lt0GmhB8klonHZvPiVLTRTLcSsc3NBopr1HL1bWsgOczwWiXSrc62oGAHUOQT/bv + vS6KIkZgez+qtCo/DCOGJrADaoJBiBCLSsZgowpzazZZDPUF7rAsfcryVCFvftK0 + wAe1OdvUG77NHrMrE1oX3zh82hTqR5azBre6Y81lNwxxug/Xl/RHjNhEOYohcsLS + /xl0m2X831fHzcGGpoISRgrfel+M4RoC7KsLrwVhrF8koCD/ZQlevfLpuRl5LNpO + s1ZtEi8ZvLliih+H+BOlBD0zUc3zZrrks/NCpm1eZba0Z6L48r4TIHW08SGlHx7o + SrXgkq3mtoM8C4uDiLwjav5KxiF7n68s/9LF82aAr7YjNXd+xYZNjsmmFlYj9CGI + lL4jVt4v4EtTONa6pbtCNv5ezOLDZ6BBcQ36xdkrWzdpjQjL2mnh3sqIAGIPu7tH + N8euQ5L1zIvIjVqYlR1eJssp96QYPWYxF7TosfML4BUhCP631IWfuD9X/K2LzDmv + B2gVZo9fbhSC+P7GYVG+tV4VLAMbspAxRXXL69+j98aeV5g59f8OFQPbGpKE/SAY + eIXtq8DD+PYUXXq3VUI2brVLv42LBVdSJpKNKG3decIBABEBAAGJAh8EGAECAAkF + AlXJ3NcCGwwACgkQuj4pM4KAskKzeg/9FxXJLV3eWwY4nn2VhwYTHnHtSUpi8usk + RzIa3Mcj6OEVjU2LZaT3UQF8h6dLM9y+CemcwyjMqm1RQ5+ogfrItby1AaBXwCvm + XCUGw2zFOAnyzSHHoDFj27sllFxDmfSiBY5KP8M+/ywHKZDkRb6EjzMPx5oKFeGW + Hmqaj5FDmTeWChSIHd1ZxobashFauOZDbS/ijRRMsVGFulU2Nb/4QJK73g3orfhY + 5mq1TMkQ5Kcbqh4OmYYYayLtJQcpa6ZVopaRhAJFe30P83zW9pM5LQDpP9JIyY+S + DjasEY4ekYtw6oCKAjpqlwaaNDjl27OkJ7R7laFKy4grZ2TSB/2KTjn/Ea3CH/pA + SrpVis1LvC90XytbBnsEKYXU55H943wmBc6oj+itQhx4WyIiv+UgtHI/DbnYbUru + 71wpfapqGBXYfu/zAra8PITngOFuizeYu+idemu55ANO3keJPKr3ZBUSBBpNFauT + VUUCSnrLt+kpSLopYESiNdsPW/aQTFgFvA4BkBJTIMQsQZXicuXUePYlg5xFzXOv + XgiqkjRA9xBI5JAIUgLRk3ulVFt2bIsTG9XgtGyphEs86Q0MOIMo0WbZGtAYDrZO + DITbm2KzVLGVLn/ZJiW11RSHPNiwgg66/puKdFWrSogYYDJdDEUJtLIhypZ+ORxe + 7oh88hTkC1w= + =UNSw + -----END PGP PUBLIC KEY BLOCK----- === added file 'examples/tests/apt_source_search_dns.yaml' --- examples/tests/apt_source_search_dns.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/apt_source_search_dns.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,21 @@ +showtrace: true +# fake up something that works in vmtets environment without dependency to external dns setup +early_commands: + xx_prep_fake_dns_search0: 'echo foo.bar > /etc/hostname' + xx_prep_fake_dns_search1: 'echo "$(getent ahosts us.archive.ubuntu.com | cut --fields=1 --delimiter " " | grep -v : | head -n 1) ubuntu-mirror" >> /etc/hosts' + xx_prep_fake_dns_search2: 'echo "$(getent ahosts security.ubuntu.com | cut --fields=1 --delimiter " " | grep -v : | head -n 1) ubuntu-security-mirror" >> /etc/hosts' +apt: + primary: + - arches: [default] + search_dns: True + security: + - arches: [default] + search_dns: True + # avoiding issues with these fake mirrors we only set them in a disabled list + # this is the default now preserve_sources_list: true + sources: + dnssearch.list.disabled: + source: | + deb $MIRROR $RELEASE multiverse + deb $PRIMARY $RELEASE universe + deb $SECURITY $RELEASE main === added file 'examples/tests/test_old_apt_features.yaml' --- examples/tests/test_old_apt_features.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/test_old_apt_features.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,10 @@ +showtrace: true +# apt_proxy gets configured by tools/launch and tests/vmtests/__init__.py +apt_mirrors: +# we need a mirror that works (even in CI) but isn't the default + ubuntu_archive: http://us.archive.ubuntu.com/ubuntu + ubuntu_security: http://archive.ubuntu.com/ubuntu +# set some key that surely is available to a non-default value +debconf_selections: + set1: | + debconf debconf/priority select low === added file 'examples/tests/test_old_apt_features_ports.yaml' --- examples/tests/test_old_apt_features_ports.yaml 1970-01-01 00:00:00 +0000 +++ examples/tests/test_old_apt_features_ports.yaml 2016-07-26 19:53:43 +0000 @@ -0,0 +1,10 @@ +showtrace: true +# apt_proxy gets configured by tools/launch and tests/vmtests/__init__.py +apt_mirrors: +# For ports there is no non-default alternative we could use + ubuntu_archive: http://ports.ubuntu.com/ubuntu-ports + ubuntu_security: http://ports.ubuntu.com/ubuntu-ports +# set some key that surely is available to a non-default value +debconf_selections: + set1: | + debconf debconf/priority select low === added file 'tests/unittests/test_apt_custom_sources_list.py' --- tests/unittests/test_apt_custom_sources_list.py 1970-01-01 00:00:00 +0000 +++ tests/unittests/test_apt_custom_sources_list.py 2016-07-26 19:53:43 +0000 @@ -0,0 +1,172 @@ +""" test_apt_custom_sources_list +Test templating of custom sources list +""" +import logging +import os +import shutil +import tempfile + +from unittest import TestCase + +import yaml +import mock +from mock import call + +from curtin import util +from curtin.commands import apt_config + +LOG = logging.getLogger(__name__) + +TARGET = "/" + +# Input and expected output for the custom template +YAML_TEXT_CUSTOM_SL = """ +preserve_sources_list: false +primary: + - arches: [default] + uri: http://test.ubuntu.com/ubuntu/ +security: + - arches: [default] + uri: http://testsec.ubuntu.com/ubuntu/ +sources_list: | + + ## Note, this file is written by curtin at install time. It should not end + ## up on the installed system itself. + # + # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to + # newer versions of the distribution. + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + # FIND_SOMETHING_SPECIAL +""" + +EXPECTED_CONVERTED_CONTENT = """ +## Note, this file is written by curtin at install time. It should not end +## up on the installed system itself. +# +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://test.ubuntu.com/ubuntu/ fakerel main restricted +deb-src http://test.ubuntu.com/ubuntu/ fakerel main restricted +deb http://test.ubuntu.com/ubuntu/ fakerel universe restricted +deb http://testsec.ubuntu.com/ubuntu/ fakerel-security multiverse +# FIND_SOMETHING_SPECIAL +""" + +# mocked to be independent to the unittest system +MOCKED_APT_SRC_LIST = """ +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""" + +EXPECTED_BASE_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""") + +EXPECTED_MIRROR_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-security main restricted +""") + +EXPECTED_PRIMSEC_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""") + + +class TestAptSourceConfigSourceList(TestCase): + """TestAptSourceConfigSourceList - Class to test sources list rendering""" + def setUp(self): + super(TestAptSourceConfigSourceList, self).setUp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + # self.patchUtils(self.new_root) + + @staticmethod + def _apt_source_list(cfg, expected): + "_apt_source_list - Test rendering from template (generic)" + + arch = util.get_architecture() + # would fail inside the unittest context + with mock.patch.object(util, 'get_architecture', + return_value=arch) as mockga: + with mock.patch.object(util, 'write_file') as mockwrite: + # keep it side effect free and avoid permission errors + with mock.patch.object(os, 'rename'): + # make test independent to executing system + with mock.patch.object(util, 'load_file', + return_value=MOCKED_APT_SRC_LIST): + with mock.patch.object(util, 'lsb_release', + return_value={'codename': + 'fakerel'}): + apt_config.handle_apt(cfg, TARGET) + + mockga.assert_called_with("/") + + cloudfile = '/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg' + cloudconf = yaml.dump({'apt_preserve_sources_list': True}, indent=1) + calls = [call(TARGET + '/etc/apt/sources.list', + expected, + mode=0o644), + call(TARGET + cloudfile, + cloudconf, + mode=0o644)] + mockwrite.assert_has_calls(calls) + + def test_apt_source_list(self): + """test_apt_source_list - Test with neither custom sources nor parms""" + cfg = {'preserve_sources_list': False} + + self._apt_source_list(cfg, EXPECTED_BASE_CONTENT) + + def test_apt_source_list_psm(self): + """test_apt_source_list_psm - Test specifying prim+sec mirrors""" + cfg = {'preserve_sources_list': False, + 'primary': [{'arches': ["default"], + 'uri': 'http://test.ubuntu.com/ubuntu/'}], + 'security': [{'arches': ["default"], + 'uri': 'http://testsec.ubuntu.com/ubuntu/'}]} + + self._apt_source_list(cfg, EXPECTED_PRIMSEC_CONTENT) + + @staticmethod + def test_apt_srcl_custom(): + """test_apt_srcl_custom - Test rendering a custom source template""" + cfg = yaml.safe_load(YAML_TEXT_CUSTOM_SL) + + arch = util.get_architecture() + # would fail inside the unittest context + with mock.patch.object(util, 'get_architecture', + return_value=arch) as mockga: + with mock.patch.object(util, 'write_file') as mockwrite: + # keep it side effect free and avoid permission errors + with mock.patch.object(os, 'rename'): + with mock.patch.object(util, 'lsb_release', + return_value={'codename': + 'fakerel'}): + apt_config.handle_apt(cfg, TARGET) + + mockga.assert_called_with("/") + cloudfile = '/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg' + cloudconf = yaml.dump({'apt_preserve_sources_list': True}, indent=1) + calls = [call(TARGET + '/etc/apt/sources.list', + EXPECTED_CONVERTED_CONTENT, + mode=0o644), + call(TARGET + cloudfile, + cloudconf, + mode=0o644)] + mockwrite.assert_has_calls(calls) + + +# vi: ts=4 expandtab === added file 'tests/unittests/test_apt_source.py' --- tests/unittests/test_apt_source.py 1970-01-01 00:00:00 +0000 +++ tests/unittests/test_apt_source.py 2016-07-26 19:53:43 +0000 @@ -0,0 +1,927 @@ +""" test_apt_source +Testing various config variations of the apt_source custom config +""" +import glob +import os +import re +import shutil +import socket +import tempfile + +from unittest import TestCase + +import mock +from mock import call + +from curtin import util +from curtin import gpg +from curtin.commands import apt_config + + +EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ +NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy +8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 +HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG +CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 +OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ +FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm +S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== +=ACB2 +-----END PGP PUBLIC KEY BLOCK-----""" + +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + +TARGET = "/" + + +def load_tfile(filename): + """ load_tfile + load file and return content after decoding + """ + try: + content = util.load_file(filename, mode="r") + except Exception as error: + print('failed to load file content for test: %s' % error) + raise + + return content + + +class TestAptSourceConfig(TestCase): + """ TestAptSourceConfig + Main Class to test apt configs + """ + def setUp(self): + super(TestAptSourceConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") + self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") + self.join = os.path.join + self.matcher = re.compile(ADD_APT_REPO_MATCH).search + + @staticmethod + def _add_apt_sources(*args, **kwargs): + with mock.patch.object(util, 'apt_update'): + apt_config.add_apt_sources(*args, **kwargs) + + @staticmethod + def _get_default_params(): + """ get_default_params + Get the most basic default mrror and release info to be used in tests + """ + params = {} + params['RELEASE'] = util.lsb_release()['codename'] + params['MIRROR'] = apt_config.get_default_mirrors()["PRIMARY"] + return params + + def _myjoin(self, *args, **kwargs): + """ _myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + + def _apt_src_basic(self, filename, cfg): + """ _apt_src_basic + Test Fix deb source string, has to overwrite mirror conf in params + """ + params = self._get_default_params() + + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "karmic-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_basic(self): + """test_apt_src_basic - Test fix deb source string""" + cfg = {self.aptlistfile: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}} + self._apt_src_basic(self.aptlistfile, cfg) + + def test_apt_src_basic_tri(self): + """test_apt_src_basic_tri - Test multiple fix deb source strings""" + cfg = {self.aptlistfile: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}, + self.aptlistfile2: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted')}, + self.aptlistfile3: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted')}} + self._apt_src_basic(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def _apt_src_replacement(self, filename, cfg): + """ apt_src_replace + Test Autoreplacement of MIRROR and RELEASE in source specs + """ + params = self._get_default_params() + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_replace(self): + """test_apt_src_replace - Test Autoreplacement of MIRROR and RELEASE""" + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}} + self._apt_src_replacement(self.aptlistfile, cfg) + + def test_apt_src_replace_fn(self): + """test_apt_src_replace_fn - Test filename being overwritten in dict""" + cfg = {'ignored': {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile}} + # second file should overwrite the dict key + self._apt_src_replacement(self.aptlistfile, cfg) + + def _apt_src_replace_tri(self, cfg): + """ _apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + generic part + """ + self._apt_src_replacement(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + params = self._get_default_params() + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "main"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "universe"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_replace_tri(self): + """test_apt_src_replace_tri - Test multiple replacements/overwrites""" + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} + self._apt_src_replace_tri(cfg) + + def _apt_src_keyid(self, filename, cfg, keynum): + """ _apt_src_keyid + Test specification of a source + keyid + """ + params = self._get_default_params() + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1234', '')) as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + # check if it added the right ammount of keys + calls = [] + for _ in range(keynum): + calls.append(call(['apt-key', 'add', '-'], data=b'fakekey 1234')) + mockobj.assert_has_calls(calls, any_order=True) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_keyid(self): + """test_apt_src_keyid - Test source + keyid with filename being set""" + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"}} + self._apt_src_keyid(self.aptlistfile, cfg, 1) + + def test_apt_src_keyid_tri(self): + """test_apt_src_keyid_tri - Test multiple src+keyid+filen overwrites""" + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"}, + 'ignored': {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial universe'), + 'keyid': "03683F77", + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial multiverse'), + 'keyid': "03683F77"}} + + self._apt_src_keyid(self.aptlistfile, cfg, 3) + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "universe"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_key(self): + """test_apt_src_key - Test source + key""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321"}} + + with mock.patch.object(util, 'subp') as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 4321') + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_keyonly(self): + """test_apt_src_keyonly - Test key without source""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'key': "fakekey 4242"}} + + with mock.patch.object(util, 'subp') as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 4242') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_keyidonly(self): + """test_apt_src_keyidonly - Test keyid without source""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'keyid': "03683F77"}} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1212', '')) as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 1212') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def apt_src_keyid_real(self, cfg, expectedkey): + """apt_src_keyid_real + Test specification of a keyid without source including + up to addition of the key (add_apt_key_raw mocked to keep the + environment as is) + """ + params = self._get_default_params() + + with mock.patch.object(apt_config, 'add_apt_key_raw') as mockkey: + with mock.patch.object(gpg, 'getkeybyid', + return_value=expectedkey) as mockgetkey: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + keycfg = cfg[self.aptlistfile] + mockgetkey.assert_called_with(keycfg['keyid'], + keycfg.get('keyserver', + 'keyserver.ubuntu.com')) + mockkey.assert_called_with(expectedkey, TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_keyid_real(self): + """test_apt_src_keyid_real - Test keyid including key add""" + keyid = "03683F77" + cfg = {self.aptlistfile: {'keyid': keyid}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_longkeyid_real(self): + """test_apt_src_longkeyid_real Test long keyid including key add""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {self.aptlistfile: {'keyid': keyid}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_longkeyid_ks_real(self): + """test_apt_src_longkeyid_ks_real Test long keyid from other ks""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {self.aptlistfile: {'keyid': keyid, + 'keyserver': 'keys.gnupg.net'}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_keyid_keyserver(self): + """test_apt_src_keyid_keyserver - Test custom keyserver""" + keyid = "03683F77" + params = self._get_default_params() + cfg = {self.aptlistfile: {'keyid': keyid, + 'keyserver': 'test.random.com'}} + + # in some test environments only *.ubuntu.com is reachable + # so mock the call and check if the config got there + with mock.patch.object(gpg, 'getkeybyid', + return_value="fakekey") as mockgetkey: + with mock.patch.object(apt_config, 'add_apt_key_raw') as mockadd: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockgetkey.assert_called_with('03683F77', 'test.random.com') + mockadd.assert_called_with('fakekey', TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_ppa(self): + """test_apt_src_ppa - Test specification of a ppa""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}} + + with mock.patch.object(util, 'subp') as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + mockobj.assert_any_call(['add-apt-repository', + 'ppa:smoser/cloud-init-test']) + + # adding ppa should ignore filename (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_ppa_tri(self): + """test_apt_src_ppa_tri - Test specification of multiple ppa's""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}, + self.aptlistfile2: {'source': 'ppa:smoser/cloud-init-test2'}, + self.aptlistfile3: {'source': 'ppa:smoser/cloud-init-test3'}} + + with mock.patch.object(util, 'subp') as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] + mockobj.assert_has_calls(calls, any_order=True) + + # adding ppa should ignore all filenames (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + self.assertFalse(os.path.isfile(self.aptlistfile2)) + self.assertFalse(os.path.isfile(self.aptlistfile3)) + + def test_mir_apt_list_rename(self): + """test_mir_apt_list_rename - Test find mirror and apt list renaming""" + pre = "/var/lib/apt/lists" + # filenames are archive dependent + arch = util.get_architecture() + if arch in apt_config.PRIMARY_ARCHES: + component = "ubuntu" + archive = "archive.ubuntu.com" + else: + component = "ubuntu-ports" + archive = "ports.ubuntu.com" + + cfg = {'primary': [{'arches': ["default"], + 'uri': + 'http://test.ubuntu.com/%s/' % component}], + 'security': [{'arches': ["default"], + 'uri': + 'http://testsec.ubuntu.com/%s/' % component}]} + post = ("%s_dists_%s-updates_InRelease" % + (component, util.lsb_release()['codename'])) + fromfn = ("%s/%s_%s" % (pre, archive, post)) + tofn = ("%s/test.ubuntu.com_%s" % (pre, post)) + + mirrors = apt_config.find_apt_mirror_info(cfg) + + self.assertEqual(mirrors['MIRROR'], + "http://test.ubuntu.com/%s/" % component) + self.assertEqual(mirrors['PRIMARY'], + "http://test.ubuntu.com/%s/" % component) + self.assertEqual(mirrors['SECURITY'], + "http://testsec.ubuntu.com/%s/" % component) + + # get_architecture would fail inside the unittest context + with mock.patch.object(util, 'get_architecture', return_value=arch): + with mock.patch.object(os, 'rename') as mockren: + with mock.patch.object(glob, 'glob', + return_value=[fromfn]): + apt_config.rename_apt_lists(mirrors, TARGET) + + mockren.assert_any_call(fromfn, tofn) + + @staticmethod + def test_apt_proxy(): + """test_mir_apt_list_rename - Test apt_*proxy configuration""" + cfg = {"proxy": "foobar1", + "http_proxy": "foobar2", + "ftp_proxy": "foobar3", + "https_proxy": "foobar4"} + + with mock.patch.object(util, 'write_file') as mockobj: + apt_config.apply_apt_proxy_config(cfg, "proxyfn", "notused") + + mockobj.assert_called_with('proxyfn', + ('Acquire::http::Proxy "foobar1";\n' + 'Acquire::http::Proxy "foobar2";\n' + 'Acquire::ftp::Proxy "foobar3";\n' + 'Acquire::https::Proxy "foobar4";\n')) + + def test_mirror(self): + """test_mirror - Test defining a mirror""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir}], + "security": [{'arches': ["default"], + "uri": smir}]} + + mirrors = apt_config.find_apt_mirror_info(cfg) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_mirror_default(self): + """test_mirror_default - Test without defining a mirror""" + default_mirrors = apt_config.get_default_mirrors() + pmir = default_mirrors["PRIMARY"] + smir = default_mirrors["SECURITY"] + mirrors = apt_config.find_apt_mirror_info({}) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_mirror_arches(self): + """test_mirror_arches - Test arches selection of mirror""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": "notthis"}, + {'arches': [util.get_architecture()], + "uri": pmir}], + "security": [{'arches': [util.get_architecture()], + "uri": smir}, + {'arches': ["default"], + "uri": "nothat"}]} + + mirrors = apt_config.find_apt_mirror_info(cfg) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_mirror_arches_default(self): + """test_mirror_arches - Test falling back to default arch""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir}, + {'arches': ["thisarchdoesntexist"], + "uri": "notthis"}], + "security": [{'arches': ["thisarchdoesntexist"], + "uri": "nothat"}, + {'arches': ["default"], + "uri": smir}]} + + mirrors = apt_config.find_apt_mirror_info(cfg) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_mirror_arches_sysdefault(self): + """test_mirror_arches - Test arches falling back to sys default""" + default_mirrors = apt_config.get_default_mirrors() + pmir = default_mirrors["PRIMARY"] + smir = default_mirrors["SECURITY"] + cfg = {"primary": [{'arches': ["thisarchdoesntexist_64"], + "uri": "notthis"}, + {'arches': ["thisarchdoesntexist"], + "uri": "notthiseither"}], + "security": [{'arches': ["thisarchdoesntexist"], + "uri": "nothat"}, + {'arches': ["thisarchdoesntexist_64"], + "uri": "nothateither"}]} + + mirrors = apt_config.find_apt_mirror_info(cfg) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_mirror_search(self): + """test_mirror_search - Test searching mirrors in a list + mock checks to avoid relying on network connectivity""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "search": ["pfailme", pmir]}], + "security": [{'arches': ["default"], + "search": ["sfailme", smir]}]} + + with mock.patch.object(apt_config, 'search_for_mirror', + side_effect=[pmir, smir]) as mocksearch: + mirrors = apt_config.find_apt_mirror_info(cfg) + + calls = [call(["pfailme", pmir]), + call(["sfailme", smir])] + mocksearch.assert_has_calls(calls) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_mirror_search_dns(self): + """test_mirror_search_dns - Test searching dns patterns""" + pmir = "phit" + smir = "shit" + cfg = {"primary": [{'arches': ["default"], + "search_dns": True}], + "security": [{'arches': ["default"], + "search_dns": True}]} + + with mock.patch.object(apt_config, 'get_mirror', + return_value="http://mocked/foo") as mockgm: + mirrors = apt_config.find_apt_mirror_info(cfg) + calls = [call(cfg, 'primary', util.get_architecture()), + call(cfg, 'security', util.get_architecture())] + mockgm.assert_has_calls(calls) + + with mock.patch.object(apt_config, 'search_for_mirror_dns', + return_value="http://mocked/foo") as mocksdns: + mirrors = apt_config.find_apt_mirror_info(cfg) + calls = [call(True, 'mirror'), + call(True, 'security-mirror')] + mocksdns.assert_has_calls(calls) + + # first return is for the non-dns call before + with mock.patch.object(apt_config, 'search_for_mirror', + side_effect=[None, pmir, None, smir]) as mockse: + with mock.patch.object(util, 'subp', + return_value=('host.sub.com', '')) as mocks: + mirrors = apt_config.find_apt_mirror_info(cfg) + + calls = [call(None), + call(['http://ubuntu-mirror.sub.com/ubuntu', + 'http://ubuntu-mirror.localdomain/ubuntu', + 'http://ubuntu-mirror/ubuntu']), + call(None), + call(['http://ubuntu-security-mirror.sub.com/ubuntu', + 'http://ubuntu-security-mirror.localdomain/ubuntu', + 'http://ubuntu-security-mirror/ubuntu'])] + mockse.assert_has_calls(calls) + mocks.assert_called_with(['hostname', '--fqdn'], capture=True, rcs=[0]) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_mirror_search_many3(self): + """test_mirror_search_many3 - Test all three mirrors specs at once""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir, + "search_dns": True, + "search": ["pfailme", "foo"]}], + "security": [{'arches': ["default"], + "uri": smir, + "search_dns": True, + "search": ["sfailme", "bar"]}]} + + # should be called once per type, despite three configs each + with mock.patch.object(apt_config, 'get_mirror', + return_value="http://mocked/foo") as mockgm: + mirrors = apt_config.find_apt_mirror_info(cfg) + calls = [call(cfg, 'primary', util.get_architecture()), + call(cfg, 'security', util.get_architecture())] + mockgm.assert_has_calls(calls) + + # should not be called, since primary is specified + with mock.patch.object(apt_config, 'search_for_mirror_dns') as mockdns: + mirrors = apt_config.find_apt_mirror_info(cfg) + mockdns.assert_not_called() + + # should not be called, since primary is specified + with mock.patch.object(apt_config, 'search_for_mirror') as mockse: + mirrors = apt_config.find_apt_mirror_info(cfg) + mockse.assert_not_called() + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_mirror_search_many2(self): + """test_mirror_search_many2 - Test the two search specs at once""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "search_dns": True, + "search": ["pfailme", pmir]}], + "security": [{'arches': ["default"], + "search_dns": True, + "search": ["sfailme", smir]}]} + + # should be called once per type, despite three configs each + with mock.patch.object(apt_config, 'get_mirror', + return_value="http://mocked/foo") as mockgm: + mirrors = apt_config.find_apt_mirror_info(cfg) + calls = [call(cfg, 'primary', util.get_architecture()), + call(cfg, 'security', util.get_architecture())] + mockgm.assert_has_calls(calls) + + # this should be the winner by priority, despite config order + with mock.patch.object(apt_config, 'search_for_mirror', + side_effect=[pmir, smir]) as mocksearch: + mirrors = apt_config.find_apt_mirror_info(cfg) + calls = [call(["pfailme", pmir]), + call(["sfailme", smir])] + mocksearch.assert_has_calls(calls) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_url_resolvable(self): + """test_url_resolvable - Test resolving urls""" + + with mock.patch.object(util, 'is_resolvable') as mockresolve: + util.is_resolvable_url("http://1.2.3.4/ubuntu") + mockresolve.assert_called_with("1.2.3.4") + + with mock.patch.object(util, 'is_resolvable') as mockresolve: + util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") + mockresolve.assert_called_with("us.archive.ubuntu.com") + + bad = [(None, None, None, "badname", ["10.3.2.1"])] + good = [(None, None, None, "goodname", ["10.2.3.4"])] + with mock.patch.object(socket, 'getaddrinfo', + side_effect=[bad, bad, good, + good]) as mocksock: + ret = util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") + ret2 = util.is_resolvable_url("http://1.2.3.4/ubuntu") + calls = [call('does-not-exist.example.com.', None, 0, 0, 1, 2), + call('example.invalid.', None, 0, 0, 1, 2), + call('us.archive.ubuntu.com', None), + call('1.2.3.4', None)] + mocksock.assert_has_calls(calls) + self.assertTrue(ret) + self.assertTrue(ret2) + + # side effect need only bad ret after initial call + with mock.patch.object(socket, 'getaddrinfo', + side_effect=[bad]) as mocksock: + ret3 = util.is_resolvable_url("http://failme.com/ubuntu") + calls = [call('failme.com', None)] + mocksock.assert_has_calls(calls) + self.assertFalse(ret3) + + def test_disable_suites(self): + """test_disable_suites - disable_suites with many configurations""" + release = "xenial" + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + + # disable nothing + cfg = {"disable_suites": []} + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # single disable release suite + cfg = {"disable_suites": ["$RELEASE"]} + expect = """\ +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # single disable other suite + cfg = {"disable_suites": ["$RELEASE-updates"]} + expect = """deb http://ubuntu.com//ubuntu xenial main +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # multi disable + cfg = {"disable_suites": ["$RELEASE-updates", "$RELEASE-security"]} + expect = """deb http://ubuntu.com//ubuntu xenial main +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial-updates main +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # multi line disable (same suite multiple times in input) + cfg = {"disable_suites": ["$RELEASE-updates", "$RELEASE-security"]} + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://UBUNTU.com//ubuntu xenial-updates main +deb http://UBUNTU.COM//ubuntu xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial-updates main +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +# suite disabled by curtin: deb http://UBUNTU.com//ubuntu xenial-updates main +# suite disabled by curtin: deb http://UBUNTU.COM//ubuntu xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # comment in input + cfg = {"disable_suites": ["$RELEASE-updates", "$RELEASE-security"]} + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +#foo +#deb http://UBUNTU.com//ubuntu xenial-updates main +deb http://UBUNTU.COM//ubuntu xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial-updates main +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +#foo +#deb http://UBUNTU.com//ubuntu xenial-updates main +# suite disabled by curtin: deb http://UBUNTU.COM//ubuntu xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # single disable custom suite + cfg = {"disable_suites": ["foobar"]} + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ foobar main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +# suite disabled by curtin: deb http://ubuntu.com/ubuntu/ foobar main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # single disable non existing suite + cfg = {"disable_suites": ["foobar"]} + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ notfoobar main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ notfoobar main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # single disable suite with option + cfg = {"disable_suites": ["$RELEASE-updates"]} + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [a=b] http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +# suite disabled by curtin: deb [a=b] http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # single disable suite with more options and auto $RELEASE expansion + cfg = {"disable_suites": ["updates"]} + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [a=b c=d] http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +# suite disabled by curtin: deb [a=b c=d] \ +http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + + # single disable suite while options at others + cfg = {"disable_suites": ["$RELEASE-security"]} + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [arch=foo] http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +deb [arch=foo] http://ubuntu.com//ubuntu xenial-updates main +# suite disabled by curtin: deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = apt_config.disable_suites(cfg, orig, release) + self.assertEqual(expect, result) + +# +# vi: ts=4 expandtab === modified file 'tests/vmtests/__init__.py' --- tests/vmtests/__init__.py 2016-07-19 19:25:08 +0000 +++ tests/vmtests/__init__.py 2016-07-26 19:53:43 +0000 @@ -355,6 +355,7 @@ extra_kern_args = None fstab_expected = {} image_store_class = ImageStore + boot_cloudconf = None install_timeout = 3000 interactive = False multipath = False @@ -363,6 +364,7 @@ recorded_errors = 0 recorded_failures = 0 uefi = False + proxy = None # these get set from base_vm_classes release = None @@ -392,7 +394,8 @@ # set up tempdir cls.td = TempDir( name=cls.__name__, - user_data=generate_user_data(collect_scripts=cls.collect_scripts)) + user_data=generate_user_data(collect_scripts=cls.collect_scripts, + boot_cloudconf=cls.boot_cloudconf)) logger.info('Using tempdir: %s , Image: %s', cls.td.tmpdir, img_verstr) cls.install_log = os.path.join(cls.td.logs, 'install-serial.log') @@ -492,11 +495,11 @@ # proxy config configs = [cls.conf_file] - proxy = get_apt_proxy() - if get_apt_proxy is not None: + cls.proxy = get_apt_proxy() + if cls.proxy is not None: proxy_config = os.path.join(cls.td.install, 'proxy.cfg') with open(proxy_config, "w") as fp: - fp.write(json.dumps({'apt_proxy': proxy}) + "\n") + fp.write(json.dumps({'apt_proxy': cls.proxy}) + "\n") configs.append(proxy_config) uefi_flags = [] @@ -731,8 +734,14 @@ # Misc functions that are useful for many tests def output_files_exist(self, files): for f in files: + logger.debug('checking file %s', f) self.assertTrue(os.path.exists(os.path.join(self.td.collect, f))) + def output_files_dont_exist(self, files): + for f in files: + logger.debug('checking file %s', f) + self.assertFalse(os.path.exists(os.path.join(self.td.collect, f))) + def check_file_strippedline(self, filename, search): with open(os.path.join(self.td.collect, filename), "r") as fp: data = list(i.strip() for i in fp.readlines()) @@ -961,7 +970,8 @@ return None -def generate_user_data(collect_scripts=None, apt_proxy=None): +def generate_user_data(collect_scripts=None, apt_proxy=None, + boot_cloudconf=None): # this returns the user data for the *booted* system # its a cloud-config-archive type, which is # just a list of parts. the 'x-shellscript' parts @@ -986,6 +996,10 @@ 'content': yaml.dump(base_cloudconfig, indent=1)}, {'type': 'text/cloud-config', 'content': ssh_keys}] + if boot_cloudconf is not None: + parts.append({'type': 'text/cloud-config', 'content': + yaml.dump(boot_cloudconf, indent=1)}) + output_dir = '/mnt/output' output_dir_macro = 'OUTPUT_COLLECT_D' output_device = '/dev/disk/by-id/virtio-%s' % OUTPUT_DISK_NAME === added file 'tests/vmtests/test_apt_config_cmd.py' --- tests/vmtests/test_apt_config_cmd.py 1970-01-01 00:00:00 +0000 +++ tests/vmtests/test_apt_config_cmd.py 2016-07-26 19:53:43 +0000 @@ -0,0 +1,55 @@ +""" test_apt_config_cmd + Collection of tests for the apt configuration features when called via the + apt-config standalone command. +""" +import textwrap + +from . import VMBaseClass +from .releases import base_vm_classes as relbase + + +class TestAptConfigCMD(VMBaseClass): + """TestAptConfigCMD - test standalone command""" + conf_file = "examples/tests/apt_config_command.yaml" + interactive = False + extra_disks = [] + fstab_expected = {} + disk_to_check = [] + collect_scripts = [textwrap.dedent(""" + cd OUTPUT_COLLECT_D + cat /etc/fstab > fstab + ls /dev/disk/by-dname > ls_dname + find /etc/network/interfaces.d > find_interfacesd + cp /etc/apt/sources.list.d/curtin-dev-ubuntu-test-archive-*.list . + cp /etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg . + apt-cache policy | grep proposed > proposed-enabled + """)] + + def test_cmd_proposed_enabled(self): + """check if proposed was enabled""" + self.output_files_exist(["proposed-enabled"]) + self.check_file_regex("proposed-enabled", + r"500.*%s-proposed" % self.release) + + def test_cmd_ppa_enabled(self): + """check if specified curtin-dev ppa was enabled""" + self.output_files_exist( + ["curtin-dev-ubuntu-test-archive-%s.list" % self.release]) + self.check_file_regex("curtin-dev-ubuntu-test-archive-%s.list" % + self.release, + (r"http://ppa.launchpad.net/" + r"curtin-dev/test-archive/ubuntu" + r" %s main" % self.release)) + + def test_cmd_preserve_source(self): + """check if cloud-init was prevented from overwriting""" + self.output_files_exist(["curtin-preserve-sources.cfg"]) + self.check_file_regex("curtin-preserve-sources.cfg", + "apt_preserve_sources_list.*true") + + +class XenialTestAptConfigCMDCMD(relbase.xenial, TestAptConfigCMD): + """ XenialTestAptSrcModifyCMD + apt feature Test for Xenial using the standalone command + """ + __test__ = True === added file 'tests/vmtests/test_apt_source.py' --- tests/vmtests/test_apt_source.py 1970-01-01 00:00:00 +0000 +++ tests/vmtests/test_apt_source.py 2016-07-26 19:53:43 +0000 @@ -0,0 +1,277 @@ +""" test_apt_source + Collection of tests for the apt configuration features +""" +import textwrap + +from . import VMBaseClass +from .releases import base_vm_classes as relbase + +from unittest import SkipTest +from curtin import util + + +class TestAptSrcAbs(VMBaseClass): + """TestAptSrcAbs - Basic tests for apt features of curtin""" + interactive = False + extra_disks = [] + fstab_expected = {} + disk_to_check = [] + collect_scripts = [textwrap.dedent(""" + cd OUTPUT_COLLECT_D + cat /etc/fstab > fstab + ls /dev/disk/by-dname > ls_dname + find /etc/network/interfaces.d > find_interfacesd + apt-key list "F430BBA5" > keyid-F430BBA5 + apt-key list "0165013E" > keyppa-0165013E + apt-key list "F470A0AC" > keylongid-F470A0AC + apt-key list "8280B242" > keyraw-8280B242 + ls -laF /etc/apt/sources.list.d/ > sources.list.d + cp /etc/apt/sources.list.d/curtin-dev-ppa.list . + cp /etc/apt/sources.list.d/my-repo2.list . + cp /etc/apt/sources.list.d/my-repo4.list . + cp /etc/apt/sources.list.d/curtin-dev-ubuntu-test-archive-*.list . + find /etc/apt/sources.list.d/ -maxdepth 1 -name "*ignore*" | wc -l > ic + apt-config dump | grep Retries > aptconf + cp /etc/apt/sources.list sources.list + cp /etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg . + """)] + mirror = "http://us.archive.ubuntu.com/ubuntu" + secmirror = "http://security.ubuntu.com/ubuntu" + + def test_output_files_exist(self): + """test_output_files_exist - Check if all output files exist""" + self.output_files_exist( + ["fstab", "ic", "keyid-F430BBA5", "keylongid-F470A0AC", + "keyraw-8280B242", "keyppa-0165013E", "aptconf", "sources.list", + "curtin-dev-ppa.list", "my-repo2.list", "my-repo4.list"]) + self.output_files_exist( + ["curtin-dev-ubuntu-test-archive-%s.list" % self.release]) + + def test_keys_imported(self): + """test_keys_imported - Check if all keys are imported correctly""" + self.check_file_regex("keyid-F430BBA5", + r"Launchpad PPA for Ubuntu Screen Profile") + self.check_file_regex("keylongid-F470A0AC", + r"Ryan Harper") + self.check_file_regex("keyppa-0165013E", + r"Launchpad PPA for curtin developers") + self.check_file_regex("keyraw-8280B242", + r"Christian Ehrhardt") + + def test_preserve_source(self): + """test_preserve_source - no clobbering sources.list by cloud-init""" + self.output_files_exist(["curtin-preserve-sources.cfg"]) + self.check_file_regex("curtin-preserve-sources.cfg", + "apt_preserve_sources_list.*true") + + def test_source_files(self): + """test_source_files - Check generated .lists for correct content""" + # hard coded deb lines + self.check_file_strippedline("curtin-dev-ppa.list", + ("deb http://ppa.launchpad.net/curtin-dev" + "/test-archive/ubuntu xenial main")) + self.check_file_strippedline("my-repo4.list", + ("deb http://ppa.launchpad.net/curtin-dev" + "/test-archive/ubuntu xenial main")) + # mirror and release replacement in deb line + self.check_file_strippedline("my-repo2.list", "deb %s %s multiverse" % + (self.mirror, self.release)) + # auto creation by apt-add-repository + self.check_file_regex("curtin-dev-ubuntu-test-archive-%s.list" % + self.release, + (r"http://ppa.launchpad.net/" + r"curtin-dev/test-archive/ubuntu" + r" %s main" % self.release)) + + def test_ignore_count(self): + """test_ignore_count - Check for files that should not be created""" + self.check_file_strippedline("ic", "0") + + def test_apt_conf(self): + """test_apt_conf - Check if the selected apt conf was set""" + self.check_file_strippedline("aptconf", 'Acquire::Retries "3";') + + +class TestAptSrcCustom(TestAptSrcAbs): + """TestAptSrcNormal - tests valid in the custom sources.list case""" + conf_file = "examples/tests/apt_source_custom.yaml" + + def test_custom_source_list(self): + """test_custom_source_list - Check custom sources with replacement""" + # check that all replacements happened + self.check_file_strippedline("sources.list", + "deb %s %s main restricted" % + (self.mirror, self.release)) + self.check_file_strippedline("sources.list", + "deb-src %s %s main restricted" % + (self.mirror, self.release)) + self.check_file_strippedline("sources.list", + "deb %s %s universe restricted" % + (self.mirror, self.release)) + self.check_file_strippedline("sources.list", + "deb %s %s-security multiverse" % + (self.secmirror, self.release)) + # check for something that guarantees us to come from our test + self.check_file_strippedline("sources.list", + "# nice line to check in test") + + +class TestAptSrcPreserve(TestAptSrcAbs): + """TestAptSrcPreserve - tests valid in the preserved sources.list case""" + conf_file = "examples/tests/apt_source_preserve.yaml" + boot_cloudconf = None + + def test_preserved_source_list(self): + """test_preserved_source_list - Check sources to be preserved as-is""" + # curtin didn't touch it, so we should find what curtin set as default + self.check_file_regex("sources.list", + r"this file is written by cloud-init") + + # overwrite inherited check to match situation here + def test_preserve_source(self): + """test_preserve_source - check apt_preserve_sources_list not set""" + self.output_files_dont_exist(["curtin-preserve-sources.cfg"]) + + +class TestAptSrcModify(TestAptSrcAbs): + """TestAptSrcModify - tests modifying sources.list""" + conf_file = "examples/tests/apt_source_modify.yaml" + + def test_modified_source_list(self): + """test_modified_source_list - Check sources with replacement""" + # we set us.archive which is non default, check for that + # this will catch if a target ever changes the expected defaults we + # have to replace in case there is no custom template + self.check_file_regex("sources.list", + r"us.archive.ubuntu.com") + self.check_file_regex("sources.list", + r"security.ubuntu.com") + + +class TestAptSrcDisablePockets(TestAptSrcAbs): + """TestAptSrcDisablePockets - tests disabling a suite in sources.list""" + conf_file = "examples/tests/apt_source_modify_disable_suite.yaml" + + def test_disabled_suite(self): + """test_disabled_suite - Check if suites were disabled""" + # two not disabled + self.check_file_regex("sources.list", + r"deb.*us.archive.ubuntu.com") + self.check_file_regex("sources.list", + r"deb.*security.ubuntu.com") + # updates disabled + self.check_file_regex("sources.list", + r"# suite disabled by curtin:.*-updates") + + +class TestAptSrcModifyArches(TestAptSrcModify): + """TestAptSrcModify - tests modifying sources.list with per arch mirror""" + # same test, just different yaml to specify the mirrors per arch + conf_file = "examples/tests/apt_source_modify_arches.yaml" + + +class TestAptSrcSearch(TestAptSrcAbs): + """TestAptSrcSearch - tests checking a list of mirror options""" + conf_file = "examples/tests/apt_source_search.yaml" + + def test_mirror_search(self): + """test_mirror_search + Check searching through a mirror list + This is checked in the test (late) intentionally. + No matter if resolution worked or failed it shouldn't fail + fatally (python error and trace). + We just can't rely on the content to be found in that case + so we skip the check then.""" + res1 = util.is_resolvable_url("http://does.not.exist/ubuntu") + res2 = util.is_resolvable_url("http://does.also.not.exist/ubuntu") + res3 = util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") + res4 = util.is_resolvable_url("http://security.ubuntu.com/ubuntu") + if res1 or res2 or not res3 or not res4: + raise SkipTest(("Name resolution not as required" + "(%s, %s, %s, %s)" % (res1, res2, res3, res4))) + + self.check_file_regex("sources.list", + r"us.archive.ubuntu.com") + self.check_file_regex("sources.list", + r"security.ubuntu.com") + + +class TestAptSrcSearchDNS(VMBaseClass): + """TestAptSrcSearchDNS - tests checking for predefined DNS names""" + interactive = False + extra_disks = [] + fstab_expected = {} + conf_file = "examples/tests/apt_source_search_dns.yaml" + disk_to_check = [] + collect_scripts = [textwrap.dedent(""" + cd OUTPUT_COLLECT_D + cat /etc/fstab > fstab + ls /dev/disk/by-dname > ls_dname + find /etc/network/interfaces.d > find_interfacesd + cp /etc/apt/sources.list.d/dnssearch.list.disabled . + """)] + + def test_output_files_exist(self): + """test_output_files_exist - Check if all output files exist""" + self.output_files_exist(["fstab", "dnssearch.list.disabled"]) + + def test_mirror_search_dns(self): + """test_mirror_search_dns - tests checking for predefined DNS names""" + # these should be the first it got resolved, so they should be in the + # sources.list file. We want to see that .lcoaldomain was not picked + # but instead what we added to the temp /etc/hosts + self.check_file_regex("dnssearch.list.disabled", + r"ubuntu-mirror/ubuntu.*multiverse") + self.check_file_regex("dnssearch.list.disabled", + r"ubuntu-mirror/ubuntu.*universe") + self.check_file_regex("dnssearch.list.disabled", + r"ubuntu-security-mirror/ubuntu.*main") + + +class XenialTestAptSrcCustom(relbase.xenial, TestAptSrcCustom): + """ XenialTestAptSrcCustom + apt feature Test for Xenial with a custom template + """ + __test__ = True + + +class XenialTestAptSrcPreserve(relbase.xenial, TestAptSrcPreserve): + """ XenialTestAptSrcPreserve + apt feature Test for Xenial with apt_preserve_sources_list enabled + """ + __test__ = True + + +class XenialTestAptSrcModify(relbase.xenial, TestAptSrcModify): + """ XenialTestAptSrcModify + apt feature Test for Xenial modifying the sources.list of the image + """ + __test__ = True + + +class XenialTestAptSrcSearch(relbase.xenial, TestAptSrcSearch): + """ XenialTestAptSrcModify + apt feature Test for Xenial searching for mirrors + """ + __test__ = True + + +class XenialTestAptSrcSearchDNS(relbase.xenial, TestAptSrcSearchDNS): + """ XenialTestAptSrcModify + apt feature Test for Xenial searching for predefined DNS names + """ + __test__ = True + + +class XenialTestAptSrcModifyArches(relbase.xenial, TestAptSrcModifyArches): + """ XenialTestAptSrcModifyArches + apt feature Test for Xenial checking per arch mirror specification + """ + __test__ = True + + +class XenialTestAptSrcDisablePockets(relbase.xenial, TestAptSrcDisablePockets): + """ XenialTestAptSrcDisablePockets + apt feature Test for Xenial disabling a suite + """ + __test__ = True === added file 'tests/vmtests/test_old_apt_features.py' --- tests/vmtests/test_old_apt_features.py 1970-01-01 00:00:00 +0000 +++ tests/vmtests/test_old_apt_features.py 2016-07-26 19:53:43 +0000 @@ -0,0 +1,80 @@ +""" testold_apt_features + Testing the former minimal apt features of curtin +""" +import re +import textwrap + +from . import VMBaseClass +from .releases import base_vm_classes as relbase + +from curtin import util + + +class TestOldAptAbs(VMBaseClass): + """TestOldAptAbs - Basic tests for old apt features of curtin""" + interactive = False + extra_disks = [] + fstab_expected = {} + disk_to_check = [] + collect_scripts = [textwrap.dedent(""" + cd OUTPUT_COLLECT_D + cat /etc/fstab > fstab + ls /dev/disk/by-dname > ls_dname + find /etc/network/interfaces.d > find_interfacesd + grep -A 3 "Name: debconf/priority" /var/cache/debconf/config.dat > debc + apt-config dump > aptconf + cp /etc/apt/apt.conf.d/90curtin-aptproxy . + cp /etc/apt/sources.list . + cp /etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg . + """)] + arch = util.get_architecture() + if arch in ['amd64', 'i386']: + conf_file = "examples/tests/test_old_apt_features.yaml" + exp_mirror = "http://us.archive.ubuntu.com/ubuntu" + exp_secmirror = "http://archive.ubuntu.com/ubuntu" + if arch in ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']: + conf_file = "examples/tests/test_old_apt_features_ports.yaml" + exp_mirror = "http://ports.ubuntu.com/ubuntu-ports" + exp_secmirror = "http://ports.ubuntu.com/ubuntu-ports" + + def test_output_files_exist(self): + """test_output_files_exist - Check if all output files exist""" + self.output_files_exist( + ["debc", "aptconf", "sources.list", "90curtin-aptproxy", + "curtin-preserve-sources.cfg"]) + + def test_preserve_source(self): + """test_preserve_source - no clobbering sources.list by cloud-init""" + self.check_file_regex("curtin-preserve-sources.cfg", + "apt_preserve_sources_list.*true") + + def test_debconf(self): + """test_debconf - Check if debconf is in place""" + self.check_file_strippedline("debc", "Value: low") + + def test_aptconf(self): + """test_aptconf - Check if apt conf for proxy is in place""" + # this gets configured by tools/launch and get_apt_proxy in + # tests/vmtests/__init__.py, so compare with those + rproxy = r"Acquire::http::Proxy \"" + re.escape(self.proxy) + r"\";" + self.check_file_regex("aptconf", rproxy) + self.check_file_regex("90curtin-aptproxy", rproxy) + + def test_mirrors(self): + """test_mirrors - Check for mirrors placed in source.list""" + + self.check_file_strippedline("sources.list", + "deb %s %s" % + (self.exp_mirror, self.release) + + " main restricted universe multiverse") + self.check_file_strippedline("sources.list", + "deb %s %s-security" % + (self.exp_secmirror, self.release) + + " main restricted universe multiverse") + + +class XenialTestOldApt(relbase.xenial, TestOldAptAbs): + """ XenialTestOldApt + Old apt features for Xenial + """ + __test__ = True