diff --git a/.coveragerc b/.coveragerc index 398ff08..62ef7f2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,3 @@ [run] branch = True +include = bsdploy/* diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..a2ac468 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,66 @@ +--- +name: "CI" + +on: + push: + +jobs: + tests: + name: "Python ${{ matrix.python-version }} (${{ matrix.tox-envs }})" + runs-on: "ubuntu-latest" + env: + PY_COLORS: 1 + + strategy: + matrix: + include: + - python-version: "2.7" + tox-envs: "py27" + continue-on-error: false + - python-version: "3.7" + tox-envs: "py37" + continue-on-error: false + - python-version: "3.8" + tox-envs: "py38" + continue-on-error: false + - python-version: "3.9" + tox-envs: "py39" + continue-on-error: false + - python-version: "3.10" + tox-envs: "py310" + continue-on-error: false + - python-version: "2.7" + tox-envs: "py27-ansible19,py27-ansible24,py27-ansible25,py27-ansible26,py27-ansible27,py27-ansible28,py27-ansible29,py27-ansible210" + continue-on-error: false + - python-version: "3.7" + tox-envs: "py37-ansible25,py37-ansible26,py37-ansible27,py37-ansible28,py37-ansible29,py37-ansible210" + continue-on-error: false + - python-version: "3.8" + tox-envs: "py38-ansible25,py38-ansible26,py38-ansible27,py38-ansible28,py38-ansible29,py38-ansible210" + continue-on-error: false + - python-version: "3.9" + tox-envs: "py39-ansible25,py39-ansible26,py39-ansible27,py39-ansible28,py39-ansible29,py39-ansible210" + continue-on-error: false + - python-version: "3.10" + tox-envs: "py310-ansible4,py310-ansible5,py310-ansible6" + continue-on-error: false + + steps: + - uses: "actions/checkout@v2" + - uses: "actions/setup-python@v2" + with: + python-version: "${{ matrix.python-version }}" + - name: "Install dependencies" + run: | + set -xe -o nounset + python -VV + python -m site + python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade virtualenv tox + + - name: "Run tox targets for ${{ matrix.python-version }}" + continue-on-error: "${{ matrix.continue-on-error }}" + run: | + set -xe -o nounset + python -m tox -a -vv + python -m tox -v -e ${{ matrix.tox-envs }} -- -v --color=yes diff --git a/.gitignore b/.gitignore index 850b37f..bab6403 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /build/ /develop-eggs/ /dist/ +/eggs/ /include/ /htmlcov/ /lib/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 14db5e3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: python -python: 2.7 -env: - - TOX_ENV=py27 - - TOX_ENV=ansible14 - - TOX_ENV=ansible15 - - TOX_ENV=ansible16 - - TOX_ENV=ansible17 -install: - - pip install setuptools-git - - pip install tox -script: - - tox -e $TOX_ENV -notifications: - irc: - - "irc.freenode.org#bsdploy" - on_success: change - on_failure: change diff --git a/CHANGES.rst b/CHANGES.rst index 6a23b54..1dbaa3a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,86 @@ -1.4 - Unreleased -================ +3.0.0 - 2022-08-17 +================== + +- [feature] support Python 3.10. + + +3.0.0b4 - 2020-09-08 +==================== + +- [feature] support ``bootstrap-password`` option. +- [feature] allow override of ``destroygeom`` via ``bootstrap-destroygeom``. +- [feature] allow override of packages installed during bootstrap via ``bootstrap-packages``. +- [fix] correct path to devfs device in mfsbsd boostrap script. + + +3.0.0b3 - 2019-06-09 +==================== + +- [feature] Python 3.x support with Ansible >= 2.4.x. +- [feature] the sysrc module supports ``dst`` option to use another file then the default ``/etc/rc.conf``. +- [change] renamed ``bootstrap-host-keys`` to ``bootstrap-ssh-host-keys``. +- [change] reintroduce ``bootstrap-ssh-fingerprints`` to allow overriding of ``ssh-fingerprints`` for bootstrapping. + + +3.0.0b2 - 2018-02-11 +==================== + +- [change] ask before automatically generating missing ssh host keys during bootstrap. +- [change] the default location for ``bootstrap-files`` changed from ``[playbooks-directory]/bootstrap-files`` to ``[playbooks-directory]/[instance-uid]/bootstrap-files``. +- [change] renamed ``firstboot-update`` to ``bootstrap-firstboot-update`` to match the other variables. + + +3.0.0b1 - 2018-02-07 +==================== + +- [change] switch to use ploy 2.0.0 and Ansible 2.4.x. +- [feature] the ``fabfile`` option is set if ``[instance-name]/fabfile.py`` exists when the more specific ``[master-name]-[instance-name]/fabfile.py`` doesn't exist. + +- [fix]: honour the ``boottrap-packages`` setting for mfsbsd. + + +2.3.0 - 2017-11-13 +================== + +- [fix] fix pf round-robin lockups. thanks to @igalic for reporting and fixing this issue +- [feature] add ed25519 support in bootstrap needed for paramiko>=2. you should check whether you have ``ssh_host_ed25519_key*`` files on your host which you might want to copy to your bootstrap files directory alongside the other ``ssh_host_*_key*`` files +- [change] removed local rsa1 host key generation + + +2.2.0 - 2016-11-08 +================== + +- [feature] add fabric helpers to keep pkg up-to-date on the host, inside jails and for the bsdploy flavour +- [feature] add support for bootstrapping on Digital Ocean by setting `bootstrap` to `digitalocean` in the `ez-master` definition +- [fix] allow setting a non-default zfs root for ezjail by setting `jails_zfs_root` in the `ez-master` definition + + +2.1.0 - 2015-07-26 +================== + +- [feature] enable jail_parallel_start in rc.conf of jail host +- [fix] import existing zpool in ``zpool`` ansible module if the name matches +- [fix] try to attach geli device first in ``zpool`` ansible module, in case it already exists, only if that fails create it from scratch +- [fix] properly handle multiple geli encrypted devices in ``zpool`` ansible module +- [fix] also honor the ``ploy_jail_host_pkg_repository`` variable during bootstrapping (not just jailhost configuration) +- [feature] files copied during bootstrap can be encrypted using the ``ploy vault`` commands. This is useful for the private ssh host keys in ``bootstrap-files``. +- [fix] fixed setting of virtualbox defaults, so they can be properly overwritten +- [feature] added new variables: ploy_jail_host_cloned_interfaces/ploy_jail_host_default_jail_interface to give more flexiblity around network interface setup +- [change] dropped support for Ansible versions < 1.8 (supports 1.8.x and 1.9.x now) +- [fix] honour proxy setting while installing ezjail itself, not just during ezjail's install run (thanks mzs114! https://github.com/ployground/bsdploy/pull/81) + + +2.0.0 - 2015-03-05 +================== -- [feature] enable `firstboot-freebsd-update `_ by default +- [feature] add support for http proxies +- [change] deactivate pkg's *auto update* feature by default +- [feature] add support for `firstboot-freebsd-update `_ (disabled by default) +- [change] [BACKWARDS INCOMPATIBLE] switched from ipfilter to pf - you must convert any existing ``ipnat_rules`` to the new ``pf_nat_rules``. - [feature] provide defaults for VirtualBox instances (less boilerplate) - [fix] set full /etc/ntp.conf instead of trying to fiddle with an existing one. +- [feature] Support configuration as non-root user (see https://github.com/ployground/bsdploy/issues/62) +- [change] switched to semantic versioning (see http://semver.org) 1.3 - 2014-11-28 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..a573784 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,12 @@ +Copyright (c) 2014, Tom Lazar +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile index c2b998a..db2d74b 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,20 @@ -# convenience Makefile to setup a development verison +# convenience Makefile to setup a development version # of bsdploy and its direct dependencies develop: .installed.cfg -.installed.cfg: bin/ansible bin/buildout buildout.cfg bin/virtualenv +.installed.cfg: bin/buildout buildout.cfg bin/virtualenv bin/buildout -v -# needed for tests -bin/ansible: bin/pip - bin/pip install ansible - bin/buildout: bin/pip - bin/pip install zc.buildout + bin/pip install -U "zc.buildout>=3dev" # needed for tox bin/virtualenv: bin/pip bin/pip install virtualenv bin/pip: - virtualenv . + virtualenv -p python2.7 . clean: git clean -dxxf diff --git a/README.rst b/README.rst index 8c1ae1f..fce94df 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ BSDploy – FreeBSD jail provisioning =================================== -BSDploy is a comprehensive tool to **provision**, **configure** and **maintain** `FreeBSD `_ `jail hosts and jails `_. +BSDploy is a comprehensive tool to remotely **provision**, **configure** and **maintain** `FreeBSD `_ `jail hosts and jails `_. Its main design goal is to lower the barrier to *repeatable jail setups*. @@ -49,7 +49,7 @@ Here's what an abbreviated bootstrapping session of a simple website inside a ja Best of both worlds ------------------- -Combining a declarative approach for setting up the initial state of a system combined with an imperative approach for providing maintenance operations on that state has significant advantages: +Combining a declarative approach for setting up the initial state of a system with an imperative approach for providing maintenance operations on that state has significant advantages: 1. Since the imperative scripts have the luxury of running against a well-defined context, you can keep them short and concise without worrying about all those edge cases. diff --git a/bsdploy/__init__.py b/bsdploy/__init__.py index 456524b..9176bef 100644 --- a/bsdploy/__init__.py +++ b/bsdploy/__init__.py @@ -1,3 +1,4 @@ +from glob import glob from os import path import argparse import logging @@ -14,7 +15,7 @@ roles=[path.join(bsdploy_path, 'roles')], library=[path.join(bsdploy_path, 'library')]) -virtualbox_defaults = { +virtualbox_instance_defaults = { 'vm-ostype': 'FreeBSD_64', 'vm-memory': '2048', 'vm-accelerate3d': 'off', @@ -22,8 +23,26 @@ 'vm-rtcuseutc': 'on', 'vm-boot1': 'disk', 'vm-boot2': 'dvd', + 'vm-nic1': 'hostonly', + 'vm-hostonlyadapter1': 'vboxnet0', } +virtualbox_hostonlyif_defaults = { + 'ip': '192.168.56.1', +} + +virtualbox_dhcpserver_defaults = { + 'ip': '192.168.56.2', + 'netmask': '255.255.255.0', + 'lowerip': '192.168.56.100', + 'upperip': '192.168.56.254', +} + +virtualbox_bootdisk_defaults = { + 'size': '102400', +} + + ez_instance_defaults = { 'ansible_python_interpreter': '/usr/local/bin/python2.7', 'fabric-shell': '/bin/sh -c', @@ -46,25 +65,81 @@ def __call__(self, argv, help): metavar="master", help="Name of the jailhost from the config.", choices=masters, - default=masters.keys()[0] if len(masters) == 1 else None) + default=list(masters.keys())[0] if len(masters) == 1 else None) parser.add_argument( "-y", "--yes", action="store_true", help="Answer yes to all questions.") + parser.add_argument( + "-p", "--http-proxy", + help="Use http proxy for bootstrapping and pkg installation") args = parser.parse_args(argv) master = args.master if len(masters) == 1 else args.master[0] instance = self.ctrl.instances[master] + instance.config.setdefault('ssh-timeout', 90) instance.hooks.before_bsdploy_bootstrap(instance) - instance.do('bootstrap', **{'bootstrap-yes': args.yes}) + bootstrap_args = {'bootstrap-yes': args.yes} + if args.http_proxy: + bootstrap_args['http_proxy'] = args.http_proxy + instance.do('bootstrap', **bootstrap_args) instance.hooks.after_bsdploy_bootstrap(instance) +def get_bootstrap_path(instance): + from ploy_ansible import get_playbooks_directory + host_defined_path = instance.config.get('bootstrap-files') + main_config = instance.master.main_config + ploy_conf_path = main_config.path + if host_defined_path is None: + playbooks_directory = get_playbooks_directory(main_config) + bootstrap_path = path.join(playbooks_directory, instance.uid, 'bootstrap-files') + else: + bootstrap_path = path.join(ploy_conf_path, host_defined_path) + return bootstrap_path + + +def get_ssh_key_paths(instance): + bootstrap_path = get_bootstrap_path(instance) + glob_path = path.join(bootstrap_path, 'ssh_host*_key.pub') + key_paths = [] + for ssh_key in glob(glob_path): + ssh_key = path.abspath(ssh_key) + key_paths.append(ssh_key) + return key_paths + + def augment_instance(instance): - from ploy_ansible import get_playbooks_directory, has_playbook + from ploy_ansible import get_playbooks_directory + from ploy_ansible import has_playbook + from ploy.config import ConfigSection + + main_config = instance.master.main_config + # provide virtualbox specific convenience defaults: if instance.master.sectiongroupname == ('vb-instance'): - for key, value in virtualbox_defaults.items(): + + # default values for virtualbox instance + for key, value in virtualbox_instance_defaults.items(): instance.config.setdefault(key, value) + # default hostonly interface + hostonlyif = main_config.setdefault('vb-hostonlyif', ConfigSection()) + vboxnet0 = hostonlyif.setdefault('vboxnet0', ConfigSection()) + for key, value in virtualbox_hostonlyif_defaults.items(): + vboxnet0.setdefault(key, value) + + # default dhcp server + dhcpserver = main_config.setdefault('vb-dhcpserver', ConfigSection()) + vboxnet0 = dhcpserver.setdefault('vboxnet0', ConfigSection()) + for key, value in virtualbox_dhcpserver_defaults.items(): + vboxnet0.setdefault(key, value) + + # default virtual disk + if 'vb-disk:defaultdisk' in instance.config.get('storage', {}): + disks = main_config.setdefault('vb-disk', ConfigSection()) + defaultdisk = disks.setdefault('defaultdisk', ConfigSection()) + for key, value in virtualbox_bootdisk_defaults.items(): + defaultdisk.setdefault(key, value) + if not instance.master.sectiongroupname.startswith('ez-'): return @@ -72,10 +147,14 @@ def augment_instance(instance): instance.config.setdefault(key, value) if 'fabfile' not in instance.config: - playbooks_directory = get_playbooks_directory(instance.master.main_config) + playbooks_directory = get_playbooks_directory(main_config) fabfile = path.join(playbooks_directory, instance.uid, 'fabfile.py') if path.exists(fabfile): instance.config['fabfile'] = fabfile + else: + fabfile = path.join(playbooks_directory, instance.id, 'fabfile.py') + if path.exists(fabfile): + instance.config['fabfile'] = fabfile if instance.master.instance is instance: # for hosts @@ -89,16 +168,9 @@ def augment_instance(instance): sys.exit(1) if not has_playbook(instance): instance.config['roles'] = 'jails_host' - if 'fingerprint' not in instance.config: - host_defined_path = instance.config.get('bootstrap-files') - ploy_conf_path = instance.master.main_config.path - if host_defined_path is None: - bootstrap_path = path.join(ploy_conf_path, '..', 'bootstrap-files') - else: - bootstrap_path = path.join(ploy_conf_path, host_defined_path) - ssh_key = path.abspath(path.join(bootstrap_path, 'ssh_host_rsa_key.pub')) - if path.exists(ssh_key): - instance.config['fingerprint'] = ssh_key + if 'ssh-host-keys' not in instance.config: + key_paths = get_ssh_key_paths(instance) + instance.config['ssh-host-keys'] = "\n".join(key_paths) else: # for jails instance.config.setdefault('startup_script', path.join( diff --git a/bsdploy/bootstrap-files/FreeBSD.conf b/bsdploy/bootstrap-files/FreeBSD.conf index 105d756..2240d0c 100644 --- a/bsdploy/bootstrap-files/FreeBSD.conf +++ b/bsdploy/bootstrap-files/FreeBSD.conf @@ -2,7 +2,7 @@ FreeBSD: { enabled: no } FreeBSD-custom: { - url: "pkg+http://pkg.freeBSD.org/${ABI}/quarterly", + url: "{{ploy_jail_host_pkg_repository}}", mirror_type: "srv", enabled: yes } diff --git a/bsdploy/bootstrap-files/daemonology-files.yml b/bsdploy/bootstrap-files/daemonology-files.yml index 011a510..c676047 100644 --- a/bsdploy/bootstrap-files/daemonology-files.yml +++ b/bsdploy/bootstrap-files/daemonology-files.yml @@ -6,7 +6,3 @@ 'FreeBSD.conf': directory: '/usr/local/etc/pkg/repos' remote: '/usr/local/etc/pkg/repos/FreeBSD.conf' -'pkg.txz': - url: 'http://pkg.freebsd.org/freebsd:9:x86:64/quarterly/Latest/pkg.txz' - directory: '/var/cache/pkg/All' - remote: '/var/cache/pkg/All/pkg.txz' diff --git a/bsdploy/bootstrap-files/files.yml b/bsdploy/bootstrap-files/files.yml index a509d46..5eef1a0 100644 --- a/bsdploy/bootstrap-files/files.yml +++ b/bsdploy/bootstrap-files/files.yml @@ -6,12 +6,12 @@ remote: '/mnt/etc/make.conf' 'pkg.conf': remote: '/mnt/usr/local/etc/pkg.conf' + use_jinja: True +'pf.conf': + remote: '/mnt/etc/pf.conf' 'FreeBSD.conf': directory: '/mnt/usr/local/etc/pkg/repos' remote: '/mnt/usr/local/etc/pkg/repos/FreeBSD.conf' + use_jinja: True 'sshd_config': remote: '/mnt/etc/ssh/sshd_config' -'pkg.txz': - url: 'http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/Latest/pkg.txz' - directory: '/mnt/var/cache/pkg/All' - remote: '/mnt/var/cache/pkg/All/pkg.txz' diff --git a/bsdploy/bootstrap-files/pf.conf b/bsdploy/bootstrap-files/pf.conf new file mode 100644 index 0000000..1f58cd6 --- /dev/null +++ b/bsdploy/bootstrap-files/pf.conf @@ -0,0 +1,2 @@ +pass in all +pass out all diff --git a/bsdploy/bootstrap-files/pkg.conf b/bsdploy/bootstrap-files/pkg.conf index d45c338..8f566f0 100644 --- a/bsdploy/bootstrap-files/pkg.conf +++ b/bsdploy/bootstrap-files/pkg.conf @@ -1 +1,8 @@ ASSUME_ALWAYS_YES: YES +REPO_AUTOUPDATE: NO +{% if http_proxy is defined %} +pkg_env: { + HTTP_PROXY: "{{http_proxy}}", + HTTPS_PROXY: "{{http_proxy}}" +} +{% endif %} diff --git a/bsdploy/bootstrap-files/rc.conf b/bsdploy/bootstrap-files/rc.conf index 99c0a79..04b78c8 100644 --- a/bsdploy/bootstrap-files/rc.conf +++ b/bsdploy/bootstrap-files/rc.conf @@ -2,6 +2,7 @@ hostname="{{hostname}}" sshd_enable="YES" syslogd_flags="-ss" zfs_enable="YES" +pf_enable="YES" {% for interface in interfaces %} ifconfig_{{interface}}="DHCP" {% endfor %} diff --git a/bsdploy/bootstrap_utils.py b/bsdploy/bootstrap_utils.py index aa1385e..06c2aec 100644 --- a/bsdploy/bootstrap_utils.py +++ b/bsdploy/bootstrap_utils.py @@ -1,9 +1,6 @@ from __future__ import print_function -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO -from bsdploy import bsdploy_path +from io import BytesIO +from bsdploy import bsdploy_path, get_bootstrap_path from fabric.api import env, local, put, quiet, run, settings from lazy import lazy from os.path import abspath, dirname, expanduser, exists, join @@ -18,6 +15,18 @@ import yaml +try: + string_types = basestring +except NameError: + string_types = str + + +try: + unicode +except NameError: + unicode = str + + class BootstrapFile(object): def __init__(self, bu, filename, **kwargs): self.bu = weakref.ref(bu) @@ -33,12 +42,12 @@ def expected_path(self): if 'url' in self.info: return self.bu().download_path else: - return self.bu().custom_template_path + return self.bu().bootstrap_path @property def raw_fallback(self): paths = self.info.get('fallback', []) - if isinstance(paths, basestring): + if isinstance(paths, string_types): paths = [paths] return paths @@ -55,7 +64,7 @@ def local(self): if self.raw_fallback: return local_path if not exists(local_path) and not self.url: - local_path = join(self.bu().default_template_path, self.filename) + local_path = join(self.bu().default_bootstrap_path, self.filename) if not exists(local_path) and not self.url: return return local_path @@ -69,27 +78,46 @@ def to_be_fetched(self): @lazy def template_from_file(self): - from ploy_ansible import inject_ansible_paths + from ploy_ansible import ANSIBLE1, inject_ansible_paths inject_ansible_paths() - from ansible.utils.template import template_from_file + if ANSIBLE1: + from ansible.utils.template import template_from_file + return template_from_file + from ansible.parsing.dataloader import DataLoader + from ansible.template import Templar + loader = DataLoader() + + def template_from_file(basedir, path, variables, vault_password=None): + templar = Templar(loader, variables=variables) + with open(path) as f: + template = f.read() + return templar.template(template) return template_from_file - def read(self, context): + def open(self, context): if self.use_jinja: result = self.template_from_file(dirname(self.local), self.local, context) if isinstance(result, unicode): result = result.encode('utf-8') - return result + return BytesIO(result) + elif self.encrypted: + vaultlib = env.instance.get_vault_lib() + with open(self.local, 'r') as f: + result = f.read() + return BytesIO(vaultlib.decrypt(result)) else: return open(self.local, 'r') + def read(self, context): + return self.open(context).read() + class BootstrapUtils: ssh_keys = frozenset([ - ('ssh_host_key', '-t rsa1 -b 1024'), ('ssh_host_rsa_key', '-t rsa'), ('ssh_host_dsa_key', '-t dsa'), - ('ssh_host_ecdsa_key', '-t ecdsa')]) + ('ssh_host_ecdsa_key', '-t ecdsa'), + ('ssh_host_ed25519_key', '-t ed25519')]) upload_authorized_keys = True bootstrap_files_yaml = 'files.yml' @@ -98,33 +126,47 @@ def ploy_conf_path(self): return env.instance.master.main_config.path @property - def default_template_path(self): + def default_bootstrap_path(self): return join(bsdploy_path, 'bootstrap-files') @property - def custom_template_path(self): - host_defined_path = env.instance.config.get('bootstrap-files') - if host_defined_path is None: - return abspath(join(self.ploy_conf_path, '..', 'bootstrap-files')) - else: - return abspath(join(self.ploy_conf_path, host_defined_path)) + def bootstrap_path(self): + return get_bootstrap_path(env.instance) + + @property + def env_vars(self): + env_vars = '' + if env.instance.config.get('http_proxy'): + env_vars = 'setenv http_proxy %s && ' % env.instance.config.get('http_proxy') + env_vars += 'setenv https_proxy %s && ' % env.instance.config.get('http_proxy') + return env_vars @property def download_path(self): return abspath(join(self.ploy_conf_path, '..', 'downloads')) def generate_ssh_keys(self): + missing = [] for ssh_key_name, ssh_keygen_args in sorted(self.ssh_keys): - if not exists(self.custom_template_path): - os.mkdir(self.custom_template_path) - ssh_key = join(self.custom_template_path, ssh_key_name) + ssh_key = join(self.bootstrap_path, ssh_key_name) if exists(ssh_key): continue + missing.append((ssh_key, ssh_key_name, ssh_keygen_args)) + if missing: + yes = env.instance.config.get('bootstrap-yes', False) + print("\n".join("Missing %s." % x[0] for x in missing)) + if not yes and not yesno("Should the above missing ssh host keys be generated in '%s'?" % self.bootstrap_path): + sys.exit(1) + if not exists(self.bootstrap_path): + os.makedirs(self.bootstrap_path) + for ssh_key, ssh_key_name, ssh_keygen_args in missing: with settings(quiet(), warn_only=True): result = local( "ssh-keygen %s -f %s -N ''" % (ssh_keygen_args, ssh_key), capture=True) if result.failed: + print("Generation of %s with '%s' failed." % ( + ssh_key_name, ssh_keygen_args)) continue with settings(quiet()): fingerprint = local( @@ -150,7 +192,7 @@ def bootstrap_files(self): For files that cannot be downloaded (authorized_keys, rc.conf etc.) we allow the user to provide their own version in a ``bootstrap-files`` folder. The location of this folder can either be explicitly provided - via the ``bootstrap-files`` key in the host definition of the config file or it defaults to ``deployment/bootstrap-files``. + via the ``bootstrap-files`` key in the host definition of the config file or it defaults to ``deployment/[instance-uid]/bootstrap-files``. User provided files can be rendered as Jinja2 templates, by providing ``use_jinja: True`` in the YAML file. They will be rendered with the instance configuration dictionary as context. @@ -160,8 +202,8 @@ def bootstrap_files(self): we look in ``~/.ssh/identity.pub``. """ bootstrap_file_yamls = [ - abspath(join(self.default_template_path, self.bootstrap_files_yaml)), - abspath(join(self.custom_template_path, self.bootstrap_files_yaml))] + abspath(join(self.default_bootstrap_path, self.bootstrap_files_yaml)), + abspath(join(self.bootstrap_path, self.bootstrap_files_yaml))] bootstrap_files = dict() if self.upload_authorized_keys: bootstrap_files['authorized_keys'] = BootstrapFile(self, 'authorized_keys', **{ @@ -193,7 +235,7 @@ def bootstrap_files(self): yes = env.instance.config.get('bootstrap-yes', False) if yes or yesno("Should we generate it using the key in '%s'?" % path): if not exists(bf.expected_path): - os.mkdir(bf.expected_path) + os.makedirs(bf.expected_path) with open(bf.local, 'wb') as out: with open(path, 'rb') as f: out.write(f.read()) @@ -216,11 +258,12 @@ def bootstrap_files(self): bootstrap_files[join(path, filename)] = BootstrapFile( self, join(path, filename), **dict( local=join(packages_path, join(path, filename)), - remote=join('/mnt/var/cache/pkg/All', filename))) + remote=join('/mnt/var/cache/pkg/All', filename), + encrypted=False)) if self.ssh_keys is not None: for ssh_key_name, ssh_key_options in list(self.ssh_keys): - ssh_key = join(self.custom_template_path, ssh_key_name) + ssh_key = join(self.bootstrap_path, ssh_key_name) if exists(ssh_key): pub_key_name = '%s.pub' % ssh_key_name pub_key = '%s.pub' % ssh_key @@ -231,12 +274,19 @@ def bootstrap_files(self): self, ssh_key_name, **dict( local=ssh_key, remote='/mnt/etc/ssh/%s' % ssh_key_name, - mode=0600)) + mode=0o600)) bootstrap_files[pub_key_name] = BootstrapFile( self, pub_key_name, **dict( local=pub_key, remote='/mnt/etc/ssh/%s' % pub_key_name, - mode=0644)) + mode=0o644)) + if hasattr(env.instance, 'get_vault_lib'): + vaultlib = env.instance.get_vault_lib() + for bf in bootstrap_files.values(): + if bf.encrypted is None and exists(bf.local): + with open(bf.local) as f: + data = f.read() + bf.info['encrypted'] = vaultlib.is_encrypted(data) return bootstrap_files def print_bootstrap_files(self): @@ -244,9 +294,14 @@ def print_bootstrap_files(self): for filename, bf in sorted(self.bootstrap_files.items()): if not bf.to_be_fetched: print('{0.local} -(template:{0.use_jinja})-> {0.remote}'.format(bf)) - print("\nThe following files will be downloaded on the host during bootstrap:") + to_be_fetched_count = 0 for filename, bf in sorted(self.bootstrap_files.items()): if bf.to_be_fetched: + if to_be_fetched_count == 0: + print("\nThe following files will be downloaded on the host during bootstrap:") + if env.instance.config.get('http_proxy'): + print("\nUsing http proxy {http_proxy}".format(**env.instance.config)) + to_be_fetched_count += 1 print('{0.url} -> {0.remote}'.format(bf)) print() @@ -262,32 +317,39 @@ def create_bootstrap_directories(self): def upload_bootstrap_files(self, context): for filename, bf in sorted(self.bootstrap_files.items()): if bf.remote and exists(bf.local): - if bf.use_jinja: - local = StringIO(bf.read(context)) - else: - local = bf.local - put(local, bf.remote, mode=bf.mode) + put(bf.open(context), bf.remote, mode=bf.mode) def install_pkg(self, root, chroot=None, packages=[]): assert isinstance(chroot, bool) - pkg = self.bootstrap_files['pkg.txz'] - if not exists(pkg.local): - run('fetch -o {0.remote} {0.url}'.format(pkg)) - run('chmod 0600 {0.remote}'.format(pkg)) - run("tar -x -C {root}{chroot} --exclude '+*' -f {0.remote}".format( - pkg, root=root, chroot=' --chroot' if chroot else '')) chroot_prefix = 'chroot %s ' % root if chroot else '' - # run pkg2ng for which the shared library path needs to be updated - run(chroot_prefix + '/etc/rc.d/ldconfig start') - run(chroot_prefix + 'pkg2ng') + + pkg = self.bootstrap_files.get('pkg.txz') + if pkg is not None: + print("\nInstalling pkg") + if not exists(pkg.local): + run(self.env_vars + 'fetch -o {0.remote} {0.url}'.format(pkg), shell=False) + run('chmod 0600 {0.remote}'.format(pkg)) + run("tar -x -C {root}{chroot} --exclude '+*' -f {0.remote}".format( + pkg, root=root, chroot=' --chroot' if chroot else '')) + # run pkg2ng for which the shared library path needs to be updated + run(chroot_prefix + '/etc/rc.d/ldconfig start') + run(chroot_prefix + 'pkg2ng') + + run(self.env_vars + chroot_prefix + 'pkg update', shell=False) + if packages: - run(chroot_prefix + 'pkg install %s' % ' '.join(packages)) + run(self.env_vars + chroot_prefix + 'pkg install %s' % ' '.join(packages), shell=False) @lazy def mounts(self): with settings(quiet()): return run('mount') + @lazy + def os_release(self): + with settings(quiet()): + return run('uname -r') + @lazy def sysctl_devices(self): with settings(quiet()): @@ -352,29 +414,34 @@ def bsd_url(self): bsd_url = env.instance.config.get('bootstrap-bsd-url') if not bsd_url: with settings(quiet(), warn_only=True): - result = run("find /cdrom/ /media/ -name 'base.txz' -exec dirname {} \;") + result = run("find /cdrom/ /media/ -name 'base.txz' -exec dirname {} \\;") if result.return_code == 0: bsd_url = result.strip() return bsd_url + @lazy + def destroygeom(self): + destroygeom = env.instance.config.get('bootstrap-destroygeom') + if destroygeom: + dest = '/root/bsdploy_destroygeom' + put(abspath(join(self.ploy_conf_path, destroygeom)), dest, mode=0o755) + return dest + else: + return 'destroygeom' + @lazy def zfsinstall(self): zfsinstall = env.instance.config.get('bootstrap-zfsinstall') if zfsinstall: - dest = '/root/bin/bsdploy_zfsinstall' - put(abspath(join(self.ploy_conf_path, zfsinstall)), dest, mode=0755) + dest = '/root/bsdploy_zfsinstall' + put(abspath(join(self.ploy_conf_path, zfsinstall)), dest, mode=0o755) return dest else: return 'zfsinstall' def _fetch_packages(self, packagesite, packages): import tarfile - try: - import lzma - except ImportError: - print("ERROR: The lzma package couldn't be imported.") - print("You most likely need to install pyliblzma in your virtualenv.") - sys.exit(1) + import lzma packageinfo = {} print("Loading package information from '%s'." % packagesite) if SafeLoader.__name__ != 'CSafeLoader': diff --git a/bsdploy/fabfile_digitalocean.py b/bsdploy/fabfile_digitalocean.py new file mode 100644 index 0000000..91bdc10 --- /dev/null +++ b/bsdploy/fabfile_digitalocean.py @@ -0,0 +1,65 @@ +# coding: utf-8 +from bsdploy.bootstrap_utils import BootstrapUtils +from fabric.api import env, sudo, run, put, task +from os import path +from time import sleep + +# a plain, default fabfile for jailhosts on digital ocean + + +env.shell = '/bin/sh -c' + + +@task +def bootstrap(**kwargs): + """Digital Oceans FreeBSD droplets are pretty much already pre-bootstrapped, + including having python2.7 and sudo etc. pre-installed. + the only thing we need to change is to allow root to login (without a password) + enable pf and ensure it is running + """ + + bu = BootstrapUtils() + # (temporarily) set the user to `freebsd` + original_host = env.host_string + env.host_string = 'freebsd@%s' % env.instance.uid + # copy DO bsdclout-init results: + if bu.os_release.startswith('10'): + sudo("""cat /etc/rc.digitalocean.d/droplet.conf > /etc/rc.conf""") + sudo("""sysrc zfs_enable=YES""") + sudo("""sysrc sshd_enable=YES""") + # enable and start pf + sudo("""sysrc pf_enable=YES""") + sudo("""sysrc -f /boot/loader.conf pfload=YES""") + sudo('kldload pf', warn_only=True) + sudo('''echo 'pass in all' > /etc/pf.conf''') + sudo('''echo 'pass out all' >> /etc/pf.conf''') + sudo('''chmod 644 /etc/pf.conf''') + sudo('service pf start') + # overwrite sshd_config, because the DO version only contains defaults + # and a line explicitly forbidding root to log in + sudo("""echo 'PermitRootLogin without-password' > /etc/ssh/sshd_config""") + # additionally, make sure the root user is unlocked! + sudo('pw unlock root') + # overwrite the authorized keys for root, because DO creates its entries to explicitly + # disallow root login + bootstrap_files = env.instance.config.get('bootstrap-files', 'bootstrap-files') + put(path.abspath(path.join(env['config_base'], bootstrap_files, 'authorized_keys')), '/tmp/authorized_keys', use_sudo=True) + sudo('''mv /tmp/authorized_keys /root/.ssh/''') + sudo('''chown root:wheel /root/.ssh/authorized_keys''') + + sudo("""service sshd fastreload""") + # revert back to root + env.host_string = original_host + # give sshd a chance to restart + sleep(2) + # clean up DO cloudinit leftovers + run("rm -f /etc/rc.d/digitalocean") + run("rm -rf /etc/rc.digitalocean.d") + run("rm -rf /usr/local/bsd-cloudinit/") + run("pkg remove -y avahi-autoipd || true") + + # allow overwrites from the commandline + env.instance.config.update(kwargs) + + bu.ssh_keys = None + bu.upload_authorized_keys = False diff --git a/bsdploy/fabfile_mfsbsd.py b/bsdploy/fabfile_mfsbsd.py index 88c6851..9522710 100644 --- a/bsdploy/fabfile_mfsbsd.py +++ b/bsdploy/fabfile_mfsbsd.py @@ -1,7 +1,8 @@ # coding: utf-8 +from __future__ import print_function from bsdploy.bootstrap_utils import BootstrapUtils from contextlib import contextmanager -from fabric.api import env, hide, run, settings +from fabric.api import env, hide, run, settings, task from ploy.common import yesno from ploy.config import value_asbool @@ -18,13 +19,34 @@ def _mfsbsd(env, kwargs={}): try: env.shell = '/bin/sh -c' - # default ssh settings for mfsbsd with possible overwrite by bootstrap-fingerprint - fingerprint = env.instance.config.get( - 'bootstrap-fingerprint', - '1f:cb:78:20:b8:97:dd:dc:3d:23:75:f0:bb:ad:84:03') - env.instance.config['fingerprint'] = fingerprint - env.instance.config['password-fallback'] = True - env.instance.config['password'] = 'mfsroot' + # default ssh settings for mfsbsd with possible overwrite by bootstrap-ssh-host-keys + env.instance.config['ssh-host-keys'] = env.instance.config.get( + 'bootstrap-ssh-host-keys', + '\n'.join([ + # mfsbsd 10.3 + 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDnxIsRrqK2Zj73DPB3doYO8eDue2mVcae9oQNAwGz1o7VBmOpAZiscxOz1kg/M/CD3VRchgT5OcbciqJGaWeNyZHzHbVpIzUCycSI28WVpG7B4jXZTcq6vGGBpD22Ms6rTczigEJmshVR3rNxHmswwImmEwR6o1KVRCOAY2gL8Ik6OOKAqWqY8mstx059MsY9usDl2FDn57T8fZ4QMd+DQBEKwhkhqHs8n2WSlJlZqCuWDBNDH0RskDizrZRz+g4ciRwAM5e2dzgaOvtlfT42WD1kxwJIVFJi/1R0O+Xw2/kGyRweJXCqdUbfynFaTm1yen+IUPzNH/jBMtxUiL25r', + # mfsbsd 10.3 se + 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqSVYJPcXOqPEv/RYV5WiDbr9K/Bz5OeU2Hayo+oBMkxwFuv9KSZGHmZ/EbJOVKhdjDtRDgenxluLU6d5F/vWyGK1M1rdzEFuWfUdfe5Htvz1KEgj/nY5x8OC1h5xme1OwCcFF7oAf7GV6YQtsKF0CZoGwSJEuGb988r8le0VqKy/u4nRiTH+pLHcZzgx6khIl1ty+mBTLgAC7tTgXhB7l83lr/HqU+ZLWZbNohbdEdDWJYVdWHWVMdETc6PG8/DISNfdKuq3YfDyQ/0uZ/uGMJKr7y/J/cabi5VRdVZvdqqbEPLW2zjDtXtRh6+yE4NZETSYx+Wu/DZcn8OsR9pr/', + # mfsbsd 11.1 + 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGV77dtuCFh+JXk3gewNaCaW7imdDIXU1xQoW0nmYXJorzEwS8iSEbsbZxN/h8u8nTumPipNy6JeTS21CHIC19A=', + 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBm28fXoW82ISHhm+T4EXc5QU5Fq0tyreJa79UohGwvw', + 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDY+mdQA1uESkvs6V/6SLs+g/8ge9om35ehSzyfZgDZldw5EXlimWzMsI2/eZHgyUJQIZkuP7V/otrWAX5D4K9paqcbUUnDkSenB27VcLFzNq67xWL3kevYMNFFb1Mfg4l9Yiq5mOO8mWDveuQlAR+0CqVo6wOAmCw857x0/raeBruh56zU6i64927sW745BQruFNd+beBW/Sr5yuML8yXWu1LoWOrc+MGVfxbGTcbNEu4CE4voYkxZ9uX+KhSkUO0Sg5fxOrxUDrNeY2oJB74os3WafHuODUaPrXNGOkBpSxF5B/BbqgfCX3H/GQ2gl3NvKT9eRzfwwyoks567Kv1f', + # mfsbsd 11.1 se + 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEvG5hTrYKjRtaIrQED+jd/y/Craqzz/11+ky1P/lyv/e8NH2iX9iPKcVLNQa4G9z2aOBVLFc3GQSyySuFlAB/Q=', + 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAuuvF9hyDyop83H/zYLDqimiR18X+NsAU9UpoDOGHVZ', + 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAI6Yj/xqZt1yrtgsOHv45mfS8gVPh5IVK+n6hjmnV+RlbwFksA7lFsjY8SRasJko4iqJmxRSmT1bJ/fuOegqFaEa2LwSaEM/J7rx9lHroKO6rtx81Ja54IY8bRscNxaxl+LFFw8F0v4F9hIfzhooLCXVaLgH/y0ScW7gjft9J6omUcfZPIvdMJuOYIHIRqLjL+HnmZSZEh0GWpMraCts3ud+na2gcHuWYMmUpbEeQIkG3FsgTsLlpkrpQLApo7fHNo1FxIbufiopdQ4zDDQFNZod9jRbV1GwUVTAHot6uOZg+oxnCKnHriKaY2/N8QISkVDsEMmGR9Ib5eQkJK9Mv', + # mfsbsd 12.0 + 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM0Y/x3g2rByYjkN8oxuHDZV2VgNCqzrZY41QzNPG+FHBbJbNlhq4zjfj550RxxefwZySWkFHfHHnBOmICihRS8=', + 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPRcWIiTYNuxkh6pIAvULxFoYXmhqsDvMWzDqRKNpC7Z', + 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBq/bd0ioNAKwVCN+p5xn4bGdg3QzN3Jqw/OyzykMT08XRSIvfTRO0TzqEKF/wDyOQCm+V6Dm0Wx0/Ybg0lxf12az/sQrRPS8VmviJmcxqOIYSLG4G//7Kn+kkAUarXS8L0NdPyon1eT63tpX2twWiawmasWpkkJS4VFt8c4Obj+96AwFW8N9sLlk6iFskj+hdAAUVZjhy8TzPkBzHY5Cljnwui5vz6RVagX9/fkJHuCSFHrGZ/ouuTJCq0S91cr75fWCifINrGSurOaFv7hAc/7l689qLlfZ4Jc8Lxt5ZTeQOMTYMoKLX4lmCWC8mgCvASzn544kLGMUHC4AkrbcD', + # mfsbsd 12.0 se + 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNJR/RUd2QKYgcBY9987ymlLBGUsQe22A/W9NtJ0P+WPFbtigbcESxi2fjZS2tOw2BRS85r9dxCSxNGlwYjw09o=', + 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICtdH4RvvStuu51nq8oiHRbyBB6UUISEA2iyMbg2t4IO', + 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD28UisgIiBqrlR+47V8v6ek5+fe58iaVIzLvsrEWDREh8QkSR01ZfxXb70oet/5hbRS5Fnnc1evw+5lNLAj3xHN0B1nGL4u3mUdoZX7w3I7llHG6A77Y0UmdgA9GF4xAxSRp75Cv5Ru7AQ4yczIc3J7KKjQgVwGFEdsbUnKENao4+yoHsFOG3eX63Zoqkxv1DphUfVT04IaUf6eyoJBOGmVhplDMWBIxkDG54JiFl/8CjyMEWpnYotmFsDfgfLkgmeyHad+lCvBsEM44QtZGO4F4nko8eFhOH2pwpDeczpgboC3CNjvuHW4ubp/6NUX+IAb812g8+IoCRafCyar1G5'])) + env.instance.config.setdefault('password-fallback', True) + env.instance.config.setdefault('password', 'mfsroot') + if 'bootstrap-ssh-fingerprints' in env.instance.config: + env.instance.config['ssh-fingerprints'] = env.instance.config['bootstrap-ssh-fingerprints'] # allow overwrites from the commandline env.instance.config.update(kwargs) @@ -41,12 +63,16 @@ def _mfsbsd(env, kwargs={}): def _bootstrap(): + if 'bootstrap-password' in env.instance.config: + print("Using password for bootstrap SSH connection") + env.instance.config['password-fallback'] = True + env.instance.config['password'] = env.instance.config['bootstrap-password'] bu = BootstrapUtils() bu.generate_ssh_keys() bu.print_bootstrap_files() # gather infos if not bu.bsd_url: - print("Found no FreeBSD system to install, please specify bootstrap-bsd-url and make sure mfsbsd is running") + print("Found no FreeBSD system to install, please use 'special edition' or specify bootstrap-bsd-url and make sure mfsbsd is running") return # get realmem here, because it may fail and we don't want that to happen # in the middle of the bootstrap @@ -57,7 +83,7 @@ def _bootstrap(): else: print("\nWARNING! Found no suitable network interface!") - template_context = {} + template_context = {"ploy_jail_host_pkg_repository": "pkg+http://pkg.freeBSD.org/${ABI}/quarterly"} # first the config, so we don't get something essential overwritten template_context.update(env.instance.config) template_context.update( @@ -65,7 +91,7 @@ def _bootstrap(): interfaces=bu.phys_interfaces, hostname=env.instance.id) - rc_conf = bu.bootstrap_files['rc.conf'].read(template_context) + rc_conf = bu.bootstrap_files['rc.conf'].read(template_context).decode('utf-8') if not rc_conf.endswith('\n'): print("\nERROR! Your rc.conf doesn't end in a newline:\n==========\n%s<<<<<<<<<<\n" % rc_conf) return @@ -81,33 +107,57 @@ def _bootstrap(): else: if not yesno("\nDidn't find an '%s' setting in rc.conf. You sure that you want to continue?" % ifconfig): return + + print("\nThe generated rc_conf:\n%s" % rc_conf) + print("bootstrap-bsd-url:", bu.bsd_url) + if 'bootstrap-destroygeom' in env.instance.config: + print("bootstrap-destroygeom:", env.instance.config['bootstrap-destroygeom']) + if 'bootstrap-zfsinstall' in env.instance.config: + print("bootstrap-zfsinstall:", env.instance.config['bootstrap-zfsinstall']) + system_pool_name = env.instance.config.get('bootstrap-system-pool-name', 'system') + print("bootstrap-system-pool-name:", system_pool_name) + data_pool_name = env.instance.config.get('bootstrap-data-pool-name', 'tank') + print("bootstrap-data-pool-name:", data_pool_name) + swap_size = env.instance.config.get('bootstrap-swap-size', '%iM' % (realmem * 2)) + print("bootstrap-swap-size:", swap_size) + system_pool_size = env.instance.config.get('bootstrap-system-pool-size', '20G') + print("bootstrap-system-pool-size:", system_pool_size) + firstboot_update = value_asbool(env.instance.config.get('bootstrap-firstboot-update', 'false')) + print("bootstrap-firstboot-update:", firstboot_update) + autoboot_delay = env.instance.config.get('bootstrap-autoboot-delay', '-1') + print("bootstrap-autoboot-delay:", autoboot_delay) + bootstrap_reboot = value_asbool(env.instance.config.get('bootstrap-reboot', 'true')) + print("bootstrap-reboot:", bootstrap_reboot) + bootstrap_packages = env.instance.config.get('bootstrap-packages', 'python27').split() + if firstboot_update: + bootstrap_packages.append('firstboot-freebsd-update') + print("bootstrap-packages:", ", ".join(bootstrap_packages)) + yes = env.instance.config.get('bootstrap-yes', False) if not (yes or yesno("\nContinuing will destroy the existing data on the following devices:\n %s\n\nContinue?" % ' '.join(bu.devices))): return # install FreeBSD in ZFS root devices_args = ' '.join('-d %s' % x for x in bu.devices) - system_pool_name = env.instance.config.get('bootstrap-system-pool-name', 'system') - data_pool_name = env.instance.config.get('bootstrap-data-pool-name', 'tank') swap_arg = '' - swap_size = env.instance.config.get('bootstrap-swap-size', '%iM' % (realmem * 2)) if swap_size: swap_arg = '-s %s' % swap_size system_pool_arg = '' - system_pool_size = env.instance.config.get('bootstrap-system-pool-size', '20G') if system_pool_size: system_pool_arg = '-z %s' % system_pool_size - run('destroygeom {devices_args} -p {system_pool_name} -p {data_pool_name}'.format( + run('{destroygeom} {devices_args} -p {system_pool_name} -p {data_pool_name}'.format( + destroygeom=bu.destroygeom, devices_args=devices_args, system_pool_name=system_pool_name, data_pool_name=data_pool_name)) - run('{zfsinstall} {devices_args} -p {system_pool_name} -V 28 -u {bsd_url} {swap_arg} {system_pool_arg}'.format( + run('{env_vars}{zfsinstall} {devices_args} -p {system_pool_name} -V 28 -u {bsd_url} {swap_arg} {system_pool_arg}'.format( + env_vars=bu.env_vars, zfsinstall=bu.zfsinstall, devices_args=devices_args, system_pool_name=system_pool_name, bsd_url=bu.bsd_url, swap_arg=swap_arg, - system_pool_arg=system_pool_arg)) + system_pool_arg=system_pool_arg), shell=False) # create partitions for data pool, but only if the system pool doesn't use # the whole disk anyway if system_pool_arg: @@ -116,32 +166,32 @@ def _bootstrap(): data_pool_name=data_pool_name, device=device)) # mount devfs inside the new system - if 'devfs on /rw/dev' not in bu.mounts: + if 'devfs on /mnt/dev' not in bu.mounts: run('mount -t devfs devfs /mnt/dev') # setup bare essentials - run('cp /etc/resolv.conf /mnt/etc/resolv.conf') + run('cp /etc/resolv.conf /mnt/etc/resolv.conf', warn_only=True) bu.create_bootstrap_directories() bu.upload_bootstrap_files(template_context) - bootstrap_packages = ['python27'] - if value_asbool(env.instance.config.get('firstboot-update', 'true')): - bootstrap_packages.append('firstboot-freebsd-update') + if firstboot_update: run('''touch /mnt/firstboot''') run('''sysrc -f /mnt/etc/rc.conf firstboot_freebsd_update_enable=YES''') + # update from config + bootstrap_packages += env.instance.config.get('bootstrap-packages', '').split() # we need to install python here, because there is no way to install it via # ansible playbooks bu.install_pkg('/mnt', chroot=True, packages=bootstrap_packages) # set autoboot delay - autoboot_delay = env.instance.config.get('bootstrap-autoboot-delay', '-1') run('echo autoboot_delay=%s >> /mnt/boot/loader.conf' % autoboot_delay) bu.generate_remote_ssh_keys() # reboot - if value_asbool(env.instance.config.get('bootstrap-reboot', 'true')): + if bootstrap_reboot: with settings(hide('warnings'), warn_only=True): run('reboot') +@task def bootstrap(**kwargs): """ bootstrap an instance booted into mfsbsd (http://mfsbsd.vx.sk) """ @@ -149,6 +199,7 @@ def bootstrap(**kwargs): _bootstrap() +@task def fetch_assets(**kwargs): """ download bootstrap assets to control host. If present on the control host they will be uploaded to the target host during bootstrapping. diff --git a/bsdploy/fabutils.py b/bsdploy/fabutils.py index b78a6a6..73f3b15 100644 --- a/bsdploy/fabutils.py +++ b/bsdploy/fabutils.py @@ -1,4 +1,4 @@ -from fabric.api import env, local, run, sudo +from fabric.api import env, local, run, sudo, task from fabric.contrib.project import rsync_project as _rsync_project from ploy.common import shjoin @@ -19,6 +19,7 @@ def rsync_project(*args, **kwargs): _rsync_project(*args, **kwargs) +@task def rsync(*args, **kwargs): """ wrapper around the rsync command. @@ -52,12 +53,27 @@ def rsync(*args, **kwargs): return local(cmd, **kwargs) +@task def service(service=None, action='status'): if service is None: exit("You must provide a service name") sudo('service %s %s' % (service, action), warn_only=True) -def freebsd_update(): - run('freebsd-update cron') - run('freebsd-update install') +@task +def pkg_upgrade(): + run('pkg-static update') + run('pkg-static install -U pkg') + run('pkg update') + run('pkg upgrade') + print("Done.") + + +@task +def update_flavour_pkg(): + """upgrade the pkg tool of the base flavour (so that newly created jails have the latest version)""" + base_cmd = 'pkg-static -r /usr/jails/flavours/bsdploy_base' + run('%s update' % base_cmd) + run('%s install -U pkg' % base_cmd) + run('%s update' % base_cmd) + print("Done.") diff --git a/bsdploy/library/sysrc b/bsdploy/library/sysrc new file mode 100644 index 0000000..c0b707e --- /dev/null +++ b/bsdploy/library/sysrc @@ -0,0 +1,80 @@ +#!/usr/bin/python +DOCUMENTATION = ''' +--- +module: sysrc +author: Florian Schulze +short_description: Manage rc.conf via sysrc command +description: + - Manage rc.conf via sysrc command +''' + + +class Sysrc(object): + + platform = 'FreeBSD' + + def __init__(self, module): + self.module = module + self.changed = False + self.state = self.module.params.pop('state') + self.name = self.module.params.pop('name') + self.value = self.module.params.pop('value') + self.dst = self.module.params.pop('dst') + self.cmd = self.module.get_bin_path('sysrc', required=True) + + def sysrc(self, *params): + params = list(params) + if self.dst: + params = ['-f', self.dst] + params + return self.module.run_command([self.cmd] + params) + + def change_needed(self, *params): + (rc, out, err) = self.sysrc('-c', *params) + return rc != 0 + + def change(self, *params): + if self.module.check_mode: + self.changed = True + return + (rc, out, err) = self.sysrc(*params) + if rc != 0: + self.module.fail_json(msg="Failed to run sysrc %s:\n%s" % (' '.join(params), err)) + self.changed = True + + def __call__(self): + result = dict(name=self.name, state=self.state) + + if self.state == 'present': + if not self.value: + self.module.fail_json(msg="When setting an rc.conf variable, a value is required.") + setting = "%s=%s" % (self.name, self.value) + if self.change_needed(setting): + self.change(setting) + elif self.state == 'absent': + if self.change_needed('-x', self.name): + self.change('-x', self.name) + result['changed'] = self.changed + return result + + +MODULE_SPECS = dict( + argument_spec=dict( + name=dict(required=True, type='str'), + value=dict(type='str'), + state=dict(default='present', choices=['present', 'absent'], type='str'), + dst=dict(type='str')), + supports_check_mode=True) + + +def main(): + module = AnsibleModule(**MODULE_SPECS) + result = Sysrc(module)() + if 'failed' in result: + module.fail_json(**result) + else: + module.exit_json(**result) + + +from ansible.module_utils.basic import * +if __name__ == "__main__": + main() diff --git a/bsdploy/library/zpool b/bsdploy/library/zpool index ff01522..1af3d36 100644 --- a/bsdploy/library/zpool +++ b/bsdploy/library/zpool @@ -33,7 +33,9 @@ class Zpool(object): def exists(self): (rc, out, err) = self.zpool('list', '-H', self.name) - return rc==0 + if self.geli and 'FAULTED' in out: + return False + return rc == 0 def get_devices(self): devices = self.devices @@ -66,20 +68,32 @@ class Zpool(object): self.module.fail_json(msg="Failed to generate '%s'.\n%s" % (self.geli_passphrase_location, err)) with open(self.geli_passphrase_location, 'w') as f: f.write(out) - os.chmod(self.geli_passphrase_location, 0600) + os.chmod(self.geli_passphrase_location, 0o600) def prepare_data_devices(self, devices): data_devices = [] - rc_lines = [] if self.geli: self.prepare_geli_passphrase() rc_devices = [] for device in devices: data_device = '{device}.eli'.format(device=device) + data_devices.append(data_device) + rc_devices.append(device) + if os.path.exists(os.path.join('/dev', data_device)): + continue + # try to attach geli device in case it's already existing + (rc, out, err) = self.module.run_command([ + 'geli', 'attach', + '-j', self.geli_passphrase_location, + device]) + if rc == 0: + continue + # create new geli device (rc, out, err) = self.module.run_command([ 'geli', 'init', '-s', '4096', '-l', '256', '-J', self.geli_passphrase_location, device]) + self.changed = True if rc != 0: self.module.fail_json(msg="Failed to init geli device '%s'.\n%s" % (data_device, err)) (rc, out, err) = self.module.run_command([ @@ -88,19 +102,24 @@ class Zpool(object): device]) if rc != 0: self.module.fail_json(msg="Failed to attach geli device '%s'.\n%s" % (data_device, err)) - data_devices.append(data_device) - rc_devices.append(device) - rc_lines.append(dict( - regexp='^geli_devices=', - line='geli_devices=\\"%s\\"' % ' '.join(rc_devices))) + (rc, out, err) = self.module.run_command([ + 'sysrc', 'geli_devices']) + if rc != 0: + geli_devices = [] + else: + geli_devices = out[14:].split() + geli_devices.extend(rc_devices) + (rc, out, err) = self.module.run_command([ + 'sysrc', 'geli_devices=%s' % ' '.join(sorted(set(geli_devices)))]) + if rc != 0: + self.module.fail_json(msg="Failed to set geli_devices in rc.conf to '%s'.\n%s" % (' '.join(geli_devices), err)) for rc_device in rc_devices: - rc_lines.append(dict( - regexp='^geli_{rc_device}_flags='.format( - rc_device=rc_device.replace('/', '_'), - geli_passphrase_location=self.geli_passphrase_location), - line='geli_{rc_device}_flags=\\"-j {geli_passphrase_location}\\"'.format( - rc_device=rc_device.replace('/', '_'), - geli_passphrase_location=self.geli_passphrase_location))) + key = 'geli_%s_flags' % rc_device.replace('/', '_') + value = '-j %s' % self.geli_passphrase_location + (rc, out, err) = self.module.run_command([ + 'sysrc', '%s=%s' % (key, value)]) + if rc != 0: + self.module.fail_json(msg="Failed to set %s in rc.conf to '%s'.\n%s" % (key, value, err)) else: for device in devices: data_device = '{device}.nop'.format(device=device) @@ -109,7 +128,7 @@ class Zpool(object): if rc != 0: self.module.fail_json(msg="Failed to create nop device '%s'.\n%s" % (data_device, err)) data_devices.append(data_device) - return data_devices, rc_lines + return data_devices def cleanup_data_devices(self, data_devices): if self.geli: @@ -133,24 +152,27 @@ class Zpool(object): return result devices = self.get_devices() raid_mode = self.get_raid_mode(devices) - data_devices, rc_lines = self.prepare_data_devices(devices) - result['rc_lines'] = rc_lines - zpool_args = ['create'] - if self.version: - zpool_args.extend(['-o', 'version=%s' % self.version]) - zpool_args.extend(['-m', 'none', self.name]) - if raid_mode: - zpool_args.append(raid_mode) - zpool_args.extend(data_devices) - (rc, out, err) = self.zpool(*zpool_args) - if rc != 0: - self.module.fail_json(msg="Failed to create zpool with the following arguments:\n%s\n%s" % (' '.join(zpool_args), err)) - self.changed = True + data_devices = self.prepare_data_devices(devices) + # check again in case the zpool wasn't active because of missing geli settings + if not self.exists(): + (rc, out, err) = self.zpool('import', '-f', self.name) + if rc != 0: + zpool_args = ['create'] + if self.version: + zpool_args.extend(['-o', 'version=%s' % self.version]) + zpool_args.extend(['-m', 'none', self.name]) + if raid_mode: + zpool_args.append(raid_mode) + zpool_args.extend(data_devices) + (rc, out, err) = self.zpool(*zpool_args) + if rc != 0: + self.module.fail_json(msg="Failed to create zpool with the following arguments:\n%s\n%s" % (' '.join(zpool_args), err)) + self.changed = True self.cleanup_data_devices(data_devices) return result def destroy(self): - raise NotImplemented + raise NotImplementedError def __call__(self): result = dict(name=self.name, state=self.state) @@ -162,8 +184,6 @@ class Zpool(object): if self.exists(): self.destroy() - if 'rc_lines' not in result: - result['rc_lines'] = [] result['changed'] = self.changed return result @@ -189,6 +209,6 @@ def main(): module.exit_json(**result) -from ansible.module_utils.basic import * +from ansible.module_utils.basic import * # noqa if __name__ == "__main__": main() diff --git a/bsdploy/roles/jails_host/defaults/main.yml b/bsdploy/roles/jails_host/defaults/main.yml index 3d4da93..ae76e6d 100644 --- a/bsdploy/roles/jails_host/defaults/main.yml +++ b/bsdploy/roles/jails_host/defaults/main.yml @@ -6,13 +6,14 @@ ploy_bootstrap_geli: no ploy_bootstrap_zpool_version: 28 ploy_ezjail_install_host: "ftp.freebsd.org" jails_dir: /usr/jails -jails_zfs_root: tank/jails +ploy_jails_zfs_root: "{{ ploy_bootstrap_data_pool_name }}/jails" zfs_checksum: fletcher4 pkg_txz_url: http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/Latest/pkg.txz jails_pkg_url: "pkg+http://pkg.freeBSD.org/${ABI}/latest" -ipnat_jail_networks: +pf_nat_jail_networks: - 10.0.0.0/8 -ipnat_rules: [] +pf_nat_rules: [] +pf_nat_interface: "{{ansible_default_ipv4.interface}}" ploy_jail_host_sshd_listenaddress: "{{ ansible_default_ipv4.address }}" ploy_jail_host_pkg_repository_kind: "quarterly" ploy_jail_host_pkg_repository_url: "pkg+http://pkg.freeBSD.org/${ABI}/{{ ploy_jail_host_pkg_repository_kind }}" @@ -22,3 +23,7 @@ ploy_jail_host_pkg_repository: | mirror_type: "srv", enabled: yes } +ploy_root_user_name: "{{ploy_user | default('root')}}" +ploy_root_home_path: "{{ '/' if ploy_root_user_name == 'root' else '/usr/home/' }}{{ploy_root_user_name}}" +ploy_jail_host_cloned_interfaces: lo1 +ploy_jail_host_default_jail_interface: lo1 diff --git a/bsdploy/roles/jails_host/files/base_flavour_motd b/bsdploy/roles/jails_host/files/base_flavour_motd index 2c9dbf2..722ae57 100644 --- a/bsdploy/roles/jails_host/files/base_flavour_motd +++ b/bsdploy/roles/jails_host/files/base_flavour_motd @@ -1 +1 @@ -Gehe nicht über Los. +Went to Jail. \ No newline at end of file diff --git a/bsdploy/roles/jails_host/files/pkg.conf b/bsdploy/roles/jails_host/files/pkg.conf deleted file mode 100644 index d45c338..0000000 --- a/bsdploy/roles/jails_host/files/pkg.conf +++ /dev/null @@ -1 +0,0 @@ -ASSUME_ALWAYS_YES: YES diff --git a/bsdploy/roles/jails_host/handlers/main.yml b/bsdploy/roles/jails_host/handlers/main.yml index 49e2448..74e0a82 100644 --- a/bsdploy/roles/jails_host/handlers/main.yml +++ b/bsdploy/roles/jails_host/handlers/main.yml @@ -3,6 +3,8 @@ raw: /etc/netstart - name: restart ezjail service: name=ezjail state=restarted +- name: reload pf + service: name=pf state=reloaded - name: restart sshd service: name=sshd state=restarted - name: restart ntpd diff --git a/bsdploy/roles/jails_host/tasks/main.yml b/bsdploy/roles/jails_host/tasks/main.yml index dd3c2ec..887f355 100644 --- a/bsdploy/roles/jails_host/tasks/main.yml +++ b/bsdploy/roles/jails_host/tasks/main.yml @@ -7,65 +7,13 @@ notify: restart sshd - { include: ntpd.yml, tags: ntpd } +- { include: obsolete-ipf.yml, tags: pf } +- { include: pf.yml, tags: pf } -- name: Enable ipfilter in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^ipfilter_enable= - line: ipfilter_enable=\"YES\" - notify: restart network -- name: Set ipfilter_rules in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^ipfilter_rules= - line: ipfilter_rules=\"/etc/ipf.rules\" - notify: restart network -- name: Setup ipf.rules - template: src=ipf.rules dest=/etc/ipf.rules - notify: restart network - -- name: Enable ipmon in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^ipmon_enable= - line: ipmon_enable=\"YES\" - notify: restart network -- name: Set ipmon_flags in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^ipmon_flags= - line: ipmon_flags=\"-Ds\" - notify: restart network - -- name: Enable ipnat in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^ipnat_enable= - line: ipnat_enable=\"YES\" - notify: restart network -- name: Set ipnat_rules in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^ipnat_rules= - line: ipnat_rules=\"/etc/ipnat.rules\" - notify: restart network -- name: Setup ipnat.rules - template: src=ipnat.rules dest=/etc/ipnat.rules - notify: restart network - tags: ipnat_rules - -- name: Enable gateway in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^gateway_enable= - line: gateway_enable=\"YES\" - notify: restart network - -- name: Add lo1 to rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^cloned_interfaces= - line: cloned_interfaces=\"lo1\" +- name: Setup cloned interfaces + sysrc: + name: cloned_interfaces + value: "{{ ploy_jail_host_cloned_interfaces }}" notify: restart network - meta: flush_handlers @@ -87,36 +35,47 @@ notify: reload sysctl tags: sysctl +- name: Ensure helper packages are installed (using http proxy) + pkgng: + name: "ezjail" + state: "present" + when: ploy_http_proxy is defined + environment: + http_proxy: "{{ploy_http_proxy}}" + https_proxy: "{{ploy_http_proxy}}" + - name: Ensure helper packages are installed - pkgng: name={{ item }} state=present - with_items: - - ezjail + pkgng: + name: "ezjail" + state: "present" + when: ploy_http_proxy is not defined - name: Set default jail interface - lineinfile: - dest: /etc/rc.conf - regexp: ^jail_interface= - line: jail_interface=\"lo1\" + sysrc: + name: jail_interface + value: "{{ ploy_jail_host_default_jail_interface }}" notify: restart ezjail - name: Set default jail parameters - lineinfile: - dest: /etc/rc.conf - regexp: ^jail_parameters= - line: jail_parameters=\"allow.raw_sockets=1 allow.sysvipc=1\" + sysrc: + name: jail_parameters + value: "allow.raw_sockets=1 allow.sysvipc=1" notify: restart ezjail - name: Set default jail exec stop - lineinfile: - dest: /etc/rc.conf - regexp: ^jail_exec_stop= - line: jail_exec_stop=\"/bin/sh /etc/rc.shutdown\" + sysrc: + name: jail_exec_stop + value: "/bin/sh /etc/rc.shutdown" notify: restart ezjail +- name: Enable jail_parallel_start + sysrc: + name: jail_parallel_start + value: "YES" - name: Enable ezjail in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^ezjail_enable= - line: ezjail_enable=\"YES\" + service: + name: ezjail + enabled: yes notify: restart ezjail + - name: Setup ezjail.conf template: src=ezjail.conf dest=/usr/local/etc/ezjail.conf notify: restart ezjail @@ -128,15 +87,6 @@ version: "{{ ploy_bootstrap_zpool_version }}" devices: "{{ ploy_bootstrap_data_pool_devices }}" raid_mode: "{{ ploy_bootstrap_raid_mode }}" - register: data_pool_result - -- name: Setup rc.conf lines for data zpool - lineinfile: - dest: /etc/rc.conf - regexp: "{{ item.regexp }}" - line: "{{ item.line }}" - with_items: data_pool_result.rc_lines - when: data_pool_result.rc_lines|default() - name: Set data zpool options zfs: @@ -147,41 +97,52 @@ - name: Jails ZFS file system zfs: - name="{{ ploy_bootstrap_data_pool_name }}/jails" + name="{{ ploy_jails_zfs_root }}" state=present mountpoint=/usr/jails +- name: Initialize ezjail (using http proxy) + command: "ezjail-admin install -h {{ ploy_ezjail_install_host }} -r {{ ploy_ezjail_install_release|default(ansible_distribution_release) }} creates=/usr/jails/basejail" + when: ploy_http_proxy is defined + environment: + http_proxy: "{{ploy_http_proxy}}" + https_proxy: "{{ploy_http_proxy}}" + - name: Initialize ezjail (may take a while) command: "ezjail-admin install -h {{ ploy_ezjail_install_host }} -r {{ ploy_ezjail_install_release|default(ansible_distribution_release) }} creates=/usr/jails/basejail" + when: ploy_http_proxy is not defined - name: Create pkg cache folder file: dest=/var/cache/pkg/All/ state=directory owner=root group=wheel -- name: Download pkg.txz - get_url: dest=/var/cache/pkg/All/pkg.txz url={{ pkg_txz_url }} - - name: Directory for jail flavour "bsdploy_base" file: dest=/usr/jails/flavours/bsdploy_base state=directory owner=root group=wheel -- name: Pkg in bsdploy_base flavour - command: "tar -x -C /usr/jails/flavours/bsdploy_base --chroot --exclude '+*' -f /var/cache/pkg/All/pkg.txz creates=/usr/jails/flavours/bsdploy_base/usr/local/sbin/pkg" - - name: The .ssh directory for root in bsdploy_base flavour - file: dest=/usr/jails/flavours/bsdploy_base/root/.ssh state=directory mode=0600 owner=root group=wheel + file: + dest: "/usr/jails/flavours/bsdploy_base/{{ploy_root_home_path}}/.ssh" + state: directory + mode: "0600" + owner: "{{ploy_root_user_name}}" + group: wheel + - name: The etc directory in bsdploy_base flavour file: dest=/usr/jails/flavours/bsdploy_base/etc state=directory owner=root group=wheel - name: The etc/ssh directory in bsdploy_base flavour file: dest=/usr/jails/flavours/bsdploy_base/etc/ssh state=directory owner=root group=wheel -- copy: src=make.conf dest=/usr/jails/flavours/bsdploy_base/etc/make.conf owner=root group=wheel -- file: dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos state=directory owner=root group=wheel -- copy: src=pkg.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg.conf owner=root group=wheel -- template: src=FreeBSD.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos/FreeBSD.conf owner=root group=wheel +- name: /etc/make.conf in bsdploy_base flavour + copy: src=make.conf dest=/usr/jails/flavours/bsdploy_base/etc/make.conf owner=root group=wheel +- name: /usr/local/etc/pkg/repos directory in bsdploy_base flavour + file: dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos state=directory owner=root group=wheel +- name: /usr/local/etc/pkg.conf in bsdploy_base flavour + template: src=pkg.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg.conf owner=root group=wheel +- name: /usr/local/etc/pkg/repos/FreeBSD.conf in bsdploy_base flavour + template: src=FreeBSD.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos/FreeBSD.conf owner=root group=wheel # # configure bsdploy_base flavour -# TODO: rename to ``bsdploy_base`` # - name: rc.conf for bsdploy_base flavour @@ -196,4 +157,4 @@ changed_when: _cp_settings_result.stdout|default() != '' with_items: - { src: "/etc/resolv.conf", dest: "/usr/jails/flavours/bsdploy_base/etc/resolv.conf" } - - { src: "/root/.ssh/authorized_keys", dest: "/usr/jails/flavours/bsdploy_base/root/.ssh/authorized_keys" } + - { src: "/{{ploy_root_home_path}}/.ssh/authorized_keys", dest: "/usr/jails/flavours/bsdploy_base{{ploy_root_home_path}}/.ssh/authorized_keys" } diff --git a/bsdploy/roles/jails_host/tasks/ntpd.yml b/bsdploy/roles/jails_host/tasks/ntpd.yml index 2ce1f6b..c4a8336 100644 --- a/bsdploy/roles/jails_host/tasks/ntpd.yml +++ b/bsdploy/roles/jails_host/tasks/ntpd.yml @@ -1,11 +1,11 @@ --- - name: Enable ntpd in rc.conf - lineinfile: - dest: /etc/rc.conf - regexp: ^ntpd_enable= - line: ntpd_enable=\"YES\" + service: + name: ntpd + enabled: yes notify: restart ntpd tags: ntpd + - name: Disable public use of ntpd copy: content: | diff --git a/bsdploy/roles/jails_host/tasks/obsolete-ipf.yml b/bsdploy/roles/jails_host/tasks/obsolete-ipf.yml new file mode 100644 index 0000000..efb31ed --- /dev/null +++ b/bsdploy/roles/jails_host/tasks/obsolete-ipf.yml @@ -0,0 +1,45 @@ +--- +- name: Check for old ipnat_rules setting + fail: + msg: "ipnat_rules is not supported anymore, see documentation on pf_nat_rules" + when: ipnat_rules is defined + tags: pf-conf + +- name: Remove ipfilter from rc.conf + lineinfile: + dest: /etc/rc.conf + regexp: ^ipfilter_enable= + state: absent + notify: restart network +- name: Remove ipfilter_rules from rc.conf + lineinfile: + dest: /etc/rc.conf + regexp: ^ipfilter_rules= + state: absent + notify: restart network + +- name: Remove ipmon from rc.conf + lineinfile: + dest: /etc/rc.conf + regexp: ^ipmon_enable= + state: absent + notify: restart network +- name: Remove ipmon_flags from rc.conf + lineinfile: + dest: /etc/rc.conf + regexp: ^ipmon_flags= + state: absent + notify: restart network + +- name: Remove ipnat from rc.conf + lineinfile: + dest: /etc/rc.conf + regexp: ^ipnat_enable= + state: absent + notify: restart network +- name: Remove ipnat_rules from rc.conf + lineinfile: + dest: /etc/rc.conf + regexp: ^ipnat_rules= + state: absent + notify: restart network diff --git a/bsdploy/roles/jails_host/tasks/pf.yml b/bsdploy/roles/jails_host/tasks/pf.yml new file mode 100644 index 0000000..f1d40d2 --- /dev/null +++ b/bsdploy/roles/jails_host/tasks/pf.yml @@ -0,0 +1,41 @@ +--- +- name: Enable pf in rc.conf + service: + name: pf + enabled: yes +- name: Check for /etc/pf.conf + stat: + path: /etc/pf.conf + register: stat_etc_pf_conf +- name: Default pf.conf + copy: + content: | + pass in all + pass out all + dest: /etc/pf.conf + when: stat_etc_pf_conf.stat.exists == False +- name: Stat of /dev/pf + stat: + path: /dev/pf + register: stat_dev_pf +- name: Checking pf + fail: + msg: "The pf service is not running, you have to start it manually" + when: stat_dev_pf.stat.exists == False +- name: Setup pf.conf + template: + src: pf.conf + dest: /etc/pf.conf + validate: '/sbin/pfctl -nf %s' + tags: pf-conf + register: setup_pf_conf_result +- name: Reload pf.conf + raw: service pf reload + when: setup_pf_conf_result.changed + tags: pf-conf + +- name: Enable gateway in rc.conf + sysrc: + name: gateway_enable + value: "YES" + notify: restart network diff --git a/bsdploy/roles/jails_host/templates/ezjail.conf b/bsdploy/roles/jails_host/templates/ezjail.conf index 9c5a035..6b26488 100644 --- a/bsdploy/roles/jails_host/templates/ezjail.conf +++ b/bsdploy/roles/jails_host/templates/ezjail.conf @@ -56,7 +56,7 @@ ezjail_use_zfs="YES" # The name of the ZFS ezjail should create jails on, it will be mounted at the ezjail_jaildir -ezjail_jailzfs="{{ jails_zfs_root }}" +ezjail_jailzfs="{{ ploy_jails_zfs_root }}" # ADVANCED, be very careful! # ezjail_zfs_properties="-o compression=lzjb -o atime=off" # ezjail_zfs_jail_properties="-o dedup=on" diff --git a/bsdploy/roles/jails_host/templates/ipf.rules b/bsdploy/roles/jails_host/templates/ipf.rules deleted file mode 100644 index 208efe4..0000000 --- a/bsdploy/roles/jails_host/templates/ipf.rules +++ /dev/null @@ -1,2 +0,0 @@ -pass in quick on {{ ansible_default_ipv4.interface }} all -pass out quick on {{ ansible_default_ipv4.interface }} all diff --git a/bsdploy/roles/jails_host/templates/ipnat.rules b/bsdploy/roles/jails_host/templates/ipnat.rules deleted file mode 100644 index 3c62ef0..0000000 --- a/bsdploy/roles/jails_host/templates/ipnat.rules +++ /dev/null @@ -1,6 +0,0 @@ -{% for network in ipnat_jail_networks %} -map {{ ansible_default_ipv4.interface }} {{ network }} -> 0/32 -{% endfor %} -{% for rule in ipnat_rules %} -{{ rule }} -{% endfor %} diff --git a/bsdploy/roles/jails_host/templates/pf.conf b/bsdploy/roles/jails_host/templates/pf.conf new file mode 100644 index 0000000..756cb1a --- /dev/null +++ b/bsdploy/roles/jails_host/templates/pf.conf @@ -0,0 +1,8 @@ +{% for network in pf_nat_jail_networks %} +nat on {{ pf_nat_interface }} from {{ network }} to any -> ({{ pf_nat_interface }}:0) +{% endfor %} +{% for rule in pf_nat_rules %} +{{ rule }} +{% endfor %} +pass in all +pass out all diff --git a/bsdploy/roles/jails_host/templates/pkg.conf b/bsdploy/roles/jails_host/templates/pkg.conf new file mode 100644 index 0000000..829c9f4 --- /dev/null +++ b/bsdploy/roles/jails_host/templates/pkg.conf @@ -0,0 +1,8 @@ +ASSUME_ALWAYS_YES: YES +REPO_AUTOUPDATE: NO +{% if ploy_http_proxy is defined %} +pkg_env: { + HTTP_PROXY: "{{ploy_http_proxy}}", + HTTPS_PROXY: "{{ploy_http_proxy}}" +} +{% endif %} diff --git a/bsdploy/roles/zfs_auto_snapshot/tasks/main.yml b/bsdploy/roles/zfs_auto_snapshot/tasks/main.yml index 297c306..483b854 100644 --- a/bsdploy/roles/zfs_auto_snapshot/tasks/main.yml +++ b/bsdploy/roles/zfs_auto_snapshot/tasks/main.yml @@ -1,6 +1,8 @@ --- - name: Ensure zfstools is installed - pkgng: name=zfstools state=present + pkgng: + name: "zfstools" + state: "present" tags: zfs_auto_snapshot - name: Setup hourly zfs-auto-snapshot diff --git a/bsdploy/startup-ansible-jail.sh b/bsdploy/startup-ansible-jail.sh index a2e2e25..8d7d6b8 100644 --- a/bsdploy/startup-ansible-jail.sh +++ b/bsdploy/startup-ansible-jail.sh @@ -3,4 +3,6 @@ exec 1>/var/log/startup.log 2>&1 chmod 0600 /var/log/startup.log set -e set -x -pkg install python27 +pkg update +pkg upgrade +pkg install python2 diff --git a/bsdploy/tests/conftest.py b/bsdploy/tests/conftest.py index ec00bd9..56272f7 100644 --- a/bsdploy/tests/conftest.py +++ b/bsdploy/tests/conftest.py @@ -1,25 +1,35 @@ +from __future__ import print_function from mock import Mock -from ploy.tests.conftest import ployconf, tempdir import pytest -(ployconf, tempdir) # shutup pyflakes - - def pytest_addoption(parser): parser.addoption( "--quickstart-bsdploy", help="Run the quickstart with this bsdploy sdist", - action="store", dest="quickstart_bsdploy") + action="store", dest="quickstart_bsdploy", default=False) parser.addoption( "--ansible-version", help="The ansible version to use for quickstart tests, defaults to newest", action="store", dest="ansible_version") -default_mounts = '\n'.join([ - '/dev/md0 on / (ufs, local, read-only)', - 'devfs on /dev (devfs, local, multilabel)', - 'tmpfs on /rw (tmpfs, local)', - 'devfs on /rw/dev (devfs, local, multilabel)']) +@pytest.fixture +def default_mounts(): + return '\n'.join([ + '/dev/md0 on / (ufs, local, read-only)', + 'devfs on /dev (devfs, local, multilabel)', + 'tmpfs on /rw (tmpfs, local)', + 'devfs on /mnt/dev (devfs, local, multilabel)']) + + +@pytest.fixture +def ctrl(ployconf, tempdir): + from ploy import Controller + import ploy.tests.dummy_plugin + ctrl = Controller(tempdir.directory) + ctrl.plugins = { + 'dummy': ploy.tests.dummy_plugin.plugin} + ctrl.configfile = ployconf.path + return ctrl @pytest.fixture @@ -33,12 +43,15 @@ class RunResult(str): pass -def run_result(out, rc): - result = RunResult(out) - result.return_code = rc - result.succeeded = rc == 0 - result.failed = rc != 0 - return result +@pytest.fixture +def run_result(): + def run_result(out, rc): + result = RunResult(out) + result.return_code = rc + result.succeeded = rc == 0 + result.failed = rc != 0 + return result + return run_result @pytest.fixture @@ -76,7 +89,10 @@ def _put(*args, **kw): for arg, earg in zip(args, eargs): if earg is object: continue - assert arg == earg + if hasattr(arg, 'name'): + assert arg.name == earg + else: + assert arg == earg assert sorted(kw.keys()) == sorted(ekw.keys()) for k in kw: if ekw[k] is object: @@ -112,13 +128,11 @@ def _local(command, **kwargs): @pytest.fixture -def env_mock(fabric_integration, monkeypatch, ployconf): - from fabric.utils import _AttributeDict - env = _AttributeDict() - env.instance = Mock() - env.instance.config = {} - env.instance.master.main_config.path = ployconf.directory - monkeypatch.setattr('bsdploy.bootstrap_utils.env', env) +def env_mock(ctrl, fabric_integration, monkeypatch, ployconf): + from fabric.api import env + ployconf.fill([ + '[dummy-instance:test_instance]']) + env.instance = ctrl.instances['test_instance'] return env @@ -140,7 +154,7 @@ def _yesno(question): expected = '', False cmd, result = expected assert question == cmd - print question + print(question) return result yesno.side_effect = _yesno diff --git a/bsdploy/tests/test_bootstrap_daemonology.py b/bsdploy/tests/test_bootstrap_daemonology.py index 80970a1..d92b3e9 100644 --- a/bsdploy/tests/test_bootstrap_daemonology.py +++ b/bsdploy/tests/test_bootstrap_daemonology.py @@ -29,11 +29,6 @@ def test_bootstrap(bootstrap, capsys, put_mock, run_mock, tempdir, yesno_mock): ("su root -c '/tmp/enable_root_login_on_daemonology.sh'", {}, ''), ('rm /tmp/enable_root_login_on_daemonology.sh', {}, ''), ('mkdir -p "/usr/local/etc/pkg/repos"', {'shell': False}, ''), - ('mkdir -p "/var/cache/pkg/All"', {'shell': False}, ''), - ('fetch -o /var/cache/pkg/All/pkg.txz http://pkg.freebsd.org/freebsd:9:x86:64/quarterly/Latest/pkg.txz', {}, ''), - ('chmod 0600 /var/cache/pkg/All/pkg.txz', {}, ''), - ("tar -x -C / --exclude '+*' -f /var/cache/pkg/All/pkg.txz", {}, ''), - ('/etc/rc.d/ldconfig start', {}, ''), - ('pkg2ng', {}, ''), - ('pkg install python27', {}, '')] + ('pkg update', {'shell': False}, ''), + ('pkg install python27', {'shell': False}, '')] bootstrap() diff --git a/bsdploy/tests/test_bootstrap_mfsbsd.py b/bsdploy/tests/test_bootstrap_mfsbsd.py index 5b1bec0..b38bf50 100644 --- a/bsdploy/tests/test_bootstrap_mfsbsd.py +++ b/bsdploy/tests/test_bootstrap_mfsbsd.py @@ -1,5 +1,4 @@ from bsdploy import bsdploy_path -from bsdploy.tests.conftest import default_mounts, run_result import pytest @@ -15,21 +14,21 @@ def bootstrap(env_mock, environ_mock, monkeypatch, put_mock, run_mock, tempdir, def create_ssh_host_keys(tempdir): - tempdir['bootstrap-files/ssh_host_dsa_key'].fill('dsa') - tempdir['bootstrap-files/ssh_host_dsa_key.pub'].fill('dsa.pub') - tempdir['bootstrap-files/ssh_host_ecdsa_key'].fill('ecdsa') - tempdir['bootstrap-files/ssh_host_ecdsa_key.pub'].fill('ecdsa.pub') - tempdir['bootstrap-files/ssh_host_key'].fill('rsa1') - tempdir['bootstrap-files/ssh_host_key.pub'].fill('rsa1.pub') - tempdir['bootstrap-files/ssh_host_rsa_key'].fill('rsa') - tempdir['bootstrap-files/ssh_host_rsa_key.pub'].fill('rsa.pub') + tempdir['default-test_instance/bootstrap-files/ssh_host_dsa_key'].fill('dsa') + tempdir['default-test_instance/bootstrap-files/ssh_host_dsa_key.pub'].fill('dsa.pub') + tempdir['default-test_instance/bootstrap-files/ssh_host_ecdsa_key'].fill('ecdsa') + tempdir['default-test_instance/bootstrap-files/ssh_host_ecdsa_key.pub'].fill('ecdsa.pub') + tempdir['default-test_instance/bootstrap-files/ssh_host_ed25519_key'].fill('ed25519') + tempdir['default-test_instance/bootstrap-files/ssh_host_ed25519_key.pub'].fill('ed25519.pub') + tempdir['default-test_instance/bootstrap-files/ssh_host_rsa_key'].fill('rsa') + tempdir['default-test_instance/bootstrap-files/ssh_host_rsa_key.pub'].fill('rsa.pub') -def test_bootstrap_ask_to_continue(bootstrap, capsys, run_mock, tempdir, yesno_mock): +def test_bootstrap_ask_to_continue(bootstrap, capsys, default_mounts, run_mock, run_result, tempdir, yesno_mock): format_info = dict( bsdploy_path=bsdploy_path, tempdir=tempdir.directory) - tempdir['bootstrap-files/authorized_keys'].fill('id_dsa') + tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') create_ssh_host_keys(tempdir) run_mock.expected = [ ('mount', {}, default_mounts), @@ -47,24 +46,22 @@ def test_bootstrap_ask_to_continue(bootstrap, capsys, run_mock, tempdir, yesno_m assert out_lines == [ "", "Using these local files for bootstrapping:", - "%(bsdploy_path)s/bootstrap-files/FreeBSD.conf -(template:False)-> /mnt/usr/local/etc/pkg/repos/FreeBSD.conf" % format_info, - "%(tempdir)s/bootstrap-files/authorized_keys -(template:False)-> /mnt/root/.ssh/authorized_keys" % format_info, + "%(bsdploy_path)s/bootstrap-files/FreeBSD.conf -(template:True)-> /mnt/usr/local/etc/pkg/repos/FreeBSD.conf" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/authorized_keys -(template:False)-> /mnt/root/.ssh/authorized_keys" % format_info, "%(bsdploy_path)s/bootstrap-files/make.conf -(template:False)-> /mnt/etc/make.conf" % format_info, - "%(bsdploy_path)s/bootstrap-files/pkg.conf -(template:False)-> /mnt/usr/local/etc/pkg.conf" % format_info, + "%(bsdploy_path)s/bootstrap-files/pf.conf -(template:False)-> /mnt/etc/pf.conf" % format_info, + "%(bsdploy_path)s/bootstrap-files/pkg.conf -(template:True)-> /mnt/usr/local/etc/pkg.conf" % format_info, "%(bsdploy_path)s/bootstrap-files/rc.conf -(template:True)-> /mnt/etc/rc.conf" % format_info, - "%(tempdir)s/bootstrap-files/ssh_host_dsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_dsa_key" % format_info, - "%(tempdir)s/bootstrap-files/ssh_host_dsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_dsa_key.pub" % format_info, - "%(tempdir)s/bootstrap-files/ssh_host_ecdsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_ecdsa_key" % format_info, - "%(tempdir)s/bootstrap-files/ssh_host_ecdsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_ecdsa_key.pub" % format_info, - "%(tempdir)s/bootstrap-files/ssh_host_key -(template:False)-> /mnt/etc/ssh/ssh_host_key" % format_info, - "%(tempdir)s/bootstrap-files/ssh_host_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_key.pub" % format_info, - "%(tempdir)s/bootstrap-files/ssh_host_rsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_rsa_key" % format_info, - "%(tempdir)s/bootstrap-files/ssh_host_rsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_rsa_key.pub" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_dsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_dsa_key" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_dsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_dsa_key.pub" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ecdsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_ecdsa_key" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ecdsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_ecdsa_key.pub" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ed25519_key -(template:False)-> /mnt/etc/ssh/ssh_host_ed25519_key" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ed25519_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_ed25519_key.pub" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_rsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_rsa_key" % format_info, + "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_rsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_rsa_key.pub" % format_info, "%(bsdploy_path)s/bootstrap-files/sshd_config -(template:False)-> /mnt/etc/ssh/sshd_config" % format_info, "", - "The following files will be downloaded on the host during bootstrap:", - "http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/Latest/pkg.txz -> /mnt/var/cache/pkg/All/pkg.txz", - "", "", "Found the following disk devices on the system:", " ada0 cd0", @@ -72,16 +69,34 @@ def test_bootstrap_ask_to_continue(bootstrap, capsys, run_mock, tempdir, yesno_m "Found the following network interfaces, now is your chance to update your rc.conf accordingly!", " em0", "", + 'The generated rc_conf:', + 'hostname="test_instance"', + 'sshd_enable="YES"', + 'syslogd_flags="-ss"', + 'zfs_enable="YES"', + 'pf_enable="YES"', + 'ifconfig_em0="DHCP"', + '', + 'bootstrap-bsd-url: /cdrom/9.2-RELEASE-amd64', + 'bootstrap-system-pool-name: system', + 'bootstrap-data-pool-name: tank', + 'bootstrap-swap-size: 1024M', + 'bootstrap-system-pool-size: 20G', + 'bootstrap-firstboot-update: False', + 'bootstrap-autoboot-delay: -1', + 'bootstrap-reboot: True', + 'bootstrap-packages: python27', + '', "Continuing will destroy the existing data on the following devices:", " ada0", "", "Continue?"] -def test_bootstrap_no_newline_at_end_of_rc_conf(bootstrap, capsys, local_mock, run_mock, tempdir): - tempdir['bootstrap-files/authorized_keys'].fill('id_dsa') +def test_bootstrap_no_newline_at_end_of_rc_conf(bootstrap, capsys, default_mounts, local_mock, run_mock, run_result, tempdir): + tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') create_ssh_host_keys(tempdir) - tempdir['bootstrap-files/rc.conf'].fill('foo') + tempdir['default-test_instance/bootstrap-files/rc.conf'].fill('foo', allow_conf=True) run_mock.expected = [ ('mount', {}, default_mounts), ('test -e /dev/cd0 && mount_cd9660 /dev/cd0 /cdrom || true', {}, '\n'), @@ -100,27 +115,28 @@ def test_bootstrap_no_newline_at_end_of_rc_conf(bootstrap, capsys, local_mock, r ''] -def test_bootstrap(bootstrap, put_mock, run_mock, tempdir, yesno_mock): +def test_bootstrap(bootstrap, default_mounts, put_mock, run_mock, run_result, tempdir, yesno_mock): format_info = dict( bsdploy_path=bsdploy_path, tempdir=tempdir.directory) - tempdir['bootstrap-files/authorized_keys'].fill('id_dsa') + tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') create_ssh_host_keys(tempdir) put_mock.expected = [ - (("%(bsdploy_path)s/bootstrap-files/FreeBSD.conf" % format_info, '/mnt/usr/local/etc/pkg/repos/FreeBSD.conf'), {'mode': None}), - (("%(tempdir)s/bootstrap-files/authorized_keys" % format_info, '/mnt/root/.ssh/authorized_keys'), {'mode': None}), + ((object, '/mnt/usr/local/etc/pkg/repos/FreeBSD.conf'), {'mode': None}), + (("%(tempdir)s/default-test_instance/bootstrap-files/authorized_keys" % format_info, '/mnt/root/.ssh/authorized_keys'), {'mode': None}), (("%(bsdploy_path)s/bootstrap-files/make.conf" % format_info, '/mnt/etc/make.conf'), {'mode': None}), - (("%(bsdploy_path)s/bootstrap-files/pkg.conf" % format_info, '/mnt/usr/local/etc/pkg.conf'), {'mode': None}), # put from upload_template + ((object, '/mnt/etc/pf.conf'), {'mode': None}), + ((object, '/mnt/usr/local/etc/pkg.conf'), {'mode': None}), ((object, '/mnt/etc/rc.conf'), {'mode': None}), - (("%(tempdir)s/bootstrap-files/ssh_host_dsa_key" % format_info, '/mnt/etc/ssh/ssh_host_dsa_key'), {'mode': 0600}), - (("%(tempdir)s/bootstrap-files/ssh_host_dsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_dsa_key.pub'), {'mode': 0644}), - (("%(tempdir)s/bootstrap-files/ssh_host_ecdsa_key" % format_info, '/mnt/etc/ssh/ssh_host_ecdsa_key'), {'mode': 0600}), - (("%(tempdir)s/bootstrap-files/ssh_host_ecdsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_ecdsa_key.pub'), {'mode': 0644}), - (("%(tempdir)s/bootstrap-files/ssh_host_key" % format_info, '/mnt/etc/ssh/ssh_host_key'), {'mode': 0600}), - (("%(tempdir)s/bootstrap-files/ssh_host_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_key.pub'), {'mode': 0644}), - (("%(tempdir)s/bootstrap-files/ssh_host_rsa_key" % format_info, '/mnt/etc/ssh/ssh_host_rsa_key'), {'mode': 0600}), - (("%(tempdir)s/bootstrap-files/ssh_host_rsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_rsa_key.pub'), {'mode': 0644}), + (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_dsa_key" % format_info, '/mnt/etc/ssh/ssh_host_dsa_key'), {'mode': 0o600}), + (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_dsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_dsa_key.pub'), {'mode': 0o644}), + (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ecdsa_key" % format_info, '/mnt/etc/ssh/ssh_host_ecdsa_key'), {'mode': 0o600}), + (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ecdsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_ecdsa_key.pub'), {'mode': 0o644}), + (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ed25519_key" % format_info, '/mnt/etc/ssh/ssh_host_ed25519_key'), {'mode': 0o600}), + (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ed25519_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_ed25519_key.pub'), {'mode': 0o644}), + (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_rsa_key" % format_info, '/mnt/etc/ssh/ssh_host_rsa_key'), {'mode': 0o600}), + (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_rsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_rsa_key.pub'), {'mode': 0o644}), (("%(bsdploy_path)s/bootstrap-files/sshd_config" % format_info, '/mnt/etc/ssh/sshd_config'), {'mode': None}), ] run_mock.expected = [ @@ -132,20 +148,13 @@ def test_bootstrap(bootstrap, put_mock, run_mock, tempdir, yesno_mock): ('sysctl -n kern.disks', {}, 'ada0 cd0\n'), ('ifconfig -l', {}, 'em0 lo0'), ('destroygeom -d ada0 -p system -p tank', {}, ''), - ('zfsinstall -d ada0 -p system -V 28 -u /cdrom/9.2-RELEASE-amd64 -s 1024M -z 20G', {}, ''), + ('zfsinstall -d ada0 -p system -V 28 -u /cdrom/9.2-RELEASE-amd64 -s 1024M -z 20G', {'shell': False}, ''), ('gpart add -t freebsd-zfs -l tank_ada0 ada0', {}, ''), - ('cp /etc/resolv.conf /mnt/etc/resolv.conf', {}, ''), + ('cp /etc/resolv.conf /mnt/etc/resolv.conf', {'warn_only': True}, ''), ('mkdir -p "/mnt/usr/local/etc/pkg/repos"', {'shell': False}, ''), ('mkdir -p "/mnt/root/.ssh" && chmod 0600 "/mnt/root/.ssh"', {'shell': False}, ''), - ('mkdir -p "/mnt/var/cache/pkg/All"', {'shell': False}, ''), - ('touch /mnt/firstboot', {}, ''), - ('sysrc -f /mnt/etc/rc.conf firstboot_freebsd_update_enable=YES', {}, ''), - ('fetch -o /mnt/var/cache/pkg/All/pkg.txz http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/Latest/pkg.txz', {}, ''), - ('chmod 0600 /mnt/var/cache/pkg/All/pkg.txz', {}, ''), - ("tar -x -C /mnt --chroot --exclude '+*' -f /mnt/var/cache/pkg/All/pkg.txz", {}, ''), - ('chroot /mnt /etc/rc.d/ldconfig start', {}, ''), - ('chroot /mnt pkg2ng', {}, ''), - ('chroot /mnt pkg install python27 firstboot-freebsd-update', {}, ''), + ('chroot /mnt pkg update', {'shell': False}, ''), + ('chroot /mnt pkg install python27', {'shell': False}, ''), ('echo autoboot_delay=-1 >> /mnt/boot/loader.conf', {}, ''), ('reboot', {}, '')] yesno_mock.expected = [ diff --git a/bsdploy/tests/test_bootstrap_utils.py b/bsdploy/tests/test_bootstrap_utils.py index bb0ef76..b2e85a0 100644 --- a/bsdploy/tests/test_bootstrap_utils.py +++ b/bsdploy/tests/test_bootstrap_utils.py @@ -1,6 +1,5 @@ from ansible.errors import AnsibleUndefinedVariable from bsdploy import bootstrap_utils -from bsdploy.tests.conftest import default_mounts, run_result import os import pytest @@ -29,7 +28,7 @@ def test_interfaces_missing(bu, run_mock): assert bu.phys_interfaces == [] -def test_devices(bu, run_mock): +def test_devices(bu, default_mounts, run_mock): run_mock.expected = [ ('sysctl -n kern.disks', {}, 'ada0 cd0\n'), ('mount', {}, default_mounts), @@ -39,7 +38,7 @@ def test_devices(bu, run_mock): assert bu.devices == set(['ada0']) -def test_devices_cdrom_mounted(bu, run_mock): +def test_devices_cdrom_mounted(bu, default_mounts, run_mock): run_mock.expected = [ ('sysctl -n kern.disks', {}, 'ada0 cd0\n'), ('mount', {}, '\n'.join([ @@ -50,7 +49,7 @@ def test_devices_cdrom_mounted(bu, run_mock): assert bu.devices == set(['ada0']) -def test_devices_usb_mounted(bu, run_mock): +def test_devices_usb_mounted(bu, default_mounts, run_mock): run_mock.expected = [ ('sysctl -n kern.disks', {}, 'ada0 da0\n'), ('mount', {}, '\n'.join([ @@ -61,7 +60,7 @@ def test_devices_usb_mounted(bu, run_mock): assert bu.devices == set(['ada0']) -def test_devices_different_cdrom(bu, run_mock, env_mock): +def test_devices_different_cdrom(bu, default_mounts, run_mock, env_mock): run_mock.expected = [ ('sysctl -n kern.disks', {}, 'ada0 cd1\n'), ('mount', {}, default_mounts), @@ -72,7 +71,7 @@ def test_devices_different_cdrom(bu, run_mock, env_mock): assert bu.devices == set(['ada0']) -def test_devices_different_usb(bu, run_mock, env_mock): +def test_devices_different_usb(bu, default_mounts, run_mock, env_mock): run_mock.expected = [ ('sysctl -n kern.disks', {}, 'ada0 cd0 da1\n'), ('mount', {}, default_mounts), @@ -83,7 +82,7 @@ def test_devices_different_usb(bu, run_mock, env_mock): assert bu.devices == set(['ada0']) -def test_devices_from_config(bu, run_mock, env_mock): +def test_devices_from_config(bu, default_mounts, run_mock, env_mock): env_mock.instance.config = {'bootstrap-system-devices': 'ada0'} run_mock.expected = [ ('sysctl -n kern.disks', {}, 'ada0 cd0\n'), @@ -94,7 +93,7 @@ def test_devices_from_config(bu, run_mock, env_mock): assert bu.devices == set(['ada0']) -def test_bsd_url(bu, run_mock): +def test_bsd_url(bu, run_mock, run_result): run_mock.expected = [ ('mount', {}, ''), ('test -e /dev/cd0 && mount_cd9660 /dev/cd0 /cdrom || true', {}, '\n'), @@ -103,7 +102,7 @@ def test_bsd_url(bu, run_mock): assert bu.bsd_url == '/cdrom/9.2-RELEASE-amd64' -def test_bsd_url_not_found(bu, run_mock): +def test_bsd_url_not_found(bu, run_mock, run_result): run_mock.expected = [ ('mount', {}, ''), ('test -e /dev/cd0 && mount_cd9660 /dev/cd0 /cdrom || true', {}, '\n'), @@ -129,7 +128,7 @@ def test_bootstrap_files_no_ssh_keys(bu, capsys, tempdir): (out, err) = capsys.readouterr() out_lines = out.splitlines() assert out_lines == [ - "Found no public key in %(tempdir)s/.ssh, you have to create '%(tempdir)s/bootstrap-files/authorized_keys' manually" % format_info] + "Found no public key in %(tempdir)s/.ssh, you have to create '%(tempdir)s/default-test_instance/bootstrap-files/authorized_keys' manually" % format_info] def test_bootstrap_files_multiple_ssh_keys_but_none_used(bu, capsys, tempdir, yesno_mock): @@ -145,7 +144,7 @@ def test_bootstrap_files_multiple_ssh_keys_but_none_used(bu, capsys, tempdir, ye (out, err) = capsys.readouterr() out_lines = out.splitlines() assert out_lines == [ - "The '%(tempdir)s/bootstrap-files/authorized_keys' file is missing." % format_info, + "The '%(tempdir)s/default-test_instance/bootstrap-files/authorized_keys' file is missing." % format_info, "Should we generate it using the key in '%(tempdir)s/.ssh/id_dsa.pub'?" % format_info, "Should we generate it using the key in '%(tempdir)s/.ssh/id_rsa.pub'?" % format_info] @@ -161,47 +160,90 @@ def test_bootstrap_files_multiple_ssh_keys_use_second(bu, capsys, run_mock, temp (out, err) = capsys.readouterr() out_lines = out.splitlines() assert out_lines == [ - "The '%(tempdir)s/bootstrap-files/authorized_keys' file is missing." % format_info, + "The '%(tempdir)s/default-test_instance/bootstrap-files/authorized_keys' file is missing." % format_info, "Should we generate it using the key in '%(tempdir)s/.ssh/id_dsa.pub'?" % format_info, "Should we generate it using the key in '%(tempdir)s/.ssh/id_rsa.pub'?" % format_info] - assert os.path.exists(tempdir['bootstrap-files/authorized_keys'].path) - with open(tempdir['bootstrap-files/authorized_keys'].path) as f: + assert os.path.exists(tempdir['default-test_instance/bootstrap-files/authorized_keys'].path) + with open(tempdir['default-test_instance/bootstrap-files/authorized_keys'].path) as f: assert f.read() == 'id_rsa' +class TestFileEncryption: + @pytest.fixture + def ctrl(self, ctrl): + import ploy_ansible + ctrl.plugins['ansible'] = ploy_ansible.plugin + return ctrl + + @pytest.fixture + def passwd_source(self, monkeypatch): + class DummySource: + id = b'dummy-id' + passwd = b'dummy-vault-password' + bytes = bytes(passwd) + + def get(self, fail_on_error=True): + return self.passwd + src = DummySource() + monkeypatch.setattr('ploy_ansible.get_vault_password_source', lambda x: src) + return src + + def test_bootstrap_files_encrypted(self, bu, env_mock, passwd_source, tempdir): + vaultlib = env_mock.instance.get_vault_lib() + tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') + tempdir['default-test_instance/bootstrap-files/secret.txt'].fill_binary(vaultlib.encrypt('test-secret')) + bindata = ''.join(chr(x) for x in range(256)) + tempdir['default-test_instance/bootstrap-files/secret.bin'].fill_binary(vaultlib.encrypt(bindata)) + with open(tempdir['default-test_instance/bootstrap-files/secret.txt'].path) as f: + assert 'test-secret' not in f.read() + tempdir['default-test_instance/bootstrap-files/files.yml'].fill([ + "'secret.txt':", + " remote: /root/secret.txt"]) + for name, bf in bu.bootstrap_files.items(): + if name == 'secret.txt': + assert bf.encrypted + assert bf.open({}).read() == b'test-secret' + elif name == 'secret.bin': + assert bf.encrypted + assert bf.open({}).read() == bindata + else: + assert not bf.encrypted + + def test_default_rc_conf(bu, tempdir): - tempdir['bootstrap-files/authorized_keys'].fill('id_dsa') + tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') bfs = bu.bootstrap_files result = bfs['rc.conf'].read({ 'hostname': 'foo', 'interfaces': []}) - assert result == '\n'.join([ - 'hostname="foo"', - 'sshd_enable="YES"', - 'syslogd_flags="-ss"', - 'zfs_enable="YES"', - '']) + assert result == b'\n'.join([ + b'hostname="foo"', + b'sshd_enable="YES"', + b'syslogd_flags="-ss"', + b'zfs_enable="YES"', + b'pf_enable="YES"', + b'']) @pytest.mark.parametrize("input, context, expected", [ - ('foo', {}, 'foo'), + ('foo', {}, b'foo'), ('{{ foo }}', {}, AnsibleUndefinedVariable), - ('{{ foo }}', {'foo': 'bar'}, 'bar'), - (' {% for x in xs %}bar\nfoo_{{x}}\n{% endfor %}\n', {'xs': []}, ' \n'), - (' {% for x in xs %}bar\nfoo_{{x}}\n{% endfor %}\n', {'xs': ['bar']}, ' bar\nfoo_bar\n'), - (' {% for x in xs -%}\nfoo_{{x}}\n{% endfor %}\n', {'xs': ['bar']}, ' foo_bar\n'), - (' {% for x in xs %}bar\n foo_{{x}}\n{% endfor %}\n', {'xs': []}, ' \n'), - (' {% for x in xs %}bar\n foo_{{x}}\n{% endfor %}\n', {'xs': ['bar']}, ' bar\n foo_bar\n'), - (' {% for x in xs -%}\n foo_{{x}}\n{% endfor %}\n', {'xs': ['bar']}, ' foo_bar\n'), + ('{{ foo }}', {'foo': 'bar'}, b'bar'), + (' {% for x in xs %}bar\nfoo_{{x}}\n{% endfor %}\n', {'xs': []}, b' \n'), + (' {% for x in xs %}bar\nfoo_{{x}}\n{% endfor %}\n', {'xs': ['bar']}, b' bar\nfoo_bar\n'), + (' {% for x in xs -%}\nfoo_{{x}}\n{% endfor %}\n', {'xs': ['bar']}, b' foo_bar\n'), + (' {% for x in xs %}bar\n foo_{{x}}\n{% endfor %}\n', {'xs': []}, b' \n'), + (' {% for x in xs %}bar\n foo_{{x}}\n{% endfor %}\n', {'xs': ['bar']}, b' bar\n foo_bar\n'), + (' {% for x in xs -%}\n foo_{{x}}\n{% endfor %}\n', {'xs': ['bar']}, b' foo_bar\n'), ]) def test_bootstrap_files_template(bu, input, context, expected, tempdir): - tempdir['bootstrap-files/authorized_keys'].fill('id_dsa') - tempdir['bootstrap-files/rc.conf'].fill(input) + tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') + tempdir['default-test_instance/bootstrap-files/rc.conf'].fill(input, allow_conf=True) bfs = bu.bootstrap_files - if isinstance(expected, basestring): + if isinstance(expected, bytes): result = bfs['rc.conf'].read(context) assert result == expected - assert not isinstance(result, unicode) + assert isinstance(result, bytes) else: with pytest.raises(expected): bfs['rc.conf'].read(context) @@ -209,7 +251,7 @@ def test_bootstrap_files_template(bu, input, context, expected, tempdir): def test_fetch_assets(bu, local_mock, tempdir): format_info = dict(tempdir=tempdir.directory) - tempdir['bootstrap-files/authorized_keys'].fill('id_dsa') + tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') local_mock.expected = [ ('wget -c -O "%(tempdir)s/downloads/pkg.txz" "http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/Latest/pkg.txz"' % format_info, {}, '')] bu.fetch_assets() @@ -219,16 +261,15 @@ def test_fetch_assets_packagesite(bu, local_mock, tempdir): from bsdploy import bsdploy_path pytest.importorskip("lzma") format_info = dict(tempdir=tempdir.directory) - tempdir['bootstrap-files/authorized_keys'].fill('id_dsa') - tempdir['bootstrap-files/files.yml'].fill([ + tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') + tempdir['default-test_instance/bootstrap-files/files.yml'].fill([ "'packagesite.txz':", " url: 'http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/packagesite.txz'", " remote: '/mnt/var/cache/pkg/packagesite.txz'"]) - with open(os.path.join(bsdploy_path, 'tests', 'packagesite.txz')) as f: - tempdir['downloads/packagesite.txz'].fill(f.read()) + with open(os.path.join(bsdploy_path, 'tests', 'packagesite.txz'), 'rb') as f: + tempdir['downloads/packagesite.txz'].fill_binary(f.read()) local_mock.expected = [ ('wget -c -O "%(tempdir)s/downloads/packagesite.txz" "http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/packagesite.txz"' % format_info, {}, ''), - ('wget -c -O "%(tempdir)s/downloads/pkg.txz" "http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/Latest/pkg.txz"' % format_info, {}, ''), ('wget -c -O "%(tempdir)s/downloads/packages/freebsd:10:x86:64/latest/All/python27-2.7.6_4.txz" "http://pkg.freebsd.org/freebsd:10:x86:64/latest/All/python27-2.7.6_4.txz"' % format_info, {}, ''), ('wget -c -O "%(tempdir)s/downloads/packages/freebsd:10:x86:64/latest/All/gettext-0.18.3.1.txz" "http://pkg.freebsd.org/freebsd:10:x86:64/latest/All/gettext-0.18.3.1.txz"' % format_info, {}, ''), ('wget -c -O "%(tempdir)s/downloads/packages/freebsd:10:x86:64/latest/All/libiconv-1.14_3.txz" "http://pkg.freebsd.org/freebsd:10:x86:64/latest/All/libiconv-1.14_3.txz"' % format_info, {}, '')] diff --git a/bsdploy/tests/test_bsdploy.py b/bsdploy/tests/test_bsdploy.py index b155752..5f274b9 100644 --- a/bsdploy/tests/test_bsdploy.py +++ b/bsdploy/tests/test_bsdploy.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import pytest @@ -25,7 +26,7 @@ def ctrl(ployconf, tempdir): def test_bootstrap_command(capsys, ctrl, monkeypatch): def do(self, *args, **kwargs): - print "do with %r, %r called!" % (args, kwargs) + print("do with %r, %r called!" % (args, kwargs)) monkeypatch.setattr('ploy_fabric.do', do) ctrl(['./bin/ploy', 'bootstrap']) (out, err) = capsys.readouterr() @@ -35,15 +36,15 @@ def do(self, *args, **kwargs): def test_augment_ezjail_master(ctrl, ployconf, tempdir): - tempdir['bootstrap-files/ssh_host_rsa_key.pub'].fill('rsa') + tempdir['jailhost/bootstrap-files/ssh_host_rsa_key.pub'].fill('rsa') config = dict(ctrl.instances['jailhost'].config) assert sorted(config.keys()) == [ - 'ansible_python_interpreter', 'fabfile', 'fabric-shell', 'fingerprint', - 'roles'] + 'ansible_python_interpreter', 'fabfile', 'fabric-shell', + 'roles', 'ssh-host-keys'] assert config['ansible_python_interpreter'] == '/usr/local/bin/python2.7' assert config['fabfile'].endswith('fabfile_mfsbsd.py') assert config['fabric-shell'] == '/bin/sh -c' - assert config['fingerprint'].endswith('bootstrap-files/ssh_host_rsa_key.pub') + assert config['ssh-host-keys'].endswith('bootstrap-files/ssh_host_rsa_key.pub') assert os.path.exists(config['fabfile']), "The fabfile '%s' doesn't exist." % config['fabfile'] assert config['roles'] == 'jails_host' @@ -129,6 +130,89 @@ def test_augment_non_ezjail_instance(ctrl, ployconf): assert dict(ctrl.instances['foo'].config) == {} +@pytest.mark.parametrize("config, expected", [ + ([], {'vboxnet0': {'ip': '192.168.56.1'}}), + (['[vb-hostonlyif:vboxnet0]'], {'vboxnet0': {'ip': '192.168.56.1'}}), + (['[vb-hostonlyif:vboxnet0]', 'ip = 192.168.57.1'], + {'vboxnet0': {'ip': '192.168.57.1'}}), + (['[vb-hostonlyif:vboxnet1]', 'ip = 192.168.57.1'], + {'vboxnet0': {'ip': '192.168.56.1'}, 'vboxnet1': {'ip': '192.168.57.1'}})]) +def test_virtualbox_hostonlyif(ctrl, config, expected, ployconf): + import ploy_virtualbox + ctrl.plugins['virtualbox'] = ploy_virtualbox.plugin + ployconf.fill([ + '[vb-instance:vb]'] + config) + # trigger augmentation + ctrl.instances['vb'] + assert ctrl.config['vb-hostonlyif'] == expected + + +@pytest.mark.parametrize("config, expected", [ + ([], + {'vboxnet0': { + 'ip': '192.168.56.2', 'netmask': '255.255.255.0', + 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}}), + (['[vb-dhcpserver:vboxnet0]'], + {'vboxnet0': { + 'ip': '192.168.56.2', 'netmask': '255.255.255.0', + 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}}), + (['[vb-dhcpserver:vboxnet0]', 'ip = 192.168.57.2'], + {'vboxnet0': { + 'ip': '192.168.57.2', 'netmask': '255.255.255.0', + 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}}), + (['[vb-dhcpserver:vboxnet0]', 'netmask = 255.255.0.0'], + {'vboxnet0': { + 'ip': '192.168.56.2', 'netmask': '255.255.0.0', + 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}}), + (['[vb-dhcpserver:vboxnet0]', 'lowerip = 192.168.56.50'], + {'vboxnet0': { + 'ip': '192.168.56.2', 'netmask': '255.255.255.0', + 'lowerip': '192.168.56.50', 'upperip': '192.168.56.254'}}), + (['[vb-dhcpserver:vboxnet0]', 'upperip = 192.168.56.200'], + {'vboxnet0': { + 'ip': '192.168.56.2', 'netmask': '255.255.255.0', + 'lowerip': '192.168.56.100', 'upperip': '192.168.56.200'}}), + (['[vb-dhcpserver:vboxnet0]', 'ip = 192.168.57.2', 'netmask = 255.255.0.0', 'lowerip = 192.168.56.50', 'upperip = 192.168.56.200'], + {'vboxnet0': { + 'ip': '192.168.57.2', 'netmask': '255.255.0.0', + 'lowerip': '192.168.56.50', 'upperip': '192.168.56.200'}}), + (['[vb-dhcpserver:vboxnet1]', 'ip = 192.168.57.2'], + {'vboxnet0': { + 'ip': '192.168.56.2', 'netmask': '255.255.255.0', + 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}, + 'vboxnet1': { + 'ip': '192.168.57.2'}})]) +def test_virtualbox_dhcpserver(ctrl, config, expected, ployconf): + import ploy_virtualbox + ctrl.plugins['virtualbox'] = ploy_virtualbox.plugin + ployconf.fill([ + '[vb-instance:vb]'] + config) + # trigger augmentation + ctrl.instances['vb'] + assert ctrl.config['vb-dhcpserver'] == expected + + +@pytest.mark.parametrize("config, expected", [ + ([], {}), + (['storage = --medium vb-disk:defaultdisk'], + {'defaultdisk': {'size': '102400'}}), + (['storage = --medium vb-disk:defaultdisk', '[vb-disk:defaultdisk]'], + {'defaultdisk': {'size': '102400'}}), + (['storage = --medium vb-disk:defaultdisk', '[vb-disk:defaultdisk]', 'size = 1024000'], + {'defaultdisk': {'size': '1024000'}}), + (['storage = --medium vb-disk:defaultdisk', '[vb-disk:otherdisk]'], + {'defaultdisk': {'size': '102400'}, + 'otherdisk': {}})]) +def test_virtualbox_defaultdisk(ctrl, config, expected, ployconf): + import ploy_virtualbox + ctrl.plugins['virtualbox'] = ploy_virtualbox.plugin + ployconf.fill([ + '[vb-instance:vb]'] + config) + # trigger augmentation + ctrl.instances['vb'] + assert ctrl.config.get('vb-disk', {}) == expected + + @pytest.mark.parametrize("instance, key, value, expected", [ ('foo', 'ansible_python_interpreter', 'python2.7', 'python2.7'), ('foo', 'startup_script', 'foo', {'path': '{tempdir}/etc/foo'}), @@ -157,8 +241,9 @@ def test_augment_overwrite(ctrl, instance, key, value, expected, ployconf, tempd if isinstance(expected, tuple): key, expected = expected if isinstance(expected, dict): - for k, v in expected.items(): - expected[k] = v.format(**format_info) + expected = { + k: v.format(**format_info) + for k, v in expected.items()} else: expected = expected.format(**format_info) assert config[key] == expected diff --git a/bsdploy/tests/test_quickstart.py b/bsdploy/tests/test_quickstart.py index 5a43abf..4d15c07 100644 --- a/bsdploy/tests/test_quickstart.py +++ b/bsdploy/tests/test_quickstart.py @@ -76,10 +76,12 @@ def parse_qs(qs_path): return result -def iter_quickstart_calls(actions, tempdir): +def iter_quickstart_calls(actions, confext, ployconf, tempdir): paths = { - 'ploy.conf': tempdir['etc/ploy.conf'], - 'etc/ploy.conf': tempdir['etc/ploy.conf'], + 'ploy.conf': ployconf, + 'etc/ploy.conf': ployconf, + 'ploy.yml': ployconf, + 'etc/ploy.yml': ployconf, 'files.yml': tempdir['bootstrap-files/files.yml'], 'jailhost.yml': tempdir['host_vars/jailhost.yml'], 'jailhost-demo_jail.yml': tempdir['jailhost-demo_jail.yml']} @@ -93,11 +95,11 @@ def iter_quickstart_calls(actions, tempdir): continue bootstrap = line.endswith('bootstrap') if bootstrap: - yield (action[0], time.sleep, (90,), {}) + yield (action[0], wait_for_ssh, ('localhost', 44003), {}) line = '%s -y' % line yield (action[0], subprocess.check_call, (line,), dict(shell=True)) if bootstrap: - yield (action[0], time.sleep, (90,), {}) + yield (action[0], wait_for_ssh, ('localhost', 44003), {}) elif action[0] == 'create': name = action[1][-1] content = list(action[2]) @@ -115,58 +117,42 @@ def iter_quickstart_calls(actions, tempdir): pytest.fail("Unknown action %s" % action[0]) -def test_quickstart_calls(qs_path, tempdir): +def test_quickstart_calls(confext, qs_path, ployconf, tempdir): calls = [] - for action, func, args, kw in iter_quickstart_calls(parse_qs(qs_path), tempdir): + for action, func, args, kw in iter_quickstart_calls(parse_qs(qs_path), confext, ployconf, tempdir): if action in ('add', 'create'): func(*args, **kw) calls.append((action, func.__self__.path)) else: calls.append((func, args)) assert calls == [ - (subprocess.check_call, ('pip install ploy_virtualbox',)), + (subprocess.check_call, ('pip install "ploy_virtualbox>=2.0.0b1"',)), (subprocess.check_call, ('mkdir ploy-quickstart',)), (subprocess.check_call, ('cd ploy-quickstart',)), - (subprocess.check_call, ('mkdir downloads',)), - (subprocess.check_call, ('ploy-download http://mfsbsd.vx.sk/files/iso/10/amd64/mfsbsd-se-10.0-RELEASE-amd64.iso 06165ce1e06ff8e4819e86c9e23e7d149f820bb4 downloads/',)), (subprocess.check_call, ('mkdir etc',)), - ('create', '%s/etc/ploy.conf' % tempdir.directory), + ('create', ('%s/etc/ploy.conf' % tempdir.directory).replace('.conf', confext)), (subprocess.check_call, ('ploy start ploy-demo',)), - ('add', '%s/etc/ploy.conf' % tempdir.directory), - (time.sleep, (90,)), + ('add', ('%s/etc/ploy.conf' % tempdir.directory).replace('.conf', confext)), + (wait_for_ssh, ('localhost', 44003)), (subprocess.check_call, ('ploy bootstrap -y',)), - (time.sleep, (90,)), - ('add', '%s/etc/ploy.conf' % tempdir.directory), + (wait_for_ssh, ('localhost', 44003)), + ('add', ('%s/etc/ploy.conf' % tempdir.directory).replace('.conf', confext)), (subprocess.check_call, ('ploy configure jailhost',)), - ('add', '%s/etc/ploy.conf' % tempdir.directory), + ('add', ('%s/etc/ploy.conf' % tempdir.directory).replace('.conf', confext)), (subprocess.check_call, ('ploy start demo_jail',)), ('create', '%s/jailhost-demo_jail.yml' % tempdir.directory), (subprocess.check_call, ('ploy configure demo_jail',)), (subprocess.check_call, ('mkdir host_vars',)), ('create', '%s/host_vars/jailhost.yml' % tempdir.directory), - (subprocess.check_call, ('ploy configure jailhost -t ipnat_rules',)), + (subprocess.check_call, ('ploy configure jailhost -t pf-conf',)), (subprocess.check_call, ("ploy ssh jailhost 'ifconfig em0'",))] - assert tempdir['etc/ploy.conf'].content().splitlines() == [ - '[vb-hostonlyif:vboxnet0]', - 'ip = 192.168.56.1', - '', - '[vb-dhcpserver:vboxnet0]', - 'ip = 192.168.56.2', - 'netmask = 255.255.255.0', - 'lowerip = 192.168.56.100', - 'upperip = 192.168.56.254', - '', + assert ployconf.content().splitlines() == [ '[vb-instance:ploy-demo]', - 'vm-nic1 = hostonly', - 'vm-hostonlyadapter1 = vboxnet0', 'vm-nic2 = nat', 'vm-natpf2 = ssh,tcp,,44003,,22', 'storage =', - ' --type dvddrive --medium ../downloads/mfsbsd-se-10.0-RELEASE-amd64.iso', - ' --medium vb-disk:boot', - '', - '[vb-disk:boot]', - 'size = 102400', + ' --medium vb-disk:defaultdisk', + ' --type dvddrive --medium https://mfsbsd.vx.sk/files/iso/12/amd64/mfsbsd-se-12.0-RELEASE-amd64.iso --medium_sha1 2fbf2be5a79cc8081d918475400581bd54bb30ae', '', '[ez-master:jailhost]', 'instance = ploy-demo', @@ -184,41 +170,60 @@ def test_quickstart_calls(qs_path, tempdir): '- hosts: jailhost-demo_jail', ' tasks:', ' - name: install nginx', - ' pkgng: name=nginx state=present', + ' pkgng:', + ' name: "nginx"', + ' state: "present"', ' - name: Setup nginx to start immediately and on boot', ' service: name=nginx enabled=yes state=started'] assert tempdir['host_vars/jailhost.yml'].content().splitlines() == [ - 'ipnat_rules:', - ' - "rdr em0 {{ ansible_em0.ipv4[0].address }}/32 port 80 -> {{ hostvars[\'jailhost-demo_jail\'][\'ploy_ip\'] }} port 80"'] + 'pf_nat_rules:', + ' - "rdr on em0 proto tcp from any to em0 port 80 -> {{ hostvars[\'jailhost-demo_jail\'][\'ploy_ip\'] }} port 80"'] @pytest.yield_fixture -def virtualenv(tempdir): +def virtualenv(monkeypatch, tempdir): origdir = os.getcwd() os.chdir(tempdir.directory) subprocess.check_output(['virtualenv', '.']) - orig_env = dict(os.environ) - os.environ.pop('PYTHONHOME', None) - os.environ['VIRTUAL_ENV'] = tempdir.directory - os.environ['PATH'] = '%s/bin:%s' % (tempdir.directory, orig_env['PATH']) + monkeypatch.delenv('PYTHONHOME', raising=False) + monkeypatch.setenv('VIRTUAL_ENV', tempdir.directory) + monkeypatch.setenv('PATH', '%s/bin:%s' % (tempdir.directory, os.environ['PATH'])) yield tempdir.directory - if 'PYTHONHOME' in orig_env: - os.environ['PYTHONHOME'] = orig_env['PYTHONHOME'] - os.environ.pop('PATH', None) - if 'PATH' in orig_env: - os.environ['PATH'] = orig_env['PATH'] - os.environ.pop('VIRTUAL_ENV', None) - if 'VIRTUAL_ENV' in orig_env: - os.environ['VIRTUAL_ENV'] = orig_env['VIRTUAL_ENV'] os.chdir(origdir) + subprocess.call(['VBoxManage', 'controlvm', 'ploy-demo', 'poweroff']) + time.sleep(5) + subprocess.call(['VBoxManage', 'unregistervm', '--delete', 'ploy-demo']) + + +def wait_for_ssh(host, port, timeout=90): + from contextlib import closing + import socket + while timeout > 0: + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + try: + s.settimeout(1) + if s.connect_ex((host, port)) == 0: + if s.recv(128).startswith(b'SSH-2'): + return + except socket.timeout: + timeout -= 1 + continue + time.sleep(1) + timeout -= 1 + raise RuntimeError( + "SSH at %s:%s didn't become accessible" % (host, port)) @pytest.mark.skipif("not config.option.quickstart_bsdploy") -def test_quickstart_functional(request, qs_path, tempdir, virtualenv): +def test_quickstart_functional(request, qs_path, confext, ployconf, tempdir, virtualenv): + if confext == '.yml': + pytest.xfail("No YML config file support yet") if not os.path.isabs(request.config.option.quickstart_bsdploy): pytest.fail("The path given by --quickstart-bsdploy needs to be absolute.") if request.config.option.ansible_version: subprocess.check_call(['pip', 'install', 'ansible==%s' % request.config.option.ansible_version]) - subprocess.check_call(['pip', 'install', '--pre', request.config.option.quickstart_bsdploy]) - for action, func, args, kw in iter_quickstart_calls(parse_qs(qs_path), tempdir): + else: + subprocess.check_call(['pip', 'install', 'ansible']) + subprocess.check_call(['pip', 'install', '-i' 'https://d.rzon.de:8141/fschulze/dev/', '--pre', request.config.option.quickstart_bsdploy]) + for action, func, args, kw in iter_quickstart_calls(parse_qs(qs_path), confext, ployconf, tempdir): func(*args, **kw) diff --git a/bsdploy/tests/test_roles.py b/bsdploy/tests/test_roles.py index 7f54a2c..53bd3de 100644 --- a/bsdploy/tests/test_roles.py +++ b/bsdploy/tests/test_roles.py @@ -10,7 +10,8 @@ def ctrl(ployconf, tempdir): import ploy_ezjail import ploy_ansible ployconf.fill([ - '[ez-master:jailhost]']) + '[ez-master:jailhost]', + 'host = jailhost']) ctrl = Controller(configpath=ployconf.directory) ctrl.plugins = { 'bsdploy': bsdploy.plugin, @@ -29,22 +30,54 @@ def get_all_roles(): return sorted(roles) +def _iter_tasks(block): + from ansible.playbook.block import Block + for task in block: + if isinstance(task, Block): + for task in _iter_tasks(task.block): + yield task + else: + yield task + + def iter_tasks(plays): - for play in plays: - for task in play.tasks(): - if task.meta: - if task.meta == 'flush_handlers': # pragma: nocover - branch coverage only on failure - continue - raise ValueError # pragma: nocover - only on failure - yield play, task + from ploy_ansible import ANSIBLE1 + if ANSIBLE1: + for play in plays: + for task in play.tasks(): + if task.meta: + if task.meta == 'flush_handlers': # pragma: nocover - branch coverage only on failure + continue + raise ValueError # pragma: nocover - only on failure + yield play, task + else: + for play in plays: + for task in _iter_tasks(play.compile()): + if task.action == 'meta': + meta_action = task.args.get('_raw_params') + if meta_action == 'flush_handlers': # pragma: nocover - branch coverage only on failure + continue + if meta_action == 'role_complete': # pragma: nocover - branch coverage only on failure + continue + raise ValueError # pragma: nocover - only on failure + yield play, task + + +def get_plays(pb, monkeypatch): + from ploy_ansible import ANSIBLE1 + if ANSIBLE1: + plays = [] + monkeypatch.setattr('ansible.playbook.PlayBook._run_play', plays.append) + pb.run() + return plays + else: + return pb.get_plays() def test_roles(ctrl, monkeypatch): instance = ctrl.instances['jailhost'] pb = instance.get_playbook() - plays = [] - monkeypatch.setattr('ansible.playbook.PlayBook._run_play', plays.append) - pb.run() + plays = get_plays(pb, monkeypatch) tasks = [] for play, task in iter_tasks(plays): tasks.append(task.name) @@ -52,40 +85,46 @@ def test_roles(ctrl, monkeypatch): 'bind host sshd to primary ip', 'Enable ntpd in rc.conf', 'Disable public use of ntpd', - 'Enable ipfilter in rc.conf', - 'Set ipfilter_rules in rc.conf', - 'Setup ipf.rules', - 'Enable ipmon in rc.conf', - 'Set ipmon_flags in rc.conf', - 'Enable ipnat in rc.conf', - 'Set ipnat_rules in rc.conf', - 'Setup ipnat.rules', + 'Check for old ipnat_rules setting', + 'Remove ipfilter from rc.conf', + 'Remove ipfilter_rules from rc.conf', + 'Remove ipmon from rc.conf', + 'Remove ipmon_flags from rc.conf', + 'Remove ipnat from rc.conf', + 'Remove ipnat_rules from rc.conf', + 'Enable pf in rc.conf', + 'Check for /etc/pf.conf', + 'Default pf.conf', + 'Stat of /dev/pf', + 'Checking pf', + 'Setup pf.conf', + 'Reload pf.conf', 'Enable gateway in rc.conf', - 'Add lo1 to rc.conf', + 'Setup cloned interfaces', 'Enable security.jail.allow_raw_sockets', 'Enable security.jail.sysvipc_allowed', + 'Ensure helper packages are installed (using http proxy)', 'Ensure helper packages are installed', 'Set default jail interface', 'Set default jail parameters', 'Set default jail exec stop', + 'Enable jail_parallel_start', 'Enable ezjail in rc.conf', 'Setup ezjail.conf', 'Setup data zpool', - 'Setup rc.conf lines for data zpool', 'Set data zpool options', 'Jails ZFS file system', + 'Initialize ezjail (using http proxy)', 'Initialize ezjail (may take a while)', 'Create pkg cache folder', - 'Download pkg.txz', 'Directory for jail flavour "bsdploy_base"', - 'Pkg in bsdploy_base flavour', 'The .ssh directory for root in bsdploy_base flavour', 'The etc directory in bsdploy_base flavour', 'The etc/ssh directory in bsdploy_base flavour', - 'copy src=make.conf dest=/usr/jails/flavours/bsdploy_base/etc/make.conf owner=root group=wheel', - 'file dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos state=directory owner=root group=wheel', - 'copy src=pkg.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg.conf owner=root group=wheel', - 'template src=FreeBSD.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos/FreeBSD.conf owner=root group=wheel', + '/etc/make.conf in bsdploy_base flavour', + '/usr/local/etc/pkg/repos directory in bsdploy_base flavour', + '/usr/local/etc/pkg.conf in bsdploy_base flavour', + '/usr/local/etc/pkg/repos/FreeBSD.conf in bsdploy_base flavour', 'rc.conf for bsdploy_base flavour', 'sshd_config for bsdploy_base flavour', 'motd for bsdploy_base flavour', @@ -96,33 +135,29 @@ def test_all_role_templates_tested(ctrl, monkeypatch, request): instance = ctrl.instances['jailhost'] instance.config['roles'] = ' '.join(get_all_roles()) pb = instance.get_playbook() - plays = [] - monkeypatch.setattr('ansible.playbook.PlayBook._run_play', plays.append) - pb.run() + plays = get_plays(pb, monkeypatch) # import after running to avoid module import issues - from ansible.utils import parse_kv, path_dwim_relative + from bsdploy.tests import test_templates templates = [] for play, task in iter_tasks(plays): - if task.module_name != 'template': + if task.action != 'template': continue - module_args_dict = task.args - if not module_args_dict and task.module_args: - module_args_dict = parse_kv(task.module_args) - template_path = path_dwim_relative( - task.module_vars['_original_file'], 'templates', - module_args_dict['src'], play.basedir) + loader = play.get_loader() + src = task.args.get('src') + template_path = loader.path_dwim_relative( + task._role._role_path, 'templates', src) if not os.path.exists(template_path): # pragma: nocover - only on failure raise ValueError - name = module_args_dict['src'].lower() + name = src.lower() for rep in ('-', '.'): name = name.replace(rep, '_') templates.append(( name, dict( - path=task.module_vars.get('_original_file'), - role_name=task.role_name, - name=module_args_dict['src'], task_name=task.name))) - test_names = [x.name for x in request.session.items] + path=task._role._role_path, + role_name=task._role.get_name(), + name=src, task_name=task.name))) + test_names = [x for x in dir(test_templates) if x.startswith('test_')] for name, info in templates: test_name = 'test_%s_%s' % (info['role_name'], name) if not any(x for x in test_names if x.startswith(test_name)): # pragma: nocover - only on failure diff --git a/bsdploy/tests/test_templates.py b/bsdploy/tests/test_templates.py index e1a2c49..5a124ba 100644 --- a/bsdploy/tests/test_templates.py +++ b/bsdploy/tests/test_templates.py @@ -2,11 +2,7 @@ def test_dhcp_host_dhclient_exit_hooks(): pass -def test_jails_host_ipf_rules(): - pass - - -def test_jails_host_ipnat_rules(): +def test_jails_host_pf_conf(): pass @@ -20,3 +16,7 @@ def test_jails_host_freebsd_conf(): def test_zfs_auto_snapshot_010_zfs_snapshot(): pass + + +def test_jails_host_pkg_conf(): + pass diff --git a/buildout.cfg b/buildout.cfg index d5a80a5..f07d542 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -9,16 +9,18 @@ auto-checkout = ploy_virtualbox parts = bsdploy +show-picked-versions = true +versions = versions -develop = . +develop = + . + src/* [bsdploy] recipe = zc.recipe.egg eggs = bsdploy [development] - ploy_virtualbox>=1.0.0 - ploy_ec2>=1.0.0 - keyring>=4.0 + keyrings.alt dependent-scripts = true [sources] @@ -28,3 +30,15 @@ ploy_ansible = git https://github.com/ployground/ploy_ansible ploy_ezjail = git https://github.com/ployground/ploy_ezjail ploy_ec2 = git https://github.com/ployground/ploy_ec2 ploy_virtualbox = git https://github.com/ployground/ploy_virtualbox + + +[versions] +Sphinx=<2 +mock=<4 +ansible=<2.9 +cryptography=<3.4 +paramiko=<2.0 +pytest=<5 +keyring=<19 +ruamel.yaml=<0.17 +ruamel.yaml.clib=<0.2.3 diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..f0f18ce --- /dev/null +++ b/default.nix @@ -0,0 +1,11 @@ +with import {}; +stdenv.mkDerivation rec { + name = "env"; + env = buildEnv { name = name; paths = buildInputs; }; + buildInputs = [ + python + python27Packages.pynacl + gnumake + openssl + ]; +} diff --git a/docs/advanced/customizing-bootstrap.rst b/docs/advanced/customizing-bootstrap.rst index 973a199..0b68742 100644 --- a/docs/advanced/customizing-bootstrap.rst +++ b/docs/advanced/customizing-bootstrap.rst @@ -39,7 +39,7 @@ For the ezjail initialization you have to add the following setting with a FreeB ansible-ploy_ezjail_install_host = http://ftp4.de.freebsd.org -The ``_mfsbsd`` context manager takes care of setting the ``bootstrap-fingerprint`` etc for mfsBSD. +The ``_mfsbsd`` context manager takes care of setting the ``bootstrap-ssh-host-keys`` etc for mfsBSD. The ``_bootstrap`` function then runs the regular bootstrapping. For the jails you can use a startup script like this: diff --git a/docs/advanced/staging.rst b/docs/advanced/staging.rst index 63d8609..6771d21 100644 --- a/docs/advanced/staging.rst +++ b/docs/advanced/staging.rst @@ -83,21 +83,9 @@ Change it to this:: And in your ``staging.conf`` you define ``fqdn_suffix`` to be i.e. ``.192.168.56.10.xip.io`` and in ``production.conf`` to an empty string. -Finally, configure the VirtualBox instance in staging to use a host-only adapter, like so (the second nic is needed for the virtual instance to access the internet):: - - [vb-hostonlyif:vboxnet0] - ip = 192.168.56.1 - - [vb-dhcpserver:vboxnet0] - ip = 192.168.56.2 - netmask = 255.255.255.0 - lowerip = 192.168.56.100 - upperip = 192.168.56.254 +Finally, configure the VirtualBox instance in staging to use a second nic (in addition to the default host-only interface) via DHCP so it can access the internet:: [vb-instance:provisioner] - vm-ostype = FreeBSD_64 - vm-nic1 = hostonly - vm-hostonlyadapter1 = vboxnet0 vm-nic2 = nat ``ploy_virtualbox`` will ensure that the virtual network ``vboxnet0`` exists (if it doesn't already). diff --git a/docs/advanced/updating.rst b/docs/advanced/updating.rst index 94a8489..b0ad23f 100644 --- a/docs/advanced/updating.rst +++ b/docs/advanced/updating.rst @@ -1,8 +1,26 @@ Updating ======== -How to best update your systems? Don't! +While in theory automated systems such as ploy allow you to create new and up-to-date instances easily and thus in theory you would never have to upgrade existing instances because you would simply replace them with newer versions, in practice both jails and host systems will often have to be upgrade in place. -Seriously, updating can be more trouble than it's worth. However, that does not mean you shouldn't stay up-to-date. It just means, that upgrading your systems may not neccessarily be the best way to achieve that. +To support this, bsdploy provides a few helper tasks which are registered by default for jailhosts. -Instead, an alternative approach is to upgrade your setup (package versions, FreeBSD version etc.) and then create a new instance with the upgraded setup, test that and repeat the process until the upgraded state is working correctly and then replace the old state with the 'upgraded' one. +If you want to use them in your own, custom fabfile you must import them their to make them available, i.e. like so:: + + from bsdploy.fabutils import * + +You can verify this by running the `do` command with `-l`: + + + # ploy do HOST -l + Available commands: + + pkg_upgrade + rsync + update_flavour_pkg + +You can use the `pkg_upgrade` task to keep the pkg and and the installed packages on the host up-to-date. + +The `update_flavour_pkg` is useful after updating the ezjail world, so that newly created jails will have an updated version of pkg from the start. (if the pkg versions become too far apart it can happen, that new jails won't bootstrap at all, because they already fail at installing python). + +See the `fabutils.py` file for more details. diff --git a/docs/installation.rst b/docs/installation.rst index 2572faa..322e179 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -7,13 +7,23 @@ BSDploy and its dependencies are written in `Python `_ and th Server requirements =================== -A FreeBSD system that wants to be managed by BSDploy will need to have `ezjail `_ installed, as well as `Python `_ and must have SSH access enabled (either for root or with ``sudo`` configured). +.. warning:: + BSDploy is intended for initial configuration of a jail host before any jails have been installed. + While technically possible, BSDploy is not intended for managing existing systems with non-BSDploy jails. + Running against hosts that have not been bootstrapped by BSDploy can result in loss of data. + +A FreeBSD system that wants to be managed by BSDploy will + +- need to have `ezjail `_ installed +- as well as `Python `_ +- must have SSH access enabled (either for root or with ``sudo`` configured). +- have ZFS support (BSDploy does not support running on non-ZFS filesystems) Strictly speaking, BSDploy only needs Python for the initial configuration of the jailhost. If you chose to perform that step yourself or use a pre-existing host, you won't need Python on the host, just ezjail. -BSDploy can take care of these requirements for you during bootstrapping but of course you can also use it to manage existing machines that already meet them. +Normally, BSDploy will take care of these requirements for you during :doc:`bootstrapping ` but in situations where this is not possible, manually providing the abovementioned requirements should allow you to :doc:`apply BSDploy's host configuration ` anyway. -BSDploy supports FreeBSD >= 9.2, including 10.0. +BSDploy supports FreeBSD >= 9.2, including 10.3. Client Installation diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 92549e1..33a2449 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -23,24 +23,17 @@ To give us the luxury of running against a well-defined context, this quickstart Since VirtualBox support is optional and BSDploy is fairly modular, you will need to install ``ploy_virtualbox`` to follow this quickstart like so:: - % pip install ploy_virtualbox + % pip install "ploy_virtualbox>=2.0.0b1" -Getting the FreeBSD installer ------------------------------ +Initializing the project environment +------------------------------------ BSDploy has the notion of an environment, which is just fancy talk for a directory with specific conventions. Let's create one:: % mkdir ploy-quickstart % cd ploy-quickstart -First, we need to download a FreeBSD boot image. BSDploy uses `mfsBSD `_ which is basically the official FreeBSD installer plus pre-configured SSH access. - -BSDploy provides a small cross-platform helper for downloading assets via HTTP which also checks the integrity of the downloaded file:: - - % mkdir downloads - % ploy-download http://mfsbsd.vx.sk/files/iso/10/amd64/mfsbsd-se-10.0-RELEASE-amd64.iso 06165ce1e06ff8e4819e86c9e23e7d149f820bb4 downloads/ - Configuring the virtual machine ------------------------------- @@ -51,32 +44,25 @@ The main configuration file is named ``ploy.conf`` and lives inside a top-level Inside it create a file named ``ploy.conf`` with the following contents:: - [vb-hostonlyif:vboxnet0] - ip = 192.168.56.1 - - [vb-dhcpserver:vboxnet0] - ip = 192.168.56.2 - netmask = 255.255.255.0 - lowerip = 192.168.56.100 - upperip = 192.168.56.254 - [vb-instance:ploy-demo] - vm-nic1 = hostonly - vm-hostonlyadapter1 = vboxnet0 vm-nic2 = nat vm-natpf2 = ssh,tcp,,44003,,22 storage = - --type dvddrive --medium ../downloads/mfsbsd-se-10.0-RELEASE-amd64.iso - --medium vb-disk:boot + --medium vb-disk:defaultdisk + --type dvddrive --medium https://mfsbsd.vx.sk/files/iso/12/amd64/mfsbsd-se-12.0-RELEASE-amd64.iso --medium_sha1 2fbf2be5a79cc8081d918475400581bd54bb30ae + - [vb-disk:boot] - size = 102400 +This creates a VirtualBox instance named ``ploy-demo``. By default BSDploy provides it with a so-called "host only interface" but since that cannot be used to connect to the internet we explicitly configure a second one using NAT (mfsBSD will configure it via DHCP) and in addtion we create a port forwarding from ``localhost`` port ``44003`` to port ``22`` on the box - in essence allowing us to SSH into it via localhost. + +Next, we assign a virtual disk named ``defaultdisk`` onto which we will install the OS. This special disk is created automatically by BSDploy if it doesn't exist yet (it's sparse by default, so it won't take up much space on your disk). + +Finally, we configure a virtual optical drive to boot from the official mfsBSD 'special edition' installation image. By providing a download URL and checksum, BSDploy will automatically download it for us. Now we can start it up:: % ploy start ploy-demo -This should fire up virtualbox and boot a VirtualBox VM into mfsBSD. +This should download the mfsBSD image, fire up VirtualBox and boot our VM into mfsBSD. Bootstrapping the host @@ -100,24 +86,32 @@ Next it will give you one last chance to abort before it commences to wipe the t To make sure that everything has worked so far, let's take a look at the host by logging into it via SSH. ``bsdploy`` provides a command for that, too:: % ploy ssh jailhost - FreeBSD 9.2-RELEASE (GENERIC) #6 r255896M: Wed Oct 9 01:45:07 CEST 2013 + FreeBSD 10.3-RELEASE (GENERIC) #0 r297264: Fri Mar 25 02:10:02 UTC 2016 + + Welcome to FreeBSD! [...] -Let's take a quick look around:: +Let's take a quick look around. First, what packages have been installed?:: root@jailhost:~ # pkg info - gettext-0.18.3.1_1 GNU gettext package - libiconv-1.14_3 Character set conversion library - python27-2.7.6_4 Interpreted object-oriented programming language + gettext-runtime-0.19.3 GNU gettext runtime libraries and programs + indexinfo-0.2.2 Utility to regenerate the GNU info page index + libffi-3.0.13_3 Foreign Function Interface + pkg-1.4.3 Package manager + python27-2.7.9 Interpreted object-oriented programming language + +Next, what's the ZFS scenario?:: + root@jailhost:~ # zpool list - NAME SIZE ALLOC FREE CAP DEDUP HEALTH ALTROOT - system 19.9G 584M 19.3G 2% 1.00x ONLINE - + NAME SIZE ALLOC FREE FRAG EXPANDSZ CAP DEDUP HEALTH ALTROOT + system 19.9G 931M 19.0G 2% - 4% 1.00x ONLINE - root@jailhost:~ # zfs list NAME USED AVAIL REFER MOUNTPOINT - system 584M 19.0G 31K none - system/root 583M 19.0G 533M / - system/root/tmp 37K 19.0G 37K /tmp - system/root/var 50.6M 19.0G 50.6M /var + system 931M 18.3G 19K none + system/root 931M 18.3G 876M / + system/root/tmp 21K 18.3G 21K /tmp + system/root/var 54.2M 18.3G 54.2M /var + root@jailhost:~ # A few things to note: @@ -153,32 +147,38 @@ Package-wise nothing much has changed – only ``ezjail`` has been installed:: root@jailhost:~ # pkg info ezjail-3.4.1 Framework to easily create, manipulate, and run FreeBSD jails - gettext-0.18.3.1_1 GNU gettext package - libiconv-1.14_3 Character set conversion library - python27-2.7.6_4 Interpreted object-oriented programming language + gettext-runtime-0.19.3 GNU gettext runtime libraries and programs + indexinfo-0.2.2 Utility to regenerate the GNU info page index + libffi-3.0.13_3 Foreign Function Interface + pkg-1.4.3 Package manager + python27-2.7.9 Interpreted object-oriented programming language + root@jailhost:~ # There is now a second zpool called ``tank`` and ``ezjail`` has been configured to use it:: root@jailhost:~ # zpool list - NAME SIZE ALLOC FREE CAP DEDUP HEALTH ALTROOT - system 19.9G 584M 19.3G 2% 1.00x ONLINE - - tank 78.5G 389M 78.1G 0% 1.00x ONLINE - + NAME SIZE ALLOC FREE FRAG EXPANDSZ CAP DEDUP HEALTH ALTROOT + system 19.9G 934M 19.0G 2% - 4% 1.00x ONLINE - + tank 75.5G 444M 75.1G - - 0% 1.00x ONLINE - root@jailhost:~ # zfs list NAME USED AVAIL REFER MOUNTPOINT - system 584M 19.0G 31K none - system/root 584M 19.0G 533M / - system/root/tmp 38K 19.0G 38K /tmp - system/root/var 50.7M 19.0G 50.7M /var - tank 389M 76.9G 144K none - tank/jails 389M 76.9G 8.05M /usr/jails - tank/jails/basejail 377M 76.9G 377M /usr/jails/basejail - tank/jails/newjail 3.58M 76.9G 3.58M /usr/jails/newjail + system 933M 18.3G 19K none + system/root 933M 18.3G 877M / + system/root/tmp 21K 18.3G 21K /tmp + system/root/var 56.6M 18.3G 56.6M /var + tank 443M 72.7G 144K none + tank/jails 443M 72.7G 10.1M /usr/jails + tank/jails/basejail 426M 72.7G 426M /usr/jails/basejail + tank/jails/newjail 6.37M 72.7G 6.37M /usr/jails/newjail + root@jailhost:~ # + But there aren't any jails configured yet:: root@jailhost:~ # ezjail-admin list STA JID IP Hostname Root Directory --- ---- --------------- ------------------------------ ------------------------ + root@jailhost:~ # Let's change that... @@ -212,10 +212,10 @@ Rather conveniently `ploy_ezjail `_ h Log out from the jailhost and run this:: # ploy ssh demo_jail - FreeBSD 9.2-RELEASE (GENERIC) #6 r255896M: Wed Oct 9 01:45:07 CEST 2013 + FreeBSD 10.3-RELEASE (GENERIC) #0 r297264: Fri Mar 25 02:10:02 UTC 2016 Gehe nicht über Los. - root@demo_jail:~ # + root@demo_jail:~ # and there you are, inside the jail. @@ -233,7 +233,9 @@ Like with the jailhost, we could assign roles to our demo jail, but another way - hosts: jailhost-demo_jail tasks: - name: install nginx - pkgng: name=nginx state=present + pkgng: + name: "nginx" + state: "present" - name: Setup nginx to start immediately and on boot service: name=nginx enabled=yes state=started @@ -255,13 +257,13 @@ To do so, make a folder named ``host_vars``:: and create the file ``jailhost.yml`` in it with the following content:: - ipnat_rules: - - "rdr em0 {{ ansible_em0.ipv4[0].address }}/32 port 80 -> {{ hostvars['jailhost-demo_jail']['ploy_ip'] }} port 80" + pf_nat_rules: + - "rdr on em0 proto tcp from any to em0 port 80 -> {{ hostvars['jailhost-demo_jail']['ploy_ip'] }} port 80" -To activate the rules, re-apply the jail host configuration. +To activate the rules, re-apply the jail host configuration with just the ``pf-conf`` tag. Ansible will figure out, that it needs to update them (and only them) and then restart the network. However, in practice running the entire configuration can take quite some time, so if you already know you only want to update some specific sub set of tasks you can pass in one or more tags. In this case for updating the ipnat rules:: - % ploy configure jailhost -t ipnat_rules + % ploy configure jailhost -t pf-conf Since the demo is running inside a host that got its IP address via DHCP we will need to find that out before we can access it in the browser. diff --git a/docs/setup/bootstrapping.rst b/docs/setup/bootstrapping.rst index ac13399..069a22b 100644 --- a/docs/setup/bootstrapping.rst +++ b/docs/setup/bootstrapping.rst @@ -9,15 +9,15 @@ The Bootstrapping process assumes that the target host has been booted into an i Bootstrapping FreeBSD 9.x ------------------------- -The default version that BSDploy assumes is 10.0. +The default version that BSDploy assumes is 10.3. If you want to install different versions, i.e. 9.2 you must: - use the iso image for that version:: % ploy-download http://mfsbsd.vx.sk/files/iso/9/amd64/mfsbsd-se-9.2-RELEASE-amd64.iso 4ef70dfd7b5255e36f2f7e1a5292c7a05019c8ce downloads/ -- set ``bootstrap-fingerprint`` to ``02:2e:b4:dd:c3:8a:b7:7b:ba:b2:4a:f0:ab:13:f4:2d`` in ``ploy.conf`` - (each mfsbsd release has it's own hardcoded fingerprint) +- set ``bootstrap-ssh-host-key`` in ``ploy.conf`` to content of ``/etc/ssh/ssh_host_rsa_key.pub`` in the mfsbsd image + (each mfsbsd release has it's own hardcoded ssh host key) - create a file named ``files.yml`` in ``bootstrap-files`` with the following contents: .. code-block:: yaml @@ -81,9 +81,32 @@ You can use the following optional parameters to configure the bootstrapping pro - ``bootstrap-bsd-url``: If you don't want to use the installation files found on the installer image (or if your boot image doesn't contain any) you can provide an explicit alternative (i.e. ``http://ftp4.de.freebsd.org/pub/FreeBSD/releases/amd64/9.2-RELEASE/``) and this will be used to fetch the system from. -- ``bootstrap-fingerprint``: Since the installer runs a different sshd configuration than the final installation, we need to provide its fingerprint explicitly. However, if you don't provide one, BSDploy will assume the (currently hardcoded) fingerprint of the 9.2 mfsBSD installer (``02:2e:b4:dd:c3:8a:b7:7b:ba:b2:4a:f0:ab:13:f4:2d``). If you are using newer versions you must update the value (for 10.0 i.e. ``1f:cb:78:20:b8:97:dd:dc:3d:23:75:f0:bb:ad:84:03``) +- ``bootstrap-ssh-host-key``: Since the installer runs a different sshd configuration than the final installation, we need to provide its ssh host key explicitly. However, if you don't provide one, BSDploy will assume the (currently hardcoded) host key of the 10.3 mfsBSD installer . If you are using newer versions you must update the value to the content of ``/etc/ssh/ssh_host_rsa_key.pub`` in the mfsbsd image. -- ``firstboot-update``: By default bootstrapping will install and enable the `firstboot-freebsd-update `_ package. This will update the installed system automatically (meaning non-interactively) to the latest patchlevel upon first boot. If for some reason you do not wish this to happen, you can disable it by setting this value to ``false``. +- ``bootstrap-ssh-fingerprints``: Another way to supply SSH fingerprints if host keys aren't an option. + +- ``bootstrap-firstboot-update``: If set install and enable the `firstboot-freebsd-update `_ package. This will update the installed system automatically (meaning non-interactively) to the latest patchlevel upon first boot. Disabled by default. + +- ``bootstrap-password``: If set will use the provided password for bootstrapping. It is used to update the known hosts. You might still have to provide it manually for the SSH command used during bootstrap. + +- ``http_proxy``: If set, that proxy will be used for all ``pkg`` operations performed on that host, as well as for downloading any assets during bootstrapping (``base.tbz`` etc.) + + +.. note:: Regarding the http proxy setting it is noteworthy, that ``pkg`` servers have rather restrictive caching policies in their response headers, so that most proxies' default configurations will produce misses. Here's an example for how to configure squid to produce better results: + + .. code-block:: sh + + # match against download urls for specific packages - their content never changes for the same url, so we cache aggressively + refresh_pattern -i (quarterly|latest)\/All\/.*(\.txz) 1440 100% 1051200 ignore-private ignore-must-revalidate override-expire ignore-no-cache + # match against meta-information - this shouldn't be cached quite so aggressively + refresh_pattern -i (quarterly|latest)\/.*(\.txz) 1440 100% 10080 ignore-private ignore-must-revalidate override-expire ignore-no-cache + + Also you will probably want to adjust the following: + + .. code-block:: sh + + maximum_object_size_in_memory 32 KB + maximum_object_size 2000 MB Bootstrap rc.conf @@ -127,6 +150,13 @@ For example, to create a custom ``rc.conf`` for a particular instance, create a Any file listed in the YAML file found inside that directory will take precedence during bootstrapping, but any file *not* found in there will be uploaded from the default location instead. +Files encrypted using ``ploy vault encrypt`` are recognized and decrypted during upload. + +SSH host keys are generated locally via ``ssh-keygen`` and stored in ``bootstrap-files``. +If your ``ssh-keygen`` doesn't support a key type (like ecdsa on OS X), then the key won't be created and FreeBSD will create it during first boot. +The generated keys are used to verify the ssh connection, so it is best to add them into version control. +Since the private ssh keys are sensitive data, you should encrypt them using ``ploy vault encrypt ...`` before adding them into version control. + Bootstrap execution ------------------- diff --git a/docs/setup/configuration.rst b/docs/setup/configuration.rst index 9ea9a12..ece0d21 100644 --- a/docs/setup/configuration.rst +++ b/docs/setup/configuration.rst @@ -19,6 +19,38 @@ Unlike bootstrapping, this final step is implemented using ansible playbooks and Among other things, this will create an additional zpool named ``tank`` (by default) which will be used to contain the jails. +Configuring as non-root +----------------------- + +While bootstrapping currently *must* be performed as ``root`` (due to the fact that mfsBSD itself requires root login) some users may not want to enable root login for their systems. + +If you want to manage a jailhost with a non-root user, you must perform the following steps manually: + +- install ``sudo`` on the jailhost +- create a user account and enable SSH access for it +- enable passwordless ``sudo`` access for it +- disable SSH login for root (currently, automatically enabled during bootstrapping) + +Additionally, you *must* configure the system using a playbook (i.e. simply assigning one or more roles won't work in this case) and in that playbook you must set the username and enable ``sudo``, i.e. like so: + +.. code-block:: yaml + + --- + - hosts: jailhost + user: tomster + sudo: yes + roles: + # apply the built-in bsdploy role jails_host + - jails_host + +And, of course, once bootstrapped, you need to set the same username in ``ploy.conf``: + +.. code-block:: ini + + [ez-master:jailhost] + user = tomster + + Full-Disk encryption with GELI ------------------------------ diff --git a/docs/setup/overview.rst b/docs/setup/overview.rst index 51cb738..4d972a3 100644 --- a/docs/setup/overview.rst +++ b/docs/setup/overview.rst @@ -25,7 +25,7 @@ Conceptually, the provider of a jailhost is a separate entity from the jailhost instance = ploy-demo [...] - [instance:webserver] + [ez-instance:webserver] master = jailhost [...] diff --git a/docs/setup/provisioning-virtualbox.rst b/docs/setup/provisioning-virtualbox.rst index f95d3bd..c19b3ae 100644 --- a/docs/setup/provisioning-virtualbox.rst +++ b/docs/setup/provisioning-virtualbox.rst @@ -3,40 +3,50 @@ Provisioning VirtualBox instances BSDploy provides automated provisioning of `VirtualBox `_ instances via the `ploy_virtualbox plugin `_. -.. Note:: The plugin is not installed by default when installing BSDploy, so you need to install it additionally like so ``pip install ploy_ec2``. +.. Note:: The plugin is not installed by default when installing BSDploy, so you need to install it additionally like so ``pip install ploy_virtualbox``. Unlike with :doc:`plain instances ` the configuration doesn't just describe existing instances but is used to create them. Consider the following entry in ``ploy.conf``:: [vb-instance:ploy-demo] - vm-ostype = FreeBSD_64 - vm-memory = 1024 - vm-accelerate3d = off - vm-acpi = on - vm-rtcuseutc = on - vm-boot1 = disk - vm-boot2 = dvd - vm-nic1 = nat + vm-nic2 = nat vm-natpf1 = ssh,tcp,,44003,,22 storage = - --type dvddrive --medium ../downloads/mfsbsd-se-9.2-RELEASE-amd64.iso - --medium vb-disk:boot + --medium vb-disk:defaultdisk + --type dvddrive --medium http://mfsbsd.vx.sk/files/iso/10/amd64/mfsbsd-se-10.3-RELEASE-amd64.iso --medium_sha1 564758b0dfebcabfa407491c9b7c4b6a09d9603e - [vb-disk:boot] - size = 102400 -VirtualBox instances are configured using the ``vb-instance`` prefix and you can set parameters of the virtual machine by prefixing them with ``vm-``. For additional details on which parameters are available and what they mean, refer to the documentation of the VirtualBox commandline tool `VBoxManage `_, in particualar for `VBoxManage createvm `_ and `VBoxManage modifyvm `_. +VirtualBox instances are configured using the ``vb-instance`` prefix and you can set parameters of the virtual machine by prefixing them with ``vm-``. For additional details on which parameters are available and what they mean, refer to `the plugin's documentation `_ and the documentation of the VirtualBox commandline tool `VBoxManage `_, in particualar for `VBoxManage createvm `_ and `VBoxManage modifyvm `_. -In addition to the ``vb-instance`` you will need to configure at least one storage device using a ``vb-disk`` entry which is essentially a wrapper for `VBoxManage createhd `_. +Having said that, BSDploy provides a number of convenience defaults for each instance, so in most cases you won't need much more than in the above example. -As you can see in the example above, you need to include the disk in the ``storage`` parameter of the ``vb-instance`` entry in order to make it available for it. -Also note, that we reference a mfsBSD boot image. Since VirtualBox won't find a bootable OS on the new drive initially, it will attempt to boot into mfsBSD. +Default hostonly network +------------------------ -To download the image, use ``ploy-download`` like so:: +Unless you configure otherwise, BSDploy will tell VirtualBox to - mkdir downloads - ploy-download http://mfsbsd.vx.sk/files/iso/9/amd64/mfsbsd-se-9.2-RELEASE-amd64.iso 4ef70dfd7b5255e36f2f7e1a5292c7a05019c8ce downloads/ +- create a host-only network interface named ``vboxnet0`` +- assign the first network interface to that +- create a DHCP server for the address range ``192.168.56.100-254`` + +This means that a) during bootstrap the VM will receive a DHCP address from that range but more importantly b) you are free to assign your own static IPs from the range *below* (i.e. ``192.168.56.10``) because the existence of the VirtualBox DHCP server will ensure that that IP is reachable from the host system. This allows you to assign known, good static IP addresses to all of your VirtualBox instances. + + +Default disk setup +------------------ + +As you can see in the example above, there is a reference to a disk named ``defaultdisk`` in the ``storage`` parameter of the ``vb-instance`` entry. If you reference a disk of that name, BSDploy will automatically provision a virtual sparse disk of 100Gb size. In practice it's often best to leave that assignment in place (it's where the OS will be installed onto during bootstrap) and instead configure additional disks for data storage. + + +Boot images +----------- + +Also note, that we reference a mfsBSD boot image above and assign it to the optical drive. By providing an external URL with a checksum, ``ploy_virtualbox`` will download that image for us (by default into ``~/.ploy/downloads/``) and connect it to the instance. + + +First Startup +------------- Unlike ``VBoxManage`` BSDploy does not provide an explicit *create* command, instead just start the instance and if it doesn't exist already, BSDploy will create it for you on-demand:: diff --git a/docs/tutorial/staging.rst b/docs/tutorial/staging.rst index c5ac8ff..6e8de7b 100644 --- a/docs/tutorial/staging.rst +++ b/docs/tutorial/staging.rst @@ -23,15 +23,6 @@ To create a 'production' environment, create an additional configuration file `` extends = ploy.conf [vb-instance:demo-production] - vm-ostype = FreeBSD_64 - vm-memory = 1024 - vm-accelerate3d = off - vm-acpi = on - vm-rtcuseutc = on - vm-boot1 = disk - vm-boot2 = dvd - vm-nic1 = hostonly - vm-hostonlyadapter1 = vboxnet0 vm-nic2 = nat vm-natpf2 = ssh,tcp,,44004,,22 storage = diff --git a/docs/tutorial/webserver.rst b/docs/tutorial/webserver.rst index b8b161d..a16db4d 100644 --- a/docs/tutorial/webserver.rst +++ b/docs/tutorial/webserver.rst @@ -86,7 +86,9 @@ For jail instances, this name contains both the name of the master instance *and - hosts: jailhost-webserver tasks: - name: install nginx - pkgng: name=nginx state=present + pkgng: + name: "nginx" + state: "present" - name: enable nginx at startup time lineinfile: dest=/etc/rc.conf state=present line='nginx_enable=YES' create=yes - name: make sure nginx is running or reloaded @@ -112,12 +114,12 @@ Eventhough the webserver is now running, we cannot reach it from the outside, we So, create or edit ``host_vars/jailhost.yml`` to look like so:: - ipnat_rules: - - "rdr em0 {{ ansible_em0.ipv4[0].address }}/32 port 80 -> {{ hostvars['jailhost-webserver']['ploy_ip'] }} port 80" + pf_nat_rules: + - "rdr on {{ ansible_default_ipv4.interface }} proto tcp from any to {{ ansible_default_ipv4.interface }} port 80 -> {{ hostvars['jailhost-webserver']['ploy_ip'] }} port 80" -To activate the rules, re-apply the jail host configuration:: +To activate the rules, re-apply the jail host configuration for just the ``pf-conf`` tag:: - ploy configure jailhost -t ipnat_rules + ploy configure jailhost -t pf-conf You should now be able to access the default nginx website at the ``http://192.168.56.100`` address. diff --git a/docs/usage/jails.rst b/docs/usage/jails.rst index b2de391..2d633b6 100644 --- a/docs/usage/jails.rst +++ b/docs/usage/jails.rst @@ -19,7 +19,11 @@ Once defined, you can start the jail straight away. There is no explicit ``creat INFO: Creating instance 'webserver' INFO: Starting instance 'webserver' with startup script, this can take a while. -You can find out about the state of a jail by running ``ploy status JAILNAME``. In addtition there are also ``stop`` and ``terminate`` which do exactly what you think they do. +You can find out about the state of a jail by running ``ploy status JAILNAME``. + +A jail can be stopped with ``ploy stop JAILNAME``. + +A jail can be completely removed with ``ploy terminate JAILNAME``. This will destroy the ZFS filesystem specific to that jail. SSH Access diff --git a/freebsd-9.2.patch b/freebsd-9.2.patch deleted file mode 100644 index cb8a3e0..0000000 --- a/freebsd-9.2.patch +++ /dev/null @@ -1,68 +0,0 @@ -diff --git a/bsdploy/tests/test_quickstart.py b/bsdploy/tests/test_quickstart.py ---- a/bsdploy/tests/test_quickstart.py -+++ b/bsdploy/tests/test_quickstart.py -@@ -128,9 +128,11 @@ def test_quickstart_calls(qs_path, tempdir): - (subprocess.check_call, ('mkdir ploy-quickstart',)), - (subprocess.check_call, ('cd ploy-quickstart',)), - (subprocess.check_call, ('mkdir downloads',)), -- (subprocess.check_call, ('ploy-download http://mfsbsd.vx.sk/files/iso/10/amd64/mfsbsd-se-10.0-RELEASE-amd64.iso 06165ce1e06ff8e4819e86c9e23e7d149f820bb4 downloads/',)), -+ (subprocess.check_call, ('ploy-download http://mfsbsd.vx.sk/files/iso/9/amd64/mfsbsd-se-9.2-RELEASE-amd64.iso 4ef70dfd7b5255e36f2f7e1a5292c7a05019c8ce downloads/',)), - (subprocess.check_call, ('mkdir etc',)), - ('create', '%s/etc/ploy.conf' % tempdir.directory), -+ (subprocess.check_call, ('mkdir bootstrap-files',)), -+ ('create', '%s/bootstrap-files/files.yml' % tempdir.directory), - (subprocess.check_call, ('ploy start ploy-demo',)), - ('add', '%s/etc/ploy.conf' % tempdir.directory), - (time.sleep, (90,)), -@@ -169,8 +171,9 @@ def test_quickstart_calls(qs_path, tempdir): - 'vm-nic2 = nat', - 'vm-natpf2 = ssh,tcp,,44003,,22', - 'storage =', -- ' --type dvddrive --medium ../downloads/mfsbsd-se-10.0-RELEASE-amd64.iso', -+ ' --type dvddrive --medium ../downloads/mfsbsd-se-9.2-RELEASE-amd64.iso', - ' --medium vb-disk:boot', -+ 'bootstrap-fingerprint = 02:2e:b4:dd:c3:8a:b7:7b:ba:b2:4a:f0:ab:13:f4:2d', - '', - '[vb-disk:boot]', - 'size = 102400', -diff --git a/docs/quickstart.rst b/docs/quickstart.rst ---- a/docs/quickstart.rst -+++ b/docs/quickstart.rst -@@ -39,7 +39,7 @@ First, we need to download a FreeBSD boot image. BSDploy uses `mfsBSD ="3.10"', + 'backports.lzma', 'PyYAML', 'jinja2', 'setuptools', - 'ploy>=1.0.2', - 'ploy_ansible>=1.1.0', - 'ploy_ezjail>=1.0.0', - 'ploy_fabric>=1.1.0', + 'ploy>=2.0.0', + 'ploy_ansible>=2.0.0', + 'ploy_ezjail>=2.0.0', + 'ploy_fabric>=2.0.0', ] setup( name="bsdploy", version=version, - description="A tool to provision, configure and maintain FreeBSD jails", + description="A tool to remotely provision, configure and maintain FreeBSD jails", long_description=README + '\n\n\nChanges\n=======\n\n' + CHANGES, author='Tom Lazar', author_email='tom@tomster.org', + maintainer='Florian Schulze', + maintainer_email='mail@florian-schulze.net', url='http://github.com/ployground/bsdploy', include_package_data=True, classifiers=[ @@ -32,26 +36,29 @@ 'Intended Audience :: System Administrators', 'Operating System :: POSIX :: BSD :: FreeBSD', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 2 :: Only', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: System :: Installation/Setup', 'Topic :: System :: Systems Administration', ], - license='Beerware Licence', + license="BSD 3-Clause License", zip_safe=False, packages=['bsdploy'], install_requires=install_requires, + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*', extras_require={ 'development': [ 'Sphinx', 'repoze.sphinx.autointerface', 'coverage', 'jarn.mkrelease', + 'ploy_virtualbox>=2dev', 'pytest >= 2.4.2', - 'pytest-flakes', - 'pytest-pep8', + 'pytest-flake8', 'tox', 'mock', - 'readline', ], }, entry_points=""" diff --git a/tox.ini b/tox.ini index 2371f7a..75da7e8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,36 +1,37 @@ [tox] -envlist = py27,ansible14,ansible15,ansible16,ansible17 +envlist = + py27{,-ansible19,-ansible24,-ansible25,-ansible26,-ansible27,-ansible28,-ansible29,-ansible210}, + py37{,-ansible25,-ansible26,-ansible27,-ansible28,-ansible29,-ansible210} + py38{,-ansible25,-ansible26,-ansible27,-ansible28,-ansible29,-ansible210} + py39{,-ansible25,-ansible26,-ansible27,-ansible28,-ansible29,-ansible210} + py310{,-ansible4,-ansible5,-ansible6} [testenv] deps = + ansible19: ansible>=1.9,<2dev + ansible24: ansible>=2.4,<2.5dev + ansible25: ansible>=2.5,<2.6dev + ansible26: ansible>=2.6,<2.7dev + ansible25,ansible26: jinja2<3.1 + ansible27: ansible>=2.7,<2.8dev + ansible28: ansible>=2.8,<2.9dev + ansible29: ansible>=2.9,<2.10dev + ansible210: ansible>=2.10,<2.11dev + ansible4: ansible>=4,<5dev + ansible5: ansible>=5,<6dev + ansible6: ansible>=6,<7dev coverage + flake8<5 mock - pyliblzma + ploy_virtualbox>=2dev pytest - pytest-pep8 - pytest-flakes - snot + pytest-cov + pytest-flake8 < 1.1.0;python_version=="2.7" + pytest-flake8;python_version!="2.7" commands = - coverage run {envbindir}/py.test {posargs} - coverage report --include bsdploy/* - coverage html --include bsdploy/* + {envbindir}/py.test --cov {envsitepackagesdir}/bsdploy/ --cov-report html:{toxinidir}/htmlcov_{envname} {posargs} {envsitepackagesdir}/bsdploy/ -[testenv:ansible14] -deps = - ansible>=1.4,<1.5 - {[testenv]deps} - -[testenv:ansible15] -deps = - ansible>=1.5,<1.6 - {[testenv]deps} - -[testenv:ansible16] -deps = - ansible>=1.6,<1.7 - {[testenv]deps} - -[testenv:ansible17] -deps = - ansible>=1.7,<1.8 - {[testenv]deps} +[pytest] +addopts = --flake8 --tb=native -W "ignore:With-statements now directly support multiple context managers:DeprecationWarning" +flake8-ignore = E501 E128 E129 +testpaths = bsdploy