From a61f113b87b4ecb3b735647e201710a51b41292b Mon Sep 17 00:00:00 2001 From: Tim Neumann Date: Thu, 20 Aug 2020 00:15:16 +0200 Subject: [PATCH] Add initial scripts (#1) --- LICENSE | 22 +++++ README.md | 223 ++++++++++++++++++++++++++++++++++++++++++ generate_ip_doc.py | 86 ++++++++++++++++ generatemac.py | 83 ++++++++++++++++ hook-post-merge.sh | 5 + inventory.py | 237 +++++++++++++++++++++++++++++++++++++++++++++ mkplaybook.py | 120 +++++++++++++++++++++++ playbook.sh | 77 +++++++++++++++ shell.nix | 23 +++++ ssh | 24 +++++ submodules | 133 +++++++++++++++++++++++++ 11 files changed, 1033 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100755 generate_ip_doc.py create mode 100755 generatemac.py create mode 100755 hook-post-merge.sh create mode 100755 inventory.py create mode 100755 mkplaybook.py create mode 100755 playbook.sh create mode 100644 shell.nix create mode 100755 ssh create mode 100755 submodules diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..31a55d7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2017-2020 The stuvus-config contributors +Copyright (c) 2020-now The ansible_config_repo_scripts contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ce54da8 --- /dev/null +++ b/README.md @@ -0,0 +1,223 @@ +# ansible_config_repo_scripts +A collection of useful scripts for ansible config repos + +This is intended to be included as a submodule in an ansible config repo. + +## Requirements + +- Ansible +- git +- Python 3 with the modules PyYAML, netaddr, and JMESPath + +### Ubuntu/Debian or other APT based systems + +```bash +sudo apt install ansible git python3 python3-netaddr python3-jmespath python3-yaml +``` + +### Nix/NixOS + +If you have [Nix](https://nixos.org/nix/), then `nix-shell` automatically makes the requirements available. +The playbook script will automatically call `nix-shell` if it's present or you can launch it in a shell yourself. +You may want to link/copy the shell.nix to the root of your config repo. + +## Usage + +Ansible config repos are typically heavily based on git submodules. +To update them (and also initialize them if they are not initialized yet), the script `submodules` may help. + +--- + +The `playbook.sh` script is designed to run playbooks from anywhere. + +``` +$ # Syntax +$ playbook.sh +$ +$ # Example +$ playbook.sh -l hypervisor01 all +``` + +The special playbook `all` is generated by `scripts/mkplaybook.py` and doesn't exist physically. + +### mkplaybook + +`mkplaybook` generates a playbook based on group memberships and `roles.yml`. +For every role in the `roles/` directory, a host group with the same name is assumed and a proper play is generated. +The facts are only gathered once. +For every role a tag is generated named `_ROLENAME`. + +The `roles.yml` contains a dict with the role name as key and settings as values. + +| `Name` | `Description` | +|------------|------------------------------------------------| +| `early` | Run this role before all other non-early roles | +| `late` | Run this role after all other non-late roles | +| `hosts` | Also run this role on these hosts/groups | +| `excludes` | Short form for `hosts: [ !HOST ]` | +| `after` | Always run this role after the specified roles | +| `tags` | Apply these additional tags | + +## Expected directory structure of parent repo + +The script in this repository expect the following directories in the parent repo: + +### hosts + +`hosts` should contain all *host files* with their variables. +Each host has an own file `hosts/.yml`. +Additionally you can add files in `hosts//`. +For example, + +```bash +hosts/myhost/first.yml +hosts/myhost/second.yml +``` + +will result in a host `myhost` with the variables from `first` and `second` merged. + +There is the special variable `_groups` which is the list of names of groups which the host is a member of. +Every other variable is just passed to Ansible as variables applying to that host. + +### groups + +`groups/` should contain *group files* which specify *group variables* +Each group has an own file, called `groups/.yml`. +Just as with hosts, you can additionally add files in `groups//` + +There are three special groups: + +- `all` contains all hosts (even though they don't list `all` in their `_groups`). + Therefore `groups/all.yml` contains the default variables which apply to all hosts. +- `ungrouped` contains all hosts without any groups. + This should however never be the case. +- `virtual` contains all hosts with a `vm` variable defined. + +### playbooks + +All Playbook files should be located in `groups/playbooks/`. +Each one should begin with a comment describing what it's supposed to do. + +### roles + +All roles (typically as submodules) used in the repo. + +### files + +Files which are used in the repo (e.g. config files used by roles). + +### modules + +Custom modules used in the repo. + +### user.yml +The file `user.yml` is used to configure user specific settings like the `ansible_user`. +This file should be in the `.gitignore` + +### roles.yml +A file containing some rules about the order of roles in the repo + +#### Example: +```yaml +--- +# All roles with non-default values. +# Please sort the keys alphabetically within the three sections. + +##### +# Early roles +##### + +apt_sources: + early: true + hosts: + - all + after: + - fstab + tags: + - common + +fstab: + early: true + hosts: + - all + tags: + - common + +sudo: + early: true + hosts: + - all + tags: + - common + +##### +# Late roles +##### + +upgrade: + late: true + hosts: + - all + +##### +# Normal roles +##### + +grub2: + after: + - crypttab + - fstab + +icinga2-client: + hosts: + - all + after: + - icinga2-master + - icinga2-scripts + tags: + - common +``` + +### ansbile.cfg +The ansible config. +It is important that the variable `inventory` is set to `/inventory.py` + +### environment +A file with some environment variables. See below. + +## Special variables + +Some important special variables are: + +- `_groups` is a mandatory variable per host and is a list of groups that host is a member of +- `ansible_host` is a mandatory variable per host and is the IP address which ansible uses to + connect to that host +- `ansible_user` is the username used to connect to the server. + +The [Ansible documentation](https://docs.ansible.com/ansible/latest/intro_inventory.html#list-of-behavioral-inventory-parameters) +lists more variables which are recognized by Ansible. + +## Environment + +To ensure consistent playbook executions, the entire environment is dropped before the Playbook is +ran. +However, as some environment variables are needed for correct playbook exection, they can be set +from files in the repository. +There are two files both located in the repository root: +`./environment` and `./environment.local`. +While `environment` must exist and has to be committed, `environment.local` is optional and must not be committed. +Both work in the same way having simple assignments in bash style. +Lines may be empty or start with `#` to indicate a comment. +Comments after assignments are treated as parts of the assignments. +Assignments may contain blank characters. + +Example: +```bash +# This is a comment followed by an empty line + +SOME_VAR=some value +LANG= +``` + +Assignments with an empty value are filled from the parent environment (your shell). +If the variable doesn't exist in the parent environment, it is ignored. diff --git a/generate_ip_doc.py b/generate_ip_doc.py new file mode 100755 index 0000000..8a3fb01 --- /dev/null +++ b/generate_ip_doc.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +import yaml +import glob +import os +from copy import deepcopy +from datetime import datetime + +# cd into stuvus_config +rpath=os.path.dirname(os.path.realpath(__file__)) +os.chdir(rpath+'/..') + +# get a list of all host configuration files +hostvar_files=glob.glob("./hosts/*.yml") +hostvar_files.extend(glob.glob("./hosts/*/*.yml")) + +data = {} +hostvar_keys_of_interest = ['ip', 'hostname', 'type', 'description', 'organisation', 'groups'] + +# build hostname from host configuration path +def get_hostname(host_config_path): + hostname = host_config_path.replace('.yml','') + hostname = hostname.replace('./hosts/','') + hostname = hostname.split('/')[0] # get the hostname not the filename (needed for hosts with multiple config files) + return hostname + +# get all ips from host configuration +def get_all_host_ips(host_config): + ips = [] + + if 'ansible_host' in host_config: + ips.append(host_config['ansible_host']) + + # go over all configured interface types + for interface_type in [ interface_type for interface_type in ['interfaces', 'bridges'] if interface_type in host_config]: + for interface in host_config[interface_type]: + if 'ip' in interface: + ips.append(interface['ip']) + if 'ips' in interface: + for ip in interface['ips']: + ips.append(ip) + ips = [ ip.split('/')[0] for ip in ips ] # get ips without CIDR + return ips + +# iterate over all hosts +for host_config_path in hostvar_files: + # parse host configuration + host_config_file = open(host_config_path) + host_config = yaml.safe_load(host_config_file) + host_config_file.close() + + # get and set the hostname + host_config['hostname'] = get_hostname(host_config_path) + + host_ips = get_all_host_ips(host_config) + for ip in host_ips: + sort_ip = ''.join([ ip_part.zfill(3) for ip_part in ip.split('.') ]) + data[sort_ip] = deepcopy(host_config) + data[sort_ip]['ip'] = ip + try: + data[sort_ip]['organisation'] = host_config['vm']['org'] + except KeyError: + data[sort_ip]['organisation'] = '___-___' + if 'vm' in data[sort_ip]: + data[sort_ip]['type'] = ' vm ' + else: + data[sort_ip]['type'] = ' hw ' + data[sort_ip]['groups'] = ", ".join(data[sort_ip]['_groups']) # pretty formate groups + +format_string = '|' +separator_string = '|' +header_string = '|' +for info_key in hostvar_keys_of_interest: + # maximum string length for relevant data + max_key_length = max([ len(data[sort_ip][info_key]) for sort_ip in data]) + format_string += ' {%s:<%d} |' % (info_key, max_key_length) # can't use .format here since i need to build a format string + separator_string += '{row_separator:{row_separator}<{length}}|'.format(row_separator = '-', length = max_key_length+2) + header_string += ' {info_key:<{length}} |'.format(info_key = info_key, length = max_key_length) + +# print table head +print("letztes Update {}\n".format(datetime.now())) +print(header_string) +print(separator_string) + +# print all host information +for sort_ip in sorted(data): + print(format_string.format(**data[sort_ip])) diff --git a/generatemac.py b/generatemac.py new file mode 100755 index 0000000..d2ff442 --- /dev/null +++ b/generatemac.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# This script generates a mac address that is not currently in use in one of the existing hostvars. Optionally, a prefix can be specified to use instead of "AA:AA:AA:AA:". Colons may be omitted. +from random import randint +from optparse import OptionParser +from os.path import join, dirname +from yaml import safe_load +from os import walk +from re import match, search + +usedMacs = set() + + +def main(): + global usedMacs + + p = OptionParser( + description= + 'This script generates a mac address that is not currently in use in one of the existing hostvars. Optionally, a prefix can be specified to use instead of "AA:AA:AA:AA:". Colons may be omitted.', + usage='%prog [options]') + p.add_option( + '--prefix', default='AA:AA:AA:AA:', type='string', dest='prefix', help='optional prefix') + options, arguments = p.parse_args() + + # Find all hostfiles + hostConfigFiles = listFiles('../hosts') + + # Parse every hostfile from yaml to dict + for hostConfigFile in hostConfigFiles: + hostConfig = readYAML(hostConfigFile) + #extract and add all used macs to the global set + addMac(hostConfig) + + # Check if prefix is valid and remove colons + asNum = int(options.prefix.replace(':', ''), 16) + prefix = options.prefix.replace(':', '') + if len(prefix) >= 12: + print(hexToMac(prefix[:12])) + return + + # Calculate how many bytes need to be generated + bytesToGenerate = 12 - len(prefix) + maxNum = pow(16, bytesToGenerate) - 1 + + # Generate an unused mac + newMac = hexToMac(prefix + '%0.2X' % randint(0, maxNum)) + while newMac in usedMacs: + newMac = hexToMac(prefix + '%0.2X' % randint(0, maxNum)) + print(newMac) + + +def listFiles(dirPath): + '''List all yml files in a directory with their path relative to this script''' + confDir = join(dirname(__file__), dirPath) + ret = next(walk(confDir))[2] + # Add path to filename and filter out non-yml files + ret = [join(confDir, fileName) for fileName in ret if match(r'^[a-z0-9_-]*\.yml$', fileName)] + return ret + + +def readYAML(path): + '''Reads the yml file in the specified path and returns an equivalent dict''' + fd = open(path, mode='r') + obj = safe_load(fd) + fd.close() + return obj + + +def addMac(config): + '''Adds all MAC addresses contained in this host configuration to the global set''' + global usedMacs + if 'interfaces' in config: + for interface in config['interfaces']: + if 'mac' in interface: + usedMacs.add(interface['mac']) + + +def hexToMac(hexString): + '''Converts a hexadecimal string to the MAC format, uppercase with colons inserted every two bytes''' + return ':'.join([hexString[i:i + 2] for i in range(0, len(hexString), 2)]).upper() + + +if __name__ == '__main__': + main() diff --git a/hook-post-merge.sh b/hook-post-merge.sh new file mode 100755 index 0000000..37798da --- /dev/null +++ b/hook-post-merge.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Hook that is run by git after merging (e.g. after a git pull) + +git submodule update --init --recursive -j 8 diff --git a/inventory.py b/inventory.py new file mode 100755 index 0000000..b86f80b --- /dev/null +++ b/inventory.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + +# This is the inventory script for Ansible. +# It reads all group and host vars from the corresponding directories and merges them. +# It expects a list of groups to be in each host as '_groups'. +# Lists are appended, dicts are merged. + +from yaml import safe_load +from copy import deepcopy +from os.path import join, dirname +from os import walk +from json import dumps +from re import match, search + + +class InventoryErrors(Exception): + def __init__(self, errors): + if not isinstance(errors, list): + errors = [errors] + self.errors = errors + + +def readYAML(paths): + ''' + Reads a list of files, and loads them as yaml content. + The files must exist. + Instead of being a list, the parameter can also be a single file + + :param list paths: List of paths of the files to read + :param str paths: Single path to read + ''' + # Ensure it's a list + if not isinstance(paths, list): + paths = [paths] + + obj = {} + for f in paths: + fd = open(f, mode='r') + mergeDict(obj, safe_load(fd)) + fd.close() + return obj + + +def mergeDict(a, b, overwrite=True): + ''' + Merges two dicts. + If the key is present in both dicts and `overwrite=True`, then the key from the + second dict is used. If the key is present in both dicts and `overwrite=False`, + then an error is raised. + This goes unless the key contains a list or a dict. For dicts and lists, + `overwrite` is ignored. For dicts, this function is called recursively. For + lists, the lists are appended. + + :param dict a: The dict to merge into + :param dict b: The dict to merge into a, overriding with the rules specified above + ''' + if not b: + return a + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + mergeDict(a[key], b[key], overwrite=overwrite) + elif isinstance(a[key], list) and isinstance(b[key], list): + a[key] = a[key] + b[key] + elif overwrite == False: + if a[key] != b[key]: + raise InventoryErrors(f'There are conflicting definitions for `{key}`: `{a[key]}` and `{b[key]}`') + else: + a[key] = b[key] + else: + a[key] = b[key] + + +def listFiles(dirPath): + ''' + List all yml files in a directory with their path relative to this script. + Only .yml files will be found. + + This returns a dict with the key being the name of the yml file, and the value + being either the path of the file relative to this file (in case of a single .yml file) + or a list of yml files if they were grouped into a directory, where the key is the name of the + directory. This resembles the behaviour of vanilla Ansible. + + :param str dirPath: Directory to scan for files + ''' + confDir = join(dirname(__file__), dirPath) + dirs = next(walk(confDir))[1] + files = next(walk(confDir))[2] + # Add path to filename and filter out non-yml files + files = [join(confDir, fileName) for fileName in files if match(r'^[a-z0-9_.-]*\.yml$', fileName)] + # Convert list to dict + ret = {} + for f in files: + name = search('([a-z0-9_.-]*)\.yml$', f).group(1) + ret[name] = f + # List directories and add directories to paths + for dir in dirs: + ret[dir] = [] + for w in walk(join(confDir, dir)): + for f in w[2]: + if match(r'^[a-z0-9_.-]*\.yml$', f): + ret[dir] += [join(join(confDir, dir), f)] + + return ret + + +def addHost(ansibleInventory, userConfig, hostName, hostConfig): + ''' + Adds a host to the ansible configuraton. + This is the core function of the script. + It adds the host to the all group and to the groups that are specified in '_groups'. + '_groups' may also be a string and may not exist. If it doesn't the host is added to the 'ungrouped' group to comply with Ansible's standards. + If a group doesn't exist, an empty one is created. + + When the host is added to all groups, the variables are merged. + First, the variables from the user config are used, then the 'all' group is used, then from each group, then from the host. + + :param dict ansibleInventory: Ansible inventory with groups. The hosts are added here + :param dict userConfig: User config that is added to all hosts + :param str hostName: Name of this host + :param dict hostConfig: Configuration dict of this host + ''' + errors = [] + + # If no ansible_host is defined, try to find an interface that has an IP defined + if 'ansible_host' not in hostConfig: + if 'stuvus_host' in hostConfig: + hostConfig['ansible_host'] = hostConfig['stuvus_host'] + elif 'interfaces' in hostConfig: + for interface in hostConfig['interfaces']: + if 'ip' in interface: + hostConfig['ansible_host'] = interface['ip'].split('/')[0] + break + + if 'ansible_host' not in hostConfig: + errors.append('No stuvus_host or interface config given for ' + hostName) + + groups = hostConfig['_groups'] if '_groups' in hostConfig else ['ungrouped'] + + # Support single-string group specifications + if isinstance(groups, str): + groups = [groups] + + if not isinstance(groups, list): + errors.append(f"Expected a list for `_groups`, found: `{groups}`") + + groups.append('all') + + # Check if the host is a virtual machine and add the virtual group if so. + if 'vm' in hostConfig: + groups.append('virtual') + + # Make elements of `groups` unique + groups = list(set(groups)) + + # This loop builds the local variable `groupsConfig` which is used below + groupsConfig = {} + for groupName in groups: + # Ensure this group exists in the global config part + if groupName not in ansibleInventory: + ansibleInventory[groupName] = {'hosts': [], 'vars': {}} + + # Add host name to group + ansibleInventory[groupName]['hosts'].append(hostName) + + # Do not merge the `all` because it is already used to initialize the + # `ansibleInventory['_meta']['hostvars'][hostName]`; see below this loop + if groupName != 'all': + # Merge group vars together, but don't allow overwrites + try: + mergeDict(groupsConfig, deepcopy(ansibleInventory[groupName]['vars']), overwrite=False) + except InventoryErrors as e: + errors.extend([ f'For host `{hostName}`: {error}' for error in e.errors ]) + + # The priorities are coded here: + mergedConfig = {} + mergeDict(mergedConfig, userConfig) + mergeDict(mergedConfig, deepcopy(ansibleInventory['all']['vars'])) + mergeDict(mergedConfig, groupsConfig) + mergeDict(mergedConfig, hostConfig) + + # Remove the _groups dict + mergedConfig.pop('_groups', None) + + ansibleInventory['_meta']['hostvars'][hostName] = mergedConfig + + if errors != []: + raise InventoryErrors(errors) + + +if __name__ == '__main__': + errors = [] + # Default config + ansibleInventory = { + '_meta': { + 'hostvars': {} + }, + 'all': { + 'hosts': [], + 'vars': {} + }, + 'ungrouped': { + 'hosts': [], + 'vars': {} + }, + 'virtual': { + 'hosts': [], + 'vars': {} + } + } + # Read user configuration + userConfig = {} + try: + userConfig = readYAML(join(dirname(__file__), '../user.yml')) + except IOError: + pass + # Find files + groupConfigFiles = listFiles('../groups') + hostConfigFiles = listFiles('../hosts') + # Parse groups + for groupName, groupConfigs in groupConfigFiles.items(): + # Add to the ansible configuration. + # The hosts list will be filled later + ansibleInventory[groupName] = {'hosts': [], 'vars': readYAML(groupConfigs)} + # Parse hosts + for hostName, hostConfigs in hostConfigFiles.items(): + hostConfig = readYAML(hostConfigs) + try: + addHost(ansibleInventory, userConfig, hostName, hostConfig) + except InventoryErrors as e: + errors.extend(e.errors) + # Check for errors + if errors != []: + e = Exception(''.join([ f'\t({i+1}):\t {e}' for i, e in enumerate(errors) ])) + raise e + # Print the result + print(dumps(ansibleInventory)) diff --git a/mkplaybook.py b/mkplaybook.py new file mode 100755 index 0000000..36c10bf --- /dev/null +++ b/mkplaybook.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +from yaml import safe_load, dump +from os.path import join, dirname +from os import listdir +from sys import exit + + +def role2task(name, config): + """Converts a role to a playbook task. + + Parameters: + name (str): Name of this role + config (dict): Dictionary containing roles.yml + + Returns: + str:The role as Ansible playbook task + """ + hosts = [name] + tags = ['_{}'.format(name)] + if name in config: + if 'hosts' in config[name]: + hosts += config[name]['hosts'] + if 'tags' in config[name]: + tags += config[name]['tags'] + if 'all' in hosts: + hosts = ['all'] + if 'excludes' in config[name]: + hosts += ['!{}'.format(e) for e in config[name]['excludes']] + + return { + 'name': 'Execute role {}'.format(role), + 'hosts': hosts, + 'become': True, + 'roles': [name], + 'gather_facts': False, + 'tags': tags, + 'pre_tasks': [{ + 'name': 'Gather facts', + 'setup': {}, + 'when': 'not ansible_facts' + }] + } + + +def resolve(name, deps, allRoles, resolved, unresolved): + """Resolves dependencies of a role recursively + + Parameters: + name (str): Name of this role + deps (list of str): Names of dependencies of this role + allRoles (dict): Dictionary of all roles for finding deps + resolved (list of str): List of resolved role names + unresolved (list of str): List of unresolved role names + """ + unresolved.append(name) + + if name in resolved: + return # Already resolved + + for dep in deps: + found = False + if dep in resolved: + found = True + if not found: + if dep in unresolved: + print('Circular dependency: {} <-> {}'.format(name, dep)) + exit(1) + resolve(dep, allRoles[dep], allRoles, resolved, unresolved) + resolved.append(name) + unresolved.remove(name) + + +if __name__ == '__main__': + config = {} # roles.yml contents + playbook = [] # Playbook to write + # These are dicts of dependency lists + earlyRoles = {} # Roles to execute at the beginning + roles = {} # Roles to execute in the middle + lateRoles = {} # Roles to execute in the end + + # Read config + with open(join(dirname(__file__), '../roles.yml')) as configFile: + config = safe_load(configFile) + + # Stat roles + for role in listdir(join(dirname(__file__), '../roles')): + early = False + late = False + after = [] + if role in config: + if 'early' in config[role]: + early = config[role]['early'] + if 'late' in config[role]: + late = config[role]['late'] + if 'after' in config[role]: + after = config[role]['after'] + if early and late: + print('Role {} has both early and late set'.format(role)) + exit(1) + if early: + earlyRoles[role] = after + elif late: + lateRoles[role] = after + else: + roles[role] = after + + # Resolve dependencies + resolved = [] + for r in [earlyRoles, roles, lateRoles]: + for name, deps in r.items(): + resolve(name, deps, r, resolved, []) + + # Build playbook + for role in resolved: + playbook.append(role2task(role, config)) + + # Write out + with open(join(dirname(__file__), '../.playbook.yml'), 'w') as out: + out.write(dump(playbook, default_flow_style=False)) diff --git a/playbook.sh b/playbook.sh new file mode 100755 index 0000000..d7fd393 --- /dev/null +++ b/playbook.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# This script is supposed to run a playbook. +# It doesn't matter where you call it from. +# `playbooks/` will automatically be prepended to the path, while `.yml` will be appended. +# This results in `$0 + +# Fail on errors and unset variables +set -e +set -o nounset + +# Try to run in nix-shell +if hash nix-shell &>/dev/null; then + if [ -z "${IN_STUVUS_NIX_SHELL:-}" ]; then + args="${0}" + for param in "${@}"; do + args="${args} \"${param}\"" + done + exec nix-shell --arg inPlaybook true --run "${args}" + fi +fi + +if [ "${#}" -lt 1 ]; then + echo "Usage: ${0} " + exit 1 +fi + +# Get the path of the playbook to execute +playbook="${!#}" +# Remove the last parameter +set -- "${@:1:$(($#-1))}" + +# Prepare playbook execution +cd "$(dirname "$(readlink -f "${0}")")/.." || exit 255 + +# Check if playbook exists +if [ "${playbook}" != all ] && [ ! -f "${PWD}/playbooks/${playbook}.yml" ]; then + echo "${PWD}/${playbook} does not exist" + exit 1 +fi + +# Build environment +declare -a environment +parseEnvironmentFile() { + while IFS= read -r line; do + # Skip empty lines + [ -z "${line}" ] && continue + # Skip comments + [[ "${line}" =~ ^[[:space:]]*#.*$ ]] && continue + # See if we need to take the variable from the parent environment + if [ "$(echo "${line}" | cut -d'=' -f2-)" = '' ]; then + # This gets the name of the variable + name="$(echo "${line}" | cut -d '=' -f1 | xargs)" + # Ignore missing variables and assign + [ -z "${!name:-}" ] || environment+=("${name}=${!name:-}") + else + # Append to environment array + environment+=("${line}") + fi + done < "${1}" +} + +# Try both the normal and local environment +parseEnvironmentFile environment +[ -f environment.local ] && parseEnvironmentFile environment.local + +if [ "${playbook}" = all ]; then + scripts/mkplaybook.py +else + # Create a temporary playbook copy at the ansible root directory. + # This workaround is needed to allow template or macro includes from tasks + cp "playbooks/${playbook}.yml" ./.playbook.yml +fi +trap 'rm ./.playbook.yml' INT TERM EXIT + +# Go! +exec env - "${environment[@]}" "${ANSIBLE_PLAYBOOK:-ansible-playbook}" "${@}" "./.playbook.yml" diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..31788b1 --- /dev/null +++ b/shell.nix @@ -0,0 +1,23 @@ +{ pkgs ? import {}, inPlaybook ? false }: + +# stdenvNoCC because we don't need a C-compiler during build or +# when using the nix-shell +pkgs.stdenvNoCC.mkDerivation { + name = "stuvus_config-shell"; + + nativeBuildInputs = with pkgs; [ + ansible + sshpass + (python3.withPackages(ps: with ps; [ + autopep8 + jmespath + pylint + pyyaml + ])) + ]; + + shellHook = '' + export IN_STUVUS_NIX_SHELL=1 + ${if !inPlaybook then "if [ -f .nix-shell-hook ]; then source .nix-shell-hook; fi" else ""} + ''; +} diff --git a/ssh b/ssh new file mode 100755 index 0000000..b55c52f --- /dev/null +++ b/ssh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# This script allows easy SSH connections to every host managed by Ansible. + +set -e +set -o pipefail +set -o nounset + +if [ "${#}" -lt 1 ]; then + echo "Usage: ${0} " + exit 1 +fi + +hostname="${!#}" +# Remove the last parameter +set -- "${@:1:$(($#-1))}" + +# Fetch infos about host +conn="$(ansible localhost -m debug -a "msg={{ hostvars[\"${hostname}\"].ansible_user }}@{{ hostvars[\"${hostname}\"].ansible_host }}" | \ + grep '^\s*"msg":' | \ + cut -d'"' -f4)" + +# Connect +exec ssh "${@}" "${conn}" diff --git a/submodules b/submodules new file mode 100755 index 0000000..0943ed1 --- /dev/null +++ b/submodules @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +# Manages git submodules. Check the help for more info. + +set -e +set -u +set -o pipefail + +# Ansi escape codes +RESET="\e[0m" +ITALIC="\e[3m" +BOLD="\e[1m" +YELLOW="\e[33m" +INDENT="\r\t\t\t\t\t" + +# root of the repository +root="$(dirname "$(dirname "$(realpath -s "${BASH_SOURCE[0]}")")")/" +cd "$root" + +# Shows help and exits with $1 +help() { + ( + echo "Usage: ${0} " + echo + echo "Subcommands are:" + echo "- help: Show this help" + echo "- fix: Try to fix all submodules by updating and initializing them" + echo " Optional parameters: Paths to submodules - omit for all submodules" + echo "- updatable: List updatable submodules" + echo "- notMaster: List submodules where the current commit is not on origin/master" + echo "- update: Update a submodule (or multiple) to origin/master. With extra force" + echo " Mandatory parameters: Paths to submodules" + ) >&2 + exit "${1}" +} + +# Runs a git command in the global repository context +runGit() { + git --git-dir="${root}/.git" --work-tree="${root}" "${@}" +} + +# Runs a git command in the context of the submodule $1 +runGitSubmodule() { + submod="${1}" + shift + git --git-dir="${root}/.git/modules/${submod}" --work-tree="${root}/${submod}" "${@}" +} + +# Tries to fix submodules +fix() { + runGit submodule absorbgitdirs + runGit submodule update --init --recursive "${@}" +} + +# Fetch all submodules parallel +fetch_submodules() { + IFS=$'\n' + echo "Fetch all submodules" + while read -r module_str; do + IFS=' ' read -ra module <<< "$module_str" + cd "${module[1]}" || exit 1 + git fetch & + cd "$root" || exit 1 + done < <(git submodule status) + wait +} + +# Check all submodules for new upstream commits +updatable() { + IFS=$'\n' + fetch_submodules + while read -r module_str; do + IFS=' ' read -ra module <<< "$module_str" + if [ "${module[0]:0:1}" = "+" ]; then + echo -e "$ITALIC$YELLOW${module[1]}$INDENT$RESET$BOLD$YELLOW is out of sync\e[0m" + fi + done < <(git submodule status) +} + +# Try to find submodules that are not on master +notMaster() { + IFS=$'\n' + fetch_submodules + while read -r module_str; do + IFS=' ' read -ra module <<< "$module_str" + if ! echo "${module[2]}"|grep -q "master"; then + echo -e "$ITALIC$YELLOW${module[1]}$INDENT$RESET$BOLD$YELLOW is not up to date\e[0m" + elif [ "${module[0]:0:1}" = "+" ]; then + echo -e "$ITALIC$YELLOW${module[1]}$INDENT$RESET$BOLD$YELLOW is out of sync\e[0m" + fi + done < <(git submodule status) +} + +# Update a submodule to origin/master +update() { + if [ "${#}" -lt 1 ]; then + help 1 + fi + for submod in "${@}"; do + runGitSubmodule "${submod}" fetch origin master + runGitSubmodule "${submod}" checkout master + runGitSubmodule "${submod}" reset --hard origin/master + runGit add "${submod}" + done +} + +# Parameter stuff +if [ "${#}" -lt 1 ]; then + help 1 +fi + +subcommand="${1}" +shift +case "${subcommand}" in + help) + help 0 + ;; + fix) + fix "${@}" + ;; + updatable) + updatable "${@}" + ;; + notMaster) + notMaster "${@}" + ;; + update) + update "${@}" + ;; + *) + help 1 + ;; +esac