First iteration of update playbooks that work for FreeBSD/Debian

ansible.cfg

@@ -0,0 +1,6 @@
+[defaults]
+#stdout_callback = yaml
+#callbacks_enabled = profile_tasks
+callbacks_enabled = community.general.unixy, profile_tasks
+stdout_callback = community.general.unixy
+vault_password_file=~/bin/ansible_vault_password.sh

debian-distupgrade.yml

@@ -0,0 +1,144 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+
+- name: Debian dist-upgrade
+  hosts: all
+  become: true
+  serial: 4
+  order: shuffle
+  vars:
+    disk_free_percentage: 20
+    required_pkgs:
+      - needrestart
+    debian_supported:
+      - bookworm
+      - bullseye
+      - buster
+    debian_upgrade_matrix:
+      buster: bullseye
+      bullseye: bookworm
+  vars_files:
+    - ~/.ansible/my_vault.yml
+
+  pre_tasks:
+    - name: Abort automated dist-upgrade non Debian systems
+      fail:
+        msg: 'Not dist-upgrading non-Debian system {{ inventory_hostname }}/{{ ansible_hostname }} ({{ansible_distribution_release}}/{{ ansible_distribution_version }}), aborting. Please upgrade to supported version'
+      when: "ansible_facts['os_family']|lower != 'debian'"
+    - name: Abort automated dist-upgrade for EOL systems
+      fail:
+        msg: 'Not dist-upgrading EOL system {{ inventory_hostname }}/{{ ansible_hostname }} ({{ansible_distribution_release}}/{{ ansible_distribution_version }}), aborting. Please upgrade to supported version'
+      when: "ansible_distribution == 'Debian' and ansible_distribution_release not in debian_supported"
+    - name: Check available space 
+      import_tasks: tasks/check-disk-free.yml
+
+  tasks:
+    - name: "Install mandatory packages"
+      apt:
+        pkg: '{{ required_pkgs }}'
+
+    - name: Tell we are not going to do dist-upgrade, if not in the upgrade matrix
+      fail:
+
+        msg: 'Upgrading {{ ansible_distribution_release }} is not possible, upgrade path not seen in upgrade matrix'
+      when: "ansible_distribution_release not in debian_upgrade_matrix"
+    - name: Tell we are going to do dist-upgrade
+      debug:
+        msg: 'Upgrade {{ ansible_distribution_release }} to {{ debian_upgrade_matrix[ansible_distribution_release] }}'
+      when: "ansible_distribution_release in debian_upgrade_matrix"
+    - name: Find apt sources mentioning Debian distribution name
+      ansible.builtin.find:
+        paths:
+          - /etc/apt/sources.list.d
+        patterns: '*.list'
+        contains: '.*\b{{ ansible_distribution_release }}\b'
+      register: apt_sources
+
+    - name: set apt_sources_files
+      ansible.builtin.set_fact:
+        apt_sources_files: '{{ ["/etc/apt/sources.list"] + (apt_sources.files | map(attribute="path") )}}'
+
+    - name: Record current selections
+      command:
+        cmd: '/usr/bin/dpkg --get-selections "*"'
+      register: dpkg_selections_all_pre
+      check_mode: false # Need to have this working in check mode
+
+    - name: Log current selections
+      ansible.builtin.copy:
+        content: |
+          Results of dpkg --get-selections "*"
+          {{ dpkg_selections_all_pre.msg }}
+
+          {% if dpkg_selections_all_pre.stdout is defined %}
+          stdout of apt
+
+          {{ dpkg_selections_all_pre.stdout }}
+          {% endif %}
+
+          {% if dpkg_selections_all_pre.stderr is defined %}
+          stderr of apt
+
+          {{ dpkg_selections_all_pre.stderr }}
+          {% endif %}
+        dest: "/var/log/apt/distupgrade-pre-{{ ansible_distribution_release }}-{{ debian_upgrade_matrix[ansible_distribution_release] }}-{{'%FT%T' | strftime}}.log"
+
+    - name: Dist upgrade on the current release to sync up and catch errors
+      import_tasks: tasks/dist_upgrade_debian.yml
+
+    - name: Show found apt source file
+      debug:
+        msg: 'Adjusting {{ item }}'
+      loop: '{{ apt_sources_files }}'
+
+    - name: Replace debian-security bullseye/updates with bullseye-security
+      ansible.builtin.replace:
+        regexp: 'debian-security buster/updates'
+        replace: 'buster-security'
+        path: '{{ item }}'
+        backup: true
+      loop: '{{ apt_sources_files }}'
+      when: "ansible_distribution == 'Debian' and ansible_distribution_release  == 'buster'"
+
+    - name: Replace dist name in apt sources
+      ansible.builtin.replace:
+        regexp: '\b{{ ansible_distribution_release }}\b'
+        replace: '{{ debian_upgrade_matrix[ansible_distribution_release] }}'
+        path: '{{ item }}'
+        backup: true
+      loop: '{{ apt_sources_files }}'
+      loop_control:
+        label: 'Replacing {{ ansible_distribution_release }} with {{ debian_upgrade_matrix[ansible_distribution_release] }} in {{ item }}'
+      register: apt_sources_files_replacements
+
+    - name: Show replacements
+      debug:
+        var: apt_sources_files_replacements
+
+    - name: Dist upgrade on to get to the new release
+      import_tasks: tasks/dist_upgrade_debian.yml
+
+    - name: Record current selections after upgrade
+      command:
+        cmd: '/usr/bin/dpkg --get-selections "*"'
+      register: dpkg_selections_all_post
+      check_mode: false # Need to have this working in check mode
+
+    - name: Log current selections
+      ansible.builtin.copy:
+        content: |
+          Results of dpkg --get-selections "*"
+          {{ dpkg_selections_all_post.msg }}
+
+          {% if dpkg_selections_all_post.stdout is defined %}
+          stdout of apt
+
+          {{ dpkg_selections_all_post.stdout }}
+          {% endif %}
+
+          {% if dpkg_selections_all_post.stderr is defined %}
+          stderr of apt
+
+          {{ dpkg_selections_all_post.stderr }}
+          {% endif %}
+        dest: "/var/log/apt/distupgrade-post-{{ ansible_distribution_release }}-{{ debian_upgrade_matrix[ansible_distribution_release] }}-{{'%FT%T' | strftime}}.log"

freebsd-update.yml

@@ -0,0 +1,65 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+
+- name: FreeBSD patches
+  hosts: all
+  become: true
+  serial: 4
+  order: shuffle
+  vars_files:
+    - ~/.ansible/my_vault.yml
+
+  tasks:
+    - name: Fetch updates
+      command:
+        cmd: /usr/sbin/freebsd-update fetch --not-running-from-cron
+      environment:
+        PAGER: cat
+      register: fetchupdates
+      # Need to have this working in check mode
+      check_mode: false
+
+    - name: show results of fetch updates
+      debug:
+        verbosity: 1
+        msg: '{{ fetchupdates.stdout }}'
+    - name: Check if updates are ready to install
+      command:
+        cmd: /usr/sbin/freebsd-update updatesready
+      register: updatesready
+      # Need to have this working in check mode
+      check_mode: false
+      # non zero exit code does not mean "failure" but "action needed"
+      ignore_errors: true
+      changed_when: updatesready.rc == 0
+      failed_when: updatesready.rc == 1
+
+    - name: show results of updatesready
+      debug:
+        verbosity: 1
+        msg: '{{ updatesready.stdout }}'
+
+    - name: Update when updates can be installed
+      block:
+        - name: Perform system updates
+          import_tasks: tasks/update_install_freebsd.yml
+
+        - name: Perform ezjail updates
+          import_tasks: tasks/update_ezjail_freebsd.yml
+
+        - name: Perform iocage updates
+          import_tasks: tasks/update_iocage_freebsd.yml
+
+        - name: Record installed kernel version
+          command:
+            cmd: /bin/freebsd-version -k
+          check_mode: false
+          register: installedkernel
+
+        - name: Reboot system if newer kernel is found
+          import_tasks: tasks/reboot_system.yml
+          when: ansible_kernel != installedkernel.stdout
+
+        - name: Perform system updates post reboot
+          import_tasks: tasks/update_install_freebsd.yml
+      when: updatesready.rc == 0

inventory/ansible_nb_device.sh

@@ -0,0 +1,19 @@
+#! /bin/sh
+NB_TOKEN=$(bw get password "Netbox RO token")
+curl -Ss -H "Authorization: Token ${NB_TOKEN}" https://netbox.niet.verweg.com/graphql/ --json '{
+    "query": "query {device_list(status: \"active\", name__n: \"null\" ) {name site {name} platform {name} device_type{ manufacturer {name} } role {name} primary_ip4 {address} primary_ip6 {address} }}"
+}' | jq -r '
+([
+	.data[][] | { (.name): {
+	    netbox_platform: .platform.name,
+		netbox_role: .device_role.name,
+		netbox_primary_ip4: .primary_ip4,
+		netbox_primary_ip6: .primary_ip6,
+	}}
+] | add ) as $hostvars |
+[([
+	.data[] | group_by(.site.name) | .[] | {
+		key: ([.[].site.name]|unique)[0],
+		value: {hosts:[(.[].name)]}}
+		] | from_entries, {_meta: {hostvars: $hostvars}})] | add
+'

inventory/ansible_nb_vm.sh

@@ -0,0 +1,21 @@
+#! /bin/sh
+NB_TOKEN=$(bw get password "Netbox RO token")
+curl -Ss -H "Authorization: Token ${NB_TOKEN}" https://netbox.niet.verweg.com/graphql/ --json '{
+    "query": "query {virtual_machine_list(status: \"active\", ) {name vcpus disk memory cluster {name} role {name} primary_ip4 {address} primary_ip6 {address} }}"
+}' | jq -r '
+([
+	.data[][] | { (.name): {
+		netbox_vcpus: .vcpus,
+		netbox_disk: .disk,
+		netbox_memory: .memory,
+		netbox_role: .role.name,
+		netbox_primary_ip4: .primary_ip4,
+		netbox_primary_ip6: .primary_ip6,
+	}}
+] | add ) as $hostvars |
+[([
+	.data[] | group_by(.cluster.name) | .[] | {
+		key: ([.[].cluster.name]|unique)[0],
+		value: {hosts:[(.[].name)]}}
+		] | from_entries, {_meta: {hostvars: $hostvars}})] | add
+'

inventory/ezjail.sh

@@ -0,0 +1,8 @@
+#! /bin/sh
+
+if [ $# -lt 1 ]; then
+	echo "Usage: $(basename $0) [ --list | --hostname <hostname> ]"
+	exit 1;
+fi
+
+ezjail-admin list

security.yaml

@@ -0,0 +1,42 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+
+- name: FreeBSD patches
+  hosts: all
+  become: true
+  serial: 4
+  order: shuffle
+  vars_files:
+    - ~/.ansible/my_vault.yml
+
+  vars:
+    restart_files: []
+    restart_files_packages: []
+    restart_services: []
+
+  tasks:
+    - block:
+        - name: Send alert to operators that patching caused alarms
+          fail:
+            msg: 'Not patching EOL system {{ inventory_hostname }}/{{ ansible_hostname }} ({{ansible_distribution_release}}/{{ ansible_distribution_version }}), aborting. Please upgrade to supported version'
+          when: "ansible_distribution == 'Debian' and ansible_distribution_release not in ['bullseye','buster','stretch']"
+
+        - name: Use update task for debian
+          import_tasks: tasks/update_all_debian.yml
+      when: "ansible_facts['os_family']|lower == 'debian'"
+
+    - block:
+        - name: Send alert to operators that patching caused alarms
+          fail:
+            msg: 'Not patching EOL system {{ inventory_hostname }}/{{ ansible_hostname }} ({{ansible_distribution_major_version}}/{{ ansible_distribution_version }}), aborting. Please upgrade to supported version'
+          when: "ansible_distribution == 'FreeBSD' and ansible_distribution_major_version not in ['13','14']"
+
+        - name: Use update task for debian
+          import_tasks: tasks/update_all_freebsd.yml
+      when: "ansible_facts['os_family']|lower == 'freebsd'"
+
+    - name: Flush handlers
+      meta: flush_handlers
+
+    - name: Check whether the remote node is still reachable
+      ansible.builtin.wait_for_connection:

tasks/check-disk-free.yml

@@ -0,0 +1,28 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+- name: test for available disk space
+  assert:
+    quiet: true
+    that:
+      - not (item.mount == '/' and ( item.size_available < item.size_total - ( item.size_total|float * ((100 - (disk_free_percentage|default(15)))/100) ) ) )
+      - not (item.mount == '/var' and ( item.size_available < item.size_total - ( item.size_total|float * ((100 - (disk_free_percentage|default(15)))/100) ) ) )
+      - not (item.mount == '/boot' and ( item.size_available < item.size_total - ( item.size_total|float * ((100 - (disk_free_percentage|default(15)))/100) ) ) )
+      - not (item.mount == '/boot/efi' and ( item.size_available < item.size_total - ( item.size_total|float * ((100 - (disk_free_percentage|default(15)))/100) ) ) )
+    success_msg: '{{ item.mount }} available space {{ item.size_available | default(0) | human_readable }} over {{ disk_free_percentage | default(15) }}% '
+    fail_msg: '{{ item.mount }} available space {{ item.size_available | default(0) | human_readable }} under {{ disk_free_percentage | default(15) }}%. Required: {{ (item.size_total - ( item.size_total|float * ((100 - (disk_free_percentage|default(15)))/100) )) | human_readable }}'
+  loop: '{{ ansible_mounts }}'
+  loop_control:
+    label: '{{ item.mount }}: Available {{ item.size_available | default(0) | human_readable }}, Total {{ item.size_total | default(0) | human_readable }}. Required: {{ (item.size_total - ( item.size_total|float * ((100 - (disk_free_percentage|default(15)))/100) )) | human_readable }} '
+  ignore_errors: true
+  register: disk_free
+
+- name: Not enough free disk space
+  fail:
+    msg: |
+      Not enough free space on system:
+      {% for failed_space in (disk_free.results | selectattr('failed')) %}
+      {{ failed_space.msg }}
+      {% endfor %}
+
+      Try cleaning up space by removing files / packages
+  when: "disk_free is failed"

tasks/dist_upgrade_debian.yml

@@ -0,0 +1,28 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+- name: Perform dist-upgrade
+  ansible.builtin.apt:
+    upgrade: dist
+    update_cache: true
+
+- name: Check restart status
+  command:
+    cmd: /usr/sbin/needrestart -pk
+  register: restart_status
+  check_mode: false # Need to have this working in check mode
+  changed_when: restart_status.rc != 0
+  failed_when: restart_status.rc > 2
+  ignore_errors: true # non zero exit code does not mean "failure" but "action needed"
+
+- name: show results of needrestart / check_restart_required
+  debug:
+    verbosity: 1
+    var: restart_status.stdout_lines
+
+- name: Restart system when allowed
+  import_tasks: tasks/reboot_system.yml
+  when: restart_status.rc > 0
+
+- name: Clean after dist-upgrade
+  ansible.builtin.apt:
+    autoremove: yes

tasks/patch_post_exec.yml

@@ -0,0 +1,12 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+  - name: Execute post patch commands
+    shell:
+      cmd: '{{ item }}'
+    loop: '{{ patch_post_exec }}'
+    register: patch_post_exec_res
+
+  - name: show results of patch_post_exec actions
+    debug:
+      verbosity: 1
+      var: patch_post_exec_res

tasks/patch_pre_exec.yml

@@ -0,0 +1,12 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+  - name: Execute pre patch commands
+    shell:
+      cmd: '{{ item }}'
+    loop: '{{ patch_pre_exec }}'
+    register: patch_pre_exec_res
+
+  - name: show results of patch_pre_exec actions
+    debug:
+      verbosity: 1
+      var: patch_pre_exec_res

tasks/reboot_system.yml

@@ -0,0 +1,11 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+- name: Reboot system for patches
+  reboot:
+    msg: 'Rebooting for patches'
+  register: system_reboot
+
+- name: show results of reboot
+  debug:
+    verbosity: 1
+    var: system_reboot

tasks/update_all_debian.yml

@@ -0,0 +1 @@
+---

tasks/update_all_freebsd.yml

@@ -0,0 +1,115 @@
+# vim:ts=2:sw=2:et:filetype=ansible
+---
+- name: Check security status
+  command:
+    cmd: /usr/sbin/pkg audit -Rjson-compact
+  register: security_status
+  # Need to have this working in check mode
+  check_mode: false
+  # non zero exit code does not mean "failure" but "action needed"
+  ignore_errors: true
+  changed_when: security_status.rc != 0
+  failed_when: security_status.rc > 2
+
+- name: show results of security_status
+  debug:
+    verbosity: 1
+    msg: '{{security_status.stdout | from_json | to_json(indent=4, sort_keys=True) }}'
+
+- name: Perform pre update commands
+  import_tasks: tasks/patch_pre_exec.yml
+  when: patch_pre_exec is defined
+
+- block:
+    - name: Update all packages to their latest version
+      command:
+        cmd: /usr/sbin/pkg upgrade -vy
+        # async: '{{ ansible_check_mode | ternary(0, (downtime_minutes | int * 60) - 60)}}'
+      register: pkg_data
+    - name: Update all packages to their latest version (dry run)
+      command:
+        cmd: /usr/sbin/pkg upgrade -vyn
+      # Need to have this working in check mode
+      check_mode: false
+      ignore_errors: true
+      changed_when: pkg_data.rc != 0
+      failed_when: "'FAILED' in pkg_data.stderr"
+      register: pkg_data
+      when: ansible_check_mode
+  rescue:
+    - name: pkg failed, try to recover if possible
+      debug:
+        msg: "Something went wrong, attempting recovery.."
+
+  always:
+    - name: Log output from pkg run
+      check_mode: false  # Need to have this working in check mode
+      ansible.builtin.copy:
+        content: |
+          Result of pkg on {{inventory_hostname}}
+
+          {% if pkg_data.msg is defined %}
+          {{ pkg_data.msg }}
+          {% endif %}
+
+          {% if pkg_data.stdout is defined %}
+          stdout of pkg
+
+          {{ pkg_data.stdout }}
+          {% endif %}
+
+          {% if pkg_data.stderr is defined %}
+          stderr of pkg
+
+          {{ pkg_data.stderr }}
+          {% endif %}
+        dest: "/var/tmp/security-patch-{{'%FT%T' | strftime}}.log"
+
+
+    - name: Perform post update commands
+      import_tasks: tasks/patch_post_exec.yml
+      when: patch_post_exec is defined
+
+    - name: Check restart status
+      command:
+        cmd: /usr/local/bin/checkrestart -j 0 --libxo json
+      register: check_restart_status
+      check_mode: false # Need to have this working in check mode
+      changed_when: check_restart_status.rc != 0
+      failed_when: check_restart_status.rc > 2
+      ignore_errors: true # non zero exit code does not mean "failure" but "action needed"
+
+    - name: set restart_files
+      ansible.builtin.set_fact:
+        restart_files: '{{ check_restart_status.stdout | from_json | community.general.json_query("checkrestart.process[].arguments") | unique}}'
+
+    - name: find packages for restart_files
+      ansible.builtin.command:
+        cmd: '/usr/sbin/pkg which -q {{ item }}'
+      register: pkg_which_output
+      loop: '{{ restart_files }}'
+
+    - name: list package contents
+      ansible.builtin.command:
+        cmd: '/usr/sbin/pkg info -ql {{ item }}'
+      register: pkg_info_output
+      loop: '{{ pkg_which_output.results  | map(attribute="stdout")}}'
+
+    - name: set services to be restarted due to stale libraries
+      ansible.builtin.set_fact:
+        restart_services: '{{ restart_services + (item)  }}'
+      loop: '{{ pkg_info_output.results | map(attribute="stdout_lines") | select("search","\/rc\.d\/([^\/]+)$") | map("basename")}}'
+      loop_control:
+        label: '{{ item }}'
+
+    - name: show services to be restarted
+      ansible.builtin.debug:
+        verbosity: 1
+        var: restart_services
+
+    - name: restart service(s)
+      ansible.builtin.service:
+        name: '{{ item }}'
+        state: restarted
+      loop: '{{ restart_services }}'
+  when: security_status.rc != 0 or ansible_check_mode

tasks/update_ezjail_freebsd.yml

@@ -0,0 +1,22 @@
+---
+- name: Check if ezjail is available
+  stat:
+    path: /usr/local/bin/ezjail-admin
+  tags:
+    - always
+  register: ezjail_available
+
+- block:
+    - name: Install updates (ezjail)
+      command:
+        cmd: /usr/local/bin/ezjail-admin update -u
+      register: installupdates_ezjail
+
+    - name: show results of install updates (ezjail)
+      debug:
+        verbosity: 1
+        msg: '{{ installupdates_ezjail.stdout }}'
+
+  # XXX etcupdate in blind mode / certificate stuff
+  #
+  when: ezjail_available.stat.exists

tasks/update_install_freebsd.yml

@@ -0,0 +1,12 @@
+---
+- name: Install updates
+  command:
+    cmd: /usr/sbin/freebsd-update install
+  environment:
+    PAGER: cat
+  register: installupdates
+
+- name: show results of install updates
+  debug:
+    verbosity: 1
+    msg: '{{ installupdates.stdout }}'

tasks/update_iocage_freebsd.yml

@@ -0,0 +1,47 @@
+---
+- name: Check if iocage is available
+  stat:
+    path: /usr/local/bin/iocage
+  tags:
+    - always
+  register: iocage_available
+
+- block:
+    - name: List iocage jails
+      command:
+        cmd: /usr/local/bin/iocage list -H
+      check_mode: false
+      register: iocage_list_jails
+
+    - name: show results of list jails (iocage)
+      debug:
+        verbosity: 1
+        var: iocage_list_jails
+
+    - name: set iocage jails
+      ansible.builtin.set_fact:
+        iocage_jails: '{{ iocage_list_jails.stdout_lines | map("split") }}'
+
+    - name: Install updates (iocage)
+      command:
+        cmd: '/usr/local/bin/iocage update {{ item.1 }}'
+      environment:
+        PAGER: cat
+      when: item.2 == 'up' and item.3 == ansible_distribution_version + '-RELEASE'
+      loop: '{{ iocage_jails }}'
+      loop_control:
+        label: 'iocage update {{ item.1 }}'
+      register: installupdates_iocage
+
+    - name: show results of install updates (iocage)
+      debug:
+        verbosity: 1
+        msg: |
+          Results of {{ item.cmd | join(' ') }}
+          {{ item.stdout | default(item.msg | default('No message')) }}
+
+      loop: '{{ installupdates_iocage.results }}'
+      loop_control:
+        label: '{{ item.cmd | join(" ") }}'
+      when: installupdates_iocage
+  when: iocage_available.stat.exists