ostree-var-lib-containers-machineconfig.yaml

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  labels:
    machineconfiguration.openshift.io/role: master
  name: 98-var-lib-containers
spec:
  config:
    ignition:
      version: 3.1.0
    storage:
      files:
        # Configuration so crio wont delete the contents of /var/lib/containers on startup
        - contents:
            source: data:text/plain;charset=utf-8;base64,W2NyaW9dCmNsZWFuX3NodXRkb3duX2ZpbGUgPSAiIgo=
          mode: 420
          path: /etc/crio/crio.conf.d/99-crio-disable-wipe.toml
        - mode: 493
          path: /usr/local/bin/create_shared_containers_dir.sh
          contents:
            source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaAojCiMgVGhpcyBzY3JpcHQgd2lsbCBjcmVhdGUgdW5kZXIgL3N5c3Jvb3QgYSBkaXJlY3RvcnkgdGhhdCB3aWxsIGJlIHVzZWQgdG8gc2hhcmUKIyBjb250YWluZXJzIGJldHdlZW4gdGhlIGRpZmZlcmVudCBzdGF0ZXJvb3RzCiMKc2V0IC14CmRpcj1jb250YWluZXJzCiMgSWYgdGhlIGRlc3RpbmF0aW9uIHNoYXJlZCBkaXIgZG9lcyBub3QgZXhpc3QsIHdlIG5lZWQgdG8gY3JlYXRlIGl0CmlmIFtbICEgLWQgL3N5c3Jvb3QvIiRkaXIiIF1dOyB0aGVuCiAgIGVjaG8gIi9zeXNyb290LyRkaXIgbm90IGV4aXN0aW5nLCBjcmVhdGluZyIKICAgZWNobyAiRnJlZWluZyBzcGFjZSBpbiBwcmV2aW91cyAvdmFyL2xpYi9jb250YWluZXJzIgogICAjIEZpcnN0IHdlIHdpbGwgZGVsZXRlIHRoZSBjb250ZW50cyBvZiB0aGUgb2xkIC92YXIvbGliL2NvbnRhaW5lcnMgdG8gbm90CiAgICMgaGF2ZSBoaWRkZW4gdXNlZCBzcGFjZSwgc2luY2Ugd2Ugd2lsbCBtb3VudCBvbiB0b3Agb2YgaXQKICAgcm0gLWZyIC92YXIvbGliL2NvbnRhaW5lcnMvKgogICB0bXA9JChta3RlbXAgLWQpCiAgICMgL3N5c3Jvb3QgaXMgcmVhZC1vbmx5LCBhbmQgaWYgd2UgcmVtb3VudCBpdCByZWFkd3JpdGUsIHdlIHdvbnQgYmUgYWJsZSB0byByZXZlcnQgaXQgYmFjayB0byByZWFkLW9ubHkKICAgIyB0byBvdmVyY29tZSB0aGlzLCB3ZSB3aWxsIGRvIGEgYmluZCBtb3VudCBzb21ld2hlcmUgZWxzZSwgYW5kIGNoYW5nZSB0aGF0IG5ldyBtb3VudCB0byByZWFkLXdyaXRlCiAgIG1vdW50IC9zeXNyb290ICR0bXAgLW8gYmluZAogICBtb3VudCAkdG1wIC1vIHJlbW91bnQscncKICAgIyBXaXRoIHNvbWUgdmVyc2lvbnMgb2Ygb3N0cmVlIHRoZSBzeXNyb290IGRpcmVjdG9yeSBoYXMgdGhlIGltbXV0YWJsZQogICAjIGF0dHJpYnV0ZSwgc28gaW4gY2FzZSBpdCBpcyBwcmVzZW50LCB3ZSBuZWVkIHRvIHJlbW92ZSBpdCwgYW5kIG9uY2Ugd2UKICAgIyBjcmVhdGVkIHRoZSBkaXJlY3RvcnksIHdlIGNhbiBzZXQgaXQgYmFjawogICBpZiBsc2F0dHIgLWQgJHRtcCB8IGN1dCAtZCAnICcgLWYgMSB8IGdyZXAgaTsgdGhlbgogICAgICAgY2hhdHRyIC1pICR0bXAKICAgICAgIG1rZGlyIC1wICIkdG1wLyRkaXIiCiAgICAgICBjaGF0dHIgK2kgJHRtcAogICBlbHNlCiAgICAgICBta2RpciAtcCAiJHRtcC8kZGlyIgogICBmaQogICB1bW91bnQgJHRtcAogICBybWRpciAkdG1wCmZpCg==

# Contents of the create_shared_containers_dir.sh script:
#
##!/usr/bin/env bash
##
## This script will create under /sysroot a directory that will be used to share
## containers between the different stateroots
##
#set -x
#dir=containers
## If the destination shared dir does not exist, we need to create it
#if [[ ! -d /sysroot/"$dir" ]]; then
#   echo "/sysroot/$dir not existing, creating"
#   echo "Freeing space in previous /var/lib/containers"
#   # First we will delete the contents of the old /var/lib/containers to not
#   # have hidden used space, since we will mount on top of it
#   rm -fr /var/lib/containers/*
#   tmp=$(mktemp -d)
#   # /sysroot is read-only, and if we remount it readwrite, we wont be able to revert it back to read-only
#   # to overcome this, we will do a bind mount somewhere else, and change that new mount to read-write
#   mount /sysroot $tmp -o bind
#   mount $tmp -o remount,rw
#   # With some versions of ostree the sysroot directory has the immutable
#   # attribute, so in case it is present, we need to remove it, and once we
#   # created the directory, we can set it back
#   if lsattr -d $tmp | cut -d ' ' -f 1 | grep i; then
#       chattr -i $tmp
#       mkdir -p "$tmp/$dir"
#       chattr +i $tmp
#   else
#       mkdir -p "$tmp/$dir"
#   fi
#   umount $tmp
#   rmdir $tmp
#fi

    # We need to run several things in the proper order. We will use systemd dependencies to achieve it
    systemd:
      units:

      # First we run the above script to create the /sysroot/containers directory if it does not exist already
      - contents: |
          [Unit]
          Description=Create /sysroot/containers
          After=var.mount
          [Service]
          Type=oneshot
          RemainAfterExit=yes
          ExecStart=/usr/local/bin/create_shared_containers_dir.sh
          [Install]
          WantedBy=local-fs.target
        enabled: true
        name: create-shared-containers.service

      # Then we can bind mount that directory in /var/lib/containers
      - contents: |
          [Unit]
          Description=Mount /sysroot/containers to /var/lib/containers
          After=create-shared-containers.service
          Wants=create-shared-containers.service
          DefaultDependencies=no
          [Mount]
          What=/sysroot/containers
          Where=/var/lib/containers
          Options=bind
          [Install]
          WantedBy=local-fs.target
        enabled: true
        name: var-lib-containers.mount

      # Since sysroot is read-only, the bind mount we just did will also be read-only.
      # We will remount it readwrite
      - contents: |
          [Unit]
          Description=Remount /var/lib/containers readwrite
          After=var-lib-containers.mount
          Wants=var-lib-containers.mount
          [Service]
          Type=oneshot
          RemainAfterExit=yes
          ExecStart=/bin/mount -o remount,rw /var/lib/containers
          [Install]
          WantedBy=local-fs.target
        enabled: true
        name: remount-var-lib-containers.service

      # And then we will have to change the selinux contexts so SELinux wont block stuff that runs inside the containers
      - contents: |
          [Unit]
          Description=Restore recursive SELinux security contexts
          DefaultDependencies=no
          After=remount-var-lib-containers.service
          Before=container-mount-namespace.service crio.service
          [Service]
          Type=oneshot
          RemainAfterExit=yes
          ExecStart=/sbin/restorecon -R /var/lib/containers/
          TimeoutSec=0
          [Install]
          WantedBy=multi-user.target graphical.target
        enabled: true
        name: restorecon-var-lib-containers.service