#!/bin/bash

set -e

LOG_LEVEL="info"
RELEASE_SEQUENCE=""
RELEASE_PATCH_SEQUENCE=""
UI_BIND_PORT="8800"
KUBERNETES_NAMESPACE="default"
PV_BASE_PATH=""
STORAGE_PROVISIONER="rook"
STORAGE_CLASS="default"
SERVICE_TYPE="NodePort"
PROXY_ADDRESS=""
NO_PROXY_ADDRESSES="10.96.0.0/12"
# replicated components registry
REGISTRY_ADDRESS_OVERRIDE=""
APP_REGISTRY_ADVERTISE_HOST=""
IP_ALLOC_RANGE="10.32.0.0/12" # default for weave
REGISTRY_CLUSTER_IP=
REPLICATED_REGISTRY_CLUSTER_IP=
CEPH_DASHBOARD_URL=
CEPH_DASHBOARD_USER=
CEPH_DASHBOARD_PASSWORD=
OBJECT_STORE_ACCESS_KEY=
OBJECT_STORE_SECRET_KEY=
OBJECT_STORE_CLUSTER_IP=
# booleans
AIRGAP=""
ENCRYPT_NETWORK=""
WEAVE_SECRET=1
REPLICATED_YAML=1
REPLICATED_PVC=1
ROOK_106_SYSTEM_YAML=0
ROOK_103_SYSTEM_YAML=0
ROOK_08_SYSTEM_YAML=0
ROOK_106_CLUSTER_YAML=0
ROOK_103_CLUSTER_YAML=0
ROOK_08_CLUSTER_YAML=0
ROOK_103_OBJECT_STORE_YAML=0
ROOK_106_OBJECT_STORE_YAML=0
ROOK_OBJECT_STORE_USER_YAML=0
STORAGE_CLASS_YAML=0
HOSTPATH_PROVISIONER_YAML=0
WEAVE_YAML=0
CONTOUR_YAML=0
DEPLOYMENT_YAML=0
REGISTRY_YAML=0
REK_OPERATOR_YAML=0
REPLICATED_REGISTRY_YAML=0
CLUSTER_ROLE_BINDING_YAML=0
NODELOCALDNS_YAML=0
BIND_DAEMON_TO_PRIMARY=0
BIND_DAEMON_HOSTNAME=
API_SERVICE_ADDRESS=""
HA_CLUSTER="0"
PURGE_DEAD_NODES="0"
CLEAR_DEAD_NODES=""
NODE_UNREACHABLE_TOLERATION=5m
MAINTAIN_ROOK_STORAGE_NODES="0"
USE_ROOK_NODES_LABEL="0"
ROOK_STORAGE_NODES_LABEL="node-role.kubernetes.io/rook=true"
REPLICATED_REGISTRY_PREFIX=
REPLICATED_VERSION="2.53.2"


#######################################
#
# common.sh
#
#######################################

GREEN='\033[0;32m'
BLUE='\033[0;94m'
LIGHT_BLUE='\033[0;34m'
YELLOW='\033[0;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color

#######################################
# Check if command exists.
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   0 if command exists
#######################################
commandExists() {
    command -v "$@" > /dev/null 2>&1
}

#######################################
# Check if replicated 1.2 is installed
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   0 if replicated 1.2 is installed
#######################################
replicated12Installed() {
    commandExists replicated && replicated --version | grep -q "Replicated version 1\.2"
}

#######################################
# Check if replicated 2.0 is installed
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   0 if replicated 2.0 is installed
#######################################
replicated2Installed() {
    commandExists /usr/local/bin/replicatedctl && /usr/local/bin/replicatedctl version >/dev/null 2>&1
}

#######################################
# Returns replicated 2.0 version
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   INSTALLED_REPLICATED_VERSION
#######################################
replicated2Version() {
    if replicated2Installed; then
        INSTALLED_REPLICATED_VERSION="$(/usr/local/bin/replicatedctl version --quiet)"
    else
        INSTALLED_REPLICATED_VERSION=""
    fi
}

#######################################
# Returns 0 if replicated will downgrade
# Globals:
#   None
# Arguments:
#   Next replicated version
# Returns:
#   0 if replicated version is less than current
#   1 if replicated version is greater than or equal to current
#######################################
isReplicatedDowngrade() {
    if ! replicated2Installed; then
        return 1
    fi

    replicated2Version
    semverCompare "$1" "$INSTALLED_REPLICATED_VERSION"
    if [ "$SEMVER_COMPARE_RESULT" -lt "0" ]; then
        return 0
    fi
    return 1
}

#######################################
# Gets curl or wget depending if cmd exits.
# Globals:
#   PROXY_ADDRESS
# Arguments:
#   None
# Returns:
#   URLGET_CMD
#######################################
URLGET_CMD=
getUrlCmd() {
    if commandExists "curl"; then
        URLGET_CMD="curl -sSL"
        if [ -n "$PROXY_ADDRESS" ]; then
            URLGET_CMD=$URLGET_CMD" -x $PROXY_ADDRESS"
        fi
    else
        URLGET_CMD="wget -qO-"
    fi
}

#######################################
# Generates a 32 char unique id.
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   GUID_RESULT
#######################################
getGuid() {
    GUID_RESULT="$(head -c 128 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)"
}

#######################################
# performs in-place sed substitution with escapting of inputs (http://stackoverflow.com/a/10467453/5344799)
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
safesed() {
    sed -i "s/$(echo $1 | sed -e 's/\([[\/.*]\|\]\)/\\&/g')/$(echo $2 | sed -e 's/[\/&]/\\&/g')/g" $3
}

#######################################
# Parses a semantic version string
# Globals:
#   None
# Arguments:
#   Version
# Returns:
#   major, minor, patch
#######################################
semverParse() {
    major="${1%%.*}"
    minor="${1#$major.}"
    minor="${minor%%.*}"
    patch="${1#$major.$minor.}"
    patch="${patch%%[-.]*}"
}

#######################################
# Compare two semvers.
# Returns -1 if A lt B, 0 if eq, 1 A gt B.
# Globals:
#   None
# Arguments:
#   Sem Version A
#   Sem Version B
# Returns:
#   SEMVER_COMPARE_RESULT
#######################################
SEMVER_COMPARE_RESULT=
semverCompare() {
    semverParse "$1"
    _a_major="${major:-0}"
    _a_minor="${minor:-0}"
    _a_patch="${patch:-0}"
    semverParse "$2"
    _b_major="${major:-0}"
    _b_minor="${minor:-0}"
    _b_patch="${patch:-0}"
    if [ "$_a_major" -lt "$_b_major" ]; then
        SEMVER_COMPARE_RESULT=-1
        return
    fi
    if [ "$_a_major" -gt "$_b_major" ]; then
        SEMVER_COMPARE_RESULT=1
        return
    fi
    if [ "$_a_minor" -lt "$_b_minor" ]; then
        SEMVER_COMPARE_RESULT=-1
        return
    fi
    if [ "$_a_minor" -gt "$_b_minor" ]; then
        SEMVER_COMPARE_RESULT=1
        return
    fi
    if [ "$_a_patch" -lt "$_b_patch" ]; then
        SEMVER_COMPARE_RESULT=-1
        return
    fi
    if [ "$_a_patch" -gt "$_b_patch" ]; then
        SEMVER_COMPARE_RESULT=1
        return
    fi
    SEMVER_COMPARE_RESULT=0
}

#######################################
# Inserts a parameter into a json file. If the file does not exist, creates it. If the parameter is already set, replaces it.
# Globals:
#   None
# Arguments:
#   path, parameter name, value
# Returns:
#   None
#######################################
insertOrReplaceJsonParam() {
    if ! [ -f "$1" ]; then
        # If settings file does not exist
        mkdir -p "$(dirname "$1")"
        echo "{\"$2\": \"$3\"}" > "$1"
    else
        # Settings file exists
        if grep -q -E "\"$2\" *: *\"[^\"]*\"" "$1"; then
            # If settings file contains named setting, replace it
            sed -i -e "s/\"$2\" *: *\"[^\"]*\"/\"$2\": \"$3\"/g" "$1"
        else
            # Insert into settings file (with proper commas)
            if [ $(wc -c <"$1") -ge 5 ]; then
                # File long enough to actually have an entry, insert "name": "value",\n after first {
                _commonJsonReplaceTmp="$(awk "NR==1,/^{/{sub(/^{/, \"{\\\"$2\\\": \\\"$3\\\", \")} 1" "$1")"
                echo "$_commonJsonReplaceTmp" > "$1"
            else
                # file not long enough to actually have contents, replace wholesale
                echo "{\"$2\": \"$3\"}" > "$1"
            fi
        fi
    fi
}

######################################
# Inserts a string array of length 1 into a json file. Fails if key is found in file.
# Globals:
#   None
# Arguments:
#   path, key, value[0]
# Returns:
#   1 if there are errors
######################################
insertJSONArray() {
	if ! [ -f "$1" ] || [ $(wc -c <"$1") -lt 5 ]; then
        mkdir -p "$(dirname "$1")"
		cat > $1 <<EOF
{
  "$2": ["$3"]
}
EOF
		return 0
	fi

	if grep -q "$2" "$1"; then
		return 1
	fi

	_commonJsonReplaceTmp="$(awk "NR==1,/^{/{sub(/^{/, \"{\\\"$2\\\": [\\\"$3\\\"], \")} 1" "$1")"
	echo "$_commonJsonReplaceTmp" > "$1"
	return 0
}

#######################################
# Splits an address in the format "host:port".
# Globals:
#   None
# Arguments:
#   address
# Returns:
#   HOST
#   PORT
#######################################
splitHostPort() {
    oIFS="$IFS"; IFS=":" read -r HOST PORT <<< "$1"; IFS="$oIFS"
}

#######################################
# Checks if Docker is installed
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   0 if Docker is installed
#######################################
isDockerInstalled() {
    commandExists "docker" && ps aux | grep -q '[d]ockerd'
}

#######################################
# Creates and sets REPLICATED_TEMP_DIR variable if not set.
# Globals:
#   None
# Arguments:
#   adNonedress
# Returns:
#   REPLICATED_TEMP_DIR
#######################################
maybeCreateTempDir() {
    if [ -z "$REPLICATED_TEMP_DIR" ]; then
        REPLICATED_TEMP_DIR="$(mktemp -d --suffix=replicated)"
    fi
}

#######################################
#
# replicated.sh
#
# require prompt.sh
#
#######################################

#######################################
# Reads a value from the /etc/replicated.conf file
# Globals:
#   None
# Arguments:
#   Variable to read
# Returns:
#   REPLICATED_CONF_VALUE
#######################################
readReplicatedConf() {
    unset REPLICATED_CONF_VALUE
    if [ -f /etc/replicated.conf ]; then
        REPLICATED_CONF_VALUE=$(cat /etc/replicated.conf | grep -o "\"$1\":\s*\"[^\"]*" | sed "s/\"$1\":\s*\"//") || true
    fi
}

#######################################
# Adds word-readable permission to conf file so the replicated container can read it
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
ensureReplicatedConfReadable() {
    if [ -f /etc/replicated.conf ]; then
        chmod a+r /etc/replicated.conf
    fi
}

#######################################
# Reads a value from REPLICATED_OPTS variable in the /etc/default/replicated file
# Globals:
#   REPLICATED_OPTS
# Arguments:
#   Variable to read
# Returns:
#   REPLICATED_OPTS_VALUE
#######################################
readReplicatedOpts() {
    unset REPLICATED_OPTS_VALUE
    REPLICATED_OPTS_VALUE="$(echo "$REPLICATED_OPTS" | grep -o "$1=[^ ]*" | cut -d'=' -f2)"
}

#######################################
# Reads a value from REPLICATED_OPERATOR_OPTS variable in the /etc/default/replicated-operator file
# Globals:
#   REPLICATED_OPTS
# Arguments:
#   Variable to read
# Returns:
#   REPLICATED_OPTS_VALUE
#######################################
readReplicatedOperatorOpts() {
    unset REPLICATED_OPTS_VALUE
    REPLICATED_OPTS_VALUE="$(echo "$REPLICATED_OPERATOR_OPTS" | grep -o "$1=[^ ]*" | cut -d'=' -f2)"
}

#######################################
# Prompts for daemon endpoint if not already set.
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   DAEMON_ENDPOINT
#######################################
DAEMON_ENDPOINT=
promptForDaemonEndpoint() {
    if [ -n "$DAEMON_ENDPOINT" ]; then
        return
    fi

    printf "Please enter the 'Daemon Address' displayed on the 'Cluster' page of your On-Prem Console.\n"
    while true; do
        printf "Daemon Address: "
        prompt
        if [ -n "$PROMPT_RESULT" ]; then
            DAEMON_ENDPOINT="$PROMPT_RESULT"
            return
        fi
    done
}

#######################################
# Prompts for daemon token if not already set.
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   DAEMON_TOKEN
#######################################
DAEMON_TOKEN=
promptForDaemonToken() {
    if [ -n "$DAEMON_TOKEN" ]; then
        return
    fi

    printf "Please enter the 'Secret Token' displayed on the 'Cluster' page of your On-Prem Console.\n"
    while true; do
        printf "Secret Token: "
        prompt
        if [ -n "$PROMPT_RESULT" ]; then
            DAEMON_TOKEN="$PROMPT_RESULT"
            return
        fi
    done
}

#######################################
# Creates user and adds to Docker group
# Globals:
#   REPLICATED_USERNAME
# Arguments:
#   None
# Returns:
#   REPLICATED_USER_ID
#######################################
REPLICATED_USER_ID=0
maybeCreateReplicatedUser() {
    # require REPLICATED_USERNAME
    if [ -z "$REPLICATED_USERNAME" ]; then
        return
    fi

    # Create the users
    REPLICATED_USER_ID=$(id -u "$REPLICATED_USERNAME" 2>/dev/null || true)
    if [ -z "$REPLICATED_USER_ID" ]; then
        useradd -g "${DOCKER_GROUP_ID:-0}" "$REPLICATED_USERNAME"
        REPLICATED_USER_ID=$(id -u "$REPLICATED_USERNAME")
    fi

    # Add the users to the docker group if needed
    # Versions older than 2.5.0 run as root
    if [ "$REPLICATED_USER_ID" != "0" ]; then
        usermod -a -G "${DOCKER_GROUP_ID:-0}" "$REPLICATED_USERNAME"
    fi
}

#######################################
# Gets the replicated image registry prefix
# Globals:
#   None
# Arguments:
#   Replicated version
# Returns:
#   REPLICATED_REGISTRY_PREFIX
#######################################
REPLICATED_REGISTRY_PREFIX=
getReplicatedRegistryPrefix() {
    REPLICATED_REGISTRY_PREFIX=replicated
    local replicated_version="$1"
    semverCompare "$replicated_version" "2.45.0"
    if [ "$SEMVER_COMPARE_RESULT" -lt "0" ]; then
        REPLICATED_REGISTRY_PREFIX=quay.io/replicated
    fi
}

#######################################
# Pull replicated and replicated-ui container images.
# Globals:
#   REGISTRY_ADDRESS_OVERRIDE
#   REGISTRY_PATH_PREFIX
#   REPLICATED_REGISTRY_PREFIX
# Arguments:
#   None
# Returns:
#   None
#######################################
pullReplicatedImages() {
    if [ -n "$REGISTRY_ADDRESS_OVERRIDE" ]; then
        docker pull "${REGISTRY_ADDRESS_OVERRIDE}/${REGISTRY_PATH_PREFIX}replicated/replicated:stable-2.53.2"
        docker pull "${REGISTRY_ADDRESS_OVERRIDE}/${REGISTRY_PATH_PREFIX}replicated/replicated-ui:stable-2.53.2"
        (set -x; docker tag "${REGISTRY_ADDRESS_OVERRIDE}/${REGISTRY_PATH_PREFIX}replicated/replicated:stable-2.53.2" "${REPLICATED_REGISTRY_PREFIX}/replicated:stable-2.53.2")
        (set -x; docker tag "${REGISTRY_ADDRESS_OVERRIDE}/${REGISTRY_PATH_PREFIX}replicated/replicated-ui:stable-2.53.2" "${REPLICATED_REGISTRY_PREFIX}/replicated-ui:stable-2.53.2")
    else
        docker pull "${REPLICATED_REGISTRY_PREFIX}/replicated:stable-2.53.2"
        docker pull "${REPLICATED_REGISTRY_PREFIX}/replicated-ui:stable-2.53.2"
    fi
}

#######################################
# Pull replicated-operator container image.
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
pullOperatorImage() {
    if [ -n "$REGISTRY_ADDRESS_OVERRIDE" ]; then
        docker pull "${REGISTRY_ADDRESS_OVERRIDE}/${REGISTRY_PATH_PREFIX}replicated/replicated-operator:stable-2.53.2"
        (set -x; docker tag "${REGISTRY_ADDRESS_OVERRIDE}/${REGISTRY_PATH_PREFIX}replicated/replicated-operator:stable-2.53.2" "${REPLICATED_REGISTRY_PREFIX}/replicated-operator:stable-2.53.2")
    else
        docker pull "${REPLICATED_REGISTRY_PREFIX}/replicated-operator:stable-2.53.2"
    fi
}

#######################################
# Tag and push replicated-operator container image to the on-prem registry.
# Globals:
#   None
# Arguments:
#   On-prem registry address
# Returns:
#   None
#######################################
tagAndPushOperatorImage()  {
    docker tag \
        "${REPLICATED_REGISTRY_PREFIX}/replicated-operator:stable-2.53.2" \
        "${1}/replicated/replicated-operator:stable-2.53.2"
    docker push "${1}/replicated/replicated-operator:stable-2.53.2"
}

#######################################
# Gets the replicated readonly docker flag if the version is greater than 2.54.1
# Globals:
#   None
# Arguments:
#   Replicated version
# Returns:
#   REPLICATED_DOCKER_READONLY_FLAG
#######################################
REPLICATED_DOCKER_READONLY_FLAG=
getReplicatedReadonlyDockerFlag() {
    REPLICATED_DOCKER_READONLY_FLAG=""
    local replicated_version="$1"
    semverCompare "$replicated_version" "2.54.1"
    if [ "$SEMVER_COMPARE_RESULT" -gt "0" ]; then
        REPLICATED_DOCKER_READONLY_FLAG="--read-only"
    fi
}
#######################################
#
# kubernetes.sh
#
# require selinux.sh common.sh docker.sh cli-script.sh
#
#######################################

UBUNTU_1604_K8S_9=ubuntu-1604-v1.9.3-20181112
UBUNTU_1604_K8S_10=ubuntu-1604-v1.10.6-20181112
UBUNTU_1604_K8S_11=ubuntu-1604-v1.11.5-20181204
UBUNTU_1604_K8S_12=ubuntu-1604-v1.12.3-20181211
UBUNTU_1604_K8S_13=ubuntu-1604-v1.13.5-20190411
UBUNTU_1604_K8S_14=ubuntu-1604-v1.14.3-20190702
UBUNTU_1604_K8S_15=ubuntu-1604-v1.15.12-20201021

UBUNTU_1804_K8S_13=ubuntu-1804-v1.13.5-20190411
UBUNTU_1804_K8S_14=ubuntu-1804-v1.14.3-20190702
UBUNTU_1804_K8S_15=ubuntu-1804-v1.15.12-20201021

RHEL7_K8S_9=rhel7-v1.9.3-20180806
RHEL7_K8S_10=rhel7-v1.10.6-20180806
RHEL7_K8S_11=rhel7-v1.11.5-20181204
RHEL7_K8S_12=rhel7-v1.12.3-20181211
RHEL7_K8S_13=rhel7-v1.13.5-20190411
RHEL7_K8S_14=rhel7-v1.14.3-20190702
RHEL7_K8S_15=rhel7-v1.15.12-20210219

DAEMON_NODE_KEY=replicated.com/daemon

#######################################
# Set the patch version in KUBERNETES_VERSION.
# Globals:
#   KUBERNETES_VERSION
# Arguments:
#   None
# Returns:
#   None
#######################################
setK8sPatchVersion() {
    semverParse "$KUBERNETES_VERSION"
    local k8sMajor="$major"
    local k8sMinor="$minor"
    local k8sPatch="$patch"

    case "$k8sMinor" in
        9)
            # 1.9.3
            k8sPatch="3"
            ;;
        11)
            # 1.11.5
            k8sPatch="5"
            ;;
        13)
            # 1.13.5
            k8sPatch="5"
            ;;
        15)
            # 1.15.12
            k8sPatch="12"
    esac
    KUBERNETES_VERSION="$k8sMajor.$k8sMinor.$k8sPatch"
}

#######################################
# Parse Kubernetes version that should be installed after the script completes
# Globals:
#   KUBERNETES_VERSION
# Arguments:
#   None
# Returns:
#   KUBERNETES_TARGET_VERSION_MAJOR
#   KUBERNETES_TARGET_VERSION_MINOR
#   KUBERNETES_TARGET_VERSION_PATCH
#######################################
parseKubernetesTargetVersion() {
    semverParse "$KUBERNETES_VERSION"
    KUBERNETES_TARGET_VERSION_MAJOR="$major"
    KUBERNETES_TARGET_VERSION_MINOR="$minor"
    KUBERNETES_TARGET_VERSION_PATCH="$patch"
}

#######################################
# Print an unsupported OS message and exit 1
# Globals:
#   LSB_DIST
#   DIST_VERSION
# Arguments:
#   None
# Returns:
#   None
#######################################
bailIfUnsupportedOS() {
    case "$LSB_DIST$DIST_VERSION" in
        ubuntu16.04|ubuntu18.04)
            ;;
        centos7.4|centos7.5|centos7.6|centos7.7|centos7.8|centos7.9|rhel7.4|rhel7.5|rhel7.6|rhel7.7|rhel7.8|rhel7.9)
            ;;
        *)
            bail "Kubernetes install is not supported on ${LSB_DIST} ${DIST_VERSION}"
            ;;
    esac
}


#######################################
# Globals:
#   None
# Arguments:
#   Message
# Returns:
#   None
#######################################
installCNIPlugins() {
    logStep "configure CNI"
    mkdir -p "$REPLICATED_TEMP_DIR/cni-plugins"
    mkdir -p /opt/cni/bin


    case "$KUBERNETES_TARGET_VERSION_MINOR" in
        13|14|15)
            if [ "$AIRGAP" = "1" ]; then
                docker load < k8s-cni-0-7-5.tar
            fi
            docker run -v "$REPLICATED_TEMP_DIR:/out" replicated/k8s-cni:0.7.5
            ;;
        *)
            if [ "$AIRGAP" = "1" ]; then
                docker load < k8s-cni.tar
            fi
            docker run -v "$REPLICATED_TEMP_DIR:/out" replicated/k8s-cni:0.6.0
            ;;
    esac

    tar zxfv "$REPLICATED_TEMP_DIR/cni.tar.gz" -C /opt/cni/bin
    mkdir -p /etc/cni/net.d
    logSuccess "CNI configured"
}

#######################################
# Lookup package tag for kubernetes version and distribution
# Globals:
#   LSB_DIST
#   DIST_VERSION
# Arguments:
#   k8sVersion - e.g. 1.9.3
# Returns:
#   pkgTag
#######################################
k8sPackageTag() {
    k8sVersion=$1

    case "$LSB_DIST$DIST_VERSION" in
        ubuntu16.04)
            case "$k8sVersion" in
                1.9.3)
                    echo "$UBUNTU_1604_K8S_9"
                    ;;
                1.10.6)
                    echo "$UBUNTU_1604_K8S_10"
                    ;;
                1.11.5)
                    echo "$UBUNTU_1604_K8S_11"
                    ;;
                1.12.3)
                    echo "$UBUNTU_1604_K8S_12"
                    ;;
                1.13.5)
                    echo "$UBUNTU_1604_K8S_13"
                    ;;
                1.14.3)
                    echo "$UBUNTU_1604_K8S_14"
                    ;;
                1.15.12)
                    echo "$UBUNTU_1604_K8S_15"
                    ;;
                *)
                    bail "Unsupported Kubernetes version $k8sVersion"
                    ;;
            esac
            ;;
        ubuntu18.04)
            case "$k8sVersion" in
                1.13.5)
                    echo "$UBUNTU_1804_K8S_13"
                    ;;
                1.14.3)
                    echo "$UBUNTU_1804_K8S_14"
                    ;;
                1.15.12)
                    echo "$UBUNTU_1804_K8S_15"
                    ;;
                *)
                    bail "Unsupported Kubernetes version $k8sVersion"
                    ;;
            esac
            ;;
        centos7.4|centos7.5|centos7.6|centos7.7|centos7.8|centos7.9|rhel7.4|rhel7.5|rhel7.6|rhel7.7|rhel7.8|rhel7.9)
            case "$k8sVersion" in
                1.9.3)
                    echo "$RHEL7_K8S_9"
                    ;;
                1.10.6)
                    echo "$RHEL7_K8S_10"
                    ;;
                1.11.5)
                    echo "$RHEL7_K8S_11"
                    ;;
                1.12.3)
                    echo "$RHEL7_K8S_12"
                    ;;
                1.13.5)
                    echo "$RHEL7_K8S_13"
                    ;;
                1.14.3)
                    echo "$RHEL7_K8S_14"
                    ;;
                1.15.12)
                    echo "$RHEL7_K8S_15"
                    ;;
                *)
                    bail "Unsupported Kubernetes version $k8sVersion"
                    ;;
            esac
            ;;
        *)
            bail "Unsupported distribution $LSB_DIST$DIST_VERSION"
            ;;
    esac

}

#######################################
# Install K8s host commands for OS
# Globals:
#   LSB_DIST
#   LSB_VERSION
# Arguments:
#   k8sVersion - e.g. 1.9.3
# Returns:
#   None
#######################################
installKubernetesComponents() {
    k8sVersion=$1

    logStep "Install kubelet, kubeadm, kubectl and cni binaries"

    if kubernetesHostCommandsOK; then
        logSuccess "Kubernetes components already installed"
        return
    fi

    prepareK8sPackageArchives $k8sVersion

    case "$LSB_DIST$DIST_VERSION" in
        ubuntu16.04|ubuntu18.04)
            export DEBIAN_FRONTEND=noninteractive
            dpkg -i --force-depends-version archives/*.deb
            ;;

        centos7.4|centos7.5|centos7.6|centos7.7|centos7.8|centos7.9|rhel7.4|rhel7.5|rhel7.6|rhel7.7|rhel7.8|rhel7.9)
            # This needs to be run on Linux 3.x nodes for Rook
            modprobe rbd
            echo 'rbd' > /etc/modules-load.d/replicated-rook.conf

            # tabs in heredoc stripped
            cat <<-EOF >  /etc/sysctl.d/k8s.conf
			net.bridge.bridge-nf-call-ip6tables = 1
			net.bridge.bridge-nf-call-iptables = 1
			net.ipv4.conf.all.forwarding = 1
			EOF

            sysctl --system

            rpm --upgrade --force --nodeps archives/*.rpm
            service docker restart
            ;;

        *)
            bail "Kubernetes install is not supported on ${LSB_DIST} ${DIST_VERSION}"
            ;;
    esac

    rm -rf archives

    if [ "$CLUSTER_DNS" != "$DEFAULT_CLUSTER_DNS" ]; then
        sed -i "s/$DEFAULT_CLUSTER_DNS/$CLUSTER_DNS/g" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
    fi
    systemctl enable kubelet && systemctl start kubelet

    logSuccess "Kubernetes components installed"
}

#######################################
# Returns 0 if all Kubernetes host commands exist
# Globals:
#   KUBERNETES_VERSION
# Arguments:
#   None
# Returns:
#   1 if any host command version is missing, 0 if all are available
#######################################
kubernetesHostCommandsOK() {
    if ! commandExists kubelet; then
        printf "kubelet command missing - will install host components\n"
        return 1
    fi
    if ! commandExists kubeadm; then
        printf "kubeadm command missing - will install host components\n"
        return 1
    fi
    if ! commandExists kubectl; then
        printf "kubectl command missing - will install host components\n"
        return 1
    fi

    return 0
}

#######################################
# Load kernel modules for kube proxy's IPVS mode
# Globals:
#   IPVS
# Arguments:
#   None
# Returns:
#   None
#######################################
loadIPVSKubeProxyModules() {
    if [ "$IPVS" != "1" ]; then
        return
    fi
    if lsmod | grep -q ip_vs ; then
        return
    fi

    kernel_major=$(uname -r | cut -d'.' -f1)
    kernel_minor=$(uname -r | cut -d'.' -f2)
    if [ "$kernel_major" -lt "4" ] || ([ "$kernel_major" -eq "4" ] && [ "$kernel_minor" -lt "19" ]); then
        modprobe nf_conntrack_ipv4
    else
        modprobe nf_conntrack
    fi

    modprobe ip_vs
    modprobe ip_vs_rr
    modprobe ip_vs_wrr
    modprobe ip_vs_sh

    echo 'nf_conntrack_ipv4' > /etc/modules-load.d/replicated-ipvs.conf
    echo 'ip_vs' >> /etc/modules-load.d/replicated-ipvs.conf
    echo 'ip_vs_rr' >> /etc/modules-load.d/replicated-ipvs.conf
    echo 'ip_vs_wrr' >> /etc/modules-load.d/replicated-ipvs.conf
    echo 'ip_vs_sh' >> /etc/modules-load.d/replicated-ipvs.conf
}

#######################################
# Unpack kubernetes images
# Globals:
#   None
# Arguments:
#   k8sVersion - e.g. 1.9.3
# Returns:
#   None
#######################################
airgapLoadKubernetesCommonImages() {
    local k8sVersion="$1"
    logStep "common images"

    # TODO if the install is 1.11.1 and is not being upgraded to 1.11.5 this will unnecessarily load the 1.11.5 images

    docker load < k8s-images-common-${k8sVersion}.tar
    case "$k8sVersion" in
        1.9.3)
            airgapLoadKubernetesCommonImages193
            ;;
        1.10.6)
            airgapLoadKubernetesCommonImages1106
            ;;
        1.11.5)
            airgapLoadKubernetesCommonImages1115
            ;;
        1.12.3)
            airgapLoadKubernetesCommonImages1123
            ;;
        1.13.5)
            airgapLoadKubernetesCommonImages1135
            ;;
        1.14.3)
            airgapLoadKubernetesCommonImages1143
            ;;
        1.15.12)
            airgapLoadKubernetesCommonImages11512
            ;;
        *)
            bail "Unsupported Kubernetes version $k8sVersion"
            ;;
    esac

    logSuccess "common images"
}

airgapLoadKubernetesCommonImages193() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-common:v1.9.3-20180809"

    # uh. its kind of insane that we have to do this. the linuxkit pkg
    # comes to us without tags, which seems super useless... we should build our own bundler maybe
    (
        set -x
        docker tag 35fdc6da5fd8 gcr.io/google_containers/kube-proxy-amd64:v1.9.3
        docker tag db76ee297b85 gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.7
        docker tag 5d049a8c4eec gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.7
        docker tag 5feec37454f4 gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.7
        docker tag 99e59f495ffa gcr.io/google_containers/pause-amd64:3.0
        docker tag 86ff1a48ce14 weaveworks/weave-kube:2.4.0
        docker tag 647ad6d59818 weaveworks/weave-npc:2.4.0
        docker tag bf0c403ea58d weaveworks/weaveexec:2.4.0
        docker tag 9c1f09fe9a86 docker.io/registry:2
        docker tag 6521ac58ca80 envoyproxy/envoy-alpine:v1.6.0
        docker tag 6a9ec4bcb60e gcr.io/heptio-images/contour:v0.5.0
        docker tag b5c343f1a3a6 rook/ceph:v0.8.1
    )
}

# only the images needed for kubeadm to upgrade from 1.9 to 1.11
airgapLoadKubernetesCommonImages1106() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-common:v1.10.6-20180809"

    (
        set -x
        docker tag 8a9a40dda603 k8s.gcr.io/kube-proxy-amd64:v1.10.6
        docker tag c2ce1ffb51ed k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.8
        docker tag 6f7f2dc7fab5 k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.8
        docker tag 80cc5ea4b547 k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.8
        docker tag da86e6ba6ca1 k8s.gcr.io/pause-amd64:3.1
    )
}

airgapLoadKubernetesCommonImages1115() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-common:v1.11.5-20181207"

    # kube-proxy is a daemon set so clusters that started at v1.11.1 will need this available on all nodes
    (
        set -x
        docker tag d5c25579d0ff k8s.gcr.io/kube-proxy-amd64:v1.11.1
        docker tag aa7b610992c0 k8s.gcr.io/kube-proxy-amd64:v1.11.5
        docker tag da86e6ba6ca1 k8s.gcr.io/pause:3.1
        docker tag b3b94275d97c k8s.gcr.io/coredns:1.1.3
        docker tag 86ff1a48ce14 weaveworks/weave-kube:2.4.0
        docker tag 647ad6d59818 weaveworks/weave-npc:2.4.0
        docker tag bf0c403ea58d weaveworks/weaveexec:2.4.0
        docker tag 9c1f09fe9a86 docker.io/registry:2
        docker tag 6521ac58ca80 docker.io/envoyproxy/envoy-alpine:v1.6.0
        docker tag 6a9ec4bcb60e gcr.io/heptio-images/contour:v0.5.0
        docker tag b5c343f1a3a6 rook/ceph:v0.8.1
        docker tag 376cb7e8748c replicated/replicated-hostpath-provisioner:cd1d272
    )
}

# only the images needed for kubeadm to upgrade from 1.11 to 1.13
airgapLoadKubernetesCommonImages1123() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-common:v1.12.3-20181207"

    docker tag ab97fa69b926 k8s.gcr.io/kube-proxy:v1.12.3
    docker tag 367cdc8433a4 k8s.gcr.io/coredns:1.2.2
}

airgapLoadKubernetesCommonImages1135() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-common:v1.13.5-20190507"

    (
        set -x
        docker tag 2ee69cad74bf k8s.gcr.io/kube-proxy:v1.13.5
        docker tag da86e6ba6ca1 k8s.gcr.io/pause:3.1
        docker tag f59dcacceff4 k8s.gcr.io/coredns:1.2.6
        docker tag 1f394ae9e226 docker.io/weaveworks/weave-kube:2.5.1
        docker tag 789b7f496034 docker.io/weaveworks/weave-npc:2.5.1
        docker tag 4cccd7ef6421 docker.io/weaveworks/weaveexec:2.5.1
        docker tag d5ef411ad932 docker.io/registry:2
        docker tag 1186b980992e docker.io/envoyproxy/envoy-alpine:v1.9.1
        docker tag 0a0aad7cff75 gcr.io/heptio-images/contour:v0.11.0
        docker tag eb6fe47e91ae docker.io/rook/ceph:v1.0.0
        docker tag 243030ce8ef0 docker.io/ceph/ceph:v14.2.0-20190410
        docker tag 376cb7e8748c replicated/replicated-hostpath-provisioner:cd1d272
    )
}

# only the images needed for kubeadm to upgrade from 1.13 to 1.15
airgapLoadKubernetesCommonImages1143() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-common:v1.14.3-20190702"

    (
        set -x
        docker tag 004666307c5b k8s.gcr.io/kube-proxy:v1.14.3
        docker tag eb516548c180 k8s.gcr.io/coredns:1.3.1
    )
}

airgapListKubernetesCommonImages11512() {
    echo "db0be43a7174 replicated/kube-proxy:v1.15.12-20211117"
    echo "da86e6ba6ca1 replicated/pause:3.1"
    echo "eb516548c180 k8s.gcr.io/coredns:1.3.1"
    echo "6acd8a755c20 replicated/weave-kube:2.6.5-20211117"
    echo "67420088f0fb replicated/weave-npc:2.6.5-20211117"
    echo "c4ca417a8cdb replicated/weaveexec:2.6.5-20211117"
    echo "6a9bdc4257f9 replicated/docker-registry:2.6.2-20211117"
    echo "16fe1d5366c5 replicated/envoy:v1.10.0-20211117"
    echo "672aff19e6e4 gcr.io/heptio-images/contour:v0.13.0"
    echo "ba01015af6de replicated/rook-ceph:v1.0.3-20211117"
    echo "9bf3ccce5f84 replicated/rook-ceph:v1.0.6-20211117"
    echo "d4702c00956b replicated/ceph:v14.2.0-20211117"
    echo "aab797bef976 replicated/ceph:v14.2.2-20211117"
    echo "376cb7e8748c replicated/replicated-hostpath-provisioner:cd1d272"
    echo "b9eb49fd9669 replicated/k8s-dns-node-cache:1.15.13-20210805"
}

airgapLoadKubernetesCommonImages11512() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-common:v1.15.12-20211118"

    while read -r image; do
        (set -x; docker tag $image)
    done <<< "$(airgapListKubernetesCommonImages11512)"
}

#######################################
# Unpack kubernetes images
# Globals:
#   None
# Arguments:
#   k8sVersion - e.g. 1.9.3
# Returns:
#   None
#######################################
airgapLoadKubernetesControlImages() {
    local k8sVersion="$1"
    logStep "control plane images"

    # TODO if the install is 1.11.1 and is not being upgraded to 1.11.5 this will unnecessarily load the 1.11.5 images

    docker load < k8s-images-control-${k8sVersion}.tar
    case "$k8sVersion" in
        1.9.3)
            airgapLoadKubernetesControlImages193
            ;;
        1.10.6)
            airgapLoadKubernetesControlImages1106
            ;;
        1.11.5)
            airgapLoadKubernetesControlImages1115
            ;;
        1.12.3)
            airgapLoadKubernetesControlImages1123
            ;;
        1.13.5)
            airgapLoadKubernetesControlImages1135
            ;;
        1.14.3)
            airgapLoadKubernetesControlImages1143
            ;;
        1.15.12)
            airgapLoadKubernetesControlImages11512
            ;;
        *)
            bail "Unsupported Kubernetes version $k8sVersion"
            ;;
    esac

    logSuccess "control plane images"
}

airgapLoadReplicatedAddonImagesSecondary() {
    semverCompare "2.53.2" "2.34.0"
    if [ "$SEMVER_COMPARE_RESULT" -lt "0" ]; then
        return
    fi

    airgapLoadReplicatedAddonImages
}

airgapLoadReplicatedAddonImages() {
    logStep "replicated addons"
    docker load < replicated-sidecar-controller.tar
    docker load < replicated-operator.tar

    if [ -f volume-mount-checker.tar ]; then
        docker load < volume-mount-checker.tar
    fi

    logSuccess "replicated addons"
}

airgapLoadKubernetesControlImages193() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-control:v1.9.3-20180809"

    (
        set -x
        docker tag 83dbda6ee810 gcr.io/google_containers/kube-controller-manager-amd64:v1.9.3
        docker tag 360d55f91cbf gcr.io/google_containers/kube-apiserver-amd64:v1.9.3
        docker tag d3534b539b76 gcr.io/google_containers/kube-scheduler-amd64:v1.9.3
        docker tag 59d36f27cceb gcr.io/google_containers/etcd-amd64:3.1.11
    )
}

airgapLoadKubernetesControlImages1106() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-control:v1.10.6-20180809"

    (
        set -x
        docker tag 6e29896cbeca k8s.gcr.io/kube-apiserver-amd64:v1.10.6
        docker tag dd246160bf59 k8s.gcr.io/kube-scheduler-amd64:v1.10.6
        docker tag 3224e7c2de11 k8s.gcr.io/kube-controller-manager-amd64:v1.10.6
        docker tag 52920ad46f5b k8s.gcr.io/etcd-amd64:3.1.12
    )
}

airgapLoadKubernetesControlImages1115() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-control:v1.11.5-20181204"

    (
        set -x
        docker tag 3a239c93cfbe k8s.gcr.io/kube-apiserver-amd64:v1.11.5
        docker tag 67fbf264abce k8s.gcr.io/kube-controller-manager-amd64:v1.11.5
        docker tag 3280f0c09d18 k8s.gcr.io/kube-scheduler-amd64:v1.11.5
        docker tag b8df3b177be2 k8s.gcr.io/etcd-amd64:3.2.18
    )
}

airgapLoadKubernetesControlImages1123() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-control:v1.12.3-20181210"

    (
        set -x
        docker tag 6b54f7bebd72 k8s.gcr.io/kube-apiserver:v1.12.3
        docker tag 5e75513787b1 k8s.gcr.io/kube-scheduler:v1.12.3
        docker tag c79022eb8bc9 k8s.gcr.io/kube-controller-manager:v1.12.3
        docker tag 3cab8e1b9802 k8s.gcr.io/etcd:3.2.24
    )
}

airgapLoadKubernetesControlImages1135() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-control:v1.13.5-20190411"

    (
        set -x
        docker tag 90332c1b9a4b k8s.gcr.io/kube-apiserver:v1.13.5
        docker tag b6b315f4f34a k8s.gcr.io/kube-controller-manager:v1.13.5
        docker tag c629ac1dae38 k8s.gcr.io/kube-scheduler:v1.13.5
        docker tag 3cab8e1b9802 k8s.gcr.io/etcd:3.2.24
    )
}

airgapLoadKubernetesControlImages1143() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-control:v1.14.3-20190702"

    (
        set -x
        docker tag 9946f563237c k8s.gcr.io/kube-apiserver:v1.14.3
        docker tag 953364a3ae7a k8s.gcr.io/kube-scheduler:v1.14.3
        docker tag ac2ce44462bc k8s.gcr.io/kube-controller-manager:v1.14.3
        docker tag 2c4adeb21b4f k8s.gcr.io/etcd:3.3.10
    )
}

airgapListKubernetesControlImages11512() {
    echo "2359bc3f498e replicated/kube-apiserver:v1.15.12-20210805"
    echo "c785c04fa84d replicated/kube-controller-manager:v1.15.12-20210805"
    echo "2abde40f76d7 replicated/kube-scheduler:v1.15.12-20210805"
    echo "3a9e5cce725f replicated/etcd:3.3.10-20200512"
    echo "9eedcb1b6638 replicated/etcd:3.4.7-20200602"
}

airgapLoadKubernetesControlImages11512() {
    docker run \
        -v /var/run/docker.sock:/var/run/docker.sock \
        "replicated/k8s-images-control:v1.15.12-20210806"

    while read -r image; do
        (set -x; docker tag $image)
    done <<< "$(airgapListKubernetesControlImages11512)"
}

function list_all_required_images() {
    local k8sVersion="$1"
    local nodeName="$2"

    case "$k8sVersion" in
        1.15.12)
            airgapListKubernetesCommonImages11512 | awk '{print $2}'
            if is_primary_node "$nodeName" ; then
                airgapListKubernetesControlImages11512 | awk '{print $2}'
            fi
            ;;
        *)
            # unsupported
            ;;
    esac
}

function patch_control_plane_images() {
    local k8sVersion="$1"

    case "$k8sVersion" in
        1.15.12)
            patch_control_plane_images_11512
            ;;
    esac
}

function patch_control_plane_images_11512() {
    # patch all control plane manifests with versioned images
    sed -i "s/image:.*kube-apiserver:.*$/image: $(echo replicated/kube-apiserver:v1.15.12-20210805 | sed 's/\//\\\//g')/" /etc/kubernetes/manifests/kube-apiserver.yaml
    sed -i "s/image:.*kube-controller-manager:.*$/image: $(echo replicated/kube-controller-manager:v1.15.12-20210805 | sed 's/\//\\\//g')/" /etc/kubernetes/manifests/kube-controller-manager.yaml
    sed -i "s/image:.*kube-scheduler:.*$/image: $(echo replicated/kube-scheduler:v1.15.12-20210805 | sed 's/\//\\\//g')/" /etc/kubernetes/manifests/kube-scheduler.yaml
}

function get_kubeadm_config_image() {
    kubeadm config images list --config=/opt/replicated/kubeadm.conf 2>/dev/null | grep "$1"
}

airgapPushReplicatedImagesToRegistry() {
    logStep "Pushing images to registry at $1"

    dockerGetRepoTagFromTar replicated.tar
    dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

    dockerGetRepoTagFromTar replicated-ui.tar
    dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

    if [ -f replicated-operator.tar ]; then
        dockerGetRepoTagFromTar replicated-operator.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    if [ -f replicated-sidecar-controller.tar ]; then
        dockerGetRepoTagFromTar replicated-sidecar-controller.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    if [ -f volume-mount-checker.tar ]; then
        dockerGetRepoTagFromTar volume-mount-checker.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    dockerGetRepoTagFromTar cmd.tar
    dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

    dockerGetRepoTagFromTar statsd-graphite.tar
    dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

    dockerGetRepoTagFromTar premkit.tar
    dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

    if [ -f debian.tar ]; then
        dockerGetRepoTagFromTar debian.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    if [ -f support-bundle.tar ]; then
        dockerGetRepoTagFromTar support-bundle.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    # these have been monocontainer'd since 2.24.0
    if [ -f retraced.tar ]; then
        dockerGetRepoTagFromTar retraced.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-postgres.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-nsqd.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    # these have been included together prior to 2.21.0
    if [ -f retraced-processor.tar ]; then
        dockerGetRepoTagFromTar retraced-processor.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-db.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-api.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-cron.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    # single retraced bundle no longer included since 2.21.0
    if [ -f retraced-bundle.tar.gz ]; then
        dockerGetRepoTagFromTar retraced-processor.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-postgres.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-nsqd.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-db.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-api.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"

        dockerGetRepoTagFromTar retraced-cron.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    # redis is included in Retraced <= 1.1.10
    if [ -f retraced-redis.tar ]; then
        dockerGetRepoTagFromTar retraced-redis.tar
        dockerRetagAndPushImageToRegistry "$REPO_TAG" "$1"
    fi

    logSuccess "Images pushed to registry at $1 successfully"
}

#######################################
# Creates an archives directory with the correct K8s packages for a given
# version of K8s on the current distribution.
# Globals:
#   AIRGAP
#   LSB_DIST
#   DIST_VERSION
# Arguments:
#   k8sVersion - e.g. 1.9.3
# Returns:
#   None
#######################################
prepareK8sPackageArchives() {
    k8sVersion=$1
    pkgTag=$(k8sPackageTag $k8sVersion)

    if [ "$AIRGAP" = "1" ]; then
        docker load < packages-kubernetes-${pkgTag}.tar
    fi
    docker run \
      -v $PWD:/out \
      "replicated/k8s-packages:${pkgTag}"
}

#######################################
# Gets node name from kubectl ignoring errors
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   primary node names
#######################################
k8sPrimaryNodeNames() {
    set +e
    _primary="$(kubectl get nodes --show-labels 2>/dev/null | grep 'node-role.kubernetes.io/master' | awk '{ print $1 }')"
    until [ -n "$_primary" ]; do
        _primary="$(kubectl get nodes --show-labels 2>/dev/null | grep 'node-role.kubernetes.io/master' | awk '{ print $1 }')"
    done
    set -e
    printf "$_primary"
}

#######################################
# Return status code 0 if a namespace exists, else 1
# Globals:
#   None
# Arguments:
#   namespace
# Returns:
#   None
#######################################
k8sNamespaceExists() {
    kubectl get namespaces | grep -q "$1"
}

#######################################
# Return status code 0 if there is a storageclass with the default annotation, else 1
# Globals:
#   None
# Arguments:
#   provisioner
# Returns:
#   None
#######################################
defaultStorageClassExists() {
    kubectl get storageclass -o=jsonpath='{.items[*].metadata.annotations}' | grep -q "storageclass.kubernetes.io/is-default-class":"true"
}


#######################################
# Display a spinner until all nodes are ready, TODO timeout
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
spinnerNodesReady()
{
    waitForNodes

    local delay=0.75
    local spinstr='|/-\'
    while true; do
        set +e
        local nodes="$(kubectl get nodes 2>/dev/null)"
        local _exit="$?"
        set -e
        if [ "$_exit" -eq "0" ]; then
            local numNodes="$(kubectl get nodes 2>/dev/null | awk 'NR > 1' | wc -l)"
            if [ "$numNodes" -gt "0" ] && ! echo "$nodes" | grep -q "NotReady"; then
                break
            fi
        fi
        local temp=${spinstr#?}
        printf " [%c]  " "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b\b\b\b"
    done
}

#######################################
# Display a spinner until the Kubernetes API /healthz endpoint returns ok.
# Kubeadm preflight checks fail unless healthz returns ok.
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
spinnerK8sAPIHealthy()
{
    local addr="${PRIVATE_ADDRESS}:6443"
    if [ -n "$LOAD_BALANCER_ADDRESS" ]; then
        addr="${LOAD_BALANCER_ADDRESS}:${LOAD_BALANCER_PORT}"
    fi

    local delay=1
    local elapsed=0
    local spinstr='|/-\'
    while [ "$(curl --noproxy "*" -sk https://$addr/healthz)" != "ok" ]; do
        elapsed=$(($elapsed + $delay))
        if [ "$elapsed" -gt 120 ]; then
            bail "Kubernetes API failed to report healthy"
        fi
        local temp=${spinstr#?}
        printf " [%c]  " "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b\b\b\b"
    done
}

#######################################
# Display a spinner until a node reaches a version
# Globals:
#   None
# Arguments:
#   node
#   k8sVersion - e.g. 1.10.6
# Returns:
#   None
#######################################
spinnerNodeVersion()
{
    node=$1
    k8sVersion=$2

    local delay=0.75
    local spinstr='|/-\'
    while true; do
        set +e
        local nout="$(kubectl get node $node 2>/dev/null)"
        local _exit="$?"
        set -e
        if [ "$_exit" -eq "0" ] && [ "$(echo "$nout" | sed '1d' | awk '{ print $5 }')" == "v$k8sVersion" ]; then
            break
        fi
        local temp=${spinstr#?}
        printf " [%c]  " "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b\b\b\b"
    done
}

#######################################
# Will wait for the get nodes call to return success
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
waitForNodes()
{
    n=0
    while ! kubectl get nodes >/dev/null 2>&1; do
        n="$(( $n + 1 ))"
        if [ "$n" -ge "120" ]; then
            # this should exit script on non-zero exit code and print error message
            kubectl get nodes 1>/dev/null
        fi
        sleep 2
    done
}

#######################################
# Display a spinner until the primary node is ready
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
spinnerPrimaryNodeReady()
{
    logStep "Await node ready"
    spinnerNodesReady
    logSuccess "Primary Node Ready!"
}

#######################################
# Replicated < 2.26 uses the replicated.com/daemon label to generate the node join script.
#
# Globals:
#   DAEMON_NODE_KEY, REPLICATED_VERSION
# Arguments:
#   None
# Returns:
#   None
#######################################
labelPrimaryNodeDeprecated()
{
    semverCompare "$REPLICATED_VERSION" "2.26.0"
    if [ "$SEMVER_COMPARE_RESULT" -ge "0" ]; then
        return
    fi
    if kubectl get nodes --show-labels | grep -q "$DAEMON_NODE_KEY" ; then
        return
    fi
    kubectl label nodes --overwrite "$(k8sPrimaryNodeNames)" "$DAEMON_NODE_KEY"=
}

#######################################
# Check if the current node is a primary
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   0 if primary node, else 1
#######################################
isCurrentNodePrimaryNode()
{
    if [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ]; then
        return 0
    else
        return 1
    fi
}


#######################################
# Check if the node is a primary running in a non-HA cluster
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   0 if single node primary, else 1
#######################################
isSingleNodePrimary()
{
    if ! isCurrentNodePrimaryNode; then
        return 1
    fi
    if [ "$HA_CLUSTER" = "1" ]; then
        return 1
    fi
    # joined primary nodes do not have HA_CLUSTER set
    if cat /opt/replicated/kubeadm.conf | grep -q 'JoinConfiguration' && cat /opt/replicated/kubeadm.conf | grep -q 'controlPlane:'; then
        return 1
    fi

    return 0
}

#######################################
# Spinner Pod Running
# Globals:
#   None
# Arguments:
#   Namespace, Pod prefix
# Returns:
#   None
#######################################
spinnerPodRunning()
{
    namespace=$1
    podPrefix=$2

    local delay=0.75
    local spinstr='|/-\'
    while ! kubectl -n "$namespace" get pods 2>/dev/null | grep "^$podPrefix" | awk '{ print $3}' | grep '^Running$' > /dev/null ; do
        local temp=${spinstr#?}
        printf " [%c]  " "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b\b\b\b"
    done
    printf "    \b\b\b\b"
}

#######################################
# Blocks until Replicated is ready
# Globals:
#   None
# Arguments:
#   Replicated version
# Returns:
#   None
#######################################
waitReplicatedReady() {
    kubectl rollout status deployment/replicated

    waitReplicatedctlReady "$REPLICATED_VERSION"
}

#######################################
# Blocks until replicatedctl is ready
# Globals:
#   None
# Arguments:
#   Replicated version
# Returns:
#   None
#######################################
waitReplicatedctlReady() {
    # TODO: spinner
    logSubstep "wait for replicated to report ready"
    for i in {1..60}; do
        if isReplicatedctlReady "$1"; then
            return 0
        fi
        sleep 2
    done
    return 1
}

#######################################
# Spinner Replicated Ready
# Globals:
#   None
# Arguments:
#   Replicated version
# Returns:
#   None
#######################################
spinnerReplicatedReady()
{
    logStep "Await replicated ready"
    sleep 2
    if ! waitReplicatedReady "$1"; then
        bail "Replicated failed to report ready"
    fi
    logSuccess "Replicated Ready!"
}

#######################################
# Return code 0 unless `replicatedctl system status` succeeds
# Globals:
#   None
# Arguments:
#   Replicated version
# Returns:
#   None
#######################################
isReplicatedctlReady() {
    semverCompare "$1" "2.26.0"
    if [ "$SEMVER_COMPARE_RESULT" -ge "0" ]; then
        /usr/local/bin/replicatedctl system status 2>/dev/null | grep -q '"ready"'
    else
        /usr/local/bin/replicatedctl task ls > /dev/null 2>&1
    fi
}

#######################################
# Spinner Rook Ready,
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
spinnerRookReady()
{
    logStep "Await rook ready"
    spinnerPodRunning rook-ceph-system rook-ceph-operator
    spinnerPodRunning rook-ceph-system rook-ceph-agent
    spinnerPodRunning rook-ceph-system rook-discover
    spinnerRookFlexVolumePluginReady
    logSuccess "Rook Ready!"
}

#######################################
# Spinner Rook FlexVolume plugin ready
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
spinnerRookFlexVolumePluginReady()
{
    local delay=0.75
    local spinstr='|/-\'
    while [ ! -e /usr/libexec/kubernetes/kubelet-plugins/volume/exec/ceph.rook.io~rook-ceph-system/rook-ceph-system ]; do
        local temp=${spinstr#?}
        printf " [%c]  " "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b\b\b\b"
    done
}

#######################################
# Spinner Hostpath Provisioner Ready,
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
spinnerHostpathProvisionerReady()
{
    logStep "Await hostpath provisioner ready"
    spinnerPodRunning kube-system replicated-hostpath-provisioner
    logSuccess "Hostpath Provisioner Ready!"
}

#######################################
# Spinner kube-system ready
# Globals:
#   None
# Arguments:
#   k8sVersion - e.g. 1.9.3
# Returns:
#   None
#######################################
spinnerKubeSystemReady()
{
    k8sVersion=$1

    logStep "Await kube-system services ready"
    spinnerPodRunning kube-system weave-net
    if [ "$k8sVersion" = "1.9.3" ]; then
        spinnerPodRunning kube-system kube-dns
    else
        spinnerPodRunning kube-system coredns
    fi
    spinnerPodRunning kube-system kube-proxy
    logStep "Kube system services ready"
}

weave_reset()
{
    BRIDGE=weave
    DATAPATH=datapath
    CONTAINER_IFNAME=ethwe

    WEAVEKUBE_IMAGE="replicated/weave-kube:2.6.5-20211117"
    WEAVEEXEC_IMAGE="replicated/weaveexec:2.6.5-20211117"
    DOCKER_BRIDGE=docker0

    # if we never unpacked/pulled the weave image, its unlikely we need to do any of this
    if ! dockerImageExists "$WEAVEKUBE_IMAGE"; then
        return
    fi

    docker pull "$WEAVEEXEC_IMAGE"

    DOCKER_BRIDGE_IP=$(docker run --rm --pid host --net host --privileged -v /var/run/docker.sock:/var/run/docker.sock --entrypoint=/usr/bin/weaveutil $WEAVEEXEC_IMAGE bridge-ip $DOCKER_BRIDGE)

    for NETDEV in $BRIDGE $DATAPATH ; do
        if [ -d /sys/class/net/$NETDEV ] ; then
            if [ -d /sys/class/net/$NETDEV/bridge ] ; then
                ip link del $NETDEV
            else
                docker run --rm --pid host --net host --privileged -v /var/run/docker.sock:/var/run/docker.sock --entrypoint=/usr/bin/weaveutil $WEAVEEXEC_IMAGE delete-datapath $NETDEV
            fi
        fi
    done

    # Remove any lingering bridged fastdp, pcap and attach-bridge veths
    for VETH in $(ip -o link show | grep -o v${CONTAINER_IFNAME}[^:@]*) ; do
        ip link del $VETH >/dev/null 2>&1 || true
    done

    if [ "$DOCKER_BRIDGE" != "$BRIDGE" ] ; then
        run_iptables -t filter -D FORWARD -i $DOCKER_BRIDGE -o $BRIDGE -j DROP 2>/dev/null || true
    fi

    run_iptables -t filter -D INPUT -i $DOCKER_BRIDGE -p udp --dport 53  -j ACCEPT  >/dev/null 2>&1 || true
    run_iptables -t filter -D INPUT -i $DOCKER_BRIDGE -p tcp --dport 53  -j ACCEPT  >/dev/null 2>&1 || true

    run_iptables -t filter -D INPUT -i $DOCKER_BRIDGE -p tcp --dst $DOCKER_BRIDGE_IP --dport $PORT          -j DROP >/dev/null 2>&1 || true
    run_iptables -t filter -D INPUT -i $DOCKER_BRIDGE -p udp --dst $DOCKER_BRIDGE_IP --dport $PORT          -j DROP >/dev/null 2>&1 || true
    run_iptables -t filter -D INPUT -i $DOCKER_BRIDGE -p udp --dst $DOCKER_BRIDGE_IP --dport $(($PORT + 1)) -j DROP >/dev/null 2>&1 || true

    run_iptables -t filter -D FORWARD -i $BRIDGE ! -o $BRIDGE -j ACCEPT 2>/dev/null || true
    run_iptables -t filter -D FORWARD -o $BRIDGE -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true
    run_iptables -t filter -D FORWARD -i $BRIDGE -o $BRIDGE -j ACCEPT 2>/dev/null || true
    run_iptables -F WEAVE-NPC >/dev/null 2>&1 || true
    run_iptables -t filter -D FORWARD -o $BRIDGE -j WEAVE-NPC 2>/dev/null || true
    run_iptables -t filter -D FORWARD -o $BRIDGE -m state --state NEW -j NFLOG --nflog-group 86 2>/dev/null || true
    run_iptables -t filter -D FORWARD -o $BRIDGE -j DROP 2>/dev/null || true
    run_iptables -X WEAVE-NPC >/dev/null 2>&1 || true

    run_iptables -F WEAVE-EXPOSE >/dev/null 2>&1 || true
    run_iptables -t filter -D FORWARD -o $BRIDGE -j WEAVE-EXPOSE 2>/dev/null || true
    run_iptables -X WEAVE-EXPOSE >/dev/null 2>&1 || true

    run_iptables -t nat -F WEAVE >/dev/null 2>&1 || true
    run_iptables -t nat -D POSTROUTING -j WEAVE >/dev/null 2>&1 || true
    run_iptables -t nat -D POSTROUTING -o $BRIDGE -j ACCEPT >/dev/null 2>&1 || true
    run_iptables -t nat -X WEAVE >/dev/null 2>&1 || true

    for LOCAL_IFNAME in $(ip link show | grep v${CONTAINER_IFNAME}pl | cut -d ' ' -f 2 | tr -d ':') ; do
        ip link del ${LOCAL_IFNAME%@*} >/dev/null 2>&1 || true
    done
}

#######################################
# Attempt to remove replicated and kubernetes from the system
# Globals:
#   none
# Arguments:
#   FORCE_RESET
# Returns:
#   None
#######################################
k8s_reset() {
    set +e

    # if FORCE_RESET is set, skip this
    if [ "$1" != 1 ]; then
        printf "${YELLOW}"
        printf "WARNING: \n"
        printf "\n"
        printf "    The \"reset\" command will attempt to remove replicated and kubernetes from this system.\n"
        printf "\n"
        printf "    This command is intended to be used only for \n"
        printf "    increasing iteration speed on development servers. It has the \n"
        printf "    potential to leave your machine in an unrecoverable state. It is \n"
        printf "    not recommended unless you will easily be able to discard this server\n"
        printf "    and provision a new one if something goes wrong.\n${NC}"
        printf "\n"
        printf "Would you like to continue? "

        if ! confirmN; then
            printf "Not resetting\n"
            exit 1
        fi
    fi

    if commandExists "kubectl" && [ -f "/opt/replicated/kubeadm.conf" ]; then
        set +e
        kubectl delete all --all
        appNSs=$(kubectl get ns | grep replicated- | awk '{ print $1 }')
        for appNS in "$appNSs"; do
            kubectl delete ns "$appNS"
        done
        nodes=$(kubectl get nodes --output=go-template --template="{{range .items}}{{.metadata.name}} {{end}}")
        for node in $nodes; do
            # continue after timeout errors
            kubectl drain "$node" --delete-local-data --force --ignore-daemonsets --grace-period=30 --timeout=300s || :
            kubectl delete node "$node"
        done
        set -e
    fi

    if commandExists "kubeadm"; then
        kubeadm reset --force
    fi

    weave_reset

    rm -rf /etc/cni
    rm -rf /etc/kubernetes
    rm -rf /opt/cni
    rm -rf /opt/replicated
    rm -f /usr/bin/kubeadm /usr/bin/kubelet /usr/bin/kubectl
    rm -rf /var/lib/etcd
    rm -rf /var/lib/kubelet
    rm -rf /var/lib/rook
    rm -rf /var/lib/weave
    rm -rf /var/lib/replicated
    kill $(ps aux | grep '[k]ubelet' | awk '{print $2}') 2> /dev/null
}

#######################################
# Get kubelet version
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   version - e.g. 1.11.5
#######################################
getKubeletVersion() {
    kubelet --version | cut -d ' ' -f 2 | sed 's/v//'
}

#######################################
# Get kubectl client command version
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   version - e.g. 1.11.5
#######################################
getKubectlVersion() {
    kubectl version | grep 'Client Version' | tr " " "\n" | grep GitVersion | cut -d'"' -f2 | sed 's/v//'
}

#######################################
# Get kubeadm command version
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   version - e.g. 1.11.5
#######################################
getKubeadmVersion() {
    kubeadm version | tr " " "\n" | grep GitVersion | cut -d'"' -f2 | sed 's/v//'
}

#######################################
# Generate kubeadm ClusterConfiguration v1beta1 and KubeProxyConfiguration v1alpha1
# Globals:
#   KUBERNETES_VERSION
#   PRIVATE_ADDRESS
#   PUBLIC_ADDRESS
#   IPVS
#   HA_CLUSTER
#   LOAD_BALANCER_ADDRESS
#   LOAD_BALANCER_PORT
# Arguments:
#   KUBERNETES_VERSION - e.g. 1.11.5
# Returns:
#   version - e.g. 1.11.5
#######################################
makeKubeadmConfig() {
    local k8sVersion="$1"
    cat << EOF >> /opt/replicated/kubeadm.conf
---
kind: ClusterConfiguration
apiVersion: kubeadm.k8s.io/v1beta1
kubernetesVersion: v$k8sVersion
networking:
  serviceSubnet: $SERVICE_CIDR
apiServer:
  extraArgs:
    service-node-port-range: "80-60000"
  certSANs:
EOF
    if [ -n "$PUBLIC_ADDRESS" ]; then
        cat <<EOF >> /opt/replicated/kubeadm.conf
  - $PUBLIC_ADDRESS
EOF
    fi
    if [ -n "$LOAD_BALANCER_ADDRESS" ] && [ -n "$LOAD_BALANCER_PORT" ]; then
        cat <<EOF >> /opt/replicated/kubeadm.conf
  - $LOAD_BALANCER_ADDRESS
controlPlaneEndpoint: "$LOAD_BALANCER_ADDRESS:$LOAD_BALANCER_PORT"
EOF
    else
        cat <<EOF >> /opt/replicated/kubeadm.conf
  - $PRIVATE_ADDRESS
EOF
    fi

    if [ "$IPVS" = "1" ]; then
        cat <<EOF >> /opt/replicated/kubeadm.conf
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
EOF
    fi
}

appendKubeadmClusterConfigV1Beta2() {
    k8sVersion=$1

    etcdVersion=$2
    local etcdImageTag=
    case $etcdVersion in
        3.3) etcdImageTag='3.3.10-20200512' ;;
        *)   etcdImageTag='3.4.7-20200602' ;;
    esac

    cat <<EOF >> /opt/replicated/kubeadm.conf
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v$k8sVersion
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
imageRepository: docker.io/replicated
controllerManager: {}
dns:
  type: CoreDNS
  imageRepository: k8s.gcr.io
etcd:
  local:
    dataDir: /var/lib/etcd
    imageRepository: docker.io/replicated
    imageTag: $etcdImageTag
networking:
  serviceSubnet: $SERVICE_CIDR
apiServer:
  extraArgs:
    service-node-port-range: "80-60000"
  certSANs:
  - $PRIVATE_ADDRESS
EOF
    if [ -n "$PUBLIC_ADDRESS" ]; then
        cat <<EOF >> /opt/replicated/kubeadm.conf
  - $PUBLIC_ADDRESS
EOF
    fi
    if [ -n "$LAST_LOAD_BALANCER_ADDRESS" ] && [ "$LAST_LOAD_BALANCER_ADDRESS" != "$LOAD_BALANCER_ADDRESS" ]; then
        cat <<EOF >> /opt/replicated/kubeadm.conf
  - $LAST_LOAD_BALANCER_ADDRESS
EOF
    fi
    if [ -n "$LOAD_BALANCER_ADDRESS" ] && [ -n "$LOAD_BALANCER_PORT" ]; then
        cat <<EOF >> /opt/replicated/kubeadm.conf
  - $LOAD_BALANCER_ADDRESS
controlPlaneEndpoint: "$LOAD_BALANCER_ADDRESS:$LOAD_BALANCER_PORT"
EOF
    fi
}

appendKubeProxyConfigV1Alpha1() {
    if [ "$IPVS" != "1" ]; then
        return
    fi
    cat <<EOF >> /opt/replicated/kubeadm.conf
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
EOF
}

appendKubeletConfigV1Beta1() {
    if [ "$IPVS" != "1" ]; then
        return
    fi
    if [ "$NODELOCAL_DNSCACHE" != "1" ]; then
        return
    fi
    cat <<EOF >> /opt/replicated/kubeadm.conf
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
clusterDNS:
- $NODELOCAL_ADDRESS
EOF
}

#######################################
# Generate kubeadm JoinConfiguration
# Globals:
#   PRIVATE_ADDRESS
#   KUBEADM_TOKEN
#   KUBEADM_TOKEN_CA_HASH
#   API_SERVICE_ADDRESS
# Arguments:
#   None
# Returns:
#   None
#######################################
makeKubeadmJoinConfig() {
    cat << EOF > /opt/replicated/kubeadm.conf
---
kind: JoinConfiguration
apiVersion: kubeadm.k8s.io/v1beta1
nodeRegistration:
  kubeletExtraArgs:
    node-ip: $PRIVATE_ADDRESS
discovery:
  bootstrapToken:
    token: $KUBEADM_TOKEN
    apiServerEndpoint: $API_SERVICE_ADDRESS
    caCertHashes:
    - $KUBEADM_TOKEN_CA_HASH
EOF
    if [ "$PRIMARY" = "1" ]; then
        cat << EOF >> /opt/replicated/kubeadm.conf
controlPlane: {}
EOF
    fi
}

#######################################
# Generate kubeadm JoinConfiguration v1beta2
# Globals:
#   PRIVATE_ADDRESS
#   KUBEADM_TOKEN
#   KUBEADM_TOKEN_CA_HASH
#   API_SERVICE_ADDRESS
#   UNSAFE_SKIP_CA_VERIFICATION
# Arguments:
#   None
# Returns:
#   None
#######################################
makeKubeadmJoinConfigV1Beta2() {
    cat << EOF > /opt/replicated/kubeadm.conf
---
kind: JoinConfiguration
apiVersion: kubeadm.k8s.io/v1beta2
nodeRegistration:
EOF
    if [ "$PRIMARY" != "1" ] || [ "$TAINT_CONTROL_PLANE" != "1" ]; then
        cat << EOF >> /opt/replicated/kubeadm.conf
  taints: []
EOF
    fi
    cat << EOF >> /opt/replicated/kubeadm.conf
  kubeletExtraArgs:
    node-ip: $PRIVATE_ADDRESS
discovery:
  bootstrapToken:
    token: $KUBEADM_TOKEN
    apiServerEndpoint: $API_SERVICE_ADDRESS
EOF
    if [ "$UNSAFE_SKIP_CA_VERIFICATION" = "1" ]; then
        cat << EOF >> /opt/replicated/kubeadm.conf
    unsafeSkipCAVerification: true
EOF
    else
        cat << EOF >> /opt/replicated/kubeadm.conf
    caCertHashes:
    - $KUBEADM_TOKEN_CA_HASH
EOF
    fi
    if [ "$PRIMARY" = "1" ]; then
        cat << EOF >> /opt/replicated/kubeadm.conf
controlPlane: {}
EOF
    fi
}

exportKubeconfig() {
    cp /etc/kubernetes/admin.conf $HOME/admin.conf
    chown $SUDO_USER:$SUDO_GID $HOME/admin.conf
    chmod 444 /etc/kubernetes/admin.conf
    if ! grep -q "kubectl completion bash" /etc/profile; then
        echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /etc/profile
        echo "source <(kubectl completion bash)" >> /etc/profile
    fi
}

#######################################
# Add insecure registry to Docker
# Globals:
#   None
# Arguments:
#   registry host
# Returns:
#   ADDED_INSECURE_REGISTRY
#######################################
ADDED_INSECURE_REGISTRY=0
addInsecureRegistry() {
    if grep -q "insecure-registry $1" /etc/systemd/system/docker.service.d/replicated-registry.conf 2>/dev/null ; then
        return
    fi
    if grep "insecure-registries" /etc/docker/daemon.json 2>/dev/null | grep -q "$1" ; then
        return
    fi

    # configure using /etc/docker/daemon.json since that does not require a docker restart
    if insertJSONArray "/etc/docker/daemon.json" "insecure-registries" "$1" ; then
        systemctl kill -s HUP --kill-who=main docker.service
        ADDED_INSECURE_REGISTRY=1
        return
    fi

    bail "Docker could not be configured to use in-cluster registry $1"
}

#######################################
# Allow scheduling on control plane nodes
# Globals:
#   None
# Arguments:
#   registry host
# Returns:
#   None
#######################################
untaintPrimary() {
    logStep "remove node-role.kubernetes.io/master:NoSchedule taint from primary node"
    kubectl taint nodes --all node-role.kubernetes.io/master:NoSchedule- || \
        echo "Taint not found or already removed. The above error can be ignored."
    logSuccess "primary taint removed"
}

#######################################
# Check if a cert has an IP or Domain in its SANs
# Globals:
#   None
# Arguments:
#   cert filepath, IP
# Returns:
#   None
#######################################
certHasSAN()
{
    openssl x509 -in "$1" -noout -text | grep -Eq "DNS:$2|IP Address:$2"
}

#######################################
# Check if a kubeconfig points to a specific endpoint
# Globals:
#   None
# Arguments:
#   cert filepath, endpoint
# Returns:
#   None
#######################################
confHasEndpoint()
{
    if [ "$(cat $1 | grep 'server:' | awk '{ print $2 }')" = "$2" ]; then
        return 0
    fi
    return 1
}


#######################################
# Check if Rook 1.0+ is installed
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None, exits 0 if Rook 1.0+ is detected
#######################################
isRook1()
{
    kubectl -n rook-ceph get cephblockpools replicapool &>/dev/null
}

#######################################
# Check if Rook 1.0.3+ is installed
# Globals:
#   ROOK_VERSION
# Arguments:
#   None
# Returns:
#   None, exits 0 if Rook 1.0.3+ is detected
#######################################
isRook103Plus()
{
    getRookVersion
    semverCompare "1.0.3" "$ROOK_VERSION"
    if [ "$SEMVER_COMPARE_RESULT" -gt "0" ]; then
        return 1
    fi
    return 0
}

#######################################
# Check if Rook is installed
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None, exits 0 if Rook is installed
#######################################
isRookInstalled()
{
    if kubectl get ns rook-ceph &>/dev/null ; then
        return 0
    else
        return 1
    fi
}

#######################################
# Check if Rook 1.0.6+ is installed
# Globals:
#   ROOK_VERSION
# Arguments:
#   None
# Returns:
#   None, exits 0 if Rook 1.0.6+ is detected
#######################################
isRook106Plus()
{
    getRookVersion
    semverCompare "1.0.6" "$ROOK_VERSION"
    if [ "$SEMVER_COMPARE_RESULT" -gt "0" ]; then
        return 1
    fi
    return 0
}

#######################################
# Check if Rook 1.0.3 is installed
# Globals:
#   ROOK_VERSION
# Arguments:
#   None
# Returns:
#   None, exits 0 if Rook 1.0.3 is detected
#######################################
isRook103()
{
    getRookVersion
    if [ "$ROOK_VERSION" = "1.0.3" ]; then
        return 0
    fi
    return 1
}

#######################################
# Get rook version
# Globals:
#   ROOK_VERSION
# Arguments:
#   None
# Returns:
#   None
#######################################
ROOK_VERSION=
getRookVersion()
{
    ROOK_VERSION="$(kubectl -n rook-ceph-system get deploy rook-ceph-operator -oyaml 2>/dev/null \
        | grep ' image: ' \
        | awk -F':' 'NR==1 { print $3 }' \
        | sed 's/v\([^-]*\).*/\1/')"
}

#######################################
# Check if Rook 0.8 is installed
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None, exits 0 if Rook 0.8 is detected
#######################################
isRook08()
{
    kubectl -n rook-ceph get pool replicapool &>/dev/null
}

#######################################
# Check if Etcd 3.3 is installed
# Globals:
#   ETCD_VERSION
# Arguments:
#   None
# Returns:
#   None, exits 0 if Etcd 3.3 is detected
#######################################
isEtcd33()
{
    getEtcdVersion
    semverCompare "3.3.0" "$ETCD_VERSION"
    if [ "$SEMVER_COMPARE_RESULT" -gt "0" ]; then
        return 1
    fi
    semverCompare "3.4.0" "$ETCD_VERSION"
    if [ "$SEMVER_COMPARE_RESULT" -le "0" ]; then
        return 1
    fi
    return 0
}

#######################################
# Get etcd version
# Globals:
#   ETCD_VERSION
# Arguments:
#   None
# Returns:
#   None
#######################################
ETCD_VERSION=
getEtcdVersion()
{
    ETCD_VERSION="$(kubectl -n kube-system get pod -l tier=control-plane -l component=etcd -oyaml 2>/dev/null \
        | grep ' image: ' \
        | awk -F':' 'NR==1 { print $3 }' \
        | sed 's/\([^-]*\).*/\1/')"
}

#######################################
# Check if Etcd is installed
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None, exits 0 if Etcd is installed
#######################################
isEtcdInstalled()
{
    if kubectl -n kube-system get pods 2>/dev/null | grep -q etcd ; then
        return 0
    else
        return 1
    fi
}

#######################################
# Wait until Ceph status is healthy
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
waitCephHealthy()
{
    if ! isRook1; then
        return
    fi

    # log output of `ceph health` once, but only if a wait is needed
    local logged=0
    while true; do
        spinnerPodRunning "rook-ceph-system" "rook-ceph-operator"
        local rookOperatorPod=$(kubectl -n rook-ceph-system get pods | grep rook-ceph-operator | awk '{ print $1 }')
        local health=$(kubectl -n rook-ceph-system exec "$rookOperatorPod" -- /bin/sh -c 'ceph health 2>/dev/null || true')
        local status=$(echo $health | awk '{ print $1 }')
        if [ "$status" = "HEALTH_OK" ]; then
            return 0
        fi
        if [ "$logged" = "0" ] && [ -n "$health" ]; then
            logStep "Waiting for Rook/Ceph to report health OK, got: $status"
            logged=1
        fi
        sleep 2
    done
}

#######################################
# disable rook operator
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
disableRookCephOperator()
{
    if isRook1 || isRook08; then
        kubectl -n rook-ceph-system scale deployment rook-ceph-operator --replicas=0
    fi

}

#######################################
# enable rook operator
# Globals:
#   None
# Arguments:
#   None
# Returns:
#   None
#######################################
enableRookCephOperator()
{
    if isRook1 || isRook08; then
        kubectl -n rook-ceph-system scale deployment rook-ceph-operator --replicas=1
    fi
}

#######################################
# bail if Docker version is incompatible with K8s version
# Globals:
#   KUBERNETES_TARGET_VESION_MINOR
# Arguments:
#   None
# Returns:
#   None
#######################################
checkDockerK8sVersion()
{
    getDockerVersion
    if [ -z "$DOCKER_VERSION" ]; then
        return
    fi

    case "$KUBERNETES_TARGET_VERSION_MINOR" in 
        14|15)
            compareDockerVersions "$DOCKER_VERSION" 1.13.1
            if [ "$COMPARE_DOCKER_VERSIONS_RESULT" -eq "-1" ]; then
                bail "Minimum Docker version for Kubernetes $KUBERNETES_VERSION is 1.13.1."
            fi
            ;;
    esac
}

#######################################
# bail if airgapped and attempting to upgrade from K8s 1.11 to Replicated 2.49.1+
# Globals:
#   KUBERNETES_TARGET_VESION_MINOR
# Arguments:
#   None
# Returns:
#   None
#######################################
checkReplicatedK8sVersion()
{
    if [ -z "$KUBERNETES_CURRENT_VERSION_MINOR" ] || [ "$KUBERNETES_CURRENT_VERSION_MINOR" -ge "13" ] || [ "$AIRGAP" != "1" ]; then
        return 0
    fi
    semverCompare "$REPLICATED_VERSION" "2.49.1"
    if [ "$SEMVER_COMPARE_RESULT" -ge "0" ]; then
        bail "Upgrade to Replicated 2.49.0 with Kubernetes 1.15 prior to installing replicated $REPLICATED_VERSION"
    fi
}

writeAKAExecStop()
{
    cat >/opt/replicated/shutdown.sh <<EOF
#!/bin/bash

KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl cordon \$(hostname | tr '[:upper:]' '[:lower:]')

EOF

    if isSingleNodePrimary; then
        cat >>/opt/replicated/shutdown.sh <<EOF
# only on primary of single-node clusters
KUBECONFIG=/etc/kubernetes/admin.conf /usr/local/bin/replicatedctl app stop || true
KUBECONFIG=/etc/kubernetes/admin.conf kubectl scale deploy replicated-shared-fs-snapshotter --replicas=0 || true

EOF
    fi

    if isCurrentNodePrimaryNode; then
        cat >>/opt/replicated/shutdown.sh <<EOF
DAEMON_SELECTOR=\$(KUBECONFIG=/etc/kubernetes/admin.conf kubectl get deploy replicated -ojsonpath='{.spec.template.spec.nodeSelector.kubernetes\.io\/hostname }')
if [ "\$DAEMON_SELECTOR" = "\$(hostname | tr '[:upper:]' '[:lower:]')" ]; then
    KUBECONFIG=/etc/kubernetes/admin.conf kubectl patch deploy replicated --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/nodeSelector", "value":{"node-role.kubernetes.io/master":""}}]'
fi

EOF
    fi


    cat >>/opt/replicated/shutdown.sh <<EOF
# delete local pods with PVCs
while read -r uid; do
        pod=\$(KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl get pods --all-namespaces -ojsonpath='{ range .items[*]}{.metadata.name}{"\\t"}{.metadata.uid}{"\\t"}{.metadata.namespace}{"\\n"}{end}' | grep \$uid )
        KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl delete pod \$(echo \$pod | awk '{ print \$1 }') --namespace=\$(echo \$pod | awk '{ print \$3 }') --wait=false
done < <(lsblk | grep '^rbd[0-9]' | awk '{ print \$7 }' | awk -F '/' '{ print \$6 }')

# delete local pods using the Ceph filesystem
while read -r uid; do
        pod=\$(KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl get pods --all-namespaces -ojsonpath='{ range .items[*]}{.metadata.name}{"\\t"}{.metadata.uid}{"\\t"}{.metadata.namespace}{"\\n"}{end}' | grep \$uid )
        KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl delete pod \$(echo \$pod | awk '{ print \$1 }') --namespace=\$(echo \$pod | awk '{ print \$3 }') --wait=false
done < <(cat /proc/mounts | grep ':6789:/' | awk '{ print \$2 }' | awk -F '/' '{ print \$6 }')

while \$(lsblk | grep -q '^rbd[0-9]'); do
        echo "Waiting for Ceph block devices to unmount"
        sleep 1
done

while \$(cat /proc/mounts | grep -q ':6789:/'); do
        echo "Waiting for Ceph shared filesystems to unmount"
        sleep 1
done

# remove ceph-operator and mds pods from this node so they can continue to service the cluster
thisHost=\$(hostname | tr '[:upper:]' '[:lower:]')
while read -r row; do
    podName=\$(echo \$row | awk '{ print \$1 }')
    ns=\$(echo \$row | awk '{ print \$2 }')

    if echo \$podName | grep -q "rook-ceph-operator"; then
        KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl -n \$ns delete pod \$podName
    fi
    if echo \$podName | grep -q "rook-ceph-mds-rook-shared-fs"; then
        KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl -n \$ns delete pod \$podName
    fi
done < <(KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl get pods --all-namespaces -ojsonpath='{ range .items[*]}{.metadata.name}{"\\t"}{.metadata.namespace}{"\\t"}{.spec.nodeName}{"\\n"}{end}' | grep -E "\${thisHost}\$")

EOF

    chmod u+x /opt/replicated/shutdown.sh
}

writeAKAExecStart()
{
    cat >/opt/replicated/start.sh <<EOF
#!/bin/bash

# wait for Kubernets API
primary=\$(cat /etc/kubernetes/kubelet.conf | grep server | awk '{ print \$2 }')
while [ "\$(curl --noproxy "*" -sk \$primary/healthz)" != "ok" ]; do
        sleep 1
done

KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl uncordon \$(hostname | tr '[:upper:]' '[:lower:]')
EOF

    chmod u+x /opt/replicated/start.sh
}

writeAKAService()
{
    cat >/etc/systemd/system/aka-reboot.service <<EOF
[Unit]
After=kubelet.service
After=docker.service

[Service]
ExecStart=/opt/replicated/start.sh
ExecStop=/opt/replicated/shutdown.sh
Type=oneshot
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOF
}

installAKAService()
{
    logStep "Install aka-reboot service"
    writeAKAExecStop
    writeAKAExecStart
    writeAKAService
    systemctl daemon-reload
    systemctl enable aka-reboot.service
    systemctl start aka-reboot.service
    logSuccess "aka-reboot service installed"
}

LOAD_BALANCER_ADDRESS_CHANGED=0
LAST_LOAD_BALANCER_ADDRESS=
promptForLoadBalancerAddress() {
    local isK8sInstalled=
    local lastLoadBalancerAddress=

    lastLoadBalancerAddress="$LAST_LOAD_BALANCER_ADDRESS"
    if kubeadm config view >/dev/null 2>&1; then
        isK8sInstalled=1
        if [ -z "$lastLoadBalancerAddress" ]; then
            lastLoadBalancerAddress="$(kubeadm config view | grep 'controlPlaneEndpoint:' | sed 's/controlPlaneEndpoint: \|"//g')"
        fi
    fi
    if [ -n "$lastLoadBalancerAddress" ]; then
        splitHostPort "$lastLoadBalancerAddress"
        LAST_LOAD_BALANCER_ADDRESS="$HOST"
        if [ "$HOST" = "$lastLoadBalancerAddress" ]; then
            lastLoadBalancerAddress="$lastLoadBalancerAddress:6443"
        fi
    fi

    if [ -n "$LOAD_BALANCER_ADDRESS" ] && [ -n "$lastLoadBalancerAddress" ]; then
        splitHostPort "$LOAD_BALANCER_ADDRESS"
        if [ "$HOST" = "$LOAD_BALANCER_ADDRESS" ]; then
            LOAD_BALANCER_ADDRESS="$LOAD_BALANCER_ADDRESS:6443"
        fi
    fi

    if [ -z "$LOAD_BALANCER_ADDRESS" ] && [ -n "$lastLoadBalancerAddress" ]; then
        LOAD_BALANCER_ADDRESS="$lastLoadBalancerAddress"
    fi

    if [ -z "$LOAD_BALANCER_ADDRESS" ]; then
        printf "Please enter a load balancer address to route external and internal traffic to the API servers.\n"
        printf "In the absence of a load balancer address, all traffic will be routed to the first primary.\n"
        printf "Load balancer address: "
        prompt
        LOAD_BALANCER_ADDRESS="$PROMPT_RESULT"
        if [ -z "$LOAD_BALANCER_ADDRESS" ]; then
            LOAD_BALANCER_ADDRESS="$PRIVATE_ADDRESS"
            LOAD_BALANCER_PORT=6443
        fi
    fi

    if [ -z "$LOAD_BALANCER_PORT" ]; then
        splitHostPort "$LOAD_BALANCER_ADDRESS"
        LOAD_BALANCER_ADDRESS="$HOST"
        LOAD_BALANCER_PORT="$PORT"
    fi
    if [ -z "$LOAD_BALANCER_PORT" ]; then
        LOAD_BALANCER_PORT=6443
    fi

    if [ "$isK8sInstalled" = "1" ] && [ -n "$LOAD_BALANCER_ADDRESS" ]; then
        if [ "$LOAD_BALANCER_ADDRESS:$LOAD_BALANCER_PORT" != "$lastLoadBalancerAddress" ]; then
            LOAD_BALANCER_ADDRESS_CHANGED=1
        fi
    fi
}

promptForToken() {
    if [ -n "$KUBEADM_TOKEN" ]; then
        return
    fi

    printf "Please enter the kubernetes discovery token.\n"
    while true; do
        printf "Kubernetes join token: "
        prompt
        if [ -n "$PROMPT_RESULT" ]; then
            KUBEADM_TOKEN="$PROMPT_RESULT"
            return
        fi
    done
}

promptForTokenCAHash() {
    if [ -n "$KUBEADM_TOKEN_CA_HASH" ]; then
        return
    fi
    if [ "$UNSAFE_SKIP_CA_VERIFICATION" = "1" ]; then
        return
    fi

    printf "Please enter the discovery token CA's hash.\n"
    while true; do
        printf "Kubernetes discovery token CA hash: "
        prompt
        if [ -n "$PROMPT_RESULT" ]; then
            KUBEADM_TOKEN_CA_HASH="$PROMPT_RESULT"
            return
        fi
    done
}

kubernetesDiscoverPrivateIp()
{
    if [ -n "$PRIVATE_ADDRESS" ]; then
        return 0
    fi
    PRIVATE_ADDRESS=$(cat /etc/kubernetes/manifests/kube-apiserver.yaml 2>/dev/null | grep advertise-address | awk -F'=' '{ print $2 }')
}

maybeSetTaintControlPlane()
{
    if [ "$TAINT_CONTROL_PLANE" = "1" ]; then
        return
    fi
    # is this a primary node
    if [ "$PRIMARY" != "1" ]; then
        return
    fi

    waitForNodes

    # do the other primary nodes have NoSchedule taints?
    if kubectl get nodes --selector=node-role.kubernetes.io/master -ojsonpath --template='{.items[*].spec.taints[?(@.key == "node-role.kubernetes.io/master")].effect}' | grep -q NoSchedule ; then
        TAINT_CONTROL_PLANE=1
    fi
}

maybeTaintControlPlaneNodeJoin()
{
    maybeSetTaintControlPlane
    if [ "$TAINT_CONTROL_PLANE" = "1" ]; then
        makeKubeadmJoinConfigV1Beta2
        kubeadm join phase control-plane-join mark-control-plane --config /opt/replicated/kubeadm.conf
    fi
}

function k8s_load_images() {
    local k8sVersion="$1"
    airgapLoadKubernetesCommonImages "$k8sVersion"
    if isCurrentNodePrimaryNode; then
        airgapLoadKubernetesControlImages "$k8sVersion"

        retag_control_images "$k8sVersion"

        # technically this is not just loading images but this will upgrade images in control plane
        # components which we cant do from the first primary
        patch_control_plane_images "$k8sVersion"
    fi
    retag_kubeproxy_image "$k8sVersion"
}

function k8s_pull_and_retag_control_images() {
    local k8sVersion="$1"

    case "$k8sVersion" in
        1.15.12)
            if [ "$AIRGAP" != "1" ]; then
                docker pull "replicated/kube-apiserver:v1.15.12-20210805"
                docker pull "replicated/kube-controller-manager:v1.15.12-20210805"
                docker pull "replicated/kube-scheduler:v1.15.12-20210805"
            fi
            retag_control_images "$k8sVersion"
            ;;
    esac
}

function k8s_pull_and_retag_kubeproxy_image() {
    local k8sVersion="$1"

    case "$k8sVersion" in
        1.15.12)
            if [ "$AIRGAP" != "1" ]; then
                docker pull "replicated/kube-proxy:v1.15.12-20211117"
            fi
            retag_kubeproxy_image "$k8sVersion"
            ;;
    esac
}

function retag_control_images() {
    local k8sVersion="$1"

    case "$k8sVersion" in
        1.15.12)
            docker tag "replicated/kube-apiserver:v1.15.12-20210805" replicated/kube-apiserver:v1.15.12
            docker tag "replicated/kube-controller-manager:v1.15.12-20210805" replicated/kube-controller-manager:v1.15.12
            docker tag "replicated/kube-scheduler:v1.15.12-20210805" replicated/kube-scheduler:v1.15.12
            ;;
    esac
}

function retag_kubeproxy_image() {
    local k8sVersion="$1"

    case "$k8sVersion" in
        1.15.12)
            docker tag "replicated/kube-proxy:v1.15.12-20211117" replicated/kube-proxy:v1.15.12
            ;;
    esac
}

function is_primary_node() {
    local nodeName="$1"
    kubectl get nodes --no-headers --show-labels "$nodeName" 2>/dev/null | grep -q 'node-role.kubernetes.io/master'
}

# if remote nodes are in the cluster and this is an airgap install, prompt the user to run the
# load-images task on all remotes before proceeding because remaining steps may cause pods to
# be scheduled on those nodes with new images.
function prompt_airgap_preload_images() {
    if [ "$AIRGAP" != "1" ]; then
        return 0
    fi

    if ! kubernetes_has_remotes; then
        return 0
    fi

    local k8sVersion="$1"
 
    while read -r node; do
        local nodeName=$(echo "$node" | awk '{ print $1 }')
        if [ "$nodeName" = "$(hostname)" ]; then
            continue
        fi
        if kubernetes_node_has_all_images "$k8sVersion" "$nodeName"; then
            continue
        fi
        printf "\nDownload the Replicated airgap bundle and run the following script on node ${GREEN}${nodeName}${NC} to load required images before proceeding:\n"
        printf "\n"
        printf "${GREEN}\tcat ./kubernetes-init.sh | sudo bash -s load-images kubernetes-version=${KUBERNETES_VERSION}${NC}"
        printf "\n"

        while true; do
            echo ""
            printf "Have images been loaded on node ${nodeName}? "
            if confirmY " "; then
                break
            fi
        done
    done <<< "$(kubectl get nodes --no-headers)"
}

function kubernetes_node_has_all_images() {
    local k8sVersion="$1"
    local nodeName="$2"

    while read -r image; do
        if ! kubernetes_node_has_image "$nodeName" "$image"; then
            printf "\n${YELLOW}Node $nodeName missing image $image${NC}\n"
            return 1
        fi
    done <<< "$(list_all_required_images "$k8sVersion" "$nodeName")"
}

function kubernetes_node_has_image() {
    local nodeName="$1"
    # docker.io/envoyproxy/envoy-alpine:v1.10.0 -> envoyproxy/envoy-alpine:v1.10.0
    local image=$(echo $2 | sed 's/^docker.io\///')

    while read -r nodeImage; do
        nodeImage=$(echo $nodeImage | sed 's/^docker.io\///')
        if [ "$nodeImage" = "$image" ]; then
            return 0
        fi
    done <<< "$(kubernetes_node_images "$nodeName")"

    return 1
}

# exit 0 if there are any remote primary or secondary nodes
function kubernetes_has_remotes() {
    if ! kubernetes_api_is_healthy; then
        # assume this is a new install
        return 1
    fi

    local count=$(kubectl get nodes --no-headers --selector="kubernetes.io/hostname!=$(hostname)" 2>/dev/null | wc -l)
    if [ "$count" -gt "0" ]; then
        return 0
    fi

    return 1
}

function kubernetes_api_is_healthy() {
    curl --noproxy "*" --fail --silent --insecure "https://$(kubernetes_api_address)/healthz" >/dev/null
}

function kubernetes_api_address() {
    if [ -n "$LOAD_BALANCER_ADDRESS" ]; then
        echo "${LOAD_BALANCER_ADDRESS}:${LOAD_BALANCER_PORT}"
        return
    fi
    echo "${PRIVATE_ADDRESS}:6443"
}

function kubernetes_node_images() {
    local nodeName="$1"

    kubectl get node "$nodeName" -ojsonpath="{range .status.images[*]}{ range .names[*] }{ @ }{'\n'}{ end }{ end }"
}

function node_name() {
    echo "$(hostname | tr '[:upper:]' '[:lower:]')"
}

function label_node() {
    local nodeName="$1"
    local label="$2"
    if kubectl get nodes --show-labels --no-headers "$nodeName" | grep -q "$label" ; then
        return
    fi
    kubectl label nodes --overwrite "$nodeName" "$label"
}

maybeCreateTempDir

while [ "$1" != "" ]; do
    _param="$(echo "$1" | cut -d= -f1)"
    _value="$(echo "$1" | grep '=' | cut -d= -f2-)"
    case $_param in
        airgap)
            AIRGAP=1
            ;;
        bind-daemon-to-primary|bind_daemon_to_primary)
            BIND_DAEMON_TO_PRIMARY=1
            ;;
        bind-daemon-to-masters|bind_daemon_to_masters) # deprecated
            BIND_DAEMON_TO_PRIMARY=1
            ;;
        bind-daemon-hostname|bind_daemon_hostname)
            BIND_DAEMON_HOSTNAME="$_value"
            ;;
        log-level|log_level)
            LOG_LEVEL="$_value"
            ;;
        release-sequence|release_sequence)
            RELEASE_SEQUENCE="$_value"
            ;;
        release-patch-sequence|release_patch_sequence)
            RELEASE_PATCH_SEQUENCE="$_value"
            ;;
        ui-bind-port|ui_bind_port)
            UI_BIND_PORT="$_value"
            ;;
        kubernetes-namespace|kubernetes_namespace)
            KUBERNETES_NAMESPACE="$_value"
            ;;
        ha)
            HA_CLUSTER=1
            ;;
        purge-dead-nodes|purge_dead_nodes)
            PURGE_DEAD_NODES=1
            ;;
        clear-dead-nodes|clear_dead_nodes)
            CLEAR_DEAD_NODES=1
            ;;
        maintain-rook-storage-nodes|maintain_rook_storage_nodes)
            MAINTAIN_ROOK_STORAGE_NODES=1
            ;;
        use-rook-nodes-label|use_rook_nodes_label)
            USE_ROOK_NODES_LABEL=1
            ;;
        api-service-address|api_service_address)
            API_SERVICE_ADDRESS="$_value"
            ;;
        pv-base-path|pv_base_path)
            PV_BASE_PATH="$_value"
            ;;
        storage-provisioner|storage_provisioner)
            STORAGE_PROVISIONER="$_value"
            ;;
        storage-class|storage_class)
            STORAGE_CLASS="$_value"
            ;;
        service-type|service_type)
            SERVICE_TYPE="$_value"
            ;;
        replicated-yaml|replicated_yaml)
            REPLICATED_YAML="$_value"
            ;;
        rook-106-system-yaml|rook_106_system_yaml)
            ROOK_106_SYSTEM_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rook-103-system-yaml|rook_103_system_yaml)
            ROOK_103_SYSTEM_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rook-08-system-yaml|rook_08_system_yaml)
            ROOK_08_SYSTEM_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rook-106-cluster-yaml|rook_106_cluster_yaml)
            ROOK_106_CLUSTER_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rook-103-cluster-yaml|rook_103_cluster_yaml)
            ROOK_103_CLUSTER_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rook-08-cluster-yaml|rook_08_cluster_yaml)
            ROOK_08_CLUSTER_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rook-103-object-store-yaml|rook_103_object_store_yaml)
            ROOK_103_OBJECT_STORE_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rook-106-object-store-yaml|rook_106_object_store_yaml)
            ROOK_106_OBJECT_STORE_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rook-object-store-user-yaml|rook_object_store_user_yaml)
            ROOK_OBJECT_STORE_USER_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        hostpath-provisioner-yaml|hostpath_provisioner_yaml)
            HOSTPATH_PROVISIONER_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        storage-class-yaml|storage_class_yaml)
            STORAGE_CLASS_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        weave-yaml|weave_yaml)
            WEAVE_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        contour-yaml|contour_yaml)
            CONTOUR_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        registry-yaml|registry_yaml)
            REGISTRY_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        rek-operator-yaml|rek_operator_yaml)
            REK_OPERATOR_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        deployment-yaml|deployment_yaml)
            DEPLOYMENT_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        replicated-registry-yaml|replicated_registry_yaml)
            REPLICATED_REGISTRY_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        registry-cluster-ip|registry_cluster_ip)
            REGISTRY_CLUSTER_IP="$_value"
            ;;
        replicated-registry-cluster-ip|replicated_registry_cluster_ip)
            REPLICATED_REGISTRY_CLUSTER_IP="$_value"
            ;;
        cluster-role-binding-yaml|cluster_role_binding_yaml)
            CLUSTER_ROLE_BINDING_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        nodelocaldns-yaml|nodelocaldns_yaml)
            NODELOCALDNS_YAML="$_value"
            REPLICATED_YAML=0
            ;;
        ip-alloc-range|ip_alloc_range)
            IP_ALLOC_RANGE="$_value"
            ;;
        http-proxy|http_proxy)
            PROXY_ADDRESS="$_value"
            ;;
        no-proxy-addresses|no_proxy_addresses)
            NO_PROXY_ADDRESSES="$_value"
            ;;
        encrypt-network|encrypt_network)
            ENCRYPT_NETWORK="$_value"
            ;;
        weave-secret|weave_secret)
            WEAVE_SECRET="$_value"
            ;;
        replicated-pvc|replicated_pvc)
            REPLICATED_PVC="$_value"
            ;;
        ceph-dashboard-url|ceph_dashboard_url)
            CEPH_DASHBOARD_URL="$_value"
            ;;
        ceph-dashboard-user|ceph_dashboard_user)
            CEPH_DASHBOARD_USER="$_value"
            ;;
        ceph-dashboard-password|ceph_dashboard_password)
            CEPH_DASHBOARD_PASSWORD="$_value"
            ;;
        registry-address-override|registry_address_override)
            REGISTRY_ADDRESS_OVERRIDE="$_value"
            ;;
        app-registry-advertise-host|app_registry_advertise_host)
            APP_REGISTRY_ADVERTISE_HOST="$_value"
            ;;
        object-store-access-key|object_store_access_key)
            OBJECT_STORE_ACCESS_KEY="$_value"
            ;;
        object-store-secret-key|object_store_secret_key)
            OBJECT_STORE_SECRET_KEY="$_value"
            ;;
        object-store-cluster-ip|object_store_cluster_ip)
            OBJECT_STORE_CLUSTER_IP="$_value"
            ;;
        replicated-registry-prefix|replicated_registry_prefix)
            REPLICATED_REGISTRY_PREFIX="$_value"
            ;;
        *)
            echo >&2 "Error: unknown parameter \"$_param\""
            exit 1
            ;;
    esac
    shift
done

render_replicated_deployment() {
    NODE_SELECTOR=
    if [ -n "$BIND_DAEMON_HOSTNAME" ]; then
        NODE_SELECTOR=$(cat <<-EOF
      nodeSelector:
        kubernetes.io/hostname: "$BIND_DAEMON_HOSTNAME"
EOF
        )
    elif [ "$BIND_DAEMON_TO_PRIMARY" = "1" ]; then
        NODE_SELECTOR=$(cat <<-EOF
      nodeSelector:
        node-role.kubernetes.io/master: ""
EOF
        )
    fi

    PROXY_ENVS=
    if [ -n "$PROXY_ADDRESS" ]; then
        PROXY_ENVS=$(cat <<-EOF
        - name: HTTP_PROXY
          value: $PROXY_ADDRESS
        - name: HTTPS_PROXY
          value: $PROXY_ADDRESS
        - name: NO_PROXY
          value: $NO_PROXY_ADDRESSES
EOF
        )
    fi

    CEPH_DASHBOARD_ENV=
    if [ -n "$CEPH_DASHBOARD_URL" ]; then
        CEPH_DASHBOARD_ENV=$(cat <<-EOF
        - name: CEPH_DASHBOARD_URL
          value: $CEPH_DASHBOARD_URL
EOF
        )
    fi
    CEPH_DASHBOARD_CREDS_ENV=
    if [ -n "$CEPH_DASHBOARD_USER" ] && [ -n "$CEPH_DASHBOARD_PASSWORD" ]; then
        CEPH_DASHBOARD_CREDS_ENV=$(cat <<-EOF
        - name: CEPH_DASHBOARD_USER
          value: "$CEPH_DASHBOARD_USER"
        - name: CEPH_DASHBOARD_PASSWORD
          value: "$CEPH_DASHBOARD_PASSWORD"
EOF
        )
    fi

    DOCKER_REGISTRY_PREFIX="$REPLICATED_REGISTRY_PREFIX"
    if [ -n "$REGISTRY_ADDRESS_OVERRIDE" ]; then
      DOCKER_REGISTRY_PREFIX="$REGISTRY_ADDRESS_OVERRIDE/replicated"
    fi

    cat <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: replicated
  labels:
    app: replicated
    tier: master
spec:
  selector:
    matchLabels:
      app: replicated
      tier: master
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: replicated
        tier: master
    spec:
$NODE_SELECTOR
      imagePullSecrets:
        - name: replicateddockerhubkey
      containers:
      - name: replicated
        image: "${DOCKER_REGISTRY_PREFIX}/replicated:stable-2.53.2"
        imagePullPolicy: IfNotPresent
        env:
        - name: SCHEDULER_ENGINE
          value: kubernetes
        - name: RELEASE_CHANNEL
          value: "stable"
        - name: RELEASE_SEQUENCE
          value: "$RELEASE_SEQUENCE"
        - name: RELEASE_PATCH_SEQUENCE
          value: "$RELEASE_PATCH_SEQUENCE"
        - name: COMPONENT_IMAGES_REGISTRY_ADDRESS_OVERRIDE
          value: "$REGISTRY_ADDRESS_OVERRIDE"
        - name: REPLICATED_TMP_PATH
          value: /var/lib/replicated-tmp
        - name: SUPPORT_BUNDLES_PATH
          value: /var/lib/replicated-support-bundles
        - name: SUPPORT_BUNDLES_HOST_PATH
          value: /var/lib/replicated/support-bundles
        - name: LOCAL_ADDRESS
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: K8S_MASTER_ADDRESS
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        - name: K8S_HOST_IP
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        - name: K8S_POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: K8S_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
EOF
    if [ -n "$APP_REGISTRY_ADVERTISE_HOST" ]; then
        cat <<EOF
        - name: REGISTRY_ADVERTISE_ADDRESS
          value: "$APP_REGISTRY_ADVERTISE_HOST:9874"
EOF
    fi
    if [ -n "$API_SERVICE_ADDRESS" ]; then
        cat <<EOF
        - name: K8S_SERVICE_ADDRESS
          value: "$API_SERVICE_ADDRESS"
EOF
    fi
    if [ "$HA_CLUSTER" = "1" ]; then
        cat <<EOF
        - name: HA_CLUSTER
          value: "true"
EOF
    fi
    cat <<EOF
        - name: K8S_STORAGECLASS
          value: "$STORAGE_CLASS"
        - name: LOG_LEVEL
          value: "$LOG_LEVEL"
        - name: AIRGAP
          value: "$AIRGAP"
        - name: MAINTAIN_ROOK_STORAGE_NODES
          value: "$MAINTAIN_ROOK_STORAGE_NODES"
$PROXY_ENVS
        ports:
        - containerPort: 9874
        - containerPort: 9876
        - containerPort: 9877
        - containerPort: 9878
        volumeMounts:
        - name: replicated-persistent
          mountPath: /var/lib/replicated
        - name: replicated-tmp
          mountPath: /var/lib/replicated-tmp
        - name: replicated-support-bundles
          mountPath: /var/lib/replicated-support-bundles
        - name: replicated-socket
          mountPath: /var/run/replicated
        - name: docker-socket
          mountPath: /host/var/run/docker.sock
        - name: replicated-conf
          mountPath: /host/etc/replicated.conf
        - name: proc
          mountPath: /host/proc
          readOnly: true
        resources:
          requests:
            cpu: 250m
            memory: 256Mi
      - name: replicated-ui
        image: "${DOCKER_REGISTRY_PREFIX}/replicated-ui:stable-2.53.2"
        imagePullPolicy: IfNotPresent
        env:
        - name: RELEASE_CHANNEL
          value: "stable"
        - name: LOG_LEVEL
          value: "$LOG_LEVEL"
$CEPH_DASHBOARD_ENV
$CEPH_DASHBOARD_CREDS_ENV
        ports:
        - containerPort: 8800
        volumeMounts:
        - name: replicated-socket
          mountPath: /var/run/replicated
        resources:
          requests:
            cpu: 10m
            memory: 64Mi
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
      volumes:
      - name: replicated-persistent
        persistentVolumeClaim:
          claimName: replicated-pv-claim
      - name: replicated-tmp
        hostPath:
          path: /var/lib/replicated/tmp
          type: DirectoryOrCreate
      - name: replicated-support-bundles
        hostPath:
          path: /var/lib/replicated/support-bundles
          type: DirectoryOrCreate
      - name: replicated-socket
      - name: docker-socket
        hostPath:
          path: /var/run/docker.sock
      - name: replicated-conf
        hostPath:
          path: /etc/replicated.conf
      - name: proc
        hostPath:
          path: /proc
EOF
}

render_replicated_pvc() {
    local size="10Gi"
    if [ "$AIRGAP" = "1" ]; then
        size="100Gi"
    fi
    cat <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: replicated-pv-claim
  labels:
    app: replicated
    tier: master
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: "$size"
  storageClassName: "$STORAGE_CLASS"
EOF
}

render_premkit_statsd_pvcs() {
    cat <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: replicated-premkit-data-volume
  labels:
    app: replicated
    tier: premkit
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
  storageClassName: "$STORAGE_CLASS"
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: replicated-statsd-graphite-storage
  labels:
    app: replicated
    tier: statsd
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: "$STORAGE_CLASS"
EOF
}

render_replicated_service() {
    cat <<EOF
---
apiVersion: v1
kind: Service
metadata:
  name: replicated
  labels:
    app: replicated
    tier: master
spec:
  selector:
    app: replicated
    tier: master
  ports:
  - name: replicated-api
    port: 9876
    protocol: TCP
  - name: replicated-iapi
    port: 9877
    protocol: TCP
  - name: replicated-snapshots
    port: 9878
    protocol: TCP
  - name: replicated-support
    port: 9881
    protocol: TCP
EOF
}

render_replicated_registry_service() {
    cat <<EOF
---
apiVersion: v1
kind: Service
metadata:
  name: replicated-registry
  labels:
    app: replicated
    tier: master
spec:
  type: ClusterIP
EOF
    if [ -n "$REPLICATED_REGISTRY_CLUSTER_IP" ]; then
      cat <<EOF
  clusterIP: ${REPLICATED_REGISTRY_CLUSTER_IP}
EOF
    fi
    cat <<EOF
  selector:
    app: replicated
    tier: master
  ports:
  - name: replicated-registry
    port: 9874
    protocol: TCP
EOF
}

render_replicated_api_service() {
    # TODO: we may want to change this to a clusterip service if we add any more routes other than
    # the pki bundle route which is unnecessary on installs other than HA.
    cat <<EOF
---
apiVersion: v1
kind: Service
metadata:
  name: replicated-api
  labels:
    app: replicated
    tier: master
spec:
  type: NodePort
  selector:
    app: replicated
    tier: master
  ports:
  - name: replicated-api
    port: 9876
    nodePort: 9876
    protocol: TCP
EOF
}

render_replicated_ui_node_port_service() {
    cat <<EOF
---
apiVersion: v1
kind: Service
metadata:
  name: replicated-ui
  labels:
    app: replicated
    tier: master
spec:
  type: NodePort
  selector:
    app: replicated
    tier: master
  ports:
  - name: replicated-ui
    port: 8800
    nodePort: ${UI_BIND_PORT}
    protocol: TCP
EOF
}

render_replicated_ui_service() {
    cat <<EOF
---
apiVersion: v1
kind: Service
metadata:
  name: replicated-ui
  labels:
    app: replicated
    tier: master
spec:
  type: "$SERVICE_TYPE"
  selector:
    app: replicated
    tier: master
  ports:
  - name: replicated-ui
    port: ${UI_BIND_PORT}
    targetPort: 8800
    protocol: TCP
EOF
}

render_cluster_role_binding() {
    cat <<EOF
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: replicated-admin
  namespace: "$KUBERNETES_NAMESPACE"
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
subjects:
  - kind: ServiceAccount
    name: default
    namespace: "$KUBERNETES_NAMESPACE"
EOF
}

render_nodelocaldns_yaml() {
  cat <<EOF
# https://github.com/kubernetes/kubernetes/blob/3995c1a54d5ab921b89ff21328fa63b9c55fd406/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml
#
# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

apiVersion: v1
kind: ServiceAccount
metadata:
  name: node-local-dns
  namespace: kube-system
  labels:
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: v1
kind: Service
metadata:
  name: kube-dns-upstream
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/name: "KubeDNSUpstream"
spec:
  ports:
  - name: dns
    port: 53
    protocol: UDP
    targetPort: 53
  - name: dns-tcp
    port: 53
    protocol: TCP
    targetPort: 53
  selector:
    k8s-app: kube-dns
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: node-local-dns
  namespace: kube-system
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
  Corefile: |
    __PILLAR__DNS__DOMAIN__:53 {
        errors
        cache {
                success 9984 30
                denial 9984 5
        }
        reload
        loop
        bind __PILLAR__LOCAL__DNS__ __PILLAR__DNS__SERVER__
        forward . __PILLAR__CLUSTER__DNS__ {
                force_tcp
        }
        prometheus :9253
        health __PILLAR__LOCAL__DNS__:8080
        }
    in-addr.arpa:53 {
        errors
        cache 30
        reload
        loop
        bind __PILLAR__LOCAL__DNS__ __PILLAR__DNS__SERVER__
        forward . __PILLAR__CLUSTER__DNS__ {
                force_tcp
        }
        prometheus :9253
        }
    ip6.arpa:53 {
        errors
        cache 30
        reload
        loop
        bind __PILLAR__LOCAL__DNS__ __PILLAR__DNS__SERVER__
        forward . __PILLAR__CLUSTER__DNS__ {
                force_tcp
        }
        prometheus :9253
        }
    .:53 {
        errors
        cache 30
        reload
        loop
        bind __PILLAR__LOCAL__DNS__ __PILLAR__DNS__SERVER__
        forward . __PILLAR__UPSTREAM__SERVERS__ {
                force_tcp
        }
        prometheus :9253
        }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-local-dns
  namespace: kube-system
  labels:
    k8s-app: node-local-dns
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 10%
  selector:
    matchLabels:
      k8s-app: node-local-dns
  template:
    metadata:
      labels:
        k8s-app: node-local-dns
      annotations:
        prometheus.io/port: "9253"
        prometheus.io/scrape: "true"
    spec:
      priorityClassName: system-node-critical
      serviceAccountName: node-local-dns
      hostNetwork: true
      dnsPolicy: Default  # Don't use cluster DNS.
      tolerations:
      - key: "CriticalAddonsOnly"
        operator: "Exists"
      - effect: "NoExecute"
        operator: "Exists"
      - effect: "NoSchedule"
        operator: "Exists"
      containers:
      - name: node-cache
        image: "replicated/k8s-dns-node-cache:1.15.13-20210805"
        resources:
          requests:
            cpu: 25m
            memory: 5Mi
        args: [ "-localip", "__PILLAR__LOCAL__DNS__,__PILLAR__DNS__SERVER__", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ]
        securityContext:
          privileged: true
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9253
          name: metrics
          protocol: TCP
        livenessProbe:
          httpGet:
            host: __PILLAR__LOCAL__DNS__
            path: /health
            port: 8080
          initialDelaySeconds: 60
          timeoutSeconds: 5
        volumeMounts:
        - mountPath: /run/xtables.lock
          name: xtables-lock
          readOnly: false
        - name: config-volume
          mountPath: /etc/coredns
        - name: kube-dns-config
          mountPath: /etc/kube-dns
      volumes:
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
      - name: kube-dns-config
        configMap:
          name: kube-dns
          optional: true
      - name: config-volume
        configMap:
          name: node-local-dns
          items:
            - key: Corefile
              path: Corefile.base
EOF
}

render_rook_storage_class() {
    cat <<EOF
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: "$STORAGE_CLASS"
   annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: ceph.rook.io/block
parameters:
  pool: replicapool
  clusterNamespace: rook-ceph
EOF
}

render_rook106_system_yaml() {
    cat <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: rook-ceph-system
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephclusters.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephCluster
    listKind: CephClusterList
    plural: cephclusters
    singular: cephcluster
  scope: Namespaced
  version: v1
  validation:
    openAPIV3Schema:
      properties:
        spec:
          properties:
            cephVersion:
              properties:
                allowUnsupported:
                  type: boolean
                image:
                  type: string
                name:
                  pattern: ^(luminous|mimic|nautilus)$
                  type: string
            dashboard:
              properties:
                enabled:
                  type: boolean
                urlPrefix:
                  type: string
                port:
                  type: integer
            dataDirHostPath:
              pattern: ^/(\S+)
              type: string
            mon:
              properties:
                allowMultiplePerNode:
                  type: boolean
                count:
                  maximum: 9
                  minimum: 1
                  type: integer
                preferredCount:
                  maximum: 9
                  minimum: 0
                  type: integer
              required:
              - count
            network:
              properties:
                hostNetwork:
                  type: boolean
            storage:
              properties:
                nodes:
                  items: {}
                  type: array
                useAllDevices: {}
                useAllNodes:
                  type: boolean
          required:
          - mon
  additionalPrinterColumns:
    - name: DataDirHostPath
      type: string
      description: Directory used on the K8s nodes
      JSONPath: .spec.dataDirHostPath
    - name: MonCount
      type: string
      description: Number of MONs
      JSONPath: .spec.mon.count
    - name: Age
      type: date
      JSONPath: .metadata.creationTimestamp
    - name: State
      type: string
      description: Current State
      JSONPath: .status.state
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephfilesystems.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephFilesystem
    listKind: CephFilesystemList
    plural: cephfilesystems
    singular: cephfilesystem
  scope: Namespaced
  version: v1
  additionalPrinterColumns:
    - name: MdsCount
      type: string
      description: Number of MDSs
      JSONPath: .spec.metadataServer.activeCount
    - name: Age
      type: date
      JSONPath: .metadata.creationTimestamp
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephnfses.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephNFS
    listKind: CephNFSList
    plural: cephnfses
    singular: cephnfs
    shortNames:
    - nfs
  scope: Namespaced
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephobjectstores.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephObjectStore
    listKind: CephObjectStoreList
    plural: cephobjectstores
    singular: cephobjectstore
  scope: Namespaced
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephobjectstoreusers.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephObjectStoreUser
    listKind: CephObjectStoreUserList
    plural: cephobjectstoreusers
    singular: cephobjectstoreuser
  scope: Namespaced
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephblockpools.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephBlockPool
    listKind: CephBlockPoolList
    plural: cephblockpools
    singular: cephblockpool
  scope: Namespaced
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: volumes.rook.io
spec:
  group: rook.io
  names:
    kind: Volume
    listKind: VolumeList
    plural: volumes
    singular: volume
    shortNames:
    - rv
  scope: Namespaced
  version: v1alpha2
---
# The cluster role for managing all the cluster-specific resources in a namespace
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: rook-ceph-cluster-mgmt
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  - secrets
  - pods
  - pods/log
  - services
  - configmaps
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - apps
  resources:
  - deployments
  - daemonsets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
---
# The role for the operator to manage resources in the system namespace
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - configmaps
  - services
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - apps
  resources:
  - daemonsets
  - statefulsets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
---
# The cluster role for managing the Rook CRDs
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: rook-ceph-global
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  # Pod access is needed for fencing
  - pods
  # Node access is needed for determining nodes where mons should run
  - nodes
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - events
    # PVs and PVCs are managed by the Rook provisioner
  - persistentvolumes
  - persistentvolumeclaims
  - endpoints
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - storage.k8s.io
  resources:
  - storageclasses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - jobs
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
- apiGroups:
  - ceph.rook.io
  resources:
  - "*"
  verbs:
  - "*"
- apiGroups:
  - rook.io
  resources:
  - "*"
  verbs:
  - "*"
---
# Aspects of ceph-mgr that require cluster-wide access
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr-cluster
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - nodes
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
---
# The rook system service account used by the operator, agent, and discovery pods
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
---
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rook-ceph-system
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-global
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-global
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# The deployment for the rook operator
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rook-ceph-operator
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
spec:
  selector:
    matchLabels:
      app: rook-ceph-operator
  replicas: 1
  template:
    metadata:
      labels:
        app: rook-ceph-operator
    spec:
      serviceAccountName: rook-ceph-system
      containers:
      - name: rook-ceph-operator
        image: "replicated/rook-ceph:v1.0.6-20211117"
        args: ["ceph", "operator"]
        volumeMounts:
        - mountPath: /var/lib/rook
          name: rook-config
        - mountPath: /etc/ceph
          name: default-config-dir
        env:
        # Allow rook to create multiple file systems. Note: This is considered
        # an experimental feature in Ceph as described at
        # http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster
        # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
        - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
          value: "false"
        # The logging level for the operator: INFO | DEBUG
        - name: ROOK_LOG_LEVEL
          value: "INFO"
        # The interval to check if every mon is in the quorum.
        - name: ROOK_MON_HEALTHCHECK_INTERVAL
          value: "45s"
        # The duration to wait before trying to failover or remove/replace the
        # current mon with a new mon (useful for compensating flapping network).
        - name: ROOK_MON_OUT_TIMEOUT
          value: "600s"
        # The duration between discovering devices in the rook-discover daemonset.
        - name: ROOK_DISCOVER_DEVICES_INTERVAL
          value: "60m"
        # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods.
        # This is necessary to workaround the anyuid issues when running on OpenShift.
        # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
        - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
          value: "false"
        # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins).
        # Disable it here if you have similar issues.
        # For more details see https://github.com/rook/rook/issues/2417
        - name: ROOK_ENABLE_SELINUX_RELABELING
          value: "true"
        # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues.
        # For more details see https://github.com/rook/rook/issues/2254
        - name: ROOK_ENABLE_FSGROUP
          value: "true"
        # Disable automatic orchestration when new devices are discovered
        - name: ROOK_DISABLE_DEVICE_HOTPLUG
          value: "true"
        # The name of the node to pass with the downward API
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        # The pod name to pass with the downward API
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        # The pod namespace to pass with the downward API
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        # Rook Agent toleration. Will tolerate all taints with all keys.
        # Choose between NoSchedule, PreferNoSchedule and NoExecute:
        - name: AGENT_TOLERATION
          value: NoSchedule
        # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
        - name: AGENT_TOLERATION_KEY
          value: node-role.kubernetes.io/master
        # Rook Discover toleration. Will tolerate all taints with all keys.
        # Choose between NoSchedule, PreferNoSchedule and NoExecute:
        - name: DISCOVER_TOLERATION
          value: NoSchedule
        # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate
        - name: DISCOVER_TOLERATION_KEY
          value: node-role.kubernetes.io/master
        - name: ROOK_LOG_LEVEL
          value: DEBUG # TODO: change this back to INFO
        # https://rook.io/docs/rook/v1.0/ceph-cluster-crd.html#cluster-wide-resources-configuration-settings
        resources:
          requests:
            cpu: 100m
            memory: 256Mi
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
      volumes:
      - name: rook-config
        emptyDir: {}
      - name: default-config-dir
        emptyDir: {}
EOF
}

render_rook103_system_yaml() {
    cat <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: rook-ceph-system
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephclusters.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephCluster
    listKind: CephClusterList
    plural: cephclusters
    singular: cephcluster
  scope: Namespaced
  version: v1
  validation:
    openAPIV3Schema:
      properties:
        spec:
          properties:
            cephVersion:
              properties:
                allowUnsupported:
                  type: boolean
                image:
                  type: string
                name:
                  pattern: ^(luminous|mimic|nautilus)$
                  type: string
            dashboard:
              properties:
                enabled:
                  type: boolean
                urlPrefix:
                  type: string
                port:
                  type: integer
            dataDirHostPath:
              pattern: ^/(\S+)
              type: string
            mon:
              properties:
                allowMultiplePerNode:
                  type: boolean
                count:
                  maximum: 9
                  minimum: 1
                  type: integer
                preferredCount:
                  maximum: 9
                  minimum: 0
                  type: integer
              required:
              - count
            network:
              properties:
                hostNetwork:
                  type: boolean
            storage:
              properties:
                nodes:
                  items: {}
                  type: array
                useAllDevices: {}
                useAllNodes:
                  type: boolean
          required:
          - mon
  additionalPrinterColumns:
    - name: DataDirHostPath
      type: string
      description: Directory used on the K8s nodes
      JSONPath: .spec.dataDirHostPath
    - name: MonCount
      type: string
      description: Number of MONs
      JSONPath: .spec.mon.count
    - name: Age
      type: date
      JSONPath: .metadata.creationTimestamp
    - name: State
      type: string
      description: Current State
      JSONPath: .status.state
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephfilesystems.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephFilesystem
    listKind: CephFilesystemList
    plural: cephfilesystems
    singular: cephfilesystem
  scope: Namespaced
  version: v1
  additionalPrinterColumns:
    - name: MdsCount
      type: string
      description: Number of MDSs
      JSONPath: .spec.metadataServer.activeCount
    - name: Age
      type: date
      JSONPath: .metadata.creationTimestamp
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephnfses.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephNFS
    listKind: CephNFSList
    plural: cephnfses
    singular: cephnfs
    shortNames:
    - nfs
  scope: Namespaced
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephobjectstores.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephObjectStore
    listKind: CephObjectStoreList
    plural: cephobjectstores
    singular: cephobjectstore
  scope: Namespaced
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephobjectstoreusers.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephObjectStoreUser
    listKind: CephObjectStoreUserList
    plural: cephobjectstoreusers
    singular: cephobjectstoreuser
  scope: Namespaced
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: cephblockpools.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: CephBlockPool
    listKind: CephBlockPoolList
    plural: cephblockpools
    singular: cephblockpool
  scope: Namespaced
  version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: volumes.rook.io
spec:
  group: rook.io
  names:
    kind: Volume
    listKind: VolumeList
    plural: volumes
    singular: volume
    shortNames:
    - rv
  scope: Namespaced
  version: v1alpha2
---
# The cluster role for managing all the cluster-specific resources in a namespace
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: rook-ceph-cluster-mgmt
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  - secrets
  - pods
  - pods/log
  - services
  - configmaps
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - apps
  resources:
  - deployments
  - daemonsets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
---
# The role for the operator to manage resources in the system namespace
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - configmaps
  - services
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - apps
  resources:
  - daemonsets
  - statefulsets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
---
# The cluster role for managing the Rook CRDs
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: rook-ceph-global
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  # Pod access is needed for fencing
  - pods
  # Node access is needed for determining nodes where mons should run
  - nodes
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - events
    # PVs and PVCs are managed by the Rook provisioner
  - persistentvolumes
  - persistentvolumeclaims
  - endpoints
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - storage.k8s.io
  resources:
  - storageclasses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - jobs
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
- apiGroups:
  - ceph.rook.io
  resources:
  - "*"
  verbs:
  - "*"
- apiGroups:
  - rook.io
  resources:
  - "*"
  verbs:
  - "*"
---
# Aspects of ceph-mgr that require cluster-wide access
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr-cluster
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - nodes
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
---
# The rook system service account used by the operator, agent, and discovery pods
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
---
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rook-ceph-system
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-global
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-global
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# The deployment for the rook operator
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rook-ceph-operator
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
spec:
  selector:
    matchLabels:
      app: rook-ceph-operator
  replicas: 1
  template:
    metadata:
      labels:
        app: rook-ceph-operator
    spec:
      serviceAccountName: rook-ceph-system
      containers:
      - name: rook-ceph-operator
        image: "replicated/rook-ceph:v1.0.3-20211117"
        args: ["ceph", "operator"]
        volumeMounts:
        - mountPath: /var/lib/rook
          name: rook-config
        - mountPath: /etc/ceph
          name: default-config-dir
        env:
        # Allow rook to create multiple file systems. Note: This is considered
        # an experimental feature in Ceph as described at
        # http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster
        # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
        - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
          value: "false"
        # The logging level for the operator: INFO | DEBUG
        - name: ROOK_LOG_LEVEL
          value: "INFO"
        # The interval to check if every mon is in the quorum.
        - name: ROOK_MON_HEALTHCHECK_INTERVAL
          value: "45s"
        # The duration to wait before trying to failover or remove/replace the
        # current mon with a new mon (useful for compensating flapping network).
        - name: ROOK_MON_OUT_TIMEOUT
          value: "600s"
        # The duration between discovering devices in the rook-discover daemonset.
        - name: ROOK_DISCOVER_DEVICES_INTERVAL
          value: "60m"
        # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods.
        # This is necessary to workaround the anyuid issues when running on OpenShift.
        # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
        - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
          value: "false"
        # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins).
        # Disable it here if you have similar issues.
        # For more details see https://github.com/rook/rook/issues/2417
        - name: ROOK_ENABLE_SELINUX_RELABELING
          value: "true"
        # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues.
        # For more details see https://github.com/rook/rook/issues/2254
        - name: ROOK_ENABLE_FSGROUP
          value: "true"
        # Disable automatic orchestration when new devices are discovered
        - name: ROOK_DISABLE_DEVICE_HOTPLUG
          value: "true"
        # The name of the node to pass with the downward API
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        # The pod name to pass with the downward API
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        # The pod namespace to pass with the downward API
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        # Rook Agent toleration. Will tolerate all taints with all keys.
        # Choose between NoSchedule, PreferNoSchedule and NoExecute:
        - name: AGENT_TOLERATION
          value: NoSchedule
        # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
        - name: AGENT_TOLERATION_KEY
          value: node-role.kubernetes.io/master
        # Rook Discover toleration. Will tolerate all taints with all keys.
        # Choose between NoSchedule, PreferNoSchedule and NoExecute:
        - name: DISCOVER_TOLERATION
          value: NoSchedule
        # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate
        - name: DISCOVER_TOLERATION_KEY
          value: node-role.kubernetes.io/master
        - name: ROOK_LOG_LEVEL
          value: INFO
        # https://rook.io/docs/rook/v1.0/ceph-cluster-crd.html#cluster-wide-resources-configuration-settings
        resources:
          requests:
            cpu: 100m
            memory: 256Mi
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
      volumes:
      - name: rook-config
        emptyDir: {}
      - name: default-config-dir
        emptyDir: {}
EOF
}

render_rook08_system_yaml() {
    cat <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: rook-ceph-system
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: clusters.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: Cluster
    listKind: ClusterList
    plural: clusters
    singular: cluster
    shortNames:
    - rcc
  scope: Namespaced
  version: v1beta1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: filesystems.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: Filesystem
    listKind: FilesystemList
    plural: filesystems
    singular: filesystem
    shortNames:
    - rcfs
  scope: Namespaced
  version: v1beta1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: objectstores.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: ObjectStore
    listKind: ObjectStoreList
    plural: objectstores
    singular: objectstore
    shortNames:
    - rco
  scope: Namespaced
  version: v1beta1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: pools.ceph.rook.io
spec:
  group: ceph.rook.io
  names:
    kind: Pool
    listKind: PoolList
    plural: pools
    singular: pool
    shortNames:
    - rcp
  scope: Namespaced
  version: v1beta1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: volumes.rook.io
spec:
  group: rook.io
  names:
    kind: Volume
    listKind: VolumeList
    plural: volumes
    singular: volume
    shortNames:
    - rv
  scope: Namespaced
  version: v1alpha2
---
# The cluster role for managing all the cluster-specific resources in a namespace
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: rook-ceph-cluster-mgmt
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  - secrets
  - pods
  - services
  - configmaps
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - extensions
  resources:
  - deployments
  - daemonsets
  - replicasets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
---
# The role for the operator to manage resources in the system namespace
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - configmaps
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - extensions
  resources:
  - daemonsets
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
---
# The cluster role for managing the Rook CRDs
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: rook-ceph-global
  labels:
    operator: rook
    storage-backend: ceph
rules:
- apiGroups:
  - ""
  resources:
  # Pod access is needed for fencing
  - pods
  # Node access is needed for determining nodes where mons should run
  - nodes
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - events
    # PVs and PVCs are managed by the Rook provisioner
  - persistentvolumes
  - persistentvolumeclaims
  verbs:
  - get
  - list
  - watch
  - patch
  - create
  - update
  - delete
- apiGroups:
  - storage.k8s.io
  resources:
  - storageclasses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - jobs
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
- apiGroups:
  - ceph.rook.io
  resources:
  - "*"
  verbs:
  - "*"
- apiGroups:
  - rook.io
  resources:
  - "*"
  verbs:
  - "*"
---
# The rook system service account used by the operator, agent, and discovery pods
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
---
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rook-ceph-system
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rook-ceph-system
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rook-ceph-global
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-global
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# The deployment for the rook operator
apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: rook-ceph-operator
  namespace: rook-ceph-system
  labels:
    operator: rook
    storage-backend: ceph
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: rook-ceph-operator
    spec:
      serviceAccountName: rook-ceph-system
      containers:
      - name: rook-ceph-operator
        image: rook/ceph:v0.8.1
        args: ["ceph", "operator"]
        volumeMounts:
        - mountPath: /var/lib/rook
          name: rook-config
        - mountPath: /etc/ceph
          name: default-config-dir
        env:
        # To disable RBAC, uncomment the following:
        # - name: RBAC_ENABLED
        #  value: "false"
        # Rook Agent toleration. Will tolerate all taints with all keys.
        # Choose between NoSchedule, PreferNoSchedule and NoExecute:
        # - name: AGENT_TOLERATION
        #  value: "NoSchedule"
        # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
        # - name: AGENT_TOLERATION_KEY
        #  value: "<KeyOfTheTaintToTolerate>"
        # Set the path where the Rook agent can find the flex volumes
        # - name: FLEXVOLUME_DIR_PATH
        #  value: "<PathToFlexVolumes>"
        # Rook Discover toleration. Will tolerate all taints with all keys.
        # Choose between NoSchedule, PreferNoSchedule and NoExecute:
        # - name: DISCOVER_TOLERATION
        #  value: "NoSchedule"
        # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate
        # - name: DISCOVER_TOLERATION_KEY
        #  value: "<KeyOfTheTaintToTolerate>"
        # Allow rook to create multiple file systems. Note: This is considered
        # an experimental feature in Ceph as described at
        # http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster
        # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
        - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
          value: "false"
        # The logging level for the operator: INFO | DEBUG
        - name: ROOK_LOG_LEVEL
          value: "INFO"
        # The interval to check if every mon is in the quorum.
        - name: ROOK_MON_HEALTHCHECK_INTERVAL
          value: "45s"
        # The duration to wait before trying to failover or remove/replace the
        # current mon with a new mon (useful for compensating flapping network).
        - name: ROOK_MON_OUT_TIMEOUT
          value: "300s"
        # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods.
        # This is necessary to workaround the anyuid issues when running on OpenShift.
        # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
        - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
          value: "false"
        # The name of the node to pass with the downward API
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        # The pod name to pass with the downward API
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        # The pod namespace to pass with the downward API
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
      volumes:
      - name: rook-config
        emptyDir: {}
      - name: default-config-dir
        emptyDir: {}
EOF
}

render_rook103_cluster_yaml() {
    PV_BASE_PATH="${PV_BASE_PATH:-"/opt/replicated/rook"}"

    cat <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: rook-ceph
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rook-ceph-osd
  namespace: rook-ceph
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-osd
  namespace: rook-ceph
rules:
- apiGroups: [""]
  resources: ["configmaps"]
  verbs: [ "get", "list", "watch", "create", "update", "delete" ]
---
# Aspects of ceph-mgr that require access to the system namespace
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr-system
  namespace: rook-ceph
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - get
  - list
  - watch
---
# Aspects of ceph-mgr that operate within the cluster's namespace
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - services
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - jobs
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
- apiGroups:
  - ceph.rook.io
  resources:
  - "*"
  verbs:
  - "*"
---
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-cluster-mgmt
  namespace: rook-ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-cluster-mgmt
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# Allow the osd pods in this namespace to work with configmaps
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-osd
  namespace: rook-ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rook-ceph-osd
subjects:
- kind: ServiceAccount
  name: rook-ceph-osd
  namespace: rook-ceph
---
# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rook-ceph-mgr
subjects:
- kind: ServiceAccount
  name: rook-ceph-mgr
  namespace: rook-ceph
---
# Allow the ceph mgr to access the rook system resources necessary for the mgr modules
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr-system
  namespace: rook-ceph-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-mgr-system
subjects:
- kind: ServiceAccount
  name: rook-ceph-mgr
  namespace: rook-ceph
---
# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr-cluster
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-mgr-cluster
subjects:
- kind: ServiceAccount
  name: rook-ceph-mgr
  namespace: rook-ceph
---
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: rook-ceph
  namespace: rook-ceph
spec:
  cephVersion:
    # image: ceph/ceph:v14.2.1 has dashboard errors but 14.2.2 should work once released
    image: "replicated/ceph:v14.2.0-20211117"
    # Whether to allow unsupported versions of Ceph.
    # Do not set to true in production.
    allowUnsupported: false
  # The path on the host where configuration files will be persisted. Must be specified.
  # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
  dataDirHostPath: /var/lib/rook
  # set the amount of mons to be started
  mon:
    count: 1
    preferredCount: 3
    allowMultiplePerNode: false
  # enable the ceph dashboard for viewing cluster status
  dashboard:
    enabled: true
    # serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy)
    urlPrefix: /ceph
    # serve the dashboard at the given port.
    port: 7000
    # serve the dashboard using SSL
    ssl: false
  network:
    # toggle to use hostNetwork
    hostNetwork: false
  rbdMirroring:
    # The number of daemons that will perform the rbd mirroring.
    # rbd mirroring must be configured with "rbd mirror" from the rook toolbox.
    workers: 0
  # To control where various services will be scheduled by kubernetes, use the placement configuration sections below.
  placement:
    all:
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
  storage: # cluster level storage configuration and selection
    # the REK operator will disable this flag and manage the list of nodes explicitly
    useAllNodes: true
    useAllDevices: false
    deviceFilter:
    location:
    config:
      # The default and recommended storeType is dynamically set to bluestore for devices and filestore for directories.
      # Set the storeType explicitly only if it is required not to use the default.
      # storeType: bluestore
      databaseSizeMB: "1024" # this value can be removed for environments with normal sized disks (100 GB or larger)
      journalSizeMB: "1024"  # this value can be removed for environments with normal sized disks (20 GB or larger)
      osdsPerDevice: "1" # this value can be overridden at the node or device level
      # encryptedDevice: "false" # the default value for this option is "false"
    directories:
    # By default create a osd in the dataDirHostPath directory. This should be removed for
    # environments where nodes have disks available for Rook to use.
    - path: "$PV_BASE_PATH"
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: replicapool
  namespace: rook-ceph
spec:
  failureDomain: host
  replicated:
    # the REK operator will scale this up to 3 as nodes join the cluster
    size: 1
EOF
}

render_rook106_cluster_yaml() {
    PV_BASE_PATH="${PV_BASE_PATH:-"/opt/replicated/rook"}"

    cat <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: rook-ceph
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rook-ceph-osd
  namespace: rook-ceph
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-osd
  namespace: rook-ceph
rules:
- apiGroups: [""]
  resources: ["configmaps"]
  verbs: [ "get", "list", "watch", "create", "update", "delete" ]
---
# Aspects of ceph-mgr that require access to the system namespace
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr-system
  namespace: rook-ceph
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - get
  - list
  - watch
---
# Aspects of ceph-mgr that operate within the cluster's namespace
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - services
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - jobs
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - delete
- apiGroups:
  - ceph.rook.io
  resources:
  - "*"
  verbs:
  - "*"
---
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-cluster-mgmt
  namespace: rook-ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-cluster-mgmt
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# Allow the osd pods in this namespace to work with configmaps
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-osd
  namespace: rook-ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rook-ceph-osd
subjects:
- kind: ServiceAccount
  name: rook-ceph-osd
  namespace: rook-ceph
---
# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rook-ceph-mgr
subjects:
- kind: ServiceAccount
  name: rook-ceph-mgr
  namespace: rook-ceph
---
# Allow the ceph mgr to access the rook system resources necessary for the mgr modules
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr-system
  namespace: rook-ceph-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-mgr-system
subjects:
- kind: ServiceAccount
  name: rook-ceph-mgr
  namespace: rook-ceph
---
# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rook-ceph-mgr-cluster
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-mgr-cluster
subjects:
- kind: ServiceAccount
  name: rook-ceph-mgr
  namespace: rook-ceph
---
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: rook-ceph
  namespace: rook-ceph
spec:
  cephVersion:
    image: "replicated/ceph:v14.2.2-20211117"
    # Whether to allow unsupported versions of Ceph.
    # Do not set to true in production.
    allowUnsupported: false
  # The path on the host where configuration files will be persisted. Must be specified.
  # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
  dataDirHostPath: /var/lib/rook
  # set the amount of mons to be started
  mon:
    count: 1
    preferredCount: 3
    allowMultiplePerNode: false
  # enable the ceph dashboard for viewing cluster status
  dashboard:
    enabled: true
    # serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy)
    urlPrefix: /ceph
    # serve the dashboard at the given port.
    port: 7000
    # serve the dashboard using SSL
    ssl: false
  network:
    # toggle to use hostNetwork
    hostNetwork: false
  rbdMirroring:
    # The number of daemons that will perform the rbd mirroring.
    # rbd mirroring must be configured with "rbd mirror" from the rook toolbox.
    workers: 0
  # To control where various services will be scheduled by kubernetes, use the placement configuration sections below.
  placement:
    all:
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
  # https://rook.io/docs/rook/v1.0/ceph-cluster-crd.html#cluster-wide-resources-configuration-settings
  # https://docs.ceph.com/docs/luminous/start/hardware-recommendations/
  resources:
    mon:
      limits:
        cpu: "2"
        memory: 4096Mi
      requests:
        cpu: 100m
        memory: 256Mi
    mgr:
      limits:
        cpu: "2"
        memory: 4096Mi
      requests:
        cpu: 100m
        memory: 256Mi
    osd:
      limits:
        cpu: "2"
        memory: 4096Mi
      requests:
        cpu: 500m
        memory: 1024Mi
  storage: # cluster level storage configuration and selection
    # the REK operator will disable this flag and manage the list of nodes explicitly
    useAllNodes: true
    useAllDevices: false
    deviceFilter:
    location:
    config:
      # The default and recommended storeType is dynamically set to bluestore for devices and filestore for directories.
      # Set the storeType explicitly only if it is required not to use the default.
      # storeType: bluestore
      databaseSizeMB: "1024" # this value can be removed for environments with normal sized disks (100 GB or larger)
      journalSizeMB: "1024"  # this value can be removed for environments with normal sized disks (20 GB or larger)
      osdsPerDevice: "1" # this value can be overridden at the node or device level
      # encryptedDevice: "false" # the default value for this option is "false"
    directories:
    # By default create a osd in the dataDirHostPath directory. This should be removed for
    # environments where nodes have disks available for Rook to use.
    - path: "$PV_BASE_PATH"
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: replicapool
  namespace: rook-ceph
spec:
  failureDomain: host
  replicated:
    # the REK operator will scale this up to 3 as nodes join the cluster
    size: 1
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: rook-config-override
  namespace: rook-ceph
data:
  config: |
    [global]
    osd pool default pg num = 64
    osd_pool_default_pg_num = 64
    osd pool default pgp num = 64
    osd_pool_default_pgp_num = 64
EOF
}

render_rook08_cluster_yaml() {
    PV_BASE_PATH="${PV_BASE_PATH:-"/opt/replicated/rook"}"

    cat <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: rook-ceph
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rook-ceph-cluster
  namespace: rook-ceph
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rook-ceph-cluster
  namespace: rook-ceph
rules:
- apiGroups: [""]
  resources: ["configmaps"]
  verbs: [ "get", "list", "watch", "create", "update", "delete" ]
---
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rook-ceph-cluster-mgmt
  namespace: rook-ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: rook-ceph-cluster-mgmt
subjects:
- kind: ServiceAccount
  name: rook-ceph-system
  namespace: rook-ceph-system
---
# Allow the pods in this namespace to work with configmaps
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rook-ceph-cluster
  namespace: rook-ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rook-ceph-cluster
subjects:
- kind: ServiceAccount
  name: rook-ceph-cluster
  namespace: rook-ceph
---
apiVersion: ceph.rook.io/v1beta1
kind: Cluster
metadata:
  name: rook-ceph
  namespace: rook-ceph
spec:
  dataDirHostPath: /var/lib/rook
  # The service account under which to run the daemon pods in this cluster if the default account is not sufficient (OSDs)
  serviceAccount: rook-ceph-cluster
  # set the amount of mons to be started
  mon:
    count: 3
    allowMultiplePerNode: true
  # enable the ceph dashboard for viewing cluster status
  dashboard:
    enabled: true
  network:
    # toggle to use hostNetwork
    hostNetwork: false
  resources:
  storage: # cluster level storage configuration and selection
    useAllNodes: true
    useAllDevices: false
    deviceFilter:
    location:
    config:
      databaseSizeMB: "1024" # this value can be removed for environments with normal sized disks (100 GB or larger)
      journalSizeMB: "1024"  # this value can be removed for environments with normal sized disks (20 GB or larger)
    directories:
    - path: "$PV_BASE_PATH"
---
apiVersion: ceph.rook.io/v1beta1
kind: Pool
metadata:
  name: replicapool
  namespace: rook-ceph
spec:
  # The failure domain will spread the replicas of the data across different failure zones
  failureDomain: osd
  # The root of the crush hierarchy that will be used for the pool. If not set, will use "default".
  crushRoot: default
  # For a pool based on raw copies, specify the number of copies. A size of 1 indicates no redundancy.
  replicated:
    size: 1
EOF
}

render_rook_103_object_store_yaml() {
    cat <<EOF
---
apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
  name: replicated
  namespace: rook-ceph
spec:
  metadataPool:
    failureDomain: host
    replicated:
      size: 1
  dataPool:
    failureDomain: host
    replicated:
      size: 1
  gateway:
    type: s3
    sslCertificateRef:
    port: 80
    securePort:
    instances: 1
    allNodes: false
    placement:
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
EOF
}

render_rook_106_object_store_yaml() {
    cat <<EOF
---
apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
  name: replicated
  namespace: rook-ceph
spec:
  metadataPool:
    failureDomain: host
    replicated:
      size: 1
  dataPool:
    failureDomain: host
    replicated:
      size: 1
  gateway:
    type: s3
    sslCertificateRef:
    port: 80
    securePort:
    instances: 1
    allNodes: false
    placement:
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
    # https://rook.io/docs/rook/v1.0/ceph-cluster-crd.html#cluster-wide-resources-configuration-settings
    # https://docs.ceph.com/docs/jewel/start/hardware-recommendations/
    resources:
      limits:
        cpu: "2"
        memory: 4096Mi
      requests:
        cpu: 250m
        memory: 256Mi
EOF
}

render_rook_object_store_user_yaml() {
    cat <<EOF
---
apiVersion: ceph.rook.io/v1
kind: CephObjectStoreUser
metadata:
  name: replicated
  namespace: rook-ceph
spec:
  store: replicated
  displayName: "Replicated"
EOF
}

render_hostpath_storage_class() {
    cat <<EOF
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: "$STORAGE_CLASS"
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: replicated.com/hostpath
EOF
}

render_hostpath_provisioner_yaml() {
    PV_BASE_PATH="${PV_BASE_PATH:-"/opt/replicated/hostpath-provisioner"}"

    cat <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: replicated-hostpath-provisioner
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      tier: controller
      kind: storage-provisioner
  template:
    metadata:
      labels:
        tier: controller
        kind: storage-provisioner
    spec:
      containers:
      - name: replicated-hostpath-provisioner
        image: "replicated/replicated-hostpath-provisioner:cd1d272"
        imagePullPolicy: IfNotPresent
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        - name: PV_BASE_PATH
          value: "$PV_BASE_PATH"
        volumeMounts:
          - name: pv-volume
            mountPath: /opt/replicated/hostpath-provisioner
        # https://github.com/rimusz/charts/blob/b3aef3bf5a3519f0a1a79c0569e8233bdec0e30d/stable/hostpath-provisioner/values.yaml#L35-L41
        resources:
          limits:
            cpu: 100m
            memory: 128Mi
          requests:
            cpu: 100m
            memory: 128Mi
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
      volumes:
      - name: pv-volume
        hostPath:
          path: "$PV_BASE_PATH"
          type: DirectoryOrCreate
      serviceAccountName: hostpath-provisioner
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: hostpath-provisioner
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: hostpath-provisioner
subjects:
- kind: ServiceAccount
  name: hostpath-provisioner
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:persistent-volume-provisioner
---
# The default system:persistent-volume-provisioner role in Kubernetes 1.11 is insufficient:
# - crud for events and endpoints
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: hostpath-provisioner-extra
rules:
- apiGroups:
  - ""
  resources:
  - endpoints
  - events
  verbs:
  - create
  - patch
  - update
  - list
  - get
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: hostpath-provisioner-extra
subjects:
- kind: ServiceAccount
  name: hostpath-provisioner
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: hostpath-provisioner-extra
EOF
}

render_weave_yaml() {
    weave_passwd_env=
    if [ "$ENCRYPT_NETWORK" != "0" ]; then
        weave_passwd_env=$(cat <<-EOF
                - name: WEAVE_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: weave-passwd
                      key: weave-passwd
EOF
        )
        if [ "$WEAVE_SECRET" != "0" ]; then
            weave_password=$(< /dev/urandom tr -dc A-Za-z0-9 | head -c9)
            cat <<EOF
---
apiVersion: v1
kind: Secret
metadata:
  name: weave-passwd
  namespace: kube-system
stringData:
  weave-passwd: $weave_password
EOF
        fi
    fi

    cat <<EOF
---
apiVersion: v1
kind: List
items:
  - apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: weave-net
      labels:
        name: weave-net
      namespace: kube-system
  - apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: ClusterRole
    metadata:
      name: weave-net
      labels:
        name: weave-net
    rules:
      - apiGroups:
          - ''
        resources:
          - pods
          - namespaces
          - nodes
        verbs:
          - get
          - list
          - watch
      - apiGroups:
          - networking.k8s.io
        resources:
          - networkpolicies
        verbs:
          - get
          - list
          - watch
      - apiGroups:
          - ''
        resources:
          - nodes/status
        verbs:
          - patch
          - update
  - apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: ClusterRoleBinding
    metadata:
      name: weave-net
      labels:
        name: weave-net
    roleRef:
      kind: ClusterRole
      name: weave-net
      apiGroup: rbac.authorization.k8s.io
    subjects:
      - kind: ServiceAccount
        name: weave-net
        namespace: kube-system
  - apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: Role
    metadata:
      name: weave-net
      labels:
        name: weave-net
      namespace: kube-system
    rules:
      - apiGroups:
          - ''
        resourceNames:
          - weave-net
        resources:
          - configmaps
        verbs:
          - get
          - update
      - apiGroups:
          - ''
        resources:
          - configmaps
        verbs:
          - create
  - apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: RoleBinding
    metadata:
      name: weave-net
      labels:
        name: weave-net
      namespace: kube-system
    roleRef:
      kind: Role
      name: weave-net
      apiGroup: rbac.authorization.k8s.io
    subjects:
      - kind: ServiceAccount
        name: weave-net
        namespace: kube-system
  - apiVersion: extensions/v1beta1
    kind: DaemonSet
    metadata:
      name: weave-net
      labels:
        name: weave-net
      namespace: kube-system
    spec:
      minReadySeconds: 5
      template:
        metadata:
          labels:
            name: weave-net
        spec:
          containers:
            - name: weave
              command:
                - /home/weave/launch.sh
              env:
                - name: HOSTNAME
                  valueFrom:
                    fieldRef:
                      apiVersion: v1
                      fieldPath: spec.nodeName
                - name: IPALLOC_RANGE
                  value: $IP_ALLOC_RANGE
                - name: EXTRA_ARGS
                  value: "--log-level=info" # default log level is debug
                - name: EXEC_IMAGE
                  value: "replicated/weaveexec:2.6.5-20211117"
$weave_passwd_env
              image: "replicated/weave-kube:2.6.5-20211117"
              livenessProbe:
                httpGet:
                  host: 127.0.0.1
                  path: /status
                  port: 6784
                initialDelaySeconds: 30
              # https://www.weave.works/docs/net/latest/kubernetes/kube-addon/#cpu-and-memory-requirements
              resources:
                requests:
                  cpu: 50m
                  memory: 200Mi
              securityContext:
                privileged: true
              volumeMounts:
                - name: weavedb
                  mountPath: /weavedb
                - name: cni-bin
                  mountPath: /host/opt
                - name: cni-bin2
                  mountPath: /host/home
                - name: cni-conf
                  mountPath: /host/etc
                - name: dbus
                  mountPath: /host/var/lib/dbus
                - name: lib-modules
                  mountPath: /lib/modules
                - name: xtables-lock
                  mountPath: /run/xtables.lock
            - name: weave-npc
              args: ["--log-level", "info"] # default log level is debug
              env:
                - name: HOSTNAME
                  valueFrom:
                    fieldRef:
                      apiVersion: v1
                      fieldPath: spec.nodeName
                - name: EXEC_IMAGE
                  value: "replicated/weaveexec:2.6.5-20211117"
              image: "replicated/weave-npc:2.6.5-20211117"
              # https://www.weave.works/docs/net/latest/kubernetes/kube-addon/#cpu-and-memory-requirements
              resources:
                requests:
                  cpu: 50m
                  memory: 200Mi
              securityContext:
                privileged: true
              volumeMounts:
                - name: xtables-lock
                  mountPath: /run/xtables.lock
          dnsPolicy: ClusterFirstWithHostNet
          hostNetwork: true
          hostPID: true
          priorityClassName: system-node-critical
          restartPolicy: Always
          securityContext:
            seLinuxOptions: {}
          serviceAccountName: weave-net
          tolerations:
            - effect: NoSchedule
              operator: Exists
            - effect: NoExecute
              operator: Exists
          volumes:
            - name: weavedb
              hostPath:
                path: /var/lib/weave
            - name: cni-bin
              hostPath:
                path: /opt
            - name: cni-bin2
              hostPath:
                path: /home
            - name: cni-conf
              hostPath:
                path: /etc
            - name: dbus
              hostPath:
                path: /var/lib/dbus
            - name: lib-modules
              hostPath:
                path: /lib/modules
            - name: xtables-lock
              hostPath:
                path: /run/xtables.lock
                type: FileOrCreate
      updateStrategy:
        type: RollingUpdate
EOF
}

render_contour_yaml() {
    cat <<EOF
# adapted from https://github.com/heptio/contour/tree/master/deployment/deployment-grpc-v2
# imagePullPolicy changed from Always to IfNotPresent
# service type changed from LoadBalancer to NodePort and add nodePorts 80 and 443
# contour image tag changed from :master to :v0.11.0
# envoy image changed to envoy-alpine
# annotations for AWS, statsd, and prometheus removed
---
apiVersion: v1
kind: Namespace
metadata:
  name: heptio-contour
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: contour
  namespace: heptio-contour
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ingressroutes.contour.heptio.com
  labels:
    component: ingressroute
spec:
  group: contour.heptio.com
  version: v1beta1
  scope: Namespaced
  names:
    plural: ingressroutes
    kind: IngressRoute
  additionalPrinterColumns:
    - name: FQDN
      type: string
      description: Fully qualified domain name
      JSONPath: .spec.virtualhost.fqdn
    - name: TLS Secret
      type: string
      description: Secret with TLS credentials
      JSONPath: .spec.virtualhost.tls.secretName
    - name: First route
      type: string
      description: First routes defined
      JSONPath: .spec.routes[0].match
    - name: Status
      type: string
      description: The current status of the IngressRoute
      JSONPath: .status.currentStatus
    - name: Status Description
      type: string
      description: Description of the current status
      JSONPath: .status.description
  validation:
    openAPIV3Schema:
      properties:
        spec:
          properties:
            virtualhost:
              properties:
                fqdn:
                  type: string
                  # This regex handles two cases:
                  # 1. A reasonably well-formed FQDN, which is allowed
                  #    a hyphen in the top-level label (not usually
                  #    the case for TLDs.) This fixes https://github.com/heptio/contour/issues/1117
                  #    This is the first option in the regex.
                  # 2. A bareword containing a hyphen, no periods. This fixes 
                  #    https://github.com/heptio/contour/issues/755 and is the
                  #    second option in the regex
                  pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[\-a-z0-9]{2,}|[\-a-z0-9]+$
                tls:
                  properties:
                    secretName:
                      type: string
                      pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?([\.\/][a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
                    minimumProtocolVersion:
                      type: string
                      enum:
                        - "1.3"
                        - "1.2"
                        - "1.1"
            strategy:
              type: string
              enum:
                - RoundRobin
                - WeightedLeastRequest
                - Random
                - Cookie
            healthCheck:
              type: object
              required:
                - path
              properties:
                path:
                  type: string
                  pattern: ^\/.*$
                intervalSeconds:
                  type: integer
                timeoutSeconds:
                  type: integer
                unhealthyThresholdCount:
                  type: integer
                healthyThresholdCount:
                  type: integer
            routes:
              type: array
              items:
                required:
                  - match
                properties:
                  match:
                    type: string
                    pattern: ^\/.*$
                  delegate:
                    type: object
                    required:
                      - name
                    properties:
                      name:
                        type: string
                        pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ # DNS-1123 subdomain
                      namespace:
                        type: string
                        pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 label
                  services:
                    type: array
                    items:
                      type: object
                      required:
                        - name
                        - port
                      properties:
                        name:
                          type: string
                          pattern: ^[a-z]([-a-z0-9]*[a-z0-9])?$ # DNS-1035 label
                        port:
                          type: integer
                        weight:
                          type: integer
                        strategy:
                          type: string
                          enum:
                            - RoundRobin
                            - WeightedLeastRequest
                            - Random
                            - Cookie
                        healthCheck:
                          type: object
                          required:
                            - path
                          properties:
                            path:
                              type: string
                              pattern: ^\/.*$
                            intervalSeconds:
                              type: integer
                            timeoutSeconds:
                              type: integer
                            unhealthyThresholdCount:
                              type: integer
                            healthyThresholdCount:
                              type: integer
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: tlscertificatedelegations.contour.heptio.com
  labels:
    component: tlscertificatedelegation
spec:
  group: contour.heptio.com
  version: v1beta1
  scope: Namespaced
  names:
    plural: tlscertificatedelegations
    kind: TLSCertificateDelegation
  validation:
    openAPIV3Schema:
      properties:
        spec:
          properties:
            delegations:
              type: array
              items:
                type: object
                required:
                  - secretName
                  - targetNamespaces
                properties:
                  match:
                    type: string
                  targetNamespaces:
                    type: array
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: contour
  name: contour
  namespace: heptio-contour
spec:
  selector:
    matchLabels:
      app: contour
  replicas: 2
  template:
    metadata:
      labels:
        app: contour
    spec:
      containers:
      - image: "gcr.io/heptio-images/contour:v0.13.0"
        imagePullPolicy: IfNotPresent
        name: contour
        command: ["contour"]
        args: ["serve", "--incluster", "--envoy-service-http-port", "8080", "--envoy-service-https-port", "8443"]
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8000
        readinessProbe:
          httpGet:
            path: /healthz
            port: 8000
        # https://projectcontour.io/guides/resource-limits/
        resources:
          requests:
            cpu: 10m
            memory: 64Mi
      - image: "replicated/envoy:v1.10.0-20211117"
        name: envoy
        ports:
        - containerPort: 8080
          name: http
        - containerPort: 8443
          name: https
        command: ["envoy"]
        args:
        - --config-path /config/contour.json
        - --service-cluster cluster0
        - --service-node node0
        - --log-level info
        readinessProbe:
          httpGet:
            path: /healthz
            port: 8002
          initialDelaySeconds: 3
          periodSeconds: 3
        volumeMounts:
        - name: contour-config
          mountPath: /config
        lifecycle:
          preStop:
            exec:
              command: ["wget", "-qO-", "--post-data=''", "http://localhost:9001/healthcheck/fail"]
        # https://projectcontour.io/guides/resource-limits/
        resources:
          requests:
            cpu: 10m
            memory: 32Mi
      initContainers:
      - image: "gcr.io/heptio-images/contour:v0.13.0"
        imagePullPolicy: IfNotPresent
        name: envoy-initconfig
        command: ["contour"]
        args:
        - bootstrap
        - /config/contour.json
        volumeMounts:
        - name: contour-config
          mountPath: /config
        env:
        - name: CONTOUR_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
      volumes:
      - name: contour-config
        emptyDir: {}
      dnsPolicy: ClusterFirst
      serviceAccountName: contour
      terminationGracePeriodSeconds: 30
      # The affinity stanza below tells Kubernetes to try hard not to place 2 of
      # these pods on the same node.
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchLabels:
                  app: contour
              topologyKey: kubernetes.io/hostname
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: contour
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: contour
subjects:
- kind: ServiceAccount
  name: contour
  namespace: heptio-contour
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: contour
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - endpoints
  - nodes
  - pods
  - secrets
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - services
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - extensions
  resources:
  - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups: ["contour.heptio.com"]
  resources: ["ingressroutes", "tlscertificatedelegations"]
  verbs:
  - get
  - list
  - watch
  - put
  - post
  - patch
---
apiVersion: v1
kind: Service
metadata:
 name: contour
 namespace: heptio-contour
spec:
 ports:
 - port: 80
   name: http
   protocol: TCP
   targetPort: 8080
   nodePort: 80
 - port: 443
   name: https
   protocol: TCP
   targetPort: 8443
   nodePort: 443
 selector:
   app: contour
 type: NodePort
EOF
}

render_rek_operator_yaml() {
    DOCKER_REGISTRY_PREFIX="$REPLICATED_REGISTRY_PREFIX"
    if [ -n "$REGISTRY_ADDRESS_OVERRIDE" ]; then
      DOCKER_REGISTRY_PREFIX="$REGISTRY_ADDRESS_OVERRIDE/replicated"
    fi

    rook_label=
    if [ "$USE_ROOK_NODES_LABEL" = "1" ]; then
      rook_label="$ROOK_STORAGE_NODES_LABEL"
    fi

    if [ "$PURGE_DEAD_NODES" = "1" ]; then
        NODE_UNREACHABLE_TOLERATION=1h
    fi

    cat <<EOF
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rek-operator
  labels:
    app: rek-operator
spec:
  selector:
    matchLabels:
      app: rek-operator
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: rek-operator
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - docker-registry
              topologyKey: "kubernetes.io/hostname"
            weight: 100
      imagePullSecrets:
        - name: replicateddockerhubkey
      containers:
      - name: rek
        image: "${DOCKER_REGISTRY_PREFIX}/replicated:stable-2.53.2"
        imagePullPolicy: IfNotPresent
        command:
        - /usr/bin/rek
        - operator
        env:
        - name: LOG_LEVEL
          value: "$LOG_LEVEL"
        - name: NODE_UNREACHABLE_TOLERATION
          value: $NODE_UNREACHABLE_TOLERATION
        - name: PURGE_DEAD_NODES
          value: "$PURGE_DEAD_NODES"
        - name: CLEAR_DEAD_NODES
          value: "$CLEAR_DEAD_NODES"
        - name: MIN_READY_MASTER_NODES
          value: "2"
        - name: MIN_READY_WORKER_NODES
          value: "0"
        - name: MAINTAIN_ROOK_STORAGE_NODES
          value: "$MAINTAIN_ROOK_STORAGE_NODES"
        - name: ROOK_STORAGE_NODES_LABEL
          value: "$rook_label"
        - name: CEPH_BLOCK_POOL
          value: replicapool
        - name: CEPH_FILESYSTEM
          value: rook-shared-fs
        - name: CEPH_OBJECT_STORE
          value: replicated
        - name: MIN_CEPH_POOL_REPLICATION
          value: "1"
        - name: MAX_CEPH_POOL_REPLICATION
          value: "3"
        - name: COMPONENT_IMAGES_REGISTRY_ADDRESS_OVERRIDE
          value: $REGISTRY_ADDRESS_OVERRIDE
        - name: NAMESPACE
          value: default
        - name: RECONCILE_INTERVAL
          value: 1m
        resources:
          requests:
            cpu: 10m
            memory: 64Mi
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
EOF
}

render_registry_object_store() {
    cat <<EOF
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: docker-registry-config
  labels:
    app: docker-registry
data:
  config.yml: |-
    http:
      addr: :5000
      headers:
        X-Content-Type-Options:
        - nosniff
    log:
      fields:
        service: registry
    storage:
      redirect:
        disable: true
      cache:
        blobdescriptor: inmemory
      s3:
        region: "us-east-1"
        regionendpoint: http://$OBJECT_STORE_CLUSTER_IP
        bucket: docker-registry
    version: 0.1
---
apiVersion: v1
kind: Secret
metadata:
  name: docker-registry-s3-secret
  labels:
    app: docker-registry
type: Opaque
stringData:
  access-key-id: $OBJECT_STORE_ACCESS_KEY
  secret-access-key: $OBJECT_STORE_SECRET_KEY
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: docker-registry
spec:
  selector:
    matchLabels:
      app: docker-registry
  replicas: 2
  template:
    metadata:
      labels:
        app: docker-registry
    spec:
      terminationGracePeriodSeconds: 30
      containers:
      - name: docker-registry
        image: "replicated/docker-registry:2.6.2-20211117"
        imagePullPolicy: IfNotPresent
        command:
        - /bin/registry
        - serve
        - /etc/docker/registry/config.yml
        ports:
        - containerPort: 5000
          protocol: TCP
        volumeMounts:
        - name: docker-registry-config
          mountPath: /etc/docker/registry
        env:
        - name: REGISTRY_HTTP_SECRET
          valueFrom:
            secretKeyRef:
              key: haSharedSecret
              name: docker-registry-secret
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef:
              key: access-key-id
              name: docker-registry-s3-secret
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              key: secret-access-key
              name: docker-registry-s3-secret
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: 5000
            scheme: HTTP
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: 5000
            scheme: HTTP
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          requests:
            cpu: 10m
            memory: 64Mi
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
      volumes:
      - name: docker-registry-config
        configMap:
          name: docker-registry-config
EOF
}

render_registry_pvc() {
    cat <<EOF
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: docker-registry-config
  labels:
    app: docker-registry
data:
  config.yml: |-
    health:
      storagedriver:
        enabled: true
        interval: 10s
        threshold: 3
    http:
      addr: :5000
      headers:
        X-Content-Type-Options:
        - nosniff
    log:
      fields:
        service: registry
    storage:
      cache:
        blobdescriptor: inmemory
    version: 0.1
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: docker-registry
spec:
  selector:
    matchLabels:
      app: docker-registry
  serviceName: "registry"
  replicas: 1
  template:
    metadata:
      labels:
        app: docker-registry
    spec:
      terminationGracePeriodSeconds: 30
      containers:
      - name: docker-registry
        image: "replicated/docker-registry:2.6.2-20211117"
        imagePullPolicy: IfNotPresent
        command:
        - /bin/registry
        - serve
        - /etc/docker/registry/config.yml
        ports:
        - containerPort: 5000
          protocol: TCP
        volumeMounts:
        - name: registry-data
          mountPath: /var/lib/registry
        - name: docker-registry-config
          mountPath: /etc/docker/registry
        env:
        - name: REGISTRY_HTTP_SECRET
          valueFrom:
            secretKeyRef:
              key: haSharedSecret
              name: docker-registry-secret
        - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
          value: /var/lib/registry
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: 5000
            scheme: HTTP
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: 5000
            scheme: HTTP
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          requests:
            cpu: 10m
            memory: 64Mi
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
        operator: Exists
      volumes:
      - name: registry-data
        persistentVolumeClaim:
          claimName: docker-registry
      - name: docker-registry-config
        configMap:
          name: docker-registry-config
  volumeClaimTemplates:
  - metadata:
      name: registry-data
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 20Gi
      storageClassName: "$STORAGE_CLASS"
EOF
}

render_registry_yaml() {
    haSharedSecret=$(< /dev/urandom tr -dc A-Za-z0-9 | head -c9)

    if [ -n "$OBJECT_STORE_CLUSTER_IP" ]; then
        render_registry_object_store
    else
        render_registry_pvc
    fi

    cat <<EOF
---
apiVersion: v1
kind: Secret
metadata:
  name: docker-registry-secret
  labels:
    app: docker-registry
type: Opaque
stringData:
  haSharedSecret: $haSharedSecret
---
apiVersion: v1
kind: Service
metadata:
  name: docker-registry
  labels:
    app: docker-registry
spec:
  type: ClusterIP
EOF
    if [ -n "$REGISTRY_CLUSTER_IP" ]; then
      cat <<EOF
  clusterIP: ${REGISTRY_CLUSTER_IP}
EOF
    fi
    cat <<EOF
  ports:
  - port: 5000
    name: registry
    targetPort: 5000
    protocol: TCP
  selector:
    app: docker-registry
EOF
}

################################################################################
# Execution starts here
################################################################################

if [ -z "$REPLICATED_REGISTRY_PREFIX" ]; then
    getReplicatedRegistryPrefix "$REPLICATED_VERSION"
fi

if [ "$WEAVE_YAML" = "1" ]; then
    render_weave_yaml
fi

if [ "$CONTOUR_YAML" = "1" ]; then
    render_contour_yaml
fi

if [ "$ROOK_106_CLUSTER_YAML" = "1" ]; then
    render_rook106_cluster_yaml
fi

if [ "$ROOK_103_CLUSTER_YAML" = "1" ]; then
    render_rook103_cluster_yaml
fi

if [ "$ROOK_08_CLUSTER_YAML" = "1" ]; then
    render_rook08_cluster_yaml
fi

if [ "$ROOK_103_OBJECT_STORE_YAML" = "1" ]; then
    render_rook_103_object_store_yaml
fi

if [ "$ROOK_106_OBJECT_STORE_YAML" = "1" ]; then
    render_rook_106_object_store_yaml
fi

if [ "$ROOK_OBJECT_STORE_USER_YAML" = "1" ]; then
    render_rook_object_store_user_yaml
fi

if [ "$ROOK_106_SYSTEM_YAML" = "1" ]; then
    render_rook106_system_yaml
fi

if [ "$ROOK_103_SYSTEM_YAML" = "1" ]; then
    render_rook103_system_yaml
fi

if [ "$ROOK_08_SYSTEM_YAML" = "1" ]; then
    render_rook08_system_yaml
fi

if [ "$HOSTPATH_PROVISIONER_YAML" = "1" ]; then
    render_hostpath_provisioner_yaml
fi

if [ "$STORAGE_CLASS_YAML" = "1" ]; then
    case "$STORAGE_PROVISIONER" in
        rook|1)
            render_rook_storage_class
            ;;
        hostpath)
            render_hostpath_storage_class
            ;;
        0|"")
            ;;
        *)
            bail "Error: unknown storage provisioner \"$STORAGE_PROVISIONER\""
            ;;
    esac
fi

if [ "$REGISTRY_YAML" = "1" ]; then
    render_registry_yaml
fi

if [ "$REK_OPERATOR_YAML" = "1" ]; then
    render_rek_operator_yaml
fi

if [ "$REPLICATED_REGISTRY_YAML" = "1" ]; then
    render_replicated_registry_service
fi

if [ "$CLUSTER_ROLE_BINDING_YAML" = "1" ]; then
  render_cluster_role_binding
fi

if [ "$NODELOCALDNS_YAML" = "1" ]; then
  render_nodelocaldns_yaml
fi

if [ "$REPLICATED_YAML" = "1" ]; then
    if [ "$REPLICATED_PVC" != "0" ]; then
        render_replicated_pvc
    fi
    render_premkit_statsd_pvcs
    render_cluster_role_binding
    render_replicated_deployment
    render_replicated_service

    if [ "$HA_CLUSTER" = "1" ]; then
        render_replicated_api_service
    fi

    if [ "$SERVICE_TYPE" = "NodePort" ]; then
        render_replicated_ui_node_port_service
    else
        render_replicated_ui_service
    fi
fi

# autoupgrades
if [ "$DEPLOYMENT_YAML" = "1" ]; then
    render_replicated_deployment
fi