#!/usr/bin/env -S bats --report-formatter junit --formatter tap
# -*-sh-*-

load ../common/test_helper_functions || exit 1
source ../common/functions || exit 1

if [ -s ../common/TEST_ENV ]; then
	source ../common/TEST_ENV
fi

setup_file() {
	TESTNAME="BOS"
	LOGDIR="${LOGDIR:-/var/log}"

	if ! command -v koomie_cf >/dev/null 2>&1; then
		ERROR "koomie_cf executable not found - please verify losf is installed."
	fi

	# koomie_cf prefixes every line of output with the host it came from,
	# so the first field of the uptime lines is the name of a compute that
	# answered. Collect them once; the tests below need the names, not just
	# how many there are.
	COMPUTES=$(koomie_cf -x "c[0-9]+\b" uptime | awk '/load average/ {print $1}')
	if [ -z "${COMPUTES}" ]; then
		ERROR "Test requires at least 1 available compute node"
	fi

	export TESTNAME LOGDIR COMPUTES
}

setup() {
	OUTPUT="$(mktemp)"
	STAMP="$(mktemp)"

	export OUTPUT STAMP
}

teardown() {
	rm -f "${OUTPUT}" "${STAMP}"
}

@test "[${TESTNAME}] increased locked memory limits" {
	[[ -n "${SIMPLE_CI}" ]] && skip "Skipping memlock settings in SIMPLE_CI mode"
	[[ "${ARCH}" == "aarch64" ]] && skip "Skipping memlock settings for ARCH=${ARCH}"

	local bound host limit limits

	# Environment Modules prints the modules it loads on every login, so
	# the output of koomie_cf holds more than the limit of each host.
	# Keep only the lines that are a host name followed by a limit, which
	# drops the header and whatever the login printed around it.
	limits='^[^[:space:]]+[[:space:]]+(unlimited|[0-9]+)$'

	for bound in SOFT HARD; do
		koomie_cf -x "c[0-9]+\b" prlimit -l -o "${bound}" |
			grep -E "${limits}" >"${OUTPUT}" ||
			flunk "no ${bound} memlock limit reported by any compute"

		while read -r host limit; do
			if [ "${limit}" != "unlimited" ]; then
				flunk "${host}: ${bound} memlock limit is ${limit}"
			fi
		done <"${OUTPUT}"
	done
}

@test "[${TESTNAME}] syslog forwarding" {
	local host missing token

	# A token of this run only, so that a message of an earlier run cannot
	# be mistaken for a fresh one. RANDOM avoids reading /dev/urandom,
	# which has deadlocked this test in the past.
	token="ohpc-syslog-${RANDOM}-${RANDOM}"

	# STAMP is the moment before the messages are sent, so that only the
	# logs written afterwards have to be searched below.
	touch "${STAMP}"

	koomie_cf -x "c[0-9]+\b" logger "${token}"

	# Where a forwarded message ends up is up to rsyslog on the head node
	# and is not the same everywhere: xCAT files the messages of the nodes
	# in /var/log/xcat/computes.log rather than in /var/log/messages. Look
	# for the token in every log written since, instead of naming one file.
	#
	# Forwarding is asynchronous, so give the messages time to arrive. A
	# compute is done as soon as one of those lines carries its name.
	for _ in $(seq 10); do
		find "${LOGDIR}" -type f -newer "${STAMP}" \
			-exec grep -hIs -- "${token}" {} + >"${OUTPUT}" || true

		missing=""
		while read -r host; do
			grep -q -- "${host}" "${OUTPUT}" ||
				missing="${missing} ${host}"
		done <<<"${COMPUTES}"

		if [ -z "${missing}" ]; then
			break
		fi

		sleep 1
	done

	if [ -n "${missing}" ]; then
		flunk "no syslog message in ${LOGDIR} from:${missing}"
	fi
}
