#!/usr/bin/env cbsd
#v11.2.0
CIXARG="jname mode"
CIXOPTARG="name suspend"
MYDESC="bhyve checkpoint"
CBSDMODULE="bhyve"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

 Checkpoint allows you to freeze a running virtual machine, saving its
 entire state to disk. You can later restore the VM to that exact state
 without rebooting. CBSD bcheckpoint command manages checkpoint creation,
 listing and removal.

 Do not confuse checkpoints with disk snapshots (cbsd bsnapshot).
 However, 'cbsd bcheckpoint suspend=1' + 'cbsd bsnapshot' can be
 used together — suspend ensures disk consistency before taking a
 snapshot.

 To restore a VM from a checkpoint, use 'cbsd bstart checkpoint=<name>'.

 ${W1_COLOR}Warning!${N0_COLOR} This feature requires vmm.ko and bhyve utilities
 (bhyve, bhyvectl) compiled with BHYVE_SNAPSHOT support:

  1) Add to /etc/src.conf:
  WITH_BHYVE_SNAPSHOT=yes

  and kernel config:
  options         BHYVE_SNAPSHOT

  Then rebuild world and kernel.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}jname=${N0_COLOR}    - target environment.
 ${N2_COLOR}mode=${N0_COLOR}     - action: create, list, destroyall:
   create     - create new checkpoint;
   list       - show available checkpoints for current domain;
   destroyall - destroy all checkpoints for current domain;
 ${N2_COLOR}name=${N0_COLOR}     - name of checkpoint. Default: 'checkpoint'.
 ${N2_COLOR}suspend=${N0_COLOR}  - set to 1 to shut down the domain immediately after
             checkpoint for disk consistency. Default: 0.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd bcheckpoint mode=create suspend=1 jname=myvm
 # cbsd bcheckpoint mode=list jname=myvm
 # cbsd bstart checkpoint=checkpoint jname=myvm

${H3_COLOR}See also${N0_COLOR}:

  cbsd bsnapshot --help
  cbsd bstart --help
  cbsd bpause --help

"

. ${subrdir}/nc.subr

suspend=0
name="checkpoint"

cixinit

# store original name before rcconf init
oname="${name}"

. ${subrdir}/rcconf.subr
[ $? -eq 1 ] && err 1 "${N1_COLOR}No such domains: ${N2_COLOR}${jname}${N0_COLOR}"
[ "${emulator}" != "bhyve" ] && log_err 1 "${N1_COLOR}Not in bhyve mode${N0_COLOR}"

# restore original name
name="${oname}"

CHECKPOINT_DIR="${jailsysdir}/${jname}/checkpoints"
[ ! -d ${CHECKPOINT_DIR} ] && ${MKDIR_CMD} -p ${CHECKPOINT_DIR}

checkpoint_create()
{
	[ ${jid} -eq 0 ] && log_err 1 "Not running"
	[ -z "${name}" ] && log_err 1 "${N1_COLOR}Empty checkpoint name${N0_COLOR}"

	CHECKPOINT="${CHECKPOINT_DIR}/${name}.ckp"

	# todo: check for .ckp extenions

	if [ -r ${CHECKPOINT} ]; then
		${ECHO} "${N1_COLOR}Prune old checkpoint...${N0_COLOR}"
		cbsdlogger NOTICE ${CIX_APP}: prune old checkpoint: ${CHECKPOINT}
		${RM_CMD} -f ${CHECKPOINT} ${CHECKPOINT}.kern ${CHECKPOINT}.meta
	fi

	if [ ${suspend} -eq 1 ]; then
		# we must shutdown VM after the checkpoint is created.
		# We guarantee that it will not work any more, freezing its condition before snapshot.
		# In this life, VM will never wake up and we can kill VM hard
		# suspend -> crash for some reason. Use checkpoint instead
		${BHYVECTL_CMD} --suspend=${CHECKPOINT} --vm=${jname}
		#${BHYVECTL_CMD} --checkpoint=${CHECKPOINT} --vm=${jname}

		# if we kill or pause VM too early, the <chk>.kern file will not be generated
		# (why does control come back before the end, bhyve bug?)
		#bpause mode=on jname=${jname} || true
		# switch to RO?
		for i in 1 2; do
			${ECHO} "${N1_COLOR}Waiting and sure that the info is written on the disk: ${N2_COLOR}${i}/5${N0_COLOR}"
			sync
			sleep 1
		done
		# kill the virtual machine immediately
		sync
		bstop jname=${jname} noacpi=1 || true
		cbsdlogger NOTICE ${CIX_APP}: stop ${jname} due to checkpoint suspend=1: ${CHECKPOINT}
	else
		# switch to dsk RO ?
		${BHYVECTL_CMD} --checkpoint ${CHECKPOINT} --vm=${jname}
	fi

	# todo:
	# check for <chk>{.kern,meta} size > 0
	# check for valid meta json

	sync
	${ECHO} "${N1_COLOR}checkpoint was created: ${N2_COLOR}${CHECKPOINT}${N0_COLOR}"
	return 0
}

checkpoint_list()
{
	${ECHO} "${N1_COLOR}Created checkpoint for ${N2_COLOR}${jname}${N1_COLOR}:${N0_COLOR}"
	${FIND_CMD} ${CHECKPOINT_DIR}/ -mindepth 1 -maxdepth 1 -name '*.ckp' -type f | ${SORT_CMD} | while read _ckp; do
		_name="${_ckp##*/}"
		${ECHO} "${_name%.ckp}"
	done
}

case "${mode}" in
	create)
		checkpoint_create
		;;
	destroyall)
		[ -d ${CHECKPOINT_DIR} ] && ${RM_CMD} -rf ${CHECKPOINT_DIR}
		;;
	list)
		checkpoint_list
		;;
	*)
		err 1 "${N1_COLOR}Unknown mode: ${N2_COLOR}${mode}${N0_COLOR}"
esac

exit 0
