#!/usr/bin/env cbsd
#v11.1.12
CIXARG=""
# should be in sync with run_qemu() func: tools/up script
CIXOPTARG="jconf inter removejconf"
# allow all qemu settings
QEMU_ARGS="relative_path jname path data rcconf host_hostname ip4_addr astart nic_hwaddr nic_ratelimit zfs_snapsrc runasap interface rctl_nice emulator imgsize imgtype vm_cpus vm_ram vm_os_type vm_efi \
iso_site iso_img register_iso_name register_iso_as vm_hostbridge qemu_flags virtio_type vm_os_profile swapsize vm_iso_path vm_iso_path2 vm_guestfs vm_vnc_port qemu_generate_acpi qemu_wire_memory \
qemu_rts_keeps_utc qemu_force_msi_irq qemu_x2apic_mode qemu_mptable_gen qemu_ignore_msr_acc cd_vnc_wait qemu_vnc_resolution qemu_vnc_tcp_bind qemu_vnc_vgaconf nic_driver \
vnc_password media_auto_eject vm_cpu_topology debug_engine xhci cd_boot_firmware jailed chrooted on_poweroff on_reboot on_crash is_cloud ci_jname ci_fqdn ci_template ci_interface ci_ip4_addr \
ci_gw4 ci_nameserver_address ci_nameserver_search ci_adjust_inteface_helper ci_user_add ci_user_pw_user ci_user_pw_root ci_user_pubkey uuid ci_interface_mtu ci_ip4_addr2 ci_gw42 ci_interface_mtu2 \
interface2 ci_interface2 machine machine_accel vga usb tablet flavor"
CIXOPTARG="${CIXOPTARG} ${QEMU_ARGS}"
MYDESC="Create QEMU domain from config file or args"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:


The QEMU VM is created according to configuration file generated by qconstruct-tui.
You can see this configuration file if you answer negatively to the
'Do you want to create jail immediately?' question at the end of the dialogue.

In addition, you can override all parameters via command line arguments.
Many parameters are set by default via profiles,
e.g: ~cbsd/etc/defaults/qemu-default-default.conf which can be overwrited via
~cbsd/etc/qemu-default-default.conf or personal profile.

When you use ZFS, you may want to adjust some properties (e.g. reservation/compression/..) by default
via ~cbsd/etc/zfs.conf ( defaults: ~cbsd/etc/defaults/zfs.conf )

You can create your own profile and specify it when creating the VM.

If you create the environment often and in large quantities, the CBSD
recommends using the CBSDfile (Vagrant style) instead of qcreate/qconstruct-tui.

To minimize the number of parameters that do not change (or individual per-hoster),
these parameters are convenient to override through the ~cbsd/etc/qemu-default-default.conf file.

For example, instead of:

 cbsd qcreate jname=myvm1 runasap=1 ci4_gw="10.0.0.1" ci_gw42="2a01:4f8:140:918b::1" interface="bridge2" default_ci_interface_mtu="1450" qemu_vnc_tcp_bind="0.0.0.0" [...other params]

Set const values:

 cat >> ~cbsd/etc/qemu-default-default.conf <<EOF
runasap=1
ci4_gw="10.0.0.1"
ci_gw42="2a01:4f8:140:918b::1"
interface="bridge2"
default_ci_interface_mtu="1450"
qemu_vnc_tcp_bind="0.0.0.0"
EOF

And now these settings can be omited:

 cbsd qcreate jname=myvm1 [...other params]

You can add custom profile directories via extra_profile_dir= args or ~cbsd/etc/qemu-default-default.conf

In order to use the SPICE instead of the VNC, please rebuild qemu-tools with SPICE support:

 make -C /usr/ports/sysutils/qemu-tools config
 make -C /usr/ports/sysutils/qemu-tools reinstall

To be able to boot the system into UEFI, you must install OVMF.fd firmware:

  for FreeBSD: pkg install edk2-qemu-x64 (for x86);
  for Linux: apt install ovmf;

Set const values:

 cat >> ~cbsd/etc/qemu-default-default.conf <<EOF
runasap=1
ci4_gw="10.0.0.1"
ci_gw42="2a01:4f8:140:918b::1"
interface="bridge2"
default_ci_interface_mtu="1450"
qemu_vnc_tcp_bind="0.0.0.0"
EOF

And now these settings can be omited:

 cbsd qcreate jname=myvm1 [...other params]

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}flavor${N0_COLOR}             - Use flavor (named group of vm_cpus/vm_ram/imgsize): see 'cbsd vm-packages';
 ${N2_COLOR}imgsize=${N0_COLOR}           - VM first/boot disk size, e.g.: '10g', '21474836480000'.
 ${N2_COLOR}inter=${N0_COLOR}             - 0 to prevent any questions and to accept answers by default.
 ${N2_COLOR}interface2=${N0_COLOR}        - <parent>, create VM with two interfaces, <parent> is uplink for nic2,
                      do not confuse with the ci_interface2 parameter.
 ${N2_COLOR}quiet=${N0_COLOR}             - 0,1: be quiet, dont output verbose message.
 ${N2_COLOR}removejconf=${N0_COLOR}       - 0,1: remove jconf after qcreate? 0 - don't remove.
 ${N2_COLOR}runasap=${N0_COLOR}           - 0,1: when 1 - run a VM immediately (atomic qcreate+qstart).
 ${N2_COLOR}vm_cpus=${N0_COLOR}           - VM CPUs cores, e.g.: '2'.
 ${N2_COLOR}vm_os_profile=${N0_COLOR}     - <name>: full config file is: vm-\${vm_os_type}-\${vm_os_profile}.conf, file
                      must be present in ~cbsd/etc/defaults/ or ~cbsd/etc/ directory.
 ${N2_COLOR}vm_os_type=${N0_COLOR}        - <name>: full config file is: vm-\${vm_os_type}-\${vm_os_profile}.conf, file
                      must be present in ~cbsd/etc/defaults/ or ~cbsd/etc/ directory.
 ${N2_COLOR}vm_ram=${N0_COLOR}            - VM RAM, e.g.: '1g', '2147483648'.
 ${N2_COLOR}zfs_snapsrc=${N0_COLOR}       - <name>: use ZFS snapshot as data source.

${H3_COLOR}Cloud-Init Options${N0_COLOR}:

 ${N2_COLOR}ci_gw4=${N0_COLOR}            - <ipv4> (cloud-init profile only): set IPv4 gateway for VM.
 ${N2_COLOR}ci_interface2=${N0_COLOR}     - configure second interface via cloud-init,
                      do not confuse with the interface2 parameter.
 ${N2_COLOR}ci_interface_mtu=${N0_COLOR}  - set MTU for NIC1, default: 1500.
 ${N2_COLOR}ci_interface_mtu2=${N0_COLOR} - set MTU for NIC2, default: 1500.
 ${N2_COLOR}ci_ip4_addr=${N0_COLOR}       - <ipv4> (cloud-init profile only): set IPv4 address for VM,
                      default is: DHCP. Can be: 'DHCPv6' or static IPv4/IPv6.
 ${N2_COLOR}ci_ip4_addr2=${N0_COLOR}      - <ipv4> (cloud-init profile only): set IPv4 address for VM
                      NIC2 (see also: ci_interface2,ci_gw42). Possible values same as ci_ip4_addr.
 ${N2_COLOR}ci_user_pubkey${N0_COLOR}     - full/relative path to authorized_keys or may contain pubkey
                      string itself, e.g: ci_user_pubkey=\"ssh-ed25519 XXXXX root@my.domain\".
                      (cloud-init profile only): set authorized_keys file for cloud-init user for VM.
 ${N2_COLOR}ci_user_pw_user=${N0_COLOR}   - set password for cloud-init user.
 ${N2_COLOR}ci_user_pw_root=${N0_COLOR}   - set password for 'root' user.

${H3_COLOR}Examples${N0_COLOR}:

 > ISO sample (see: \"cbsd get-profiles src=iso | grep linux\" )

 # cbsd qcreate jname=vm1 vm_os_type=linux vm_os_profile=Rocky-10-x86_64 vm_ram=1g vm_cpus=1 runasap=1 imgsize=10g
 # cbsd qcreate jname=vm1 vm_os_type=linux vm_os_profile=Debian-13-x86_64 vm_ram=1g vm_cpus=1 runasap=1 imgsize=10g
 # cbsd qcreate jname=vm1 vm_os_type=freebsd vm_os_profile=FreeBSD-15.1-x86_64 vm_ram=1g vm_cpus=1 imgsize=10g qemu_vnc_tcp_bind=\"0.0.0.0\"

 > CLOUD sample (see: \"cbsd get-profiles src=cloud | grep freebsd\" )

 # cbsd qcreate jname=c1 vm_ram=4g vm_cpus=2 vm_os_type=freebsd vm_os_profile=cloud-FreeBSD-15.1-ufs-x86_64 imgsize=20g ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3
 # cbsd qcreate jname=gateway flavor=small1 vm_os_type=linux vm_os_profile=cloud-Debian-x86-13 ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3 ci_interface2=bridge2 ci_ip4_addr2=192.168.0.2 ci_gw42=192.168.0.1

# When qemu-system-aarch64/riscv installed, (Non-?)-native ARCH also possible:

 # cbsd qcreate jname=vm1 vm_os_type=linux vm_os_profile=Debian-aarch64-13 vm_ram=4g vm_cpus=1 imgsize=10g runasap=1
 # cbsd qcreate jname=vm1 vm_os_type=freebsd vm_os_profile=FreeBSD-aarch64-14.5 vm_ram=8g vm_cpus=8 imgsize=10g runasap=1 qemu_vnc_tcp_bind=\"0.0.0.0\"

# (Non-?)-native aarch64 CLOUD images:

 # cbsd qcreate jname=vm1 flavor=small1 vm_os_type=linux vm_os_profile=cloud-Debian-aarch64-13 ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3 runasap=1
 # cbsd qcreate jname=vm1 flavor=small1 vm_os_type=freebsd vm_os_profile=cloud-FreeBSD-ufs-aarch64-14.5 ci_ip4_addr=10.0.1.88 ci_gw4=10.0.1.3 runasap=1

${H3_COLOR}See also${N0_COLOR}:

 cbsd qconstruct-tui --help
 cbsd up --help
 cbsd get-profiles --help
 cbsd vm-packages --help
 cbsd jail2iso --help
 cat ~cbsd/etc/defaults/qemu-default-default.conf
 cat ~cbsd/etc/defaults/zfs.conf

"
CBSDMODULE="qemu"
EXTHELP="wf_qcreate"

. ${subrdir}/nc.subr

nic_flags=
nic2_flags=
quiet=0

# temporary hack for second nic
interface2=
ci_ip4_addr2=
ci_gw42=
ci_interface_mtu2=
ci_interface2=

readconf buildworld.conf
readconf zfs.conf

oextra_profile_dir=
extra_profile_dir=

flavor=
oflavor=
quiet=0
oquiet=0

. ${subrdir}/universe.subr
. ${subrdir}/freebsd_world.subr
. ${subrdir}/qemu.subr

progress_state_file=
export CIX_INIT_SAVE2FILE="new"
cixinit

# Migrate H_CBSD_* env vars into params.
# The cbsd binary passes key=value args via H_CBSD_* env vars, not as positional params ($nparam=0).
# cixinit only reads from shellparam.p, so we need to load H_CBSD_* vars explicitly
# and persist them into the CIX_INIT_CONF file.
for _i in ${MYOPTARG}; do
	_tmp_env="H_CBSD_${_i}"
	eval _tmpval="\${${_tmp_env}}"
	if [ -n "${_tmpval}" ]; then
		eval "${_i}=\"\${_tmpval}\""
		export "o${_i}=${_tmpval}"
		updateconf "${CIX_INIT_CONF}" "${_i}=\"${_tmpval}\""
	fi
done

readconf qemu-default-default.conf

. ${tools}
. ${system}
. ${mdtools}
. ${jfs}

if [ -z "${jconf}" ]; then
	jconf="${CIX_INIT_CONF}"
	: "${removejconf:=1}"
fi

[ -z "${jconf}" -a -z "${jname}" ] && err 1 "${N1_COLOR}Please set for qcreate: ${N2_COLOR}${jconf}${N0_COLOR}"
[ -n "${removejconf}" ] && oremovejconf="${removejconf}"

cbsd_queue_remove()
{
	if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
		[ -n "${cbsd_bhyve_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_bhyve_queue_name} id=${jname} cmd=bremove status=2 emulator=${emulator}
	fi
}

really_create_base()
{
	if [ "${vm_os_type}" = "freebsd" -a "${from_jail}" = "1" ]; then
		case ${vm_os_profile} in
			"FreeBSD-bsdinstall-jail")
				export UNAME_r="${ver}-RELEASE"
				export DISTRIBUTIONS="kernel.txz base.txz"
				bsdinstall jail ${data}
				unset UNAME_r
				nobase=1
				;;
			*)
				nobase=0 # 0 for jail2iso
				init_target_arch
				init_basedir
				init_kerneldir
				get_base -v ${ver}
				get_kernel
				${ECHO} "${N1_COLOR}Stage1: ${N2_COLOR}jcreate...${N0_COLOR}"
				echo "jcreate jconf=${temprcconf} pkg_bootstrap=0"
				jcreate jconf=${temprcconf} pkg_bootstrap=0
				customskel
				;;
		esac
	fi
}

really_create_vm()
{
	local _res= _msg=

	# do not create disk
	[ "${imgtype}" = "none" ] && return 0

	capture _res substr --pos=0 --len=5 --str="${imgtype}"
	if [ "${_res}" = "/dev/" ]; then
		cbsdlogger NOTICE ${CIX_APP}: qcreate for ${jname}: imgtype = raw: ${imgtype}
		 [ ! -c "${imgtype}" ] && log_err 1 "${N1_COLOR}${CIX_APP}: no such character device: ${N2_COLOR}${imgtype}${N0_COLOR}"
		oimgtype="raw"
		raw_path_dsk0="${imgtype}"
		return 0
	fi

	#test for imgtype
	case ${zfsfeat} in
		1)
			;;
		*)
			# force switch imgtype to md when no zfsfeat
			imgtype="md"
	esac

	if [ "${from_jail}" = "1" ]; then
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Stage2: ${N2_COLOR}jail2iso...${N0_COLOR}"
		[ -z "${swapsize}" ] && swapsize="0"

		mountbase -o "" -p "" -d "" -c "" -s ""
		#echo "jail2iso jname=${jname} nameserver=${jnameserver} ip4_addr=${ip4_addr} gw4=${gw4} dstname=${jname}.$$.img swapsize=${swapsize} freesize=${imgsize} dstdir=/tmp host_hostname="${host_hostname}" media=qemu quiet=1 prunelist=0 nobase=${nobase} vm_guestfs=${vm_guestfs} fromfile=${temprcconf}"
		jail2iso jname=${jname} nameserver=${jnameserver} ip4_addr=${ip4_addr} gw4=${gw4} dstname=${jname}.$$.img swapsize=${swapsize} freesize=${imgsize} dstdir=/tmp host_hostname="${host_hostname}" media=qemu quiet=1 vm_guestfs=${vm_guestfs}
		# fromfile=${temprcconf} addmod=0

		jremove ${jname}
		create_fs ${data}

		#test for zfs mounted & mount if not
		case ${zfsfeat} in
			1)
				. ${subrdir}/zfs.subr
				zfsmnt ${data}
				[ $? -eq 2 ] && ${ZFS_CMD} mount "${ZPOOL}"
				;;
		esac

		updateconf ${jconf} ver="${ver}"
		${MV_CMD} /tmp/${jname}.$$.img ${data}/${defdsk}
	else
		capture _msg virtual_create_dsk -p ${data}/${defdsk} -s ${imgsize} -f 1 -t ${imgtype}
		_res=$?
		if [ ${_res} -ne 0 ]; then
			qremove ${jname}
			err 1 "qcreate error: Couldn't create the image file. ${_msg}"
		fi
	fi
}

### MAIN
[ ! -f "${jconf}" ] && err 1 "${N1_COLOR}no such jconf file${N0_COLOR}";

if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

gettime st_time
over="${ver}"
oarch="${arch}"
capture jconf ${REALPATH_CMD} ${jconf}

# several defaults
[ -z "${mnt_start}" ] && mnt_start="0"
[ -z "${mnt_stop}" ] && mnt_stop="0"

# make tmp jconf and work with it, insofar CBSD can
# change content of jconf in the course of their work
# we must leave the original file intact
jconf_tmp="${jconf}.tmp"
${CP_CMD} -a ${jconf} ${jconf_tmp}

if [ -z "${delpkglist}" ]; then
	delpkglist=0
else
	delpkglist=1
fi

temprcconf="${ftmpdir}/jcreate_jconf.$$"

# TRIM DOS CRLF
${CAT_CMD} ${jconf} | ${TR_CMD} -d \\r > ${temprcconf}

# read jname
. ${temprcconf}

[ -z "${jname}" ] && err 1 "${N1_COLOR}${CIX_APP}: please set: ${N2_COLOR}jname=${N0_COLOR}"

# map flavor
[ -n "${oflavor}" ] && flavor="${oflavor}"
if [ -n "${flavor}" ]; then
	unset oimgsize imgsize ovm_ram vm_ram ovm_cpus vm_cpus
	capture flavors_available cbsdsqlro ${dbdir}/local.sqlite "SELECT name FROM vmpackages ORDER BY name"
	flavor_exist=0
	for i in ${flavors_available}; do
		[ "${flavor}" = "${i}" ] && flavor_exist=1 && break
	done

	[ ${flavor_exist} -eq 0 ] && log_err 1 "${N1_COLOR}${CIX_APP}: no such flavor [${flavor}], available flavors: ${N2_COLOR}${flavors_available}${N0_COLOR}"

	capture _res cbsdsqlro ${dbdir}/local.sqlite "SELECT pkg_vm_cpus,pkg_vm_ram,pkg_vm_disk FROM vmpackages WHERE name='${flavor}' LIMIT 1"

	[ -z "${_res}" ] && log_err 1 "${N1_COLOR}${CIX_APP}: unable to get vm_cpus/vm_ram/imgsize for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	sqllistdelimer="|"
	sqllist "${_res}" ovm_cpus ovm_ram oimgsize
	unset sqllistdelimer
	${ECHO} "${N1_COLOR}${CIX_APP}: flavor ${flavor} -> vm_cpus/vm_ram/imgsize: ${N2_COLOR}[${_res}]${N0_COLOR}"

	[ -z "${ovm_ram}" ] && log_err 1 "${N1_COLOR}${CIX_APP}: unable to get 'vm_ram' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	[ -z "${oimgsize}" ] && log_err 1 "${N1_COLOR}${CIX_APP}: unable to get 'imgsize' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"
	[ -z "${ovm_cpus}" ] && log_err 1 "${N1_COLOR}${CIX_APP}: unable to get 'vm_cpus' for flavor: ${N2_COLOR}${flavor}${N0_COLOR}"

	vm_ram="${ovm_ram}"
	imgsize="${oimgsize}"
	vm_cpus="${ovm_cpus}"

	unset flavor oflavor flavor_exist flavors_available _res
fi

# if some of params specified via args, restore them from temporary vars
SYSRC_TEMPRCCONF=
for i in ${QEMU_ARGS}; do
	eval _mytest=\$o$i
	# Adjust some missed optional args
	if [ -z "${_mytest}" ]; then
		case "${i}" in
			emulator)
				_mytest="qemu"
				;;
			host_hostname)
				_mytest="${jname}.my.domain"
				;;
			path)
				_mytest="${jaildir}/${jname}"
				;;
			data)
				_mytest="${jaildatadir}/${jname}-${jaildatapref}"
				;;
			rcconf)
				_mytest="${jailrcconfdir}/rc.conf_${jname}"
				;;
			vm_os_type)
				capture _mytest ${TR_CMD} '[:upper:]' '[:lower:]' <<< "${_mytest}"
				;;
			*)
				# skip unknown args
				continue
				;;
		esac
	fi
	SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ${i}=${_mytest}"
done

[ -n "${SYSRC_TEMPRCCONF}" ] && updateconf ${temprcconf} ${SYSRC_TEMPRCCONF}
SYSRC_TEMPRCCONF=

if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_bhyve_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_bhyve_queue_name} id=${jname} cmd=qcreate vm_ram=${vm_ram} vm_cpus=${vm_cpus} vm_os_type=${vm_os_type} astart=${astart} protected=${protected} emulator=${emulator} vnc_port='0' status=1
fi

jstatus jname=${jname} > /dev/null 2>&1
if [ $? -ne 0 ]; then
	cbsd_queue_remove
	log_err 1 "${N1_COLOR}VM already exist: ${N2_COLOR}${jname}${N0_COLOR}"
fi

capture vm_os_type ${TR_CMD} '[:upper:]' '[:lower:]' <<< "${vm_os_type}"
updateconf ${temprcconf} vm_os_type="${vm_os_type}"

# profile
if [ -z "${vm_os_profile}" ]; then
	cbsd_queue_remove
	log_err 1 "${N1_COLOR}No such vm_os_profile=${N0_COLOR}"
fi
if [ -z "${vm_os_type}" ]; then
	cbsd_queue_remove
	log_err 1 "${N1_COLOR}No such vm_os_type=${N0_COLOR}"
fi

template_profile=
template_profile="vm-${vm_os_type}-${vm_os_profile}.conf"

[ -n "${oextra_profile_dir}" ] && extra_profile_dir="${oextra_profile_dir}"

users_template_profile="${etcdir}/${template_profile}"

if [ ! -r ${etcdir}/defaults/${template_profile} -a ! -r ${etcdir}/${template_profile} ]; then
	[ -z "${extra_profile_dir}" ] && err 1 "${N1_COLOR}${CIX_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
	if [ -d "${extra_profile_dir}" ]; then
		if [ -r ${extra_profile_dir}/${template_profile} ]; then
			users_template_profile="${etcdir}/${template_profile} ${extra_profile_dir}/${template_profile}"
		else
			err 1 "${N1_COLOR}${CIX_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
		fi
	else
		${ECHO} "${W1_COLOR}${CIX_APP} warning: ${N1_COLOR}no such extra_profile_dir: ${N2_COLOR}${extra_profile_dir}${N0_COLOR}" >&2
		err 1 "${N1_COLOR}${CIX_APP} error: no such profile: ${N2_COLOR}${template_profile}${N0_COLOR}"
	fi
fi

[ -z "${mnt_start}" ] && mnt_start="0"

[ -n "${ointer}" ] && inter="${ointer}"
[ "${inter}" = "0" ] && export NOINTER=1

if [ "${mnt_start}" != "0" ]; then
	if [ ! -r "${mnt_start}" -o ! -x "${mnt_start}" ]; then
		err 1 "mnt_start script not exist or not executable: ${mnt_start}"
	fi
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Execute mnt_start script: ${N2_COLOR}${mnt_start}${N0_COLOR}..."
	# external mount, reset zfsfeat
	zfsfeat=0
	[ ! -d "${data}" ] && ${MKDIR_CMD} -m 0770 ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${mnt_start} -d ${data} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}
	_ret=$?
	if [ ${_ret} -ne 0 ]; then
		err 1 "${W1_COLOR}error: ${N1_COLOR}mnt_start script failed: ${N2_COLOR}${mnt_start} -d ${data} -f ${jailfstabdir} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}${N0_COLOR}"
	fi
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${data} ${jailsysdir}/${jname}

	create_fs ${data}
	[ $? -ne 0 ] && err 1 "${N1_COLOR}create_fs failed${N0_COLOR}"
	vm_zfs_guid="0"
fi

if [ "${zfsfeat}" = "1" -a "${mnt_start}" = "0" ]; then
	readconf zfs.conf
	. ${subrdir}/zfs.subr
	capture DATA ${ZFS_CMD} get -Ho value name ${jaildatadir}
	_msg=$( ${ZFS_CMD} create -o mountpoint=${workdir}/vm/${jname} ${DATA}/${jname} )
	_res=$?
	if [ ${_res} -ne 0 ]; then
		echo "${_msg}"
		exit ${_res}
	fi

	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${workdir}/vm/${jname}
	${CHMOD_CMD} 0770 ${workdir}/vm/${jname}

	capture vm_zfs_guid ${ZFS_CMD} get -Ho value guid ${DATA}/${jname}
	[ -z "${vm_zfs_guid}" ] && vm_zfs_guid="0"

	[ -z "${data}" ] && data="${jaildatadir}/${jname}-${jaildatapref}"
	olddata="${data}"
	data="${workdir}/vm/${jname}"
	SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} data=\"${data}\""
	${LN_CMD} -sf ${data} ${olddata}
	${LN_CMD} -sf ${data} ${jailsysdir}/${jname}
else
	# no mnt_start and no ZFS
	[ ! -d "${data}" ] && ${MKDIR_CMD} -m 0770 ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${data} ${jailsysdir}/${jname}
	vm_zfs_guid="0"
fi

. ${subrdir}/build.subr
. ${temprcconf}

case "${platform}" in
	Linux)
		capture conf_owner ${STAT_CMD} -c "%u" ${jconf_tmp}
		capture conf_group ${STAT_CMD} -c "%g" ${jconf_tmp}
		;;
	*)
		capture conf_owner ${STAT_CMD} -f "%u" ${jconf_tmp}
		capture conf_group ${STAT_CMD} -f "%g" ${jconf_tmp}
		;;
esac

eval $( ${miscdir}/merge -o ${jconf_tmp} ${etcdir}/defaults/qemu-default-default.conf ${etcdir}/defaults/${template_profile} ${etcdir}/qemu-default-default.conf ${etcdir}/${template_profile} ${temprcconf} 2>/dev/null )

# minimal config allows not to specify rcconf variable
# e.g: minimal config
# jname="freebsd1"
# imgsize="10g";
# vm_os_profile="FreeBSD-x64-12.0";
# vm_profile="FreeBSD-x64-12.0"
[ -z "${rcconf}" ] && rcconf="${jailrcconfdir}/rc.conf_${jname}"

[ -n "${oremovejconf}" ] && removejconf="${oremovejconf}"

if [ "${removejconf}" = "1" ]; then
	trap "${RM_CMD} -f ${temprcconf} ${jconf} ${jconf_tmp}" HUP INT ABRT BUS TERM  EXIT
else
	trap "${RM_CMD} -f ${temprcconf} ${jconf_tmp}" HUP INT ABRT BUS TERM  EXIT
fi

[ -z "${jname}" ] && log_err 1 "${N1_COLOR}No such jname variable${N0_COLOR}"

if [ "${vm_os_type}" = "freebsd" -a "${from_jail}" = "1" ]; then
	# change emulator type for jcreate
	updateconf ${jconf_tmp} emulator="jail"

	if [ -n "${jprofile}" ]; then
		. ${subrdir}/settings-tui.subr
		if [ -r "${etcdir}/jail-freebsd-${jprofile}.conf" ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Use profile: ${N2_COLOR}${etcdir}/jail-freebsd-${jprofile}.conf${N0_COLOR}"
			eval $( ${miscdir}/merge ${etcdir}/jail-freebsd-${jprofile}.conf ${jconf_tmp} 2>/dev/null )
		elif [ -r "${etcdir}/defaults/jail-freebsd-${jprofile}.conf" ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Use profile: ${N2_COLOR}${etcdir}/defaults/jail-freebsd-${jprofile}.conf${N0_COLOR}"
			eval $( ${miscdir}/merge ${etcdir}/defaults/jail-freebsd-${jprofile}.conf ${jconf_tmp} 2>/dev/null )
		fi
	fi

	updateconf ${jconf_tmp} emulator="qemu"
fi

# adjust some missed variabled
readconf vnc.conf
readconf spice.conf
[ -z "${qemu_vnc_resolution}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} qemu_vnc_resolution=${default_vnc_width}x${default_vnc_height}"
[ -z "${qemu_vnc_tcp_bind}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} qemu_vnc_tcp_bind=${default_vnc_tcp_bind}"
[ -z "${qemu_spice_resolution}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} qemu_spice_resolution=${default_spicy_width}x${default_spicy_height}"
[ -z "${qemu_spice_tcp_bind}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} qemu_spice_tcp_bind=${default_spicy_tcp_bind}"

if [ -z "${cd_vnc_wait}" ]; then
	case "${default_vnc_wait}" in
		auto|1)
			SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} cd_vnc_wait=1"
			;;
		*)
			SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} cd_vnc_wait=0"
			;;
	esac
fi

[ -z "${vnc_password}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} vnc_password=${default_vnc_password}"
[ -z "${spicy_password}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} spicy_password=${default_spicy_password}"

updateconf ${jconf_tmp} ${SYSRC_TEMPRCCONF}
SYSRC_TEMPRCCONF=

# re-read jail params and apply personal after profile
. ${jconf_tmp}

# extra check
# args deps:
#  - interface2 required for ci_interface2
#  - ci_interface2 required for ci_ip4_addr2
# when ci_interface2 not empty - inherit interface2 settings
[ -n "${oci_interface2}" -a "${oci_interface2}" != "0" ] && ci_interface2="${oci_interface2}"
[ -n "${ointerface2}" -a "${ointerface2}" != "0" ] && interface2="${ointerface2}"
[ -n "${ci_ip4_addr2}" -a -z "${ci_interface2}" ] && log_err 1 "${N1_COLOR}${CIX_APP}: ci_ip4_addr2 set, but ci_interface2 is empty${N0_COLOR}"
[ -z "${interface2}" -a -n "${ci_interface2}" ] && interface2="${ci_interface2}"

# apply pkglist from tpl_pkglist
[ -n "${tpl_pkglist}" ] && updateconf ${jconf_tmp} pkglist="${tpl_pkglist}"

if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_qemu_queue_name}" ] && ${cbsd_queue_backend} cbsd_qemu_queue_name=${cbsd_qemu_queue_name} id=${jname} cmd=qcreate vm_ram=${vm_ram} vm_cpus=${vm_cpus} vm_os_type=${vm_os_type} astart=${astart} protected=${protected} vnc_port='0' status=1
fi

# imgsize validate
if [ -n "${imgsize_min}" -a -n "${imgsize}" ]; then

	if is_number "${imgsize_min}"; then
		imgsize_min_bytes="${imgsize_min}"
	else
		capture imgsize_min_bytes ${miscdir}/conv2human "${imgsize_min}"
	fi

	[ -z "${imgsize_min_bytes}" -o "${imgsize_min_bytes}" = "0" ] && err 1 "${N1_COLOR}${CIX_APP}: unable to convert imgsize to bytes: ${N2_COLOR}${imgsize_min}${N0_COLOR}"

	imgsize_min_human_mb=$(( imgsize_min_bytes / 1024 / 1024 ))

	if is_number "${imgsize}"; then
		imgsize_bytes="${imgsize}"
	else
		capture imgsize_bytes ${miscdir}/conv2human "${imgsize}"
	fi

	imgsize_human_mb=$(( imgsize_bytes / 1024 / 1024 ))

	if [ ${imgsize_bytes} -lt ${imgsize_min_bytes} ]; then
		# not fatal?
#		if [ "${zfsfeat}" = "1" ]; then
#			${ZFS_CMD} destroy ${DATA}/${jname}
#		fi
#		err 1 "${N1_COLOR}imgsize too small: ${N2_COLOR}${imgsize_bytes} ( ${imgsize_human_mb}mb ) < ${imgsize_min_bytes} ( ${imgsize_min_human_mb}mb )${N0_COLOR}"
		${ECHO} "${W1_COLOR}imgsize too small: ${N2_COLOR}${imgsize_bytes} ( ${imgsize_human_mb}mb ) < ${imgsize_min_bytes} ( ${imgsize_min_human_mb}mb )${N0_COLOR}"
	fi
fi

# vm_ram validate
if [ -n "${vm_ram_min}" -a -n "${vm_ram}" ]; then

	if is_number "${vm_ram_min}"; then
		vm_ram_min_bytes="${vm_ram_min}"
	else
		capture vm_ram_min_bytes ${miscdir}/conv2human "${vm_ram_min}"
	fi

	vm_ram_min_human_mb=$(( vm_ram_min_bytes / 1024 / 1024 ))

	if is_number "${vm_ram}"; then
		vm_ram_bytes="${vm_ram}"
	else
		capture vm_ram_bytes ${miscdir}/conv2human "${vm_ram}"
	fi

	vm_ram_human_mb=$(( vm_ram_bytes / 1024 / 1024 ))

	if [ ${vm_ram_bytes} -lt ${vm_ram_min_bytes} ]; then
		if [ "${zfsfeat}" = "1" ]; then
			${ZFS_CMD} destroy ${DATA}/${jname}
		fi
		err 1 "${N1_COLOR}vm_ram too small: ${N2_COLOR}${vm_ram_bytes} ( ${vm_ram_human_mb}mb ) < ${vm_ram_min_bytes} ( ${vm_ram_min_human_mb}mb )${N0_COLOR}"
	fi
fi

defdsk="dsk1.vhd"
defnic="nic1.vhd"

# cached sqlite3
if [ ! -r "${tmpdir}/qemu-local.sqlite" ]; then
	qsqlite-cache-create file=${tmpdir}/qemu-local.sqlite
	_ret=$?
	[ ${_ret} -ne 0 ] && err 1 "${W1_COLOR}${CIX_APP} error: ${N2_COLOR}qsqlite-cache-create file=${tmpdir}/qemu-local.sqlite${N0_COLOR}"
fi
${CP_CMD} -a ${tmpdir}/qemu-local.sqlite ${jailsysdir}/${jname}/local.sqlite

if [ -z "${zfs_snapsrc}" ]; then
	really_create_base
	really_create_vm
	${CP_CMD} ${jconf_tmp} ${rcconf}
else
	if [ ! -d ${data} ]; then
		create_fs ${data}
		[ $? -ne 0 ] && err 1 "${N1_COLOR}create_fs failed${N0_COLOR}"
	fi
fi

[ ! -d "${data}" ] && log_err 1 "Can't create datadir ${data}"
[ ! -d ${jailfstabdir}  ] && ${MKDIR_CMD} -m 0770 -p ${jailfstabdir}

[ -n "${fstablocal}" -a -f "${fstablocal}" ] && ${CP_CMD} ${fstablocal} ${jailfstabdir}/${jailfstabpref}${jname}.local

${CP_CMD} ${jconf_tmp} ${rcconf}
. ${rcconf}

# Finnaly export to SQLite
jregister jname=${jname} mode=new

_res=$?

if [ ${_res} -ne 0 ]; then
	if [ ${quiet} -ne 1 ]; then
		${ECHO}
		${ECHO} "${N1_COLOR}Creating ${jname} failed: ${N2_COLOR}cbsd jregister${N0_COLOR}"
		${ECHO} "${N1_COLOR}Please review bad config file: ${N2_COLOR}/tmp/rc.conf_${jname}${N0_COLOR}"
	fi
	${MV_CMD} ${rcconf} /tmp
	#cleanup
	[ -f "${mount_fstab}" ] && ${RM_CMD} -f ${mount_fstab}
	remove_data_dir ${data}
	exit 1
fi

# copy BHYVE_UEFI_VARS.fd
if [ -n "${vars_img}" -a -r "${srcdir}/iso/${vars_img}" ]; then
	${CP_CMD} "${srcdir}/iso/${vars_img}" ${jailsysdir}/${jname}/BHYVE_UEFI_VARS.fd
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Custom UEFI VARS file: ${N2_COLOR}${srcdir}/iso/${vars_img}${N0_COLOR}"
elif [ -r ${CIX_DISTDIR}/upgrade/patch/BHYVE_UEFI_VARS.fd ]; then
	${CP_CMD} ${CIX_DISTDIR}/upgrade/patch/BHYVE_UEFI_VARS.fd ${jailsysdir}/${jname}/BHYVE_UEFI_VARS.fd
fi

echo
if [ "${vm_zfs_guid}" != "0" ]; then
	[ -z "${NOINTER}" -o ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Global VM ZFS guid: ${N2_COLOR}${vm_zfs_guid}${N0_COLOR}"
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE settings SET vm_zfs_guid='${vm_zfs_guid}'"
fi

if [ "${uuid}" = "0" -o -z "${uuid}" ]; then
	capture uuid ${UUIDGEN_CMD}
	#[ -z "${NOINTER}" -o ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Global VM UUID: ${N2_COLOR}${uuid}${N0_COLOR}"
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE settings SET uuid='${uuid}'"
	updateconf ${jailsysdir}/${jname}/rc.conf_${jname} uuid="${uuid}"
fi

if [ ${quiet} -ne 1 -o -z "${NOINTER}" ]; then
	${ECHO} "${N1_COLOR}To edit VM properties use: ${N2_COLOR}cbsd qconfig jname=${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To start VM use: ${N2_COLOR}cbsd qstart ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To stop VM use: ${N2_COLOR}cbsd qstop ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}To remove VM use: ${N2_COLOR}cbsd qdestroy ${jname}${N0_COLOR}"
	${ECHO} "${N1_COLOR}For attach VM console use: ${N2_COLOR}cbsd qlogin ${jname}${N0_COLOR}"
	echo
	${ECHO} "${N1_COLOR}Creating ${jname} complete: ${N2_COLOR}Enjoy!${N0_COLOR}"
fi

${RM_CMD} -f ${rcconf}

if [ -z "${sectorsize}" ]; then
	sectorsize="${default_sectorsize}"
	[ -z "${sectorsize}" ] && sectorsize="4096"
fi

if [ ${zfsfeat} -eq 1 -a "${imgtype}" = "zvol" ]; then
	. ${subrdir}/zfs.subr
	capture dsk_zfs_guid get_dsk_zfs_guid -p ${data}/${defdsk}
	_ret=$?
	[ ${_ret} -ne 0 ] && dsk_zfs_guid="0"
fi

[ -z "${dsk_zfs_guid}" ] && dsk_zfs_guid="0"

dsk_bsize=0

if ! is_number "${imgsize}"; then
	capture dsk_bsize ${miscdir}/conv2human ${imgsize}
	[ -z "${dsk_bsize}" ] && dsk_bsize=0
else
	dsk_bsize="${imgsize}"
fi

# qemudsk/qemunic inserts + settings update — batch into SQLITE_LOCAL
SQLITE_LOCAL=

if [ -n "${qemu_virtio_type}" ]; then
	SQLITE_LOCAL="${SQLITE_LOCAL}; INSERT INTO qemudsk ( jname,dsk_controller,dsk_path,dsk_slot,dsk_size,dsk_zfs_guid ) VALUES ( '${jname}','${qemu_virtio_type}','${defdsk}','0','${dsk_bsize}','${dsk_zfs_guid}' )"
else
	SQLITE_LOCAL="${SQLITE_LOCAL}; INSERT INTO qemudsk ( jname,dsk_path,dsk_slot,dsk_size,dsk_zfs_guid ) VALUES ( '${jname}','${defdsk}','0','${dsk_bsize}','${dsk_zfs_guid}' )"
fi

if [ -n "${nic_driver}" ]; then
	SQLITE_LOCAL="${SQLITE_LOCAL}; INSERT INTO qemunic ( jname,nic_driver,nic_parent ) VALUES ( '${jname}', '${nic_driver}', '${interface}' )"
else
	SQLITE_LOCAL="${SQLITE_LOCAL}; INSERT INTO qemunic ( jname,nic_parent ) VALUES ( '${jname}', '${interface}' )"
fi

# update state_time
SQLITE_LOCAL="${SQLITE_LOCAL}; UPDATE settings SET state_time=(strftime('%s','now')); UPDATE settings SET created=(strftime('%s','now'))"

# check bootable + vnc_password
SQLITE_LOCAL="${SQLITE_LOCAL}; UPDATE qemudsk SET bootable='true' WHERE dsk_path='${defdsk}'"
[ "${vnc_password}" = "0" ] && SQLITE_LOCAL="${SQLITE_LOCAL}; UPDATE settings SET vnc_password=''"

cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "${SQLITE_LOCAL}"
SQLITE_LOCAL=

# Check if SIZE is valid: convert to bytes if needed
if ! is_number "${vm_ram}"; then
	# not a number (e.g. "1g"), try to convert to bytes
	if capture _vm_ram_bytes ${miscdir}/conv2human "${vm_ram}"; then
		vm_ram="${_vm_ram_bytes}"
	else
		log_err 1 "${vm_ram} is not a number and we can't convert via conv2human"
	fi
fi

if [ -n "${register_iso_as}" -a -n "${register_iso_name}" ]; then
	cd_name="${register_iso_as}"
	cd_path="${srcdir}/iso/${register_iso_name}"

	cbsdsqlro_vars storage_media "SELECT COUNT(path) FROM media WHERE name='${cd_name}' AND path='${cd_path}' AND type='iso' AND jname='-'" cd_rec_num

	if [ "${cd_rec_num}" = "0" ]; then
		# register new ISO with assignment to this VM
		capture _res media mode=register name="${register_iso_as}" path="${srcdir}/iso/${register_iso_name}" type=iso jname=${jname}
		_ret=$?
		[ ${_ret} -ne 0 ] && err 1 "${N1_COLOR}qcreate: media register error: ${_res}${N0_COLOR}"
	else
		# we have free/unassignent CD. link to this VM
		cbsdsqlrw storage_media "UPDATE media SET jname='${jname}' WHERE jname='-' AND type='iso' AND name='${cd_name}' AND path='${cd_path}'"
	fi
fi

if [ ! -d ${jailsysdir}/${jname} ]; then
	${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${jailsysdir}/${jname}
fi

if [ -d "${jailsysskeldir}" ]; then
	# we have custom skeldir. copy
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Applying custom skel system dir template from: ${N2_COLOR}${jailsysskeldir}${N0_COLOR}"
	${RSYNC_CMD} -a ${jailsysskeldir}/ ${jailsysdir}/${jname}/
fi

[ ! -d ${jailsysdir}/${jname}/etc ] && ${MKDIR_CMD} -m 0770 -p ${jailsysdir}/${jname}/etc

system_dir="create.d \
facts.d \
master_create.d \
master_poststart.d \
master_poststop.d \
master_prestart.d \
master_prestop.d \
master_reboot.d \
remove.d \
start.d \
stop.d"

for i in ${system_dir}; do
	if [ -n "${systemskeldir}" -a "${systemskeldir}/${i}" ]; then
		[ ! -d ${jailsysdir}/${jname}/${i} ] && ${MKDIR_CMD} -m 0775 -p ${jailsysdir}/${jname}/${i}
		${RSYNC_CMD} -az ${systemskeldir}/${i}/ ${jailsysdir}/${jname}/${i}/
	else
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}qcreate: warning: no such dir: ${N2_COLOR}${systemskeldir}/${i}${N0_COLOR}"
			continue
	fi
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${jailsysdir}/${jname}/${i}
done

# is cloud-init-based ?
if [ ${is_cloud} -eq 1 ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}auto-generate cloud-init settings: ${N2_COLOR}${jailsysdir}/${jname}/cloud-init${N0_COLOR}" 1>&2
	# auto adjust some missed settings
	SYSRC_TEMPRCCONF=
	if [ -z "${ci_interface}" ]; then
		ci_interface="${interface}"
		SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_interface=\"${ci_interface}\""
	fi
	[ -z "${ci_jname}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_jname=\"${jname}\""
	[ -z "${ci_fqdn}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_fqdn=\"${host_hostname}\""

	if [ -z "${ci_ip4_addr}" ]; then
		${ECHO} "${W1_COLOR}warning: ${N1_COLOR}cloud-init based profile but ci_ip4_addr not set. Force to: ${N2_COLOR}DHCP${N0_COLOR}" 2>&1
		ci_ip4_addr="DHCP"
	fi
	if [ -z "${ci_gw4}" ]; then
		case "${ci_ip4_addr}" in
			[Rr][Ee][Aa][Ll][Dd][Hh][Cc][Pp])
				true
				;;
			*)
				${ECHO} "${W1_COLOR}warning: ${N1_COLOR}cloud-init based profile but ci_gw4 not set. VM without gateway?${N0_COLOR}" 2>&1
				;;
		esac
	fi

	OIFS="${IFS}"
	IFS=","

	normalize_ip4_addr=
	normalize_ci_ip4_addr=

	# normalize IP addresses
	for _pureip in ${ci_ip4_addr}; do
		IFS="${OIFS}"

		case "${_pureip}" in
			[Dd][Hh][Cc][Pp])
				capture _pureip dhcpd
				if [ $? -eq 2 ]; then
					cbsdlogger NOTICE ${CIX_APP}: no free IP address for DHCP in nodeippool
					err 1 "${N1_COLOR}${CIX_APP}: no free IP address for DHCP in nodeippool${N0_COLOR}"
				fi

				_pureip="${_pureip}/24"		# todo: hardcoded mask
				_mod=1
				;;
			[Dd][Hh][Cc][Pp][vV]6)
				capture _pureip dhcpdv6
				if [ $? -eq 2 ]; then
					cbsdlogger NOTICE ${CIX_APP}: no free IP address for DHCPv6 in nodeip6pool
					err 1 "${N1_COLOR}${CIX_APP}: no free IP address for DHCPv6 in nodeip6pool${N0_COLOR}"
				fi

				_pureip="${_pureip}/64"		# todo: hardcoded mask
				_mod=1
				;;
		esac

		iptype ${_pureip}
		_ret=$?

		if [ -z "${normalize_ip4_addr}" ]; then
			normalize_ip4_addr="${IWM}"
		else
			normalize_ip4_addr="${normalize_ip4_addr},${IWM}"
		fi
		if [ -z "${normalize_ci_ip4_addr}" ]; then
			normalize_ci_ip4_addr="${IWM}/24"
		else
			normalize_ci_ip4_addr="${normalize_ci_ip4_addr},${IWM}/24"
		fi

		IFS=","
	done

	IFS="${OIFS}"

	ci_ip4_addr="${normalize_ci_ip4_addr}"
	[ -n "${ci_gw42}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_gw42=\"${ci_gw42}\""

	if [ -n "${ci_ip4_addr2}" ]; then
		#todo: normalize
		SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_ip4_addr2=\"${ci_ip4_addr2}\""
		normalize_ip4_addr="${normalize_ip4_addr},${ci_ip4_addr2}"
	fi

	SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_ip4_addr=\"${normalize_ci_ip4_addr}\""
	qset jname="${jname}" ip4_addr="${normalize_ip4_addr}" > /dev/null 2>&1

	readconf cloud-init.conf

	[ -z "${ci_nameserver_address}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_nameserver_address=\"${default_ci_nameserver_address}\""
	[ -z "${ci_nameserver_search}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_nameserver_search=\"${default_ci_nameserver_search}\""
	[ -z "${ci_interface_name}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_interface_name=\"${default_ci_interface_name}\""
	[ -z "${ci_user_pw_user}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_user_pw_user=\"${ci_user_pw_user}\""
	[ -z "${ci_user_pw_root}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_user_pw_root=\"${ci_user_pw_root}\""
	[ -z "${ci_user_pubkey}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_user_pubkey=\"${ci_user_pubkey}\""
	[ -z "${ci_interface2}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_interface2=\"${ci_interface2}\""

	# GET MAC
	cbsdsqlro_vars "${jailsysdir}/${jname}/local.sqlite" "SELECT nic_hwaddr FROM qemunic LIMIT 1" nic_hwaddr0

	if [ "${nic_hwaddr0}" = "0" ]; then
		# gen MAC
		capture nic_hwaddr0 mac_gen 00:a0:98
		cbsdlogger NOTICE ${CIX_APP}: qcreate for ${jname}: MAC address randomized and updated for nic id 1: ${nic_hwaddr0}
		cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE qemunic SET nic_hwaddr='${nic_hwaddr0}' WHERE id='1'"
	fi

	[ -z "${ci_nic_hwaddr0}" ] && SYSRC_TEMPRCCONF="${SYSRC_TEMPRCCONF} ci_nic_hwaddr0=\"${nic_hwaddr0}\""

	#${CIX_DISTDIR}/misc/cbsdsysrc -qf ${jailsysdir}/${jname}/rc.conf_${jname} ip4_addr="${ip4_addr}" > /dev/null 2>&1

	updateconf ${jailsysdir}/${jname}/rc.conf_${jname} ${SYSRC_TEMPRCCONF}
	SYSRC_TEMPRCCONF=

	case "${vm_os_type}" in
		windows)
			cloudengine="cloudinit-base"
			;;
		*)
			cloudengine="cloud-init"
			;;
	esac

	cbsdlogger NOTICE ${CIX_APP}: cloudinit mode=gen fromfile=${jailsysdir}/${jname}/rc.conf_${jname} jname=${jname} cloudengine="${cloudengine}"
	cloudinit mode=gen fromfile=${jailsysdir}/${jname}/rc.conf_${jname} jname=${jname} cloudengine="${cloudengine}"
fi

# mark the first disk bootable + vnc_password
SQLITE_LOCAL="UPDATE qemudsk SET bootable='true' WHERE dsk_path='${defdsk}'"
[ "${vnc_password}" = "0" ] && SQLITE_LOCAL="${SQLITE_LOCAL}; UPDATE settings SET vnc_password=''"
cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "${SQLITE_LOCAL}"
SQLITE_LOCAL=

# set default nice from rctl_nice
[ -n "${rctl_nice}" ] && nice="${rctl_nice}"

# rctl
. ${distsharedir}/rctl.conf
for i in ${RCTL} ${RCTL_EXTRA}; do
	_val=
	eval _val="\$rctl_${i}"
	if [ -n "${_val}" ]; then
		jrctl jname=${jname} mode=set ${i}=${_val} > /dev/null 2>&1 || /usr/bin/true
	fi
done

# store profile in jail system dir
storeconf vm-${vm_os_type}-${vm_os_profile}.conf ${jailsysdir}/${jname}/etc/vm-${vm_os_type}-${vm_os_profile}.conf

# create ascii rc.conf file for overwriting values
${CAT_CMD} > ${jailsysdir}/${jname}/etc/rc.conf <<EOF
# The parameters in this file can overwrite the settings from SQLite3 or global params
# e.g:
# qemu_flags="-p 1:1 -p 2:2"
EOF

geniplist		${ip4_addr}		# for ipvX_first_public-like vars
. ${subrdir}/jcreate.subr
export_qemu_data_for_external_hook
external_exec_master_script "master_create.d"

# API compat: ~cbsd/jails-system/info*.*
${miscdir}/daemonize ${toolsdir}/save-jail-info jname=${jname}

jcleanup jname=${jname}

if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_bhyve_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_bhyve_queue_name} id="${jname}" cmd=qcreate status=2 data_status=0 emulator=${emulator}
fi

# expose
if [ -n "${app_frontend}" ]; then
	case "${app_frontend}" in
		mybqt)
			${TOUCH_CMD} ${jailsysdir}/${jname}/mybqt
			;;
		*)
			true
			;;
	esac
fi

gettime end_time
cbsdlogger NOTICE ${CIX_APP}: vm ${jname} has been created in $(( end_time - st_time ))s

[ "${runasap}" = "1" ] && qstart inter=0 quiet=${quiet} jname=${jname}

exit 0
