ark1668ed-bsp/linux/arch/loongarch/kvm/switch.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
 */

#include <linux/linkage.h>
#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/loongarch.h>
#include <asm/regdef.h>
#include <asm/unwind_hints.h>

#define HGPR_OFFSET(x)		(PT_R0 + 8*x)
#define GGPR_OFFSET(x)		(KVM_ARCH_GGPR + 8*x)

.macro kvm_save_host_gpr base
	.irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
	st.d	$r\n, \base, HGPR_OFFSET(\n)
	.endr
.endm

.macro kvm_restore_host_gpr base
	.irp n,1,2,3,22,23,24,25,26,27,28,29,30,31
	ld.d	$r\n, \base, HGPR_OFFSET(\n)
	.endr
.endm

/*
 * Save and restore all GPRs except base register,
 * and default value of base register is a2.
 */
.macro kvm_save_guest_gprs base
	.irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
	st.d	$r\n, \base, GGPR_OFFSET(\n)
	.endr
.endm

.macro kvm_restore_guest_gprs base
	.irp n,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
	ld.d	$r\n, \base, GGPR_OFFSET(\n)
	.endr
.endm

/*
 * Prepare switch to guest, save host regs and restore guest regs.
 * a2: kvm_vcpu_arch, don't touch it until 'ertn'
 * t0, t1: temp register
 */
.macro kvm_switch_to_guest
	/* Set host ECFG.VS=0, all exceptions share one exception entry */
	csrrd		t0, LOONGARCH_CSR_ECFG
	bstrins.w	t0, zero, CSR_ECFG_VS_SHIFT_END, CSR_ECFG_VS_SHIFT
	csrwr		t0, LOONGARCH_CSR_ECFG

	/* Load up the new EENTRY */
	ld.d	t0, a2, KVM_ARCH_GEENTRY
	csrwr	t0, LOONGARCH_CSR_EENTRY

	/* Set Guest ERA */
	ld.d	t0, a2, KVM_ARCH_GPC
	csrwr	t0, LOONGARCH_CSR_ERA

	/* Save host PGDL */
	csrrd	t0, LOONGARCH_CSR_PGDL
	st.d	t0, a2, KVM_ARCH_HPGD

	/* Switch to kvm */
	ld.d	t1, a2, KVM_VCPU_KVM - KVM_VCPU_ARCH

	/* Load guest PGDL */
	li.w    t0, KVM_GPGD
	ldx.d   t0, t1, t0
	csrwr	t0, LOONGARCH_CSR_PGDL

	/* Mix GID and RID */
	csrrd		t1, LOONGARCH_CSR_GSTAT
	bstrpick.w	t1, t1, CSR_GSTAT_GID_SHIFT_END, CSR_GSTAT_GID_SHIFT
	csrrd		t0, LOONGARCH_CSR_GTLBC
	bstrins.w	t0, t1, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT
	csrwr		t0, LOONGARCH_CSR_GTLBC

	/*
	 * Enable intr in root mode with future ertn so that host interrupt
	 * can be responsed during VM runs
	 * Guest CRMD comes from separate GCSR_CRMD register
	 */
	ori	t0, zero, CSR_PRMD_PIE
	csrxchg	t0, t0,   LOONGARCH_CSR_PRMD

	/* Set PVM bit to setup ertn to guest context */
	ori	t0, zero, CSR_GSTAT_PVM
	csrxchg	t0, t0,   LOONGARCH_CSR_GSTAT

	/* Load Guest GPRs */
	kvm_restore_guest_gprs a2
	/* Load KVM_ARCH register */
	ld.d	a2, a2,	(KVM_ARCH_GGPR + 8 * REG_A2)

	ertn /* Switch to guest: GSTAT.PGM = 1, ERRCTL.ISERR = 0, TLBRPRMD.ISTLBR = 0 */
.endm

	/*
	 * Exception entry for general exception from guest mode
	 *  - IRQ is disabled
	 *  - kernel privilege in root mode
	 *  - page mode keep unchanged from previous PRMD in root mode
	 *  - Fixme: tlb exception cannot happen since registers relative with TLB
	 *  -        is still in guest mode, such as pgd table/vmid registers etc,
	 *  -        will fix with hw page walk enabled in future
	 * load kvm_vcpu from reserved CSR KVM_VCPU_KS, and save a2 to KVM_TEMP_KS
	 */
	.text
	.cfi_sections	.debug_frame
SYM_CODE_START(kvm_exc_entry)
	UNWIND_HINT_UNDEFINED
	csrwr	a2,   KVM_TEMP_KS
	csrrd	a2,   KVM_VCPU_KS
	addi.d	a2,   a2, KVM_VCPU_ARCH

	/* After save GPRs, free to use any GPR */
	kvm_save_guest_gprs a2
	/* Save guest A2 */
	csrrd	t0,	KVM_TEMP_KS
	st.d	t0,	a2,	(KVM_ARCH_GGPR + 8 * REG_A2)

	/* A2 is kvm_vcpu_arch, A1 is free to use */
	csrrd	s1,   KVM_VCPU_KS
	ld.d	s0,   s1, KVM_VCPU_RUN

	csrrd	t0,   LOONGARCH_CSR_ESTAT
	st.d	t0,   a2, KVM_ARCH_HESTAT
	csrrd	t0,   LOONGARCH_CSR_ERA
	st.d	t0,   a2, KVM_ARCH_GPC
	csrrd	t0,   LOONGARCH_CSR_BADV
	st.d	t0,   a2, KVM_ARCH_HBADV
	csrrd	t0,   LOONGARCH_CSR_BADI
	st.d	t0,   a2, KVM_ARCH_HBADI

	/* Restore host ECFG.VS */
	csrrd	t0, LOONGARCH_CSR_ECFG
	ld.d	t1, a2, KVM_ARCH_HECFG
	or	t0, t0, t1
	csrwr	t0, LOONGARCH_CSR_ECFG

	/* Restore host EENTRY */
	ld.d	t0, a2, KVM_ARCH_HEENTRY
	csrwr	t0, LOONGARCH_CSR_EENTRY

	/* Restore host pgd table */
	ld.d    t0, a2, KVM_ARCH_HPGD
	csrwr   t0, LOONGARCH_CSR_PGDL

	/*
	 * Disable PGM bit to enter root mode by default with next ertn
	 */
	ori	t0, zero, CSR_GSTAT_PVM
	csrxchg	zero, t0, LOONGARCH_CSR_GSTAT

	/*
	 * Clear GTLBC.TGID field
	 *       0: for root  tlb update in future tlb instr
	 *  others: for guest tlb update like gpa to hpa in future tlb instr
	 */
	csrrd	t0, LOONGARCH_CSR_GTLBC
	bstrins.w	t0, zero, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT
	csrwr	t0, LOONGARCH_CSR_GTLBC
	ld.d	tp, a2, KVM_ARCH_HTP
	ld.d	sp, a2, KVM_ARCH_HSP
	/* restore per cpu register */
	ld.d	u0, a2, KVM_ARCH_HPERCPU
	addi.d	sp, sp, -PT_SIZE

	/* Prepare handle exception */
	or	a0, s0, zero
	or	a1, s1, zero
	ld.d	t8, a2, KVM_ARCH_HANDLE_EXIT
	jirl	ra, t8, 0

	or	a2, s1, zero
	addi.d	a2, a2, KVM_VCPU_ARCH

	/* Resume host when ret <= 0 */
	blez	a0, ret_to_host

	/*
         * Return to guest
         * Save per cpu register again, maybe switched to another cpu
         */
	st.d	u0, a2, KVM_ARCH_HPERCPU

	/* Save kvm_vcpu to kscratch */
	csrwr	s1, KVM_VCPU_KS
	kvm_switch_to_guest

ret_to_host:
	ld.d    a2, a2, KVM_ARCH_HSP
	addi.d  a2, a2, -PT_SIZE
	kvm_restore_host_gpr    a2
	jr      ra

SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL)
SYM_CODE_END(kvm_exc_entry)

/*
 * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu)
 *
 * @register_param:
 *  a0: kvm_run* run
 *  a1: kvm_vcpu* vcpu
 */
SYM_FUNC_START(kvm_enter_guest)
	/* Allocate space in stack bottom */
	addi.d	a2, sp, -PT_SIZE
	/* Save host GPRs */
	kvm_save_host_gpr a2

	addi.d	a2, a1, KVM_VCPU_ARCH
	st.d	sp, a2, KVM_ARCH_HSP
	st.d	tp, a2, KVM_ARCH_HTP
	/* Save per cpu register */
	st.d	u0, a2, KVM_ARCH_HPERCPU

	/* Save kvm_vcpu to kscratch */
	csrwr	a1, KVM_VCPU_KS
	kvm_switch_to_guest
SYM_INNER_LABEL(kvm_enter_guest_end, SYM_L_LOCAL)
SYM_FUNC_END(kvm_enter_guest)

SYM_FUNC_START(kvm_save_fpu)
	fpu_save_csr	a0 t1
	fpu_save_double a0 t1
	fpu_save_cc	a0 t1 t2
	jr              ra
SYM_FUNC_END(kvm_save_fpu)

SYM_FUNC_START(kvm_restore_fpu)
	fpu_restore_double a0 t1
	fpu_restore_csr    a0 t1 t2
	fpu_restore_cc	   a0 t1 t2
	jr                 ra
SYM_FUNC_END(kvm_restore_fpu)

#ifdef CONFIG_CPU_HAS_LSX
SYM_FUNC_START(kvm_save_lsx)
	fpu_save_csr    a0 t1
	fpu_save_cc     a0 t1 t2
	lsx_save_data   a0 t1
	jr              ra
SYM_FUNC_END(kvm_save_lsx)

SYM_FUNC_START(kvm_restore_lsx)
	lsx_restore_data a0 t1
	fpu_restore_cc   a0 t1 t2
	fpu_restore_csr  a0 t1 t2
	jr               ra
SYM_FUNC_END(kvm_restore_lsx)
#endif

#ifdef CONFIG_CPU_HAS_LASX
SYM_FUNC_START(kvm_save_lasx)
	fpu_save_csr    a0 t1
	fpu_save_cc     a0 t1 t2
	lasx_save_data  a0 t1
	jr              ra
SYM_FUNC_END(kvm_save_lasx)

SYM_FUNC_START(kvm_restore_lasx)
	lasx_restore_data a0 t1
	fpu_restore_cc    a0 t1 t2
	fpu_restore_csr   a0 t1 t2
	jr                ra
SYM_FUNC_END(kvm_restore_lasx)
#endif
	.section ".rodata"
SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)

#ifdef CONFIG_CPU_HAS_LBT
STACK_FRAME_NON_STANDARD kvm_restore_fpu
#ifdef CONFIG_CPU_HAS_LSX
STACK_FRAME_NON_STANDARD kvm_restore_lsx
#endif
#ifdef CONFIG_CPU_HAS_LASX
STACK_FRAME_NON_STANDARD kvm_restore_lasx
#endif
#endif