diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index ae172fcba725a9efd0154eaf3425870e89cfe221..86615df90ae9a59254b4a7519be8b88d7f03b0a1 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -840,6 +840,12 @@ struct kvm_vcpu_arch { } pvsched; #endif + /* Guest PV idle time state */ + struct { + u64 base; + struct gfn_to_hva_cache cache; + } pv_idle_time; + /* Per-vcpu CCSIDR override or NULL */ u32 *ccsidr; @@ -1356,6 +1362,14 @@ static inline bool kvm_arm_is_pvsched_enabled(struct kvm_vcpu_arch *vcpu_arch) } #endif +static inline void kvm_arm_pv_idle_time_vcpu_init(struct kvm_vcpu_arch *vcpu_arch) +{ + vcpu_arch->pv_idle_time.base = 0; +} + +long kvm_smccc_register_pv_idle_time(struct kvm_vcpu *vcpu); +bool kvm_arch_is_vcpu_pv_idle(struct kvm_vcpu *vcpu); + void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 syndrome); struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr); diff --git a/arch/arm64/include/asm/paravirt.h b/arch/arm64/include/asm/paravirt.h index 1a5c3f1624ec667b1b629521d1cc15edcd8282c8..183440af2cb2bd4caaab6cb85de042bd47f2fb46 100644 --- a/arch/arm64/include/asm/paravirt.h +++ b/arch/arm64/include/asm/paravirt.h @@ -71,10 +71,20 @@ static inline void pv_kick(int cpu) #endif /* CONFIG_PARAVIRT_SCHED */ +int __init pv_idle_time_init(void); +int pv_idle_time_cpu_online(unsigned int cpu); +void pv_idle_time_enter(void); +void pv_idle_time_exit(void); + #else #define pv_time_init() do {} while (0) #define pv_sched_init() do {} while (0) +#define pv_idle_time_init() do {} while (0) +#define pv_idle_time_cpu_online(cpu) do {} while (0) + +static inline void pv_idle_time_enter(void) {} +static inline void pv_idle_time_exit(void) {} #endif // CONFIG_PARAVIRT diff --git a/arch/arm64/include/asm/pv_idle_time-abi.h b/arch/arm64/include/asm/pv_idle_time-abi.h new file mode 100644 index 0000000000000000000000000000000000000000..4aaf3c927bcc07767e94d344fa750d5958acf74a --- /dev/null +++ b/arch/arm64/include/asm/pv_idle_time-abi.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Paravirtualised vCPU idle hint shared structure ABI. + * + * The guest publishes the idle state of a vCPU into a per-vCPU shared + * page so that the host KVM can know whether the vCPU is currently idle + * without forcing a VM exit on every idle entry/exit. + */ + +#ifndef __ASM_PV_IDLE_TIME_ABI_H +#define __ASM_PV_IDLE_TIME_ABI_H + +#include + +/* Values for the flag field. */ +#define KVM_PV_VCPU_RUNNING 0 +#define KVM_PV_VCPU_IDLE (1U << 0) + +/* + * When the guest enters its idle loop it sets ``flag`` to KVM_PV_VCPU_IDLE + * and clears it back to KVM_PV_VCPU_RUNNING on exit. The host can read + * ``flag`` at any time to cheaply decide whether the vCPU is currently idle. + * + * ``idle_accum`` is accumulated idle time in nanoseconds of this vcpu. + */ +struct kvm_idle_time { + __le64 flag; + __le64 idle_accum; + /* Structure must be 64 byte aligned, pad to that size. */ + __u32 pad[12]; +} __packed; + +#endif /* __ASM_PV_IDLE_TIME_ABI_H */ diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index cbc35121febeccc18b672b793c1e7226773dc602..b88e14837fd512822ba640e64409d5609f100ecd 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -383,6 +383,7 @@ enum { * current sequence, add in sequence. */ KVM_REG_ARM_VENDOR_HYP_BIT_IPIV = 4, + KVM_REG_ARM_VENDOR_HYP_BIT_PV_IDLE_TIME = 5, #ifdef __KERNEL__ KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_COUNT, #endif diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c index c17e9d0393ac3e4ca8163d4df9ee0625b20e737a..dee827a48f3c2d11525827d370c443f046337639 100644 --- a/arch/arm64/kernel/paravirt.c +++ b/arch/arm64/kernel/paravirt.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -384,3 +385,119 @@ int __init pv_sched_init(void) return 0; } #endif /* CONFIG_PARAVIRT_SCHED */ + +/* + * Guest PV vCPU idle time + * + * Each vCPU registers a per-cpu shared page with the host via SMCCC. + * On idle entry/exit the guest writes the idle flag into that page. + */ + +DEFINE_PER_CPU(struct kvm_idle_time, kvm_idle_time_region) __aligned(64); + +DEFINE_STATIC_KEY_FALSE(kvm_pv_idle_time_enabled); + +struct kvm_idle { + ktime_t idle_start; +}; +static DEFINE_PER_CPU(struct kvm_idle, kvm_idle); + +void pv_idle_time_enter(void) +{ + struct kvm_idle_time *state = this_cpu_ptr(&kvm_idle_time_region); + struct kvm_idle *idle = this_cpu_ptr(&kvm_idle); + + if (!static_branch_unlikely(&kvm_pv_idle_time_enabled)) + return; + + WRITE_ONCE(state->flag, cpu_to_le64(KVM_PV_VCPU_IDLE)); + + idle->idle_start = ktime_get(); +} + +void pv_idle_time_exit(void) +{ + struct kvm_idle_time *state = this_cpu_ptr(&kvm_idle_time_region); + struct kvm_idle *idle = this_cpu_ptr(&kvm_idle); + s64 idle_time; + u64 idle_accum; + + if (!static_branch_unlikely(&kvm_pv_idle_time_enabled)) + return; + + WRITE_ONCE(state->flag, cpu_to_le64(KVM_PV_VCPU_RUNNING)); + + idle_time = ktime_to_ns(ktime_sub(ktime_get(), idle->idle_start)); + idle_accum = le64_to_cpu(state->idle_accum) + idle_time; + WRITE_ONCE(state->idle_accum, cpu_to_le64(idle_accum)); +} + +static int pvidle_cpu_down_prepare(unsigned int cpu) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(ARM_SMCCC_HV_REGISTER_PV_IDLE_TIME, + 0, &res); + if (res.a0 != SMCCC_RET_SUCCESS) + pr_warn("%s: Failed to unregister pv idle time by SMCCC.\n", __func__); + + return 0; +} + +int pv_idle_time_cpu_online(unsigned int cpu) +{ + struct kvm_idle_time *state = this_cpu_ptr(&kvm_idle_time_region); + struct arm_smccc_res res; + + if (!static_branch_unlikely(&kvm_pv_idle_time_enabled)) + return 0; + + /* Register per-cpu shared page with host via SMCCC. */ + WRITE_ONCE(state->flag, cpu_to_le64(KVM_PV_VCPU_RUNNING)); + WRITE_ONCE(state->idle_accum, cpu_to_le64(0)); + arm_smccc_1_1_invoke(ARM_SMCCC_HV_REGISTER_PV_IDLE_TIME, + virt_to_phys(state) | ARM_SMCCC_KVM_ENABLED, &res); + if (res.a0 != SMCCC_RET_SUCCESS) + pr_warn("%s: Failed to register PV idle time by SMCCC.\n", __func__); + + return 0; +} + +static bool kvm_has_pv_idle_time(void) +{ + struct arm_smccc_res res; + + /* To detect the presence of PV idle time support we require SMCCC 1.1+ */ + if (arm_smccc_1_1_get_conduit() == SMCCC_CONDUIT_NONE) + return false; + + arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, + ARM_SMCCC_HV_PV_IDLE_TIME_FEATURES, &res); + + return (res.a0 == SMCCC_RET_SUCCESS); +} + +int __init pv_idle_time_init(void) +{ + int ret; + + if (is_hyp_mode_available()) + return 0; + + if (!kvm_has_pv_idle_time()) + return 0; + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "hypervisor/arm/pv_idle_time:starting", + pv_idle_time_cpu_online, + pvidle_cpu_down_prepare); + if (ret < 0) { + pr_warn("PV idle time init failed\n"); + return ret; + } + + static_branch_enable(&kvm_pv_idle_time_enabled); + pr_info("using PV idle time\n"); + + return 0; +} diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index c3b83023851f5ac6d02fd05505a2e9c058de75b9..91bed55b52bcfb70ed7e665037cfc500e03090e3 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -56,6 +56,7 @@ #include #include #include +#include #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include @@ -76,6 +77,11 @@ void __noreturn arch_cpu_idle_dead(void) } #endif +void arch_cpu_idle_exit(void) +{ + pv_idle_time_exit(); +} + /* * Called by kexec, immediately prior to machine_kexec(). * diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 59dc93e9bf1f34c6376267728677a04a84977d7f..db4271d36300f939be32ea3402db4142866f10b4 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -400,6 +400,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) #ifdef CONFIG_PARAVIRT_SPINLOCKS pv_qspinlock_init(); #endif + + pv_idle_time_init(); } static inline bool cpu_can_disable(unsigned int cpu) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 369fa7b767a1660d6413ccaddf709906ae8f8f5c..13caa77f0f42a86a2a4ebe2e710074045a116243 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -472,6 +472,8 @@ void __init smp_prepare_boot_cpu(void) kasan_init_hw_tags(); /* Init percpu seeds for random tags after cpus are set up. */ kasan_init_sw_tags(); + + pv_idle_time_cpu_online(smp_processor_id()); } /* diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 96a8adfdd83127c586c4132f365382fea2083183..86dbd8cd1573efa0c809ea79a83158cf38ff6237 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -25,6 +25,7 @@ #include #include #include +#include static unsigned int cpufreq_khz; @@ -280,6 +281,8 @@ void arch_cpu_idle_enter(void) { unsigned int cpu = smp_processor_id(); + pv_idle_time_enter(); + if (!amu_fie_cpu_supported(cpu)) return; diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 145a1dc41bec6b3deb7300e0e8e64e3bbfe819be..103f88d5068b2181677a75030c1475c1223ecd56 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -10,7 +10,7 @@ include $(srctree)/virt/kvm/Makefile.kvm obj-$(CONFIG_KVM) += kvm.o kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o pvsched.o \ - inject_fault.o handle_exit.o pv_cpufreq.o \ + inject_fault.o handle_exit.o pv_cpufreq.o pv_idle_time.o \ guest.o debug.o reset.o sys_regs.o \ vgic-sys-reg-v3.o fpsimd.o \ arch_timer.o trng.o vmid.o emulate-nested.o nested.o at.o \ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 91846ed5a076a323e600a551b2bd802d072a50cc..73c8e3a25d13264440074241e27426a2400d623e 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -500,6 +500,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kvm_arm_pvsched_vcpu_init(&vcpu->arch); + kvm_arm_pv_idle_time_vcpu_init(&vcpu->arch); + vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; /* @@ -1648,6 +1650,8 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, kvm_arm_pvsched_vcpu_init(&vcpu->arch); + kvm_arm_pv_idle_time_vcpu_init(&vcpu->arch); + return 0; } diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index b07d4d8739f410a3e538248ac7f8fd4b40c0cd55..0abb6498040bd32e79e411cae5e1fe23504b6fa4 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -131,6 +131,9 @@ static bool kvm_smccc_test_fw_bmap(struct kvm_vcpu *vcpu, u32 func_id) case ARM_SMCCC_HV_PV_CPU_FREQ_GET: return test_bit(KVM_REG_ARM_VENDOR_HYP_BIT_PV_CPU_FREQ, &smccc_feat->vendor_hyp_bmap); + case ARM_SMCCC_HV_REGISTER_PV_IDLE_TIME: + return test_bit(KVM_REG_ARM_VENDOR_HYP_BIT_PV_IDLE_TIME, + &smccc_feat->vendor_hyp_bmap); #ifdef CONFIG_ARM64_HISI_IPIV case ARM_SMCCC_VENDOR_PV_SGI_FEATURES: case ARM_SMCCC_VENDOR_PV_SGI_ENABLE: @@ -378,6 +381,11 @@ int kvm_smccc_call_handler(struct kvm_vcpu *vcpu) &smccc_feat->vendor_hyp_bmap)) val[0] = SMCCC_RET_SUCCESS; break; + case ARM_SMCCC_HV_PV_IDLE_TIME_FEATURES: + if (test_bit(KVM_REG_ARM_VENDOR_HYP_BIT_PV_IDLE_TIME, + &smccc_feat->vendor_hyp_bmap)) + val[0] = SMCCC_RET_SUCCESS; + break; } break; case ARM_SMCCC_HV_PV_TIME_FEATURES: @@ -431,6 +439,10 @@ int kvm_smccc_call_handler(struct kvm_vcpu *vcpu) case ARM_SMCCC_HV_PV_CPU_FREQ_GET: val[0] = kvm_pv_cpu_freq_get(vcpu); break; + case ARM_SMCCC_HV_REGISTER_PV_IDLE_TIME: { + val[0] = kvm_smccc_register_pv_idle_time(vcpu); + break; + } case ARM_SMCCC_TRNG_VERSION: case ARM_SMCCC_TRNG_FEATURES: case ARM_SMCCC_TRNG_GET_UUID: diff --git a/arch/arm64/kvm/pv_idle_time.c b/arch/arm64/kvm/pv_idle_time.c new file mode 100644 index 0000000000000000000000000000000000000000..13b4b3c3b4c48798f509b1448a195dc87df6dbf7 --- /dev/null +++ b/arch/arm64/kvm/pv_idle_time.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PV vCPU idle time host support. + * + * The guest registers a per-vCPU shared page (kvm_idle_time) via + * SMCCC. The guest writes the idle flag into that page on idle entry + * and clears it on idle exit. The host can then peek at that flag to + * know whether the vCPU is currently in its idle loop. + */ + +#include +#include + +#include +#include +#include + +#define KVM_PV_IDLE_TIME_ALIGNMENT_BITS 5 +#define KVM_PV_IDLE_TIME_VALID_BITS ((-1ULL << (KVM_PV_IDLE_TIME_ALIGNMENT_BITS + 1))) +#define KVM_PV_IDLE_TIME_RESERVED_MASK \ + (((1 << KVM_PV_IDLE_TIME_ALIGNMENT_BITS) - 1) << 1) + +long kvm_smccc_register_pv_idle_time(struct kvm_vcpu *vcpu) +{ + struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_idle_time.cache; + u64 data = smccc_get_arg1(vcpu); + gpa_t gpa = data & KVM_PV_IDLE_TIME_VALID_BITS; + int ret, idx; + + /* We rely on the fact that it fits in a single page. */ + BUILD_BUG_ON((sizeof(struct kvm_idle_time) - 1) & KVM_PV_IDLE_TIME_VALID_BITS); + + if (data & KVM_PV_IDLE_TIME_RESERVED_MASK) + return SMCCC_RET_INVALID_PARAMETER; + + if (data & ARM_SMCCC_KVM_ENABLED) { + idx = srcu_read_lock(&vcpu->kvm->srcu); + ret = kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, + sizeof(struct kvm_idle_time)); + srcu_read_unlock(&vcpu->kvm->srcu, idx); + if (ret) + return SMCCC_RET_INVALID_PARAMETER; + } + + vcpu->arch.pv_idle_time.base = data; + + return SMCCC_RET_SUCCESS; +} + +static bool kvm_pv_idle_time_enabled(struct kvm_vcpu *vcpu) +{ + return (vcpu->arch.pv_idle_time.base & ARM_SMCCC_KVM_ENABLED); +} + +bool kvm_arch_is_vcpu_pv_idle(struct kvm_vcpu *vcpu) +{ + struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_idle_time.cache; + __le64 flag; + int ret, idx; + + if (!kvm_pv_idle_time_enabled(vcpu)) + return false; + + idx = srcu_read_lock(&vcpu->kvm->srcu); + ret = kvm_read_guest_offset_cached(vcpu->kvm, ghc, &flag, + offsetof(struct kvm_idle_time, flag), + sizeof(flag)); + srcu_read_unlock(&vcpu->kvm->srcu, idx); + if (ret) + return false; + + return !!(le64_to_cpu(flag) & KVM_PV_VCPU_IDLE); +} diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index 8623225a17dfd3a19ca16615096719a418ebf7b9..3c549890a5b9e09f6c8523b3ace5182e21c42d4f 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -166,6 +166,21 @@ ARM_SMCCC_OWNER_VENDOR_HYP, \ 0xc1) +#define ARM_SMCCC_KVM_ENABLED 1 + +/* Paravirtualized vCPU idle hint (KVM vendor hyp services) */ +#define ARM_SMCCC_HV_PV_IDLE_TIME_FEATURES \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_VENDOR_HYP, \ + 0xd0) + +#define ARM_SMCCC_HV_REGISTER_PV_IDLE_TIME \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_VENDOR_HYP, \ + 0xd1) + /* TRNG entropy source calls (defined by ARM DEN0098) */ #define ARM_SMCCC_TRNG_VERSION \ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \