From b87ed72e14a8e8158decf81d4c781e5487bf0525 Mon Sep 17 00:00:00 2001 From: David Dai Date: Mon, 03 Oct 2022 17:24:41 -0700 Subject: [PATCH] BACKPORT: FROMLIST: cpufreq: add Android-V virtual-cpufreq driver Introduce a virtualized cpufreq driver on Android-V for guest kernels to improve performance and power of workloads within VMs. This driver does two main things: 1. Sends the frequency of vCPUs as a hint to the host. The host uses the hint to schedule the vCPU threads and decide physical CPU frequency. 2. If a VM does not support a virtualized FIE(like AMUs), it queries the host CPU frequency by reading a MMIO region of a virtual cpufreq device to update the guest's frequency scaling factor periodically. This enables accurate Per-Entity Load Tracking for tasks running in the guest. Bug: 284368385 Link: https://lore.kernel.org/all/20240127004321.1902477-3-davidai@google.com/ Change-Id: I53047a30fa2843a25bba57c7373fc1829dd08f00 Signed-off-by: David Dai (cherry picked from commit 9fc999beb76abbbf3e65903d83892a94a2090532) --- diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 093a7e7..c996963f 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -224,6 +224,21 @@ If in doubt, say N. +config ANDROID_V_CPUFREQ_VIRT + tristate "Android-V Virtual cpufreq driver" + depends on OF && GENERIC_ARCH_TOPOLOGY + select PM_OPP + help + This adds a virtualized cpufreq driver for guest kernels that + read/writes to a MMIO region for a virtualized cpufreq device to + communicate with the host. It sends frequency updates to the host + which gets used as a hint to schedule vCPU threads and select CPU + frequency. If a VM does not support a virtualized FIE such as AMUs, + it updates the frequency scaling factor by polling host CPU frequency + to enable accurate Per-Entity Load Tracking for tasks running in the guest. + + If in doubt, say N. + config CPUFREQ_DT_PLATDEV tristate "Generic DT based cpufreq platdev driver" depends on OF diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 2b5647b..6312cdf 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -19,6 +19,7 @@ obj-$(CONFIG_CPUFREQ_DT) += cpufreq-dt.o obj-$(CONFIG_CPUFREQ_DT_PLATDEV) += cpufreq-dt-platdev.o +obj-$(CONFIG_ANDROID_V_CPUFREQ_VIRT) += android-v-virt-cpufreq.o obj-$(CONFIG_CPUFREQ_DUMMY) += dummy-cpufreq.o diff --git a/drivers/cpufreq/android-v-virt-cpufreq.c b/drivers/cpufreq/android-v-virt-cpufreq.c new file mode 100644 index 0000000..97ef67d --- /dev/null +++ b/drivers/cpufreq/android-v-virt-cpufreq.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 Google LLC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REG_CUR_FREQ_KHZ_OFFSET 0x0 +#define REG_SET_FREQ_KHZ_OFFSET 0x4 +#define PER_CPU_OFFSET 0x8 + +static void __iomem *base; + +static void android_v_virt_scale_freq_tick(void) +{ + int cpu = smp_processor_id(); + u32 max_freq = (u32)cpufreq_get_hw_max_freq(cpu); + u64 cur_freq; + long scale; + + cur_freq = (u64)readl_relaxed(base + cpu * PER_CPU_OFFSET + + REG_CUR_FREQ_KHZ_OFFSET); + + cur_freq <<= SCHED_CAPACITY_SHIFT; + scale = (long)div_u64(cur_freq, max_freq); + scale = min(scale, SCHED_CAPACITY_SCALE); + + this_cpu_write(arch_freq_scale, scale); +} + +static struct scale_freq_data virt_sfd = { + .source = SCALE_FREQ_SOURCE_VIRT, + .set_freq_scale = android_v_virt_scale_freq_tick, +}; + +static unsigned int android_v_vcpufreq_set_perf(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + writel_relaxed(target_freq, + base + policy->cpu * PER_CPU_OFFSET + REG_SET_FREQ_KHZ_OFFSET); + return 0; +} + +static unsigned int android_v_vcpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + android_v_vcpufreq_set_perf(policy, target_freq); + return target_freq; +} + +static int android_v_vcpufreq_target_index(struct cpufreq_policy *policy, + unsigned int index) +{ + return android_v_vcpufreq_set_perf(policy, + policy->freq_table[index].frequency); +} + +static int android_v_vcpufreq_cpu_init(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *table; + struct device *cpu_dev; + int ret; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) + return -ENODEV; + + ret = dev_pm_opp_of_add_table(cpu_dev); + if (ret) + return ret; + + ret = dev_pm_opp_get_opp_count(cpu_dev); + if (ret <= 0) { + dev_err(cpu_dev, "OPP table can't be empty\n"); + return -ENODEV; + } + + ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &table); + if (ret) { + dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); + return ret; + } + + policy->freq_table = table; + + /* + * To simplify and improve latency of handling frequency requests on + * the host side, this ensures that the vCPU thread triggering the MMIO + * abort is the same thread whose performance constraints (Ex. uclamp + * settings) need to be updated. This simplifies the VMM (Virtual + * Machine Manager) having to find the correct vCPU thread and/or + * facing permission issues when configuring other threads. + */ + policy->dvfs_possible_from_any_cpu = false; + policy->fast_switch_possible = true; + + /* + * Using the default SCALE_FREQ_SOURCE_CPUFREQ is insufficient since + * the actual physical CPU frequency may not match requested frequency + * from the vCPU thread due to frequency update latencies or other + * inputs to the physical CPU frequency selection. This additional FIE + * source allows for more accurate freq_scale updates and only takes + * effect if another FIE source such as AMUs have not been registered. + */ + topology_set_scale_freq_source(&virt_sfd, policy->cpus); + + return 0; +} + +static int android_v_vcpufreq_cpu_exit(struct cpufreq_policy *policy) +{ + struct device *cpu_dev; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) + return -ENODEV; + + topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_VIRT, policy->related_cpus); + dev_pm_opp_free_cpufreq_table(cpu_dev, &policy->freq_table); + return 0; +} + +static int android_v_vcpufreq_online(struct cpufreq_policy *policy) +{ + /* Nothing to restore. */ + return 0; +} + +static int android_v_vcpufreq_offline(struct cpufreq_policy *policy) +{ + /* Dummy offline() to avoid exit() being called and freeing resources. */ + return 0; +} + +static struct cpufreq_driver cpufreq_android_v_virt_driver = { + .name = "andr-v-vcpufreq", + .init = android_v_vcpufreq_cpu_init, + .exit = android_v_vcpufreq_cpu_exit, + .online = android_v_vcpufreq_online, + .offline = android_v_vcpufreq_offline, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = android_v_vcpufreq_target_index, + .fast_switch = android_v_vcpufreq_fast_switch, + .attr = cpufreq_generic_attr, +}; + +static int android_v_vcpufreq_driver_probe(struct platform_device *pdev) +{ + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + ret = cpufreq_register_driver(&cpufreq_android_v_virt_driver); + if (ret) { + dev_err(&pdev->dev, "Android-V Virt Cpufreq driver failed to register: %d\n", ret); + return ret; + } + + dev_dbg(&pdev->dev, "Android-V Virt CPUFreq driver initialized\n"); + return 0; +} + +static int android_v_vcpufreq_driver_remove(struct platform_device *pdev) +{ + cpufreq_unregister_driver(&cpufreq_android_v_virt_driver); + return 0; +} + +static const struct of_device_id android_v_vcpufreq_match[] = { + { .compatible = "virtual,android-v-only-cpufreq", .data = NULL}, + {} +}; +MODULE_DEVICE_TABLE(of, android_v_vcpufreq_match); + +static struct platform_driver android_v_vcpufreq_driver = { + .probe = android_v_vcpufreq_driver_probe, + .remove = android_v_vcpufreq_driver_remove, + .driver = { + .name = "andr-v-vcpufreq", + .of_match_table = android_v_vcpufreq_match, + }, +}; + +static int __init android_v_vcpufreq_init(void) +{ + return platform_driver_register(&android_v_vcpufreq_driver); +} +postcore_initcall(android_v_vcpufreq_init); + +static void __exit android_v_vcpufreq_exit(void) +{ + platform_driver_unregister(&android_v_vcpufreq_driver); +} +module_exit(android_v_vcpufreq_exit); + +MODULE_DESCRIPTION("Virtual cpufreq driver only for Android-V"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 589f2a2..838780aa 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -42,6 +42,7 @@ enum scale_freq_source { SCALE_FREQ_SOURCE_CPUFREQ = 0, SCALE_FREQ_SOURCE_ARCH, SCALE_FREQ_SOURCE_CPPC, + SCALE_FREQ_SOURCE_VIRT, }; struct scale_freq_data {