Initial commit: Linux kernel 4.19 for Unplugged OS

This commit is contained in:
ziv committed 2025-10-06 13:52:42 +00:00
commit d30235b04c
80519 files changed
+35800681

No files matched your search

+27
View File
@@ -0,0 +1,27 @@
config MTK_PERF_COMMON
bool "MTK system performance common feature"
help
CONFIG_MTK_PERF_COMMON provides common entrypoint to support
multiple the system performance features and it can easily
extend. The trigger period is aligned to sched-tick. If you are
not sure about whether to enable it or not, please set n.
config MTK_PERF_TRACKER
bool "MTK system performance index support"
depends on MTK_PERF_COMMON
help
CONFIG_PERF_TRACKER define if system performance index should
be built-in or not. It supports system related performance index
via trace. If you are not sure about whether to enable it or not,
please set n.
config MTK_CORE_CTL
bool "MTK Core Control with CPU Isolation"
depends on MTK_SCHED_EXTENSION
help
This config is for the core control policy with CPU isolation.
The core control online and offline CPU cores based on task
utilizations from scheduler information automatically.
If you are not sure about whether to enable it or not,
please set n.
@@ -0,0 +1,38 @@
#
# Copyright (C) 2018 MediaTek Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See http://www.gnu.org/licenses/gpl-2.0.html for more details.
#
MTK_PLATFORM := $(subst ",,$(CONFIG_MTK_PLATFORM))
LINUXINCLUDE += -include $(srctree)/kernel/sched/sched.h
ccflags-y += -I$(src) # needed for trace events
ccflags-y += -I$(srctree)/drivers/misc/mediatek/dramc/$(MTK_PLATFORM)/
ccflags-y += -I$(srctree)/drivers/misc/mediatek/base/power/include/vcorefs_v3/
ccflags-y += -I$(srctree)/drivers/devfreq/
ifneq (,$(filter $(CONFIG_MTK_PLATFORM), "mt6768" "mt6781" "mt6785" "mt6833" "mt6853" "mt6877" "mt6885" "mt6893" "mt6873"))
ccflags-y += -I$(srctree)/drivers/misc/mediatek/base/power/qos/$(MTK_PLATFORM)/
else
ccflags-y += -I$(srctree)/drivers/misc/mediatek/qos/
endif
subdir-ccflags-y += -I$(srctree)/drivers/misc/mediatek/include
subdir-ccflags-y += -I$(srctree)/drivers/misc/mediatek/include/mt-plat/
ccflags-$(CONFIG_MTK_CORE_CTL) += -I$(srctree)/drivers/misc/mediatek/base/power/include/ppm_v3/
ccflags-$(CONFIG_MTK_CORE_CTL) += -I$(srctree)/drivers/misc/mediatek/include/mt-plat/
obj-y += perf_common.o
# a tracker for performance index
obj-$(CONFIG_MTK_PERF_TRACKER) += perf_tracker.o
# RQ stats for TLP estimation
obj-$(CONFIG_MTK_CORE_CTL) += rq_stats.o
obj-$(CONFIG_MTK_CORE_CTL) += sched_avg.o
obj-$(CONFIG_MTK_CORE_CTL) += core_ctl.o
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,46 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (c) 2022 MediaTek Inc.
*/
enum mt_cpu_dvfs_id {
MT_CPU_DVFS_LL,
MT_CPU_DVFS_L,
MT_CPU_DVFS_B,
MT_CPU_DVFS_CCI,
NR_MT_CPU_DVFS,
};
extern void set_overutil_threshold(int index, int val);
extern int get_immediate_tslvts1_1_wrap(void); /* CPU7 TS */
extern int sched_get_nr_overutil_avg(int cluster_id,
int *l_avg,
int *h_avg,
int *sum_nr_overutil_l,
int *sum_nr_overutil_h,
int *max_nr);
extern int update_userlimit_cpu_freq(int kicker,
int num_cluster,
struct ppm_limit_data *freq_limit);
extern int sched_isolate_cpu(int cpu);
extern int sched_unisolate_cpu(int cpu);
extern int sched_unisolate_cpu_unlocked(int cpu);
#ifdef CONFIG_MTK_CPU_FREQ
extern unsigned int mt_cpufreq_get_freq_by_idx(enum mt_cpu_dvfs_id id, int idx);
extern unsigned int mt_cpufreq_get_cur_freq(enum mt_cpu_dvfs_id id);
extern unsigned int mt_cpufreq_find_close_freq(unsigned int cluster_id, unsigned int freq);
extern int mt_cpufreq_set_by_schedule_load_cluster(unsigned int cluster_id, unsigned int freq);
#else
static unsigned int mt_cpufreq_get_freq_by_idx(enum mt_cpu_dvfs_id id, int idx) {return 0; }
static unsigned int mt_cpufreq_get_cur_freq(enum mt_cpu_dvfs_id id) {return 0; }
static unsigned int mt_cpufreq_find_close_freq(unsigned int cluster_id, unsigned int freq)
{
return 0;
}
static int mt_cpufreq_set_by_schedule_load_cluster(unsigned int cluster_id, unsigned int freq)
{
return 0;
}
#endif
@@ -0,0 +1,143 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 MediaTek Inc.
*/
#undef TRACE_SYSTEM
#define TRACE_SYSTEM core_ctl
#if !defined(_CORE_CTL_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
#define _CORE_CTL_TRACE_H
#include <linux/types.h>
#include <linux/tracepoint.h>
TRACE_EVENT(core_ctl_info,
TP_PROTO(
int ts_cpu,
unsigned int btask_thresh,
int util,
unsigned int isolate_cpus,
unsigned int *need_cpus),
TP_ARGS(ts_cpu, btask_thresh, util, isolate_cpus, need_cpus),
TP_STRUCT__entry(
__field(int, ts_cpu)
__field(unsigned int, btask_thresh)
__field(int, util)
__field(u32, isolate_cpus)
__array(unsigned int, need_cpus, 3)
),
TP_fast_assign(
__entry->ts_cpu = ts_cpu;
__entry->btask_thresh = btask_thresh;
__entry->util = util;
__entry->isolate_cpus = isolate_cpus;
memcpy(__entry->need_cpus, need_cpus, sizeof(unsigned int)*3);
),
TP_printk("ts_cpu=%d btask_thresh=%u util=%d isolate_cpus=%x need_cpus=%u|%u|%u",
__entry->ts_cpu,
__entry->btask_thresh, __entry->util,
__entry->isolate_cpus,
__entry->need_cpus[0], __entry->need_cpus[1], __entry->need_cpus[2])
);
TRACE_EVENT(core_ctl_demand_eval,
TP_PROTO(
unsigned int cid,
unsigned int old_need,
unsigned int new_need,
unsigned int active_cpus,
unsigned int min_cpus,
unsigned int max_cpus,
unsigned int boost,
unsigned int enable,
unsigned int updated),
TP_ARGS(
cid,
old_need,
new_need,
active_cpus,
min_cpus,
max_cpus,
boost,
enable,
updated),
TP_STRUCT__entry(
__field(u32, cid)
__field(u32, old_need)
__field(u32, new_need)
__field(u32, active_cpus)
__field(u32, min_cpus)
__field(u32, max_cpus)
__field(u32, boost)
__field(u32, enable)
__field(u32, updated)
),
TP_fast_assign(
__entry->cid = cid;
__entry->old_need = old_need;
__entry->new_need = new_need;
__entry->active_cpus = active_cpus;
__entry->min_cpus = min_cpus;
__entry->max_cpus = max_cpus;
__entry->boost = boost;
__entry->enable = enable;
__entry->updated = updated;
),
TP_printk("cid=%u, old=%u, new=%u, act=%u min=%u max=%u bst=%u enbl=%u update=%u",
__entry->cid,
__entry->old_need,
__entry->new_need,
__entry->active_cpus,
__entry->min_cpus,
__entry->max_cpus,
__entry->boost,
__entry->enable,
__entry->updated)
);
TRACE_EVENT(core_ctl_update_nr_btask,
TP_PROTO(
int *nr_up,
int *nr_down,
int *max_nr_down),
TP_ARGS(nr_up, nr_down, max_nr_down),
TP_STRUCT__entry(
__array(int, nr_up, 3)
__array(int, nr_down, 3)
__array(int, max_nr_down, 3)
),
TP_fast_assign(
memcpy(__entry->nr_up, nr_up, sizeof(unsigned int) * 3);
memcpy(__entry->nr_down, nr_down, sizeof(unsigned int) * 3);
memcpy(__entry->max_nr_down, max_nr_down, sizeof(unsigned int) * 3);
),
TP_printk("nr_up=%d|%d|%d nr_down=%d|%d|%d max_nr_down=%d|%d|%d ",
__entry->nr_up[0], __entry->nr_up[1], __entry->nr_up[2],
__entry->nr_down[0], __entry->nr_down[1], __entry->nr_down[2],
__entry->max_nr_down[0], __entry->max_nr_down[1], __entry->max_nr_down[2])
);
#endif /*_CORE_CTL_TRACE_H */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE core_ctl_trace
/* This part must be outside protection */
#include <trace/define_trace.h>
@@ -0,0 +1,109 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 MediaTek Inc.
*/
#include <linux/spinlock.h>
#include <linux/time.h>
#include <mt-plat/perf_common.h>
#include <perf_tracker.h>
#include <linux/cpu.h>
#include <linux/topology.h>
#ifdef CONFIG_MTK_CORE_CTL
#include <mt-plat/core_ctl.h>
#endif
static u64 checked_timestamp;
static bool long_trace_check_flag;
static DEFINE_SPINLOCK(check_lock);
static int perf_common_init;
#ifdef CONFIG_MTK_PERF_TRACKER
int cluster_nr = -1;
#endif
static inline bool perf_do_check(u64 wallclock)
{
bool do_check = false;
unsigned long flags;
/* check interval */
spin_lock_irqsave(&check_lock, flags);
if ((s64)(wallclock - checked_timestamp)
>= (s64)(2 * NSEC_PER_MSEC)) {
checked_timestamp = wallclock;
long_trace_check_flag = !long_trace_check_flag;
do_check = true;
}
spin_unlock_irqrestore(&check_lock, flags);
return do_check;
}
bool hit_long_check(void)
{
bool do_check = false;
unsigned long flags;
spin_lock_irqsave(&check_lock, flags);
if (long_trace_check_flag)
do_check = true;
spin_unlock_irqrestore(&check_lock, flags);
return do_check;
}
void perf_common(u64 wallclock)
{
long mm_available = -1, mm_free = -1;
if (!perf_do_check(wallclock))
return;
#ifdef CONFIG_MTK_CORE_CTL
/* period is 8ms */
if (hit_long_check())
core_ctl_tick(wallclock);
#endif
if (unlikely(!perf_common_init))
return;
perf_tracker(wallclock, mm_available, mm_free);
}
static struct attribute *perf_attrs[] = {
#ifdef CONFIG_MTK_PERF_TRACKER
&perf_tracker_enable_attr.attr,
#endif
NULL,
};
static struct attribute_group perf_attr_group = {
.attrs = perf_attrs,
};
static int init_perf_common(void)
{
int ret = 0;
struct kobject *kobj = NULL;
perf_common_init = 1;
#ifdef CONFIG_MTK_PERF_TRACKER
cluster_nr = arch_nr_clusters();
if (unlikely(cluster_nr <= 0 || cluster_nr > 3))
cluster_nr = 3;
#endif
kobj = kobject_create_and_add("perf", &cpu_subsys.dev_root->kobj);
if (kobj) {
ret = sysfs_create_group(kobj, &perf_attr_group);
if (ret)
kobject_put(kobj);
else
kobject_uevent(kobj, KOBJ_ADD);
}
return 0;
}
late_initcall_sync(init_perf_common);
@@ -0,0 +1,230 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2019 MediaTek Inc.
*/
#include <trace/events/sched.h>
#ifdef CONFIG_MTK_DRAMC
#include <mtk_dramc.h>
#endif
#include <linux/mm.h>
#include <linux/swap.h>
#include <helio-dvfsrc.h>
#define CREATE_TRACE_POINTS
#include <perf_tracker_trace.h>
#ifdef CONFIG_MTK_QOS_FRAMEWORK
#include <mtk_qos_sram.h>
#endif
static int perf_tracker_on;
static DEFINE_MUTEX(perf_ctl_mutex);
static struct mtk_btag_mictx_iostat_struct iostat;
#ifdef CONFIG_MTK_BLOCK_TAG
void __attribute__((weak)) mtk_btag_mictx_enable(int enable) {}
int __attribute__((weak)) mtk_btag_mictx_get_data(
struct mtk_btag_mictx_iostat_struct *io)
{
return -1;
}
#endif
int perf_tracker_enable(int on)
{
perf_tracker_on = on;
return (perf_tracker_on == on) ? 0 : -1;
}
EXPORT_SYMBOL(perf_tracker_enable);
unsigned int __attribute__((weak)) get_dram_data_rate(void)
{
return 0;
}
u32 __attribute__((weak)) qos_sram_read(u32 offset)
{
return 0;
}
int __attribute__((weak)) get_cur_vcore_uv(void)
{
return 0;
}
int __attribute__((weak)) get_cur_ddr_khz(void)
{
return 0;
}
unsigned int __attribute__((weak)) qos_rec_get_hist_bw(unsigned int idx, unsigned int type)
{
return 0;
}
unsigned int __attribute__((weak)) qos_rec_get_hist_data_bw(unsigned int idx, unsigned int type)
{
return 0;
}
unsigned int __attribute__((weak)) qos_rec_get_hist_idx(void)
{
return 0xFFFF;
}
static inline u32 cpu_stall_ratio(int cpu)
{
#ifdef CM_STALL_RATIO_OFFSET
return qos_sram_read(CM_STALL_RATIO_OFFSET + cpu * 4);
#else
return 0;
#endif
}
#define K(x) ((x) << (PAGE_SHIFT - 10))
#define max_cpus 8
#define bw_hist_nums 8
#define bw_record_nums 32
void perf_tracker(u64 wallclock,
long mm_available,
long mm_free)
{
int dram_rate = 0;
struct mtk_btag_mictx_iostat_struct *iostat_ptr = &iostat;
int bw_c = 0, bw_g = 0, bw_mm = 0, bw_total = 0, bw_idx = 0;
u32 bw_record = 0, bw_data[bw_record_nums] = {0};
int vcore_uv = 0;
int i;
int stall[max_cpus] = {0};
unsigned int sched_freq[3] = {0};
int cid;
if (!perf_tracker_on)
return;
/* dram freq */
dram_rate = get_cur_ddr_khz();
dram_rate = dram_rate / 1000;
if (dram_rate <= 0)
dram_rate = get_dram_data_rate();
/* vcore */
vcore_uv = get_cur_vcore_uv();
#ifdef CONFIG_MTK_QOS_FRAMEWORK
/* emi */
bw_c = qos_sram_read(QOS_DEBUG_1);
bw_g = qos_sram_read(QOS_DEBUG_2);
bw_mm = qos_sram_read(QOS_DEBUG_3);
bw_total = qos_sram_read(QOS_DEBUG_0);
#endif
/* emi history */
bw_idx = qos_rec_get_hist_idx();
if (bw_idx != 0xFFFF) {
for (bw_record = 0; bw_record < bw_record_nums; bw_record += 8) {
/* occupied bw history */
bw_data[bw_record] = qos_rec_get_hist_bw(bw_idx, 0);
bw_data[bw_record+1] = qos_rec_get_hist_bw(bw_idx, 1);
bw_data[bw_record+2] = qos_rec_get_hist_bw(bw_idx, 2);
bw_data[bw_record+3] = qos_rec_get_hist_bw(bw_idx, 3);
/* data bw history */
bw_data[bw_record+4] = qos_rec_get_hist_data_bw(bw_idx, 0);
bw_data[bw_record+5] = qos_rec_get_hist_data_bw(bw_idx, 1);
bw_data[bw_record+6] = qos_rec_get_hist_data_bw(bw_idx, 2);
bw_data[bw_record+7] = qos_rec_get_hist_data_bw(bw_idx, 3);
bw_idx -= 1;
if (bw_idx < 0)
bw_idx = bw_idx + bw_hist_nums;
}
/* trace for short bin */
trace_perf_index_sbin(bw_data, bw_record);
}
/* sched: cpu freq */
for (cid = 0; cid < cluster_nr; cid++)
sched_freq[cid] = mt_cpufreq_get_cur_freq(cid);
/* trace for short msg */
trace_perf_index_s(
sched_freq[0], sched_freq[1], sched_freq[2],
dram_rate, bw_c, bw_g, bw_mm, bw_total,
vcore_uv);
if (!hit_long_check())
return;
/* free mem */
if (mm_free == -1) {
mm_free = global_zone_page_state(NR_FREE_PAGES);
mm_available = si_mem_available();
}
#ifdef CONFIG_MTK_BLOCK_TAG
/* If getting I/O stat fail, fallback to zero value. */
if (mtk_btag_mictx_get_data(iostat_ptr))
memset(iostat_ptr, 0,
sizeof(struct mtk_btag_mictx_iostat_struct));
#endif
/* cpu stall ratio */
for (i = 0; i < nr_cpu_ids || i < max_cpus; i++)
stall[i] = cpu_stall_ratio(i);
/* trace for long msg */
trace_perf_index_l(
K(mm_free),
K(mm_available),
iostat_ptr,
stall
);
}
/*
* make perf tracker on
* /sys/devices/system/cpu/perf/enable
* 1: on
* 0: off
*/
static ssize_t show_perf_enable(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
unsigned int len = 0;
unsigned int max_len = 4096;
len += snprintf(buf, max_len, "enable = %d\n",
perf_tracker_on);
return len;
}
static ssize_t store_perf_enable(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf,
size_t count)
{
int val = 0;
mutex_lock(&perf_ctl_mutex);
if (sscanf(buf, "%iu", &val) != 0) {
val = (val > 0) ? 1 : 0;
perf_tracker_on = val;
#ifdef CONFIG_MTK_BLOCK_TAG
mtk_btag_mictx_enable(val);
#endif
}
mutex_unlock(&perf_ctl_mutex);
return count;
}
struct kobj_attribute perf_tracker_enable_attr =
__ATTR(enable, 0600, show_perf_enable, store_perf_enable);
@@ -0,0 +1,49 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 MediaTek Inc.
*/
#ifndef _PERF_TRACKER_H
#define _PERF_TRACKER_H
#include <linux/types.h>
#include <linux/module.h>
#ifdef CONFIG_MTK_PERF_TRACKER
/* cpufreq section */
extern int cluster_nr;
#ifdef CONFIG_MTK_CPU_FREQ
extern unsigned int mt_cpufreq_get_cur_freq(int id);
#else
static inline int mt_cpufreq_get_cur_freq(int id) { return 0; }
#endif
/* storage I/O section */
#ifdef CONFIG_MTK_BLOCK_TAG
#include <mt-plat/mtk_blocktag.h>
#else
struct mtk_btag_mictx_iostat_struct {
__u64 duration; /* duration time for below performance data (ns) */
__u32 tp_req_r; /* throughput (per-request): read (KB/s) */
__u32 tp_req_w; /* throughput (per-request): write (KB/s) */
__u32 tp_all_r; /* throughput (overlapped) : read (KB/s) */
__u32 tp_all_w; /* throughput (overlapped) : write (KB/s) */
__u32 reqsize_r; /* request size : read (Bytes) */
__u32 reqsize_w; /* request size : write (Bytes) */
__u32 reqcnt_r; /* request count: read */
__u32 reqcnt_w; /* request count: write */
__u16 wl; /* storage device workload (%) */
__u16 q_depth; /* storage cmdq queue depth */
};
#endif
extern struct kobj_attribute perf_tracker_enable_attr;
extern void perf_tracker(u64 wallclock,
long mm_available,
long mm_free);
extern bool hit_long_check(void);
#else
static inline void perf_tracker(u64 wallclock,
long mm_available,
long mm_free) {}
#endif /* CONFIG_MTK_PERF_TRACKER */
#endif /* _PERF_TRACKER_H */
@@ -0,0 +1,18 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 MediaTek Inc.
*/
#ifndef _PERF_TRACKER_INTERNAL_H
#define _PERF_TRACKER_INTERNAL_H
#ifdef CONFIG_MTK_GPU_SWPM_SUPPORT
extern void MTKGPUPower_model_start(unsigned int interval_ns);
extern void MTKGPUPower_model_stop(void);
extern void MTKGPUPower_model_suspend(void);
extern void MTKGPUPower_model_resume(void);
void perf_update_gpu_counter(unsigned int gpu_pmu[], unsigned int len) { };
#endif
#endif /* _PERF_TRACKER_INTERNAL_H */
@@ -0,0 +1,166 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2019 MediaTek Inc.
*/
#undef TRACE_SYSTEM
#define TRACE_SYSTEM perf_tracker
#if !defined(_PERF_TRACKER_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
#define _PERF_TRACKER_TRACE_H
#include <linux/string.h>
#include <linux/types.h>
#include <linux/tracepoint.h>
#include <perf_tracker.h>
TRACE_EVENT(perf_index_s,
TP_PROTO(
unsigned int sf0,
unsigned int sf1,
unsigned int sf2,
int dram_freq,
int bw_c,
int bw_g,
int bw_mm,
int bw_total,
int vcore_uv
),
TP_ARGS(sf0, sf1, sf2, dram_freq, bw_c, bw_g, bw_mm, bw_total, vcore_uv),
TP_STRUCT__entry(
__field(unsigned int, sf0)
__field(unsigned int, sf1)
__field(unsigned int, sf2)
__field(int, dram_freq)
__field(int, bw_c)
__field(int, bw_g)
__field(int, bw_mm)
__field(int, bw_total)
__field(int, vcore_uv)
),
TP_fast_assign(
__entry->sf0 = sf0;
__entry->sf1 = sf1;
__entry->sf2 = sf2;
__entry->dram_freq = dram_freq;
__entry->bw_c = bw_c;
__entry->bw_g = bw_g;
__entry->bw_mm = bw_mm;
__entry->bw_total = bw_total;
__entry->vcore_uv = vcore_uv;
),
TP_printk("sched_freq=%d|%d|%d dram_freq=%d bw=%d|%d|%d|%d vcore=%d",
__entry->sf0,
__entry->sf1,
__entry->sf2,
__entry->dram_freq,
__entry->bw_c,
__entry->bw_g,
__entry->bw_mm,
__entry->bw_total,
__entry->vcore_uv
)
);
TRACE_EVENT(perf_index_l,
TP_PROTO(
long free_mem,
long avail_mem,
struct mtk_btag_mictx_iostat_struct *iostatptr,
int *stall
),
TP_ARGS(free_mem,
avail_mem,
iostatptr,
stall
),
TP_STRUCT__entry(
__field(long, free_mem)
__field(long, avail_mem)
__field(int, io_wl)
__field(int, io_req_r)
__field(int, io_all_r)
__field(int, io_reqsz_r)
__field(int, io_reqc_r)
__field(int, io_req_w)
__field(int, io_all_w)
__field(int, io_reqsz_w)
__field(int, io_reqc_w)
__field(int, io_dur)
__field(int, io_q_dept)
__array(int, stall, 8)
),
TP_fast_assign(
__entry->free_mem = free_mem;
__entry->avail_mem = avail_mem;
__entry->io_wl = iostatptr->wl;
__entry->io_req_r = iostatptr->tp_req_r;
__entry->io_all_r = iostatptr->tp_all_r;
__entry->io_reqsz_r = iostatptr->reqsize_r;
__entry->io_reqc_r = iostatptr->reqcnt_r;
__entry->io_req_w = iostatptr->tp_req_w;
__entry->io_all_w = iostatptr->tp_all_w;
__entry->io_reqsz_w = iostatptr->reqsize_w;
__entry->io_reqc_w = iostatptr->reqcnt_w;
__entry->io_dur = iostatptr->duration;
__entry->io_q_dept = iostatptr->q_depth;
memcpy(__entry->stall, stall, sizeof(int)*8);
),
TP_printk(
"free_mem=%ld avail_mem=%ld iostats=%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d stall=%d|%d|%d|%d|%d|%d|%d|%d",
__entry->free_mem,
__entry->avail_mem,
__entry->io_wl,
__entry->io_req_r,
__entry->io_all_r,
__entry->io_reqsz_r,
__entry->io_reqc_r,
__entry->io_req_w,
__entry->io_all_w,
__entry->io_reqsz_w,
__entry->io_reqc_w,
__entry->io_dur,
__entry->io_q_dept,
__entry->stall[0],
__entry->stall[1],
__entry->stall[2],
__entry->stall[3],
__entry->stall[4],
__entry->stall[5],
__entry->stall[6],
__entry->stall[7]
)
);
TRACE_EVENT(perf_index_sbin,
TP_PROTO(u32 *raw_data, u32 lens),
TP_ARGS(raw_data, lens),
TP_STRUCT__entry(
__dynamic_array(u32, raw_data, lens)
__field(u32, lens)
),
TP_fast_assign(
memcpy(__get_dynamic_array(raw_data), raw_data,
lens * sizeof(u32));
__entry->lens = lens;
),
TP_printk("data=%s", __print_array(__get_dynamic_array(raw_data),
__entry->lens, sizeof(u32)))
);
#endif /*_PERF_TRACKER_TRACE_H */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE perf_tracker_trace
/* This part must be outside protection */
#include <trace/define_trace.h>
@@ -0,0 +1,132 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2022 MediaTek Inc.
*/
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/kobject.h>
#include <linux/sysfs.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <rq_stats.h>
#define OVERUTIL_THRESHOLD_SIZE 2
/* cap(LL)*0.8, cap(BL)*0.7 */
static unsigned int overutil_thres[OVERUTIL_THRESHOLD_SIZE] = {19, 60};
unsigned int get_overutil_threshold(int index)
{
if (index >= 0 && index < OVERUTIL_THRESHOLD_SIZE)
return overutil_thres[index];
return 100;
}
EXPORT_SYMBOL(get_overutil_threshold);
void set_overutil_threshold(unsigned int index, unsigned int val)
{
if (index >= OVERUTIL_THRESHOLD_SIZE)
return;
if (val <= 100) {
overutil_thres[index] = (int)val;
overutil_thresh_chg_notify();
}
}
EXPORT_SYMBOL(set_overutil_threshold);
int inc_nr_heavy_running(int invoker, struct task_struct *p,
int inc, bool ack_cap)
{
sched_update_nr_heavy_prod(invoker, p,
cpu_of(task_rq(p)), inc, ack_cap);
return 0;
}
EXPORT_SYMBOL(inc_nr_heavy_running);
/* For read/write utilization related settings */
static ssize_t store_overutil(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
unsigned int val[OVERUTIL_THRESHOLD_SIZE];
int ret, i;
ret = sscanf(buf, "%u %u\n", &val[0], &val[1]);
if (ret <= 0 || OVERUTIL_THRESHOLD_SIZE <= ret)
return -EINVAL;
if (ret == 1) {
for (i = 0; i < OVERUTIL_THRESHOLD_SIZE; i++)
overutil_thres[i] =
val[0] < 100 ? val[0] : overutil_thres[i];
} else {
for (i = 0; i < OVERUTIL_THRESHOLD_SIZE; i++)
overutil_thres[i] =
val[i] < 100 ? val[i] : overutil_thres[i];
}
overutil_thresh_chg_notify();
return count;
}
static ssize_t show_overutil(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
unsigned int len = 0, i;
unsigned int max_len = 4096;
for (i = 0; i < OVERUTIL_THRESHOLD_SIZE; i++) {
len += snprintf(buf+len,
max_len-len, "over_util threshold[%u]=%d max=100\n",
i, overutil_thres[i]);
}
len += get_overutil_stats(buf+len, max_len-len);
return len;
}
static struct kobj_attribute over_util_attr =
__ATTR(over_util, 0600 /* S_IWUSR | S_IRUSR*/, show_overutil,
store_overutil);
static struct attribute *rq_attrs[] = {
&over_util_attr.attr,
NULL,
};
static struct attribute_group rq_attr_group = {
.attrs = rq_attrs,
};
static int init_rq_attribs(void)
{
int ret = 0;
struct kobject *kobj = NULL;
/* Create /sys/devices/system/cpu/rq-stats/... */
kobj = kobject_create_and_add("rq-stats",
&cpu_subsys.dev_root->kobj);
if (!kobj)
return -ENOMEM;
ret = sysfs_create_group(kobj, &rq_attr_group);
if (ret)
kobject_put(kobj);
else
kobject_uevent(kobj, KOBJ_ADD);
return ret;
}
static int __init rq_stats_init(void)
{
int ret = 0;
ret = init_rq_attribs();
return ret;
}
late_initcall(rq_stats_init);
@@ -0,0 +1,15 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (c) 2022 MediaTek Inc.
*/
extern void get_task_util(struct task_struct *p, unsigned long *util);
/* For heavy task detection */
extern void sched_update_nr_heavy_prod(int invoker,
struct task_struct *p, int cpu, int heavy_nr_inc, bool ack_cap);
extern void overutil_thresh_chg_notify(void);
extern int get_overutil_stats(char *buf, int buf_size);
extern unsigned long get_cpu_orig_capacity(unsigned int cpu);
unsigned int get_overutil_threshold(int index);
@@ -0,0 +1,681 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2022 MediaTek Inc.
*/
/*
* Scheduler hook for average runqueue determination
*/
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/hrtimer.h>
#include <linux/sched.h>
#include <linux/math64.h>
#include <linux/types.h>
#include <linux/sched/clock.h>
#include <linux/topology.h>
#include <linux/arch_topology.h>
#include <rq_stats.h>
enum overutil_type_t {
NO_OVERUTIL = 0,
L_OVERUTIL,
H_OVERUTIL
};
static DEFINE_PER_CPU(u64, last_time);
static DEFINE_PER_CPU(u64, nr);
static DEFINE_PER_CPU(u64, nr_heavy);
#ifdef CONFIG_MTK_CORE_CTL
static DEFINE_PER_CPU(u64, nr_max);
#endif
static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock);
static DEFINE_PER_CPU(spinlock_t, nr_heavy_lock) =
__SPIN_LOCK_UNLOCKED(nr_heavy_lock);
static int init_heavy;
struct overutil_stats_t {
int nr_overutil_l;
int nr_overutil_h;
int overutil_thresh_l;
int overutil_thresh_h;
u64 nr_overutil_l_prod_sum;
u64 nr_overutil_h_prod_sum;
u64 l_last_update_time;
u64 h_last_update_time;
int max_task_util;
int max_task_pid;
};
static DEFINE_PER_CPU(struct overutil_stats_t, cpu_overutil_state);
struct cluster_heavy_tbl_t {
u64 last_get_heavy_time;
u64 last_get_overutil_time;
u64 max_capacity;
};
struct cluster_heavy_tbl_t *cluster_heavy_tbl;
static int init_heavy_tlb(void);
static int arch_get_nr_clusters(void)
{
return arch_nr_clusters();
}
static int arch_get_cluster_id(unsigned int cpu)
{
return arch_cpu_cluster_id(cpu);
}
enum overutil_type_t is_task_overutil(struct task_struct *p)
{
struct overutil_stats_t *cpu_overutil;
unsigned long task_util;
int cpu, cid;
int cluster_nr = arch_get_nr_clusters();
if (!p)
return NO_OVERUTIL;
cpu = cpu_of(task_rq(p));
cid = arch_get_cluster_id(cpu);
if (cid < 0 || cid >= cluster_nr) {
printk_deferred_once("[%s] invalid cluster id %d\n",
__func__, cid);
return NO_OVERUTIL;
}
get_task_util(p, &task_util);
cpu_overutil = &per_cpu(cpu_overutil_state, cpu);
/* track task with max utilization */
if (task_util > cpu_overutil->max_task_util) {
cpu_overutil->max_task_util = task_util;
cpu_overutil->max_task_pid = p->pid;
}
/* check if task is overutilized */
if (task_util >= cpu_overutil->overutil_thresh_h)
return H_OVERUTIL;
else if (task_util >= cpu_overutil->overutil_thresh_l)
return L_OVERUTIL;
else
return NO_OVERUTIL;
}
#ifdef CONFIG_MTK_CORE_CTL
#define MAX_UTIL_TRACKER_PERIODIC_MS 8
static unsigned int fpsgo_update_max_util_windows;
#else
#define MAX_UTIL_TRACKER_PERIODIC_MS 32
#endif
static int gb_task_util;
static int gb_task_pid;
static int gb_task_cpu;
static DEFINE_SPINLOCK(gb_max_util_lock);
void (*fpsgo_sched_nominate_fp)(pid_t *id, int *utl);
void sched_max_util_task_tracking(void)
{
int cpu;
struct overutil_stats_t *cpu_overutil;
int max_util = 0;
int max_cpu = 0;
int max_task_pid = 0;
ktime_t now = ktime_get();
unsigned long flag;
static ktime_t max_util_tracker_last_update;
pid_t tasks[NR_CPUS] = {0};
int utils[NR_CPUS] = {0};
#ifdef CONFIG_MTK_CORE_CTL
bool skip_mintop = false;
#endif
spin_lock_irqsave(&gb_max_util_lock, flag);
if (ktime_before(now, ktime_add_ms(
max_util_tracker_last_update, MAX_UTIL_TRACKER_PERIODIC_MS))) {
spin_unlock_irqrestore(&gb_max_util_lock, flag);
return;
}
/* update last update time for tracker */
max_util_tracker_last_update = now;
#ifdef CONFIG_MTK_CORE_CTL
fpsgo_update_max_util_windows += 1;
/* periodic: 32ms */
if (fpsgo_update_max_util_windows < 4)
skip_mintop = true;
else
fpsgo_update_max_util_windows = 0;
#endif
spin_unlock_irqrestore(&gb_max_util_lock, flag);
for_each_possible_cpu(cpu) {
cpu_overutil = &per_cpu(cpu_overutil_state, cpu);
if (cpu_online(cpu) &&
(cpu_overutil->max_task_util > max_util)) {
max_util = cpu_overutil->max_task_util;
max_cpu = cpu;
max_task_pid = cpu_overutil->max_task_pid;
}
if (fpsgo_sched_nominate_fp) {
tasks[cpu] = cpu_overutil->max_task_pid;
utils[cpu] = cpu_overutil->max_task_util;
}
cpu_overutil->max_task_util = 0;
cpu_overutil->max_task_pid = 0;
}
gb_task_util = max_util;
gb_task_pid = max_task_pid;
gb_task_cpu = max_cpu;
#ifdef CONFIG_MTK_CORE_CTL
if (skip_mintop)
return;
#endif
if (fpsgo_sched_nominate_fp)
fpsgo_sched_nominate_fp(&tasks[0], &utils[0]);
}
void sched_max_util_task(int *cpu, int *pid, int *util, int *boost)
{
if (cpu)
*cpu = gb_task_cpu;
if (pid)
*pid = gb_task_pid;
if (util)
*util = gb_task_util;
}
EXPORT_SYMBOL(sched_max_util_task);
// sched_avg.c
void overutil_thresh_chg_notify(void)
{
int cpu;
unsigned long flags;
struct task_struct *p;
enum overutil_type_t over_type = NO_OVERUTIL;
int nr_overutil_l, nr_overutil_h;
struct overutil_stats_t *cpu_overutil;
int cid;
unsigned int overutil_threshold = 100;
int cluster_nr = arch_nr_clusters();
for_each_possible_cpu(cpu) {
u64 curr_time = sched_clock();
nr_overutil_l = 0;
nr_overutil_h = 0;
cid = arch_get_cluster_id(cpu);
cpu_overutil = &per_cpu(cpu_overutil_state, cpu);
if (cid < 0 || cid >= cluster_nr) {
printk_deferred_once("%s: cid=%d is out of nr=%d\n",
__func__, cid, cluster_nr);
continue;
}
raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags); /* rq-lock */
/* update threshold */
spin_lock(&per_cpu(nr_heavy_lock, cpu)); /* heavy-lock */
if (cid == 0)
cpu_overutil->overutil_thresh_l = INT_MAX;
else {
overutil_threshold = get_overutil_threshold(cid-1);
cpu_overutil->overutil_thresh_l =
(int)(cluster_heavy_tbl[cluster_nr-1].max_capacity*
overutil_threshold)/100;
}
if (cid == cluster_nr-1)
cpu_overutil->overutil_thresh_h = INT_MAX;
else {
overutil_threshold = get_overutil_threshold(cid);
cpu_overutil->overutil_thresh_h =
(int)(cluster_heavy_tbl[cluster_nr-1].max_capacity*
overutil_threshold)/100;
}
spin_unlock(&per_cpu(nr_heavy_lock, cpu)); /* heavy-unlock */
/* pick next cpu if not online */
if (!cpu_online(cpu)) {
/* rq-unlock */
raw_spin_unlock_irqrestore(&cpu_rq(cpu)->lock, flags);
continue;
}
/* re-calculate overutil counting by updated threshold */
list_for_each_entry(p, &cpu_rq(cpu)->cfs_tasks, se.group_node) {
over_type = is_task_overutil(p);
if (over_type) {
if (over_type == H_OVERUTIL) {
nr_overutil_l++;
nr_overutil_h++;
} else {
nr_overutil_l++;
}
}
}
/* Threshold for heavy is changed. Need to reset stats */
spin_lock(&per_cpu(nr_heavy_lock, cpu)); /* heavy-lock */
cpu_overutil->nr_overutil_h = nr_overutil_h;
cpu_overutil->nr_overutil_h_prod_sum = 0;
cpu_overutil->h_last_update_time = curr_time;
cpu_overutil->nr_overutil_l = nr_overutil_l;
cpu_overutil->nr_overutil_l_prod_sum = 0;
cpu_overutil->l_last_update_time = curr_time;
spin_unlock(&per_cpu(nr_heavy_lock, cpu)); /* heavy-unlock */
/* rq-unlock */
raw_spin_unlock_irqrestore(&cpu_rq(cpu)->lock, flags);
}
}
static void arch_get_cluster_cpus(struct cpumask *cpus, int cid)
{
unsigned int cpu;
cpumask_clear(cpus);
for_each_possible_cpu(cpu) {
struct cpu_topology *cpu_topo = &cpu_topology[cpu];
#ifdef CONFIG_ARM64
if (cpu_topo->package_id == cid)
#else
if (cpu_topo->socket_id == cid)
#endif
cpumask_set_cpu(cpu, cpus);
}
}
int sched_get_nr_overutil_avg(int cluster_id,
int *l_avg,
int *h_avg,
int *sum_nr_overutil_l,
int *sum_nr_overutil_h,
int *max_nr)
{
u64 curr_time = sched_clock();
s64 diff;
u64 l_tmp_avg = 0, h_tmp_avg = 0;
u32 cpumask = 0;
bool clk_faulty = 0;
unsigned long flags;
int cpu = 0;
int cluster_nr;
struct cpumask cls_cpus;
u64 tmp_max_nr = 0;
/* Need to make sure initialization done. */
if (!init_heavy) {
*l_avg = *h_avg = *max_nr = 0;
*sum_nr_overutil_l = *sum_nr_overutil_h = 0;
return -1;
}
/* cluster_id need reasonale. */
cluster_nr = arch_nr_clusters();
if (cluster_id < 0 || cluster_id >= cluster_nr) {
printk_deferred_once("[%s] invalid cluster id %d\n",
__func__, cluster_id);
return -1;
}
/* Time diff can't be zero/negative. */
diff = (s64)(curr_time -
cluster_heavy_tbl[cluster_id].last_get_overutil_time);
if (diff <= 0) {
*l_avg = *h_avg = *max_nr = 0;
*sum_nr_overutil_l = *sum_nr_overutil_h = 0;
return -1;
}
arch_get_cluster_cpus(&cls_cpus, cluster_id);
cluster_heavy_tbl[cluster_id].last_get_overutil_time = curr_time;
/* visit all cpus of this cluster */
for_each_cpu(cpu, &cls_cpus) {
struct overutil_stats_t *cpu_overutil;
spin_lock_irqsave(&per_cpu(nr_heavy_lock, cpu), flags);
cpu_overutil = &per_cpu(cpu_overutil_state, cpu);
if ((s64) (curr_time - cpu_overutil->l_last_update_time < 0)) {
clk_faulty = 1;
cpumask |= 1 << cpu;
spin_unlock_irqrestore(&per_cpu(nr_heavy_lock, cpu), flags);
break;
}
/* get max_nr */
tmp_max_nr = per_cpu(nr_max, cpu);
if (tmp_max_nr > *max_nr)
*max_nr = tmp_max_nr;
/* reset max_nr value */
per_cpu(nr_max, cpu) = per_cpu(nr, cpu);
/* get sum of nr_overutil */
*sum_nr_overutil_l += cpu_overutil->nr_overutil_l;
*sum_nr_overutil_h += cpu_overutil->nr_overutil_h;
/* get prod sum */
l_tmp_avg += cpu_overutil->nr_overutil_l_prod_sum;
l_tmp_avg += cpu_overutil->nr_overutil_l *
(curr_time - cpu_overutil->l_last_update_time);
h_tmp_avg += cpu_overutil->nr_overutil_h_prod_sum;
h_tmp_avg += cpu_overutil->nr_overutil_h *
(curr_time - cpu_overutil->h_last_update_time);
/* update last update time */
cpu_overutil->l_last_update_time = curr_time;
cpu_overutil->h_last_update_time = curr_time;
/* clear prod_sum */
cpu_overutil->nr_overutil_l_prod_sum = 0;
cpu_overutil->nr_overutil_h_prod_sum = 0;
spin_unlock_irqrestore(&per_cpu(nr_heavy_lock, cpu), flags);
}
if (clk_faulty) {
*l_avg = *h_avg = *max_nr = 0;
*sum_nr_overutil_l = *sum_nr_overutil_h = 0;
return -1;
}
*l_avg = (int)div64_u64(l_tmp_avg * 100, (u64) diff);
*h_avg = (int)div64_u64(h_tmp_avg * 100, (u64) diff);
return 0;
}
EXPORT_SYMBOL(sched_get_nr_overutil_avg);
/**
* sched_update_nr_prod
* @cpu: The core id of the nr running driver.
* @nr: Updated nr running value for cpu.
* @inc: Whether we are increasing or decreasing the count
* @return: N/A
*
* Update average with latest nr_running value for CPU
*/
void sched_update_nr_prod(int cpu, unsigned long nr_running, int inc)
{
s64 diff;
u64 curr_time;
unsigned long flags;
spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);
curr_time = sched_clock();
diff = (s64) (curr_time - per_cpu(last_time, cpu));
/* skip this problematic clock violation */
if (diff < 0) {
spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
return;
}
/* ////////////////////////////////////// */
per_cpu(last_time, cpu) = curr_time;
per_cpu(nr, cpu) = nr_running + inc;
if (per_cpu(nr, cpu) < 0)
printk_deferred_once("assertion failed at %s:%d\n",
__FILE__,
__LINE__);
#ifdef CONFIG_MTK_CORE_CTL
spin_lock(&per_cpu(nr_heavy_lock, cpu));
if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu))
per_cpu(nr_max, cpu) = per_cpu(nr, cpu);
spin_unlock(&per_cpu(nr_heavy_lock, cpu));
#endif
spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
}
EXPORT_SYMBOL(sched_update_nr_prod);
int get_overutil_stats(char *buf, int buf_size)
{
int cpu;
int len = 0;
struct overutil_stats_t *cpu_overutil;
for_each_possible_cpu(cpu) {
cpu_overutil = &per_cpu(cpu_overutil_state, cpu);
len += snprintf(buf+len, buf_size-len,
"cpu=%d capacity=%lu overutil_l=%d overutil_h=%d\n",
cpu, cpu_rq(cpu)->cpu_capacity_orig,
cpu_overutil->overutil_thresh_l,
cpu_overutil->overutil_thresh_h);
}
for_each_possible_cpu(cpu) {
cpu_overutil = &per_cpu(cpu_overutil_state, cpu);
len += snprintf(buf+len, buf_size-len,
"cpu=%d nr_overutil_l=%d nr_overutil_h=%d\n",
cpu, cpu_overutil->nr_overutil_l,
cpu_overutil->nr_overutil_h);
}
return len;
}
void sched_update_nr_heavy_prod(int invoker, struct task_struct *p,
int cpu, int heavy_nr_inc, bool ack_cap_req)
{
s64 diff;
u64 curr_time;
unsigned long flags;
enum overutil_type_t over_type = NO_OVERUTIL;
if (!init_heavy) {
init_heavy_tlb();
if (!init_heavy) {
printk_deferred_once("assertion failed at %s:%d\n",
__FILE__,
__LINE__);
return;
}
}
spin_lock_irqsave(&per_cpu(nr_heavy_lock, cpu), flags);
curr_time = sched_clock();
over_type = is_task_overutil(p);
if (over_type) {
struct overutil_stats_t *cpu_overutil;
cpu_overutil = &per_cpu(cpu_overutil_state, cpu);
if (over_type == H_OVERUTIL) {
/* H_OVERUTIL */
diff = (s64) (curr_time -
cpu_overutil->l_last_update_time);
/* update overutil for degrading threshold */
if (diff >= 0) {
cpu_overutil->l_last_update_time = curr_time;
cpu_overutil->nr_overutil_l_prod_sum +=
cpu_overutil->nr_overutil_l*diff;
cpu_overutil->nr_overutil_l +=
heavy_nr_inc;
}
diff = (s64) (curr_time -
cpu_overutil->h_last_update_time);
/* update overutil for upgrading threshold */
if (diff >= 0) {
cpu_overutil->h_last_update_time = curr_time;
cpu_overutil->nr_overutil_h_prod_sum +=
cpu_overutil->nr_overutil_h*diff;
cpu_overutil->nr_overutil_h +=
heavy_nr_inc;
}
} else {/* L_OVERUTIL */
diff = (s64) (curr_time -
cpu_overutil->l_last_update_time);
if (diff >= 0) {
cpu_overutil->l_last_update_time = curr_time;
cpu_overutil->nr_overutil_l_prod_sum +=
cpu_overutil->nr_overutil_l*diff;
cpu_overutil->nr_overutil_l +=
heavy_nr_inc;
}
}
}
spin_unlock_irqrestore(&per_cpu(nr_heavy_lock, cpu), flags);
}
EXPORT_SYMBOL(sched_update_nr_heavy_prod);
#define MAX_CLUSTER_NR 3
/*
* if all core ids are set correctly, then return true.
* That is meaning that there is no repeat core id in same
* cluster.
*/
static inline bool is_all_cpu_parsed(void)
{
#ifdef CONFIG_ARM64
int cpu = 0, core_id = 0, cluster_id = 0;
bool all_parsed = true;
struct cpumask cluster_mask[MAX_CLUSTER_NR];
struct cpumask *tmp_cpumask = NULL;
for (cluster_id = 0; cluster_id < MAX_CLUSTER_NR; cluster_id++)
cpumask_clear(&cluster_mask[cluster_id]);
for_each_possible_cpu(cpu) {
core_id = cpu_topology[cpu].core_id;
cluster_id = cpu_topology[cpu].package_id;
if (core_id < 0 || cluster_id < 0) {
all_parsed = false;
break;
}
tmp_cpumask = &cluster_mask[cluster_id];
if (cpumask_test_cpu(core_id, tmp_cpumask)) {
all_parsed = false;
break;
}
cpumask_set_cpu(core_id, tmp_cpumask);
}
return all_parsed;
#else
return true;
#endif
}
static int init_heavy_tlb(void)
{
if (!is_all_cpu_parsed())
return init_heavy;
if (!init_heavy) {
/* init variables */
int tmp_cpu, cluster_nr;
int i;
int cid;
struct cpumask cls_cpus;
struct overutil_stats_t *cpu_overutil;
unsigned int overutil_threshold = 100;
printk_deferred_once("%s start.\n", __func__);
gb_task_util = 0;
gb_task_pid = 0;
gb_task_cpu = 0;
/* allocation for clustser information */
cluster_nr = arch_nr_clusters();
if (cluster_nr <= 0)
return 0;
cluster_heavy_tbl = kcalloc(cluster_nr,
sizeof(struct cluster_heavy_tbl_t), GFP_ATOMIC);
if (!cluster_heavy_tbl)
return 0;
for (i = 0; i < cluster_nr; i++) {
arch_get_cluster_cpus(&cls_cpus, i);
tmp_cpu = cpumask_first(&cls_cpus);
/* replace cpu_rq(cpu)->cpu_capacity_orig by
* get_cpu_orig_capacity()
*/
cluster_heavy_tbl[i].max_capacity =
capacity_orig_of(tmp_cpu);
}
for_each_possible_cpu(tmp_cpu) {
cpu_overutil = &per_cpu(cpu_overutil_state, tmp_cpu);
cid = arch_get_cluster_id(tmp_cpu);
/* reset nr_heavy */
per_cpu(nr_heavy, tmp_cpu) = 0;
/* reset nr_overutil */
cpu_overutil->nr_overutil_l = 0;
cpu_overutil->nr_overutil_h = 0;
cpu_overutil->max_task_util = 0;
cpu_overutil->max_task_pid = 0;
if (cid < 0 || cid >= cluster_nr) {
printk_deferred_once("%s: cid=%d is out of nr=%d\n", __func__,
cid, cluster_nr);
continue;
}
/*
* Initialize threshold for up/down
* over-utilization tracking
*/
if (cid == 0)
cpu_overutil->overutil_thresh_l = INT_MAX;
else {
overutil_threshold = get_overutil_threshold(cid-1);
cpu_overutil->overutil_thresh_l =
(int)(cluster_heavy_tbl[cluster_nr-1].max_capacity*
overutil_threshold)/100;
}
if (cid == cluster_nr-1)
cpu_overutil->overutil_thresh_h = INT_MAX;
else {
overutil_threshold = get_overutil_threshold(cid);
cpu_overutil->overutil_thresh_h =
(int)(cluster_heavy_tbl[cluster_nr-1].max_capacity*
overutil_threshold)/100;
}
printk_deferred_once("%s: cpu=%d thresh_l=%d thresh_h=%d max_capaicy=%lu\n",
__func__, tmp_cpu,
cpu_overutil->overutil_thresh_l,
cpu_overutil->overutil_thresh_h,
(unsigned long)
cluster_heavy_tbl[cid].max_capacity);
}
init_heavy = 1;
}
return init_heavy;
}