248 lines
6.6 KiB
C
248 lines
6.6 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_SCHED_TOPOLOGY_H
|
|
#define _LINUX_SCHED_TOPOLOGY_H
|
|
|
|
#include <linux/topology.h>
|
|
#include <linux/android_kabi.h>
|
|
|
|
#include <linux/sched/idle.h>
|
|
|
|
/*
|
|
* sched-domains (multiprocessor balancing) declarations:
|
|
*/
|
|
#ifdef CONFIG_SMP
|
|
|
|
#define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */
|
|
#define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */
|
|
#define SD_BALANCE_EXEC 0x0004 /* Balance on exec */
|
|
#define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */
|
|
#define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */
|
|
#define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */
|
|
#define SD_ASYM_CPUCAPACITY 0x0040 /* Domain members have different CPU capacities */
|
|
#define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share CPU capacity */
|
|
#define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */
|
|
#define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share CPU pkg resources */
|
|
#define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */
|
|
#define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */
|
|
#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */
|
|
#define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */
|
|
#define SD_NUMA 0x4000 /* cross-node balancing */
|
|
|
|
#ifdef CONFIG_SCHED_SMT
|
|
static inline int cpu_smt_flags(void)
|
|
{
|
|
return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_SCHED_MC
|
|
static inline int cpu_core_flags(void)
|
|
{
|
|
return SD_SHARE_PKG_RESOURCES;
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_NUMA
|
|
static inline int cpu_numa_flags(void)
|
|
{
|
|
return SD_NUMA;
|
|
}
|
|
#endif
|
|
|
|
extern int arch_asym_cpu_priority(int cpu);
|
|
|
|
struct sched_domain_attr {
|
|
int relax_domain_level;
|
|
};
|
|
|
|
#define SD_ATTR_INIT (struct sched_domain_attr) { \
|
|
.relax_domain_level = -1, \
|
|
}
|
|
|
|
extern int sched_domain_level_max;
|
|
|
|
struct sched_group;
|
|
|
|
struct sched_domain_shared {
|
|
atomic_t ref;
|
|
atomic_t nr_busy_cpus;
|
|
int has_idle_cores;
|
|
|
|
bool overutilized;
|
|
};
|
|
|
|
struct sched_domain {
|
|
/* These fields must be setup */
|
|
struct sched_domain *parent; /* top domain must be null terminated */
|
|
struct sched_domain *child; /* bottom domain must be null terminated */
|
|
struct sched_group *groups; /* the balancing groups of the domain */
|
|
unsigned long min_interval; /* Minimum balance interval ms */
|
|
unsigned long max_interval; /* Maximum balance interval ms */
|
|
unsigned int busy_factor; /* less balancing by factor if busy */
|
|
unsigned int imbalance_pct; /* No balance until over watermark */
|
|
unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */
|
|
unsigned int busy_idx;
|
|
unsigned int idle_idx;
|
|
unsigned int newidle_idx;
|
|
unsigned int wake_idx;
|
|
unsigned int forkexec_idx;
|
|
unsigned int smt_gain;
|
|
|
|
int nohz_idle; /* NOHZ IDLE status */
|
|
int flags; /* See SD_* */
|
|
int level;
|
|
|
|
/* Runtime fields. */
|
|
unsigned long last_balance; /* init to jiffies. units in jiffies */
|
|
unsigned int balance_interval; /* initialise to 1. units in ms. */
|
|
unsigned int nr_balance_failed; /* initialise to 0 */
|
|
|
|
/* idle_balance() stats */
|
|
u64 max_newidle_lb_cost;
|
|
unsigned long next_decay_max_lb_cost;
|
|
|
|
u64 avg_scan_cost; /* select_idle_sibling */
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
/* load_balance() stats */
|
|
unsigned int lb_count[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_failed[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_balanced[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_gained[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES];
|
|
|
|
/* Active load balancing */
|
|
unsigned int alb_count;
|
|
unsigned int alb_failed;
|
|
unsigned int alb_pushed;
|
|
|
|
/* SD_BALANCE_EXEC stats */
|
|
unsigned int sbe_count;
|
|
unsigned int sbe_balanced;
|
|
unsigned int sbe_pushed;
|
|
|
|
/* SD_BALANCE_FORK stats */
|
|
unsigned int sbf_count;
|
|
unsigned int sbf_balanced;
|
|
unsigned int sbf_pushed;
|
|
|
|
/* try_to_wake_up() stats */
|
|
unsigned int ttwu_wake_remote;
|
|
unsigned int ttwu_move_affine;
|
|
unsigned int ttwu_move_balance;
|
|
#endif
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
char *name;
|
|
#endif
|
|
union {
|
|
void *private; /* used during construction */
|
|
struct rcu_head rcu; /* used during destruction */
|
|
};
|
|
struct sched_domain_shared *shared;
|
|
|
|
unsigned int span_weight;
|
|
|
|
ANDROID_KABI_RESERVE(1);
|
|
ANDROID_KABI_RESERVE(2);
|
|
|
|
/*
|
|
* Span of all CPUs in this domain.
|
|
*
|
|
* NOTE: this field is variable length. (Allocated dynamically
|
|
* by attaching extra space to the end of the structure,
|
|
* depending on how many CPUs the kernel has booted up with)
|
|
*/
|
|
unsigned long span[0];
|
|
};
|
|
|
|
static inline struct cpumask *sched_domain_span(struct sched_domain *sd)
|
|
{
|
|
return to_cpumask(sd->span);
|
|
}
|
|
|
|
extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
|
|
struct sched_domain_attr *dattr_new);
|
|
|
|
/* Allocate an array of sched domains, for partition_sched_domains(). */
|
|
cpumask_var_t *alloc_sched_domains(unsigned int ndoms);
|
|
void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);
|
|
|
|
bool cpus_share_cache(int this_cpu, int that_cpu);
|
|
|
|
typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
|
|
typedef int (*sched_domain_flags_f)(void);
|
|
|
|
#define SDTL_OVERLAP 0x01
|
|
|
|
struct sd_data {
|
|
struct sched_domain *__percpu *sd;
|
|
struct sched_domain_shared *__percpu *sds;
|
|
struct sched_group *__percpu *sg;
|
|
struct sched_group_capacity *__percpu *sgc;
|
|
};
|
|
|
|
struct sched_domain_topology_level {
|
|
sched_domain_mask_f mask;
|
|
sched_domain_flags_f sd_flags;
|
|
int flags;
|
|
int numa_level;
|
|
struct sd_data data;
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
char *name;
|
|
#endif
|
|
};
|
|
|
|
extern void set_sched_topology(struct sched_domain_topology_level *tl);
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
# define SD_INIT_NAME(type) .name = #type
|
|
#else
|
|
# define SD_INIT_NAME(type)
|
|
#endif
|
|
|
|
#ifndef arch_scale_cpu_capacity
|
|
static __always_inline
|
|
unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu)
|
|
{
|
|
if (sd && (sd->flags & SD_SHARE_CPUCAPACITY) && (sd->span_weight > 1))
|
|
return sd->smt_gain / sd->span_weight;
|
|
|
|
return SCHED_CAPACITY_SCALE;
|
|
}
|
|
#endif
|
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
struct sched_domain_attr;
|
|
|
|
static inline void
|
|
partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
|
|
struct sched_domain_attr *dattr_new)
|
|
{
|
|
}
|
|
|
|
static inline bool cpus_share_cache(int this_cpu, int that_cpu)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
#ifndef arch_scale_cpu_capacity
|
|
static __always_inline
|
|
unsigned long arch_scale_cpu_capacity(void __always_unused *sd, int cpu)
|
|
{
|
|
return SCHED_CAPACITY_SCALE;
|
|
}
|
|
#endif
|
|
|
|
#endif /* !CONFIG_SMP */
|
|
|
|
static inline int task_node(const struct task_struct *p)
|
|
{
|
|
return cpu_to_node(task_cpu(p));
|
|
}
|
|
|
|
#endif /* _LINUX_SCHED_TOPOLOGY_H */
|