/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_SCHED_TOPOLOGY_H #define _LINUX_SCHED_TOPOLOGY_H #include #include #include /* * sched-domains (multiprocessor balancing) declarations: */ #ifdef CONFIG_SMP #define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */ #define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */ #define SD_BALANCE_EXEC 0x0004 /* Balance on exec */ #define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */ #define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */ #define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */ #define SD_ASYM_CPUCAPACITY 0x0040 /* Domain members have different CPU capacities */ #define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share CPU capacity */ #define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */ #define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share CPU pkg resources */ #define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */ #define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ #define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ #define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ #define SD_NUMA 0x4000 /* cross-node balancing */ #ifdef CONFIG_SCHED_SMT static inline int cpu_smt_flags(void) { return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES; } #endif #ifdef CONFIG_SCHED_MC static inline int cpu_core_flags(void) { return SD_SHARE_PKG_RESOURCES; } #endif #ifdef CONFIG_NUMA static inline int cpu_numa_flags(void) { return SD_NUMA; } #endif extern int arch_asym_cpu_priority(int cpu); struct sched_domain_attr { int relax_domain_level; }; #define SD_ATTR_INIT (struct sched_domain_attr) { \ .relax_domain_level = -1, \ } extern int sched_domain_level_max; struct sched_group; struct sched_domain_shared { atomic_t ref; atomic_t nr_busy_cpus; int has_idle_cores; bool overutilized; }; struct sched_domain { /* These fields must be setup */ struct sched_domain *parent; /* top domain must be null terminated */ struct sched_domain *child; /* bottom domain must be null terminated */ struct sched_group *groups; /* the balancing groups of the domain */ unsigned long min_interval; /* Minimum balance interval ms */ unsigned long max_interval; /* Maximum balance interval ms */ unsigned int busy_factor; /* less balancing by factor if busy */ unsigned int imbalance_pct; /* No balance until over watermark */ unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */ unsigned int busy_idx; unsigned int idle_idx; unsigned int newidle_idx; unsigned int wake_idx; unsigned int forkexec_idx; unsigned int smt_gain; int nohz_idle; /* NOHZ IDLE status */ int flags; /* See SD_* */ int level; /* Runtime fields. */ unsigned long last_balance; /* init to jiffies. units in jiffies */ unsigned int balance_interval; /* initialise to 1. units in ms. */ unsigned int nr_balance_failed; /* initialise to 0 */ /* idle_balance() stats */ u64 max_newidle_lb_cost; unsigned long next_decay_max_lb_cost; u64 avg_scan_cost; /* select_idle_sibling */ #ifdef CONFIG_SCHEDSTATS /* load_balance() stats */ unsigned int lb_count[CPU_MAX_IDLE_TYPES]; unsigned int lb_failed[CPU_MAX_IDLE_TYPES]; unsigned int lb_balanced[CPU_MAX_IDLE_TYPES]; unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES]; unsigned int lb_gained[CPU_MAX_IDLE_TYPES]; unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES]; unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES]; unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES]; /* Active load balancing */ unsigned int alb_count; unsigned int alb_failed; unsigned int alb_pushed; /* SD_BALANCE_EXEC stats */ unsigned int sbe_count; unsigned int sbe_balanced; unsigned int sbe_pushed; /* SD_BALANCE_FORK stats */ unsigned int sbf_count; unsigned int sbf_balanced; unsigned int sbf_pushed; /* try_to_wake_up() stats */ unsigned int ttwu_wake_remote; unsigned int ttwu_move_affine; unsigned int ttwu_move_balance; #endif #ifdef CONFIG_SCHED_DEBUG char *name; #endif union { void *private; /* used during construction */ struct rcu_head rcu; /* used during destruction */ }; struct sched_domain_shared *shared; unsigned int span_weight; ANDROID_KABI_RESERVE(1); ANDROID_KABI_RESERVE(2); /* * Span of all CPUs in this domain. * * NOTE: this field is variable length. (Allocated dynamically * by attaching extra space to the end of the structure, * depending on how many CPUs the kernel has booted up with) */ unsigned long span[0]; }; static inline struct cpumask *sched_domain_span(struct sched_domain *sd) { return to_cpumask(sd->span); } extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], struct sched_domain_attr *dattr_new); /* Allocate an array of sched domains, for partition_sched_domains(). */ cpumask_var_t *alloc_sched_domains(unsigned int ndoms); void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); bool cpus_share_cache(int this_cpu, int that_cpu); typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); typedef int (*sched_domain_flags_f)(void); #define SDTL_OVERLAP 0x01 struct sd_data { struct sched_domain *__percpu *sd; struct sched_domain_shared *__percpu *sds; struct sched_group *__percpu *sg; struct sched_group_capacity *__percpu *sgc; }; struct sched_domain_topology_level { sched_domain_mask_f mask; sched_domain_flags_f sd_flags; int flags; int numa_level; struct sd_data data; #ifdef CONFIG_SCHED_DEBUG char *name; #endif }; extern void set_sched_topology(struct sched_domain_topology_level *tl); #ifdef CONFIG_SCHED_DEBUG # define SD_INIT_NAME(type) .name = #type #else # define SD_INIT_NAME(type) #endif #ifndef arch_scale_cpu_capacity static __always_inline unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) { if (sd && (sd->flags & SD_SHARE_CPUCAPACITY) && (sd->span_weight > 1)) return sd->smt_gain / sd->span_weight; return SCHED_CAPACITY_SCALE; } #endif #else /* CONFIG_SMP */ struct sched_domain_attr; static inline void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], struct sched_domain_attr *dattr_new) { } static inline bool cpus_share_cache(int this_cpu, int that_cpu) { return true; } #ifndef arch_scale_cpu_capacity static __always_inline unsigned long arch_scale_cpu_capacity(void __always_unused *sd, int cpu) { return SCHED_CAPACITY_SCALE; } #endif #endif /* !CONFIG_SMP */ static inline int task_node(const struct task_struct *p) { return cpu_to_node(task_cpu(p)); } #endif /* _LINUX_SCHED_TOPOLOGY_H */