| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Christoph Lameter | 2681 | 31.84% | 37 | 11.56% |
| Mel Gorman | 2174 | 25.82% | 31 | 9.69% |
| Kemi Wang | 514 | 6.10% | 2 | 0.62% |
| Andrew Morton | 445 | 5.29% | 22 | 6.88% |
| Motohiro Kosaki | 237 | 2.81% | 7 | 2.19% |
| Linus Torvalds (pre-git) | 190 | 2.26% | 18 | 5.62% |
| Hugh Dickins | 176 | 2.09% | 4 | 1.25% |
| Pavel Tatashin | 157 | 1.86% | 4 | 1.25% |
| Sebastian Andrzej Siewior | 125 | 1.48% | 3 | 0.94% |
| Michal Hocko | 119 | 1.41% | 9 | 2.81% |
| JoonSoo Kim | 113 | 1.34% | 4 | 1.25% |
| Johannes Weiner | 97 | 1.15% | 11 | 3.44% |
| Roman Gushchin | 87 | 1.03% | 2 | 0.62% |
| Geert Uytterhoeven | 77 | 0.91% | 1 | 0.31% |
| Alexey Dobriyan | 74 | 0.88% | 6 | 1.88% |
| MinChan Kim | 69 | 0.82% | 7 | 2.19% |
| Konstantin Khlebnikov | 65 | 0.77% | 4 | 1.25% |
| David Rientjes | 65 | 0.77% | 5 | 1.56% |
| Nicholas Piggin | 42 | 0.50% | 4 | 1.25% |
| Kirill A. Shutemov | 40 | 0.48% | 9 | 2.81% |
| Song Muchun | 40 | 0.48% | 1 | 0.31% |
| Huang Ying | 39 | 0.46% | 7 | 2.19% |
| Kamezawa Hiroyuki | 36 | 0.43% | 2 | 0.62% |
| Uros Bizjak | 34 | 0.40% | 1 | 0.31% |
| Nitin Gupta | 33 | 0.39% | 2 | 0.62% |
| Ingo Molnar | 31 | 0.37% | 2 | 0.62% |
| Vinayak Menon | 27 | 0.32% | 1 | 0.31% |
| Michael Rubin | 27 | 0.32% | 1 | 0.31% |
| Toshi Kani | 25 | 0.30% | 2 | 0.62% |
| Tim Chen | 23 | 0.27% | 1 | 0.31% |
| Nikita Danilov | 23 | 0.27% | 1 | 0.31% |
| Dave Hansen | 23 | 0.27% | 4 | 1.25% |
| Yang Yang | 20 | 0.24% | 3 | 0.94% |
| Andrea Arcangeli | 20 | 0.24% | 1 | 0.31% |
| Neil Brown | 19 | 0.23% | 1 | 0.31% |
| Thomas Gleixner | 18 | 0.21% | 3 | 0.94% |
| Linus Torvalds | 15 | 0.18% | 5 | 1.56% |
| Miaohe Lin | 15 | 0.18% | 3 | 0.94% |
| Shakeel Butt | 15 | 0.18% | 3 | 0.94% |
| Oscar Salvador | 15 | 0.18% | 1 | 0.31% |
| Eric W. Biedermann | 14 | 0.17% | 4 | 1.25% |
| Hao Lee | 14 | 0.17% | 1 | 0.31% |
| Andi Kleen | 13 | 0.15% | 1 | 0.31% |
| Rohit Seth | 13 | 0.15% | 1 | 0.31% |
| Suren Baghdasaryan | 13 | 0.15% | 1 | 0.31% |
| Marcelo Tosatti | 12 | 0.14% | 1 | 0.31% |
| Baoquan He | 12 | 0.14% | 1 | 0.31% |
| Jiang Liu | 12 | 0.14% | 2 | 0.62% |
| Hideaki Yoshifuji / 吉藤英明 | 12 | 0.14% | 1 | 0.31% |
| Heiko Carstens | 12 | 0.14% | 3 | 0.94% |
| Kefeng Wang | 10 | 0.12% | 1 | 0.31% |
| Wen Yang | 10 | 0.12% | 1 | 0.31% |
| Anshuman Khandual | 9 | 0.11% | 2 | 0.62% |
| Saravanan D | 9 | 0.11% | 1 | 0.31% |
| Lee Schermerhorn | 9 | 0.11% | 3 | 0.94% |
| Christoph Hellwig | 8 | 0.10% | 1 | 0.31% |
| Liu Shixin | 8 | 0.10% | 1 | 0.31% |
| Anton Blanchard | 8 | 0.10% | 1 | 0.31% |
| Adam Litke | 7 | 0.08% | 1 | 0.31% |
| Vlastimil Babka | 7 | 0.08% | 3 | 0.94% |
| Arun K S | 6 | 0.07% | 1 | 0.31% |
| Sami Tolvanen | 6 | 0.07% | 1 | 0.31% |
| Jens Axboe | 6 | 0.07% | 2 | 0.62% |
| Janne Huttunen | 6 | 0.07% | 1 | 0.31% |
| David Hildenbrand | 6 | 0.07% | 1 | 0.31% |
| Kaiyang Zhao | 6 | 0.07% | 1 | 0.31% |
| Michal Nazarewicz | 5 | 0.06% | 1 | 0.31% |
| Greg Kroah-Hartman | 5 | 0.06% | 2 | 0.62% |
| Rik Van Riel | 5 | 0.06% | 1 | 0.31% |
| Liangcai Fan | 5 | 0.06% | 1 | 0.31% |
| Cédric Le Goater | 5 | 0.06% | 1 | 0.31% |
| Fengguang Wu | 5 | 0.06% | 1 | 0.31% |
| John Hubbard | 4 | 0.05% | 1 | 0.31% |
| Song Liu | 4 | 0.05% | 1 | 0.31% |
| Pekka J Enberg | 4 | 0.05% | 1 | 0.31% |
| Yury Norov | 4 | 0.05% | 1 | 0.31% |
| Jann Horn | 4 | 0.05% | 1 | 0.31% |
| David Howells | 4 | 0.05% | 1 | 0.31% |
| Helge Deller | 4 | 0.05% | 1 | 0.31% |
| Yang Shi | 4 | 0.05% | 1 | 0.31% |
| Barry Song | 4 | 0.05% | 1 | 0.31% |
| Yisheng Xie | 4 | 0.05% | 1 | 0.31% |
| Jiang Biao | 3 | 0.04% | 1 | 0.31% |
| Namhyung Kim | 3 | 0.04% | 1 | 0.31% |
| Cody P Schafer | 3 | 0.04% | 1 | 0.31% |
| Dimitri Sivanich | 3 | 0.04% | 2 | 0.62% |
| Matthew Wilcox | 3 | 0.04% | 1 | 0.31% |
| Uwe Kleine-König | 3 | 0.04% | 1 | 0.31% |
| Adrian Bunk | 3 | 0.04% | 1 | 0.31% |
| Dmitry Torokhov | 2 | 0.02% | 1 | 0.31% |
| Rusty Russell | 2 | 0.02% | 1 | 0.31% |
| Arjan van de Ven | 2 | 0.02% | 1 | 0.31% |
| Domenico Cerasuolo | 2 | 0.02% | 1 | 0.31% |
| Li Zhijian | 2 | 0.02% | 1 | 0.31% |
| Shaohua Li | 2 | 0.02% | 1 | 0.31% |
| Joel Granados | 2 | 0.02% | 1 | 0.31% |
| Usama Arif | 2 | 0.02% | 1 | 0.31% |
| Peter Xu | 2 | 0.02% | 1 | 0.31% |
| Yosry Ahmed | 2 | 0.02% | 1 | 0.31% |
| Matthew Cassell | 2 | 0.02% | 1 | 0.31% |
| Ying Han | 2 | 0.02% | 1 | 0.31% |
| Davidlohr Bueso A | 2 | 0.02% | 1 | 0.31% |
| Lai Jiangshan | 2 | 0.02% | 1 | 0.31% |
| Sasikantha babu | 1 | 0.01% | 1 | 0.31% |
| Srivatsa S. Bhat | 1 | 0.01% | 1 | 0.31% |
| Peter Zijlstra | 1 | 0.01% | 1 | 0.31% |
| Joe Perches | 1 | 0.01% | 1 | 0.31% |
| Tejun Heo | 1 | 0.01% | 1 | 0.31% |
| Jianyu Zhan | 1 | 0.01% | 1 | 0.31% |
| Lin Feng | 1 | 0.01% | 1 | 0.31% |
| SeongJae Park | 1 | 0.01% | 1 | 0.31% |
| Total | 8420 | 320 |
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337
// SPDX-License-Identifier: GPL-2.0-only /* * linux/mm/vmstat.c * * Manages VM statistics * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds * * zoned VM statistics * Copyright (C) 2006 Silicon Graphics, Inc., * Christoph Lameter <christoph@lameter.com> * Copyright (C) 2008-2014 Christoph Lameter */ #include <linux/fs.h> #include <linux/mm.h> #include <linux/err.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/cpu.h> #include <linux/cpumask.h> #include <linux/vmstat.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/debugfs.h> #include <linux/sched.h> #include <linux/math64.h> #include <linux/writeback.h> #include <linux/compaction.h> #include <linux/mm_inline.h> #include <linux/page_owner.h> #include <linux/sched/isolation.h> #include "internal.h" #ifdef CONFIG_NUMA int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; /* zero numa counters within a zone */ static void zero_zone_numa_counters(struct zone *zone) { int item, cpu; for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) { atomic_long_set(&zone->vm_numa_event[item], 0); for_each_online_cpu(cpu) { per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item] = 0; } } } /* zero numa counters of all the populated zones */ static void zero_zones_numa_counters(void) { struct zone *zone; for_each_populated_zone(zone) zero_zone_numa_counters(zone); } /* zero global numa counters */ static void zero_global_numa_counters(void) { int item; for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) atomic_long_set(&vm_numa_event[item], 0); } static void invalid_numa_statistics(void) { zero_zones_numa_counters(); zero_global_numa_counters(); } static DEFINE_MUTEX(vm_numa_stat_lock); int sysctl_vm_numa_stat_handler(const struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) { int ret, oldval; mutex_lock(&vm_numa_stat_lock); if (write) oldval = sysctl_vm_numa_stat; ret = proc_dointvec_minmax(table, write, buffer, length, ppos); if (ret || !write) goto out; if (oldval == sysctl_vm_numa_stat) goto out; else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) { static_branch_enable(&vm_numa_stat_key); pr_info("enable numa statistics\n"); } else { static_branch_disable(&vm_numa_stat_key); invalid_numa_statistics(); pr_info("disable numa statistics, and clear numa counters\n"); } out: mutex_unlock(&vm_numa_stat_lock); return ret; } #endif #ifdef CONFIG_VM_EVENT_COUNTERS DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; EXPORT_PER_CPU_SYMBOL(vm_event_states); static void sum_vm_events(unsigned long *ret) { int cpu; int i; memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); for_each_online_cpu(cpu) { struct vm_event_state *this = &per_cpu(vm_event_states, cpu); for (i = 0; i < NR_VM_EVENT_ITEMS; i++) ret[i] += this->event[i]; } } /* * Accumulate the vm event counters across all CPUs. * The result is unavoidably approximate - it can change * during and after execution of this function. */ void all_vm_events(unsigned long *ret) { cpus_read_lock(); sum_vm_events(ret); cpus_read_unlock(); } EXPORT_SYMBOL_GPL(all_vm_events); /* * Fold the foreign cpu events into our own. * * This is adding to the events on one processor * but keeps the global counts constant. */ void vm_events_fold_cpu(int cpu) { struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); int i; for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { count_vm_events(i, fold_state->event[i]); fold_state->event[i] = 0; } } #endif /* CONFIG_VM_EVENT_COUNTERS */ /* * Manage combined zone based / global counters * * vm_stat contains the global counters */ atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp; EXPORT_SYMBOL(vm_zone_stat); EXPORT_SYMBOL(vm_node_stat); #ifdef CONFIG_NUMA static void fold_vm_zone_numa_events(struct zone *zone) { unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, }; int cpu; enum numa_stat_item item; for_each_online_cpu(cpu) { struct per_cpu_zonestat *pzstats; pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0); } for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) zone_numa_event_add(zone_numa_events[item], zone, item); } void fold_vm_numa_events(void) { struct zone *zone; for_each_populated_zone(zone) fold_vm_zone_numa_events(zone); } #endif #ifdef CONFIG_SMP int calculate_pressure_threshold(struct zone *zone) { int threshold; int watermark_distance; /* * As vmstats are not up to date, there is drift between the estimated * and real values. For high thresholds and a high number of CPUs, it * is possible for the min watermark to be breached while the estimated * value looks fine. The pressure threshold is a reduced value such * that even the maximum amount of drift will not accidentally breach * the min watermark */ watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); threshold = max(1, (int)(watermark_distance / num_online_cpus())); /* * Maximum threshold is 125 */ threshold = min(125, threshold); return threshold; } int calculate_normal_threshold(struct zone *zone) { int threshold; int mem; /* memory in 128 MB units */ /* * The threshold scales with the number of processors and the amount * of memory per zone. More memory means that we can defer updates for * longer, more processors could lead to more contention. * fls() is used to have a cheap way of logarithmic scaling. * * Some sample thresholds: * * Threshold Processors (fls) Zonesize fls(mem)+1 * ------------------------------------------------------------------ * 8 1 1 0.9-1 GB 4 * 16 2 2 0.9-1 GB 4 * 20 2 2 1-2 GB 5 * 24 2 2 2-4 GB 6 * 28 2 2 4-8 GB 7 * 32 2 2 8-16 GB 8 * 4 2 2 <128M 1 * 30 4 3 2-4 GB 5 * 48 4 3 8-16 GB 8 * 32 8 4 1-2 GB 4 * 32 8 4 0.9-1GB 4 * 10 16 5 <128M 1 * 40 16 5 900M 4 * 70 64 7 2-4 GB 5 * 84 64 7 4-8 GB 6 * 108 512 9 4-8 GB 6 * 125 1024 10 8-16 GB 8 * 125 1024 10 16-32 GB 9 */ mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); /* * Maximum threshold is 125 */ threshold = min(125, threshold); return threshold; } /* * Refresh the thresholds for each zone. */ void refresh_zone_stat_thresholds(void) { struct pglist_data *pgdat; struct zone *zone; int cpu; int threshold; /* Zero current pgdat thresholds */ for_each_online_pgdat(pgdat) { for_each_online_cpu(cpu) { per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; } } for_each_populated_zone(zone) { struct pglist_data *pgdat = zone->zone_pgdat; unsigned long max_drift, tolerate_drift; threshold = calculate_normal_threshold(zone); for_each_online_cpu(cpu) { int pgdat_threshold; per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold = threshold; /* Base nodestat threshold on the largest populated zone. */ pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = max(threshold, pgdat_threshold); } /* * Only set percpu_drift_mark if there is a danger that * NR_FREE_PAGES reports the low watermark is ok when in fact * the min watermark could be breached by an allocation */ tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); max_drift = num_online_cpus() * threshold; if (max_drift > tolerate_drift) zone->percpu_drift_mark = high_wmark_pages(zone) + max_drift; } } void set_pgdat_percpu_threshold(pg_data_t *pgdat, int (*calculate_pressure)(struct zone *)) { struct zone *zone; int cpu; int threshold; int i; for (i = 0; i < pgdat->nr_zones; i++) { zone = &pgdat->node_zones[i]; if (!zone->percpu_drift_mark) continue; threshold = (*calculate_pressure)(zone); for_each_online_cpu(cpu) per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold = threshold; } } /* * For use when we know that interrupts are disabled, * or when we know that preemption is disabled and that * particular counter cannot be updated from interrupt context. */ void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, long delta) { struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; s8 __percpu *p = pcp->vm_stat_diff + item; long x; long t; /* * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels, * atomicity is provided by IRQs being disabled -- either explicitly * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables * CPU migrations and preemption potentially corrupts a counter so * disable preemption. */ preempt_disable_nested(); x = delta + __this_cpu_read(*p); t = __this_cpu_read(pcp->stat_threshold); if (unlikely(abs(x) > t)) { zone_page_state_add(x, zone, item); x = 0; } __this_cpu_write(*p, x); preempt_enable_nested(); } EXPORT_SYMBOL(__mod_zone_page_state); void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, long delta) { struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; s8 __percpu *p = pcp->vm_node_stat_diff + item; long x; long t; if (vmstat_item_in_bytes(item)) { /* * Only cgroups use subpage accounting right now; at * the global level, these items still change in * multiples of whole pages. Store them as pages * internally to keep the per-cpu counters compact. */ VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); delta >>= PAGE_SHIFT; } /* See __mod_node_page_state */ preempt_disable_nested(); x = delta + __this_cpu_read(*p); t = __this_cpu_read(pcp->stat_threshold); if (unlikely(abs(x) > t)) { node_page_state_add(x, pgdat, item); x = 0; } __this_cpu_write(*p, x); preempt_enable_nested(); } EXPORT_SYMBOL(__mod_node_page_state); /* * Optimized increment and decrement functions. * * These are only for a single page and therefore can take a struct page * * argument instead of struct zone *. This allows the inclusion of the code * generated for page_zone(page) into the optimized functions. * * No overflow check is necessary and therefore the differential can be * incremented or decremented in place which may allow the compilers to * generate better code. * The increment or decrement is known and therefore one boundary check can * be omitted. * * NOTE: These functions are very performance sensitive. Change only * with care. * * Some processors have inc/dec instructions that are atomic vs an interrupt. * However, the code must first determine the differential location in a zone * based on the processor number and then inc/dec the counter. There is no * guarantee without disabling preemption that the processor will not change * in between and therefore the atomicity vs. interrupt cannot be exploited * in a useful way here. */ void __inc_zone_state(struct zone *zone, enum zone_stat_item item) { struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; s8 __percpu *p = pcp->vm_stat_diff + item; s8 v, t; /* See __mod_node_page_state */ preempt_disable_nested(); v = __this_cpu_inc_return(*p); t = __this_cpu_read(pcp->stat_threshold); if (unlikely(v > t)) { s8 overstep = t >> 1; zone_page_state_add(v + overstep, zone, item); __this_cpu_write(*p, -overstep); } preempt_enable_nested(); } void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) { struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; s8 __percpu *p = pcp->vm_node_stat_diff + item; s8 v, t; VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); /* See __mod_node_page_state */ preempt_disable_nested(); v = __this_cpu_inc_return(*p); t = __this_cpu_read(pcp->stat_threshold); if (unlikely(v > t)) { s8 overstep = t >> 1; node_page_state_add(v + overstep, pgdat, item); __this_cpu_write(*p, -overstep); } preempt_enable_nested(); } void __inc_zone_page_state(struct page *page, enum zone_stat_item item) { __inc_zone_state(page_zone(page), item); } EXPORT_SYMBOL(__inc_zone_page_state); void __inc_node_page_state(struct page *page, enum node_stat_item item) { __inc_node_state(page_pgdat(page), item); } EXPORT_SYMBOL(__inc_node_page_state); void __dec_zone_state(struct zone *zone, enum zone_stat_item item) { struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; s8 __percpu *p = pcp->vm_stat_diff + item; s8 v, t; /* See __mod_node_page_state */ preempt_disable_nested(); v = __this_cpu_dec_return(*p); t = __this_cpu_read(pcp->stat_threshold); if (unlikely(v < - t)) { s8 overstep = t >> 1; zone_page_state_add(v - overstep, zone, item); __this_cpu_write(*p, overstep); } preempt_enable_nested(); } void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) { struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; s8 __percpu *p = pcp->vm_node_stat_diff + item; s8 v, t; VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); /* See __mod_node_page_state */ preempt_disable_nested(); v = __this_cpu_dec_return(*p); t = __this_cpu_read(pcp->stat_threshold); if (unlikely(v < - t)) { s8 overstep = t >> 1; node_page_state_add(v - overstep, pgdat, item); __this_cpu_write(*p, overstep); } preempt_enable_nested(); } void __dec_zone_page_state(struct page *page, enum zone_stat_item item) { __dec_zone_state(page_zone(page), item); } EXPORT_SYMBOL(__dec_zone_page_state); void __dec_node_page_state(struct page *page, enum node_stat_item item) { __dec_node_state(page_pgdat(page), item); } EXPORT_SYMBOL(__dec_node_page_state); #ifdef CONFIG_HAVE_CMPXCHG_LOCAL /* * If we have cmpxchg_local support then we do not need to incur the overhead * that comes with local_irq_save/restore if we use this_cpu_cmpxchg. * * mod_state() modifies the zone counter state through atomic per cpu * operations. * * Overstep mode specifies how overstep should handled: * 0 No overstepping * 1 Overstepping half of threshold * -1 Overstepping minus half of threshold */ static inline void mod_zone_state(struct zone *zone, enum zone_stat_item item, long delta, int overstep_mode) { struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; s8 __percpu *p = pcp->vm_stat_diff + item; long n, t, z; s8 o; o = this_cpu_read(*p); do { z = 0; /* overflow to zone counters */ /* * The fetching of the stat_threshold is racy. We may apply * a counter threshold to the wrong the cpu if we get * rescheduled while executing here. However, the next * counter update will apply the threshold again and * therefore bring the counter under the threshold again. * * Most of the time the thresholds are the same anyways * for all cpus in a zone. */ t = this_cpu_read(pcp->stat_threshold); n = delta + (long)o; if (abs(n) > t) { int os = overstep_mode * (t >> 1) ; /* Overflow must be added to zone counters */ z = n + os; n = -os; } } while (!this_cpu_try_cmpxchg(*p, &o, n)); if (z) zone_page_state_add(z, zone, item); } void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, long delta) { mod_zone_state(zone, item, delta, 0); } EXPORT_SYMBOL(mod_zone_page_state); void inc_zone_page_state(struct page *page, enum zone_stat_item item) { mod_zone_state(page_zone(page), item, 1, 1); } EXPORT_SYMBOL(inc_zone_page_state); void dec_zone_page_state(struct page *page, enum zone_stat_item item) { mod_zone_state(page_zone(page), item, -1, -1); } EXPORT_SYMBOL(dec_zone_page_state); static inline void mod_node_state(struct pglist_data *pgdat, enum node_stat_item item, int delta, int overstep_mode) { struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; s8 __percpu *p = pcp->vm_node_stat_diff + item; long n, t, z; s8 o; if (vmstat_item_in_bytes(item)) { /* * Only cgroups use subpage accounting right now; at * the global level, these items still change in * multiples of whole pages. Store them as pages * internally to keep the per-cpu counters compact. */ VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); delta >>= PAGE_SHIFT; } o = this_cpu_read(*p); do { z = 0; /* overflow to node counters */ /* * The fetching of the stat_threshold is racy. We may apply * a counter threshold to the wrong the cpu if we get * rescheduled while executing here. However, the next * counter update will apply the threshold again and * therefore bring the counter under the threshold again. * * Most of the time the thresholds are the same anyways * for all cpus in a node. */ t = this_cpu_read(pcp->stat_threshold); n = delta + (long)o; if (abs(n) > t) { int os = overstep_mode * (t >> 1) ; /* Overflow must be added to node counters */ z = n + os; n = -os; } } while (!this_cpu_try_cmpxchg(*p, &o, n)); if (z) node_page_state_add(z, pgdat, item); } void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, long delta) { mod_node_state(pgdat, item, delta, 0); } EXPORT_SYMBOL(mod_node_page_state); void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) { mod_node_state(pgdat, item, 1, 1); } void inc_node_page_state(struct page *page, enum node_stat_item item) { mod_node_state(page_pgdat(page), item, 1, 1); } EXPORT_SYMBOL(inc_node_page_state); void dec_node_page_state(struct page *page, enum node_stat_item item) { mod_node_state(page_pgdat(page), item, -1, -1); } EXPORT_SYMBOL(dec_node_page_state); #else /* * Use interrupt disable to serialize counter updates */ void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, long delta) { unsigned long flags; local_irq_save(flags); __mod_zone_page_state(zone, item, delta); local_irq_restore(flags); } EXPORT_SYMBOL(mod_zone_page_state); void inc_zone_page_state(struct page *page, enum zone_stat_item item) { unsigned long flags; struct zone *zone; zone = page_zone(page); local_irq_save(flags); __inc_zone_state(zone, item); local_irq_restore(flags); } EXPORT_SYMBOL(inc_zone_page_state); void dec_zone_page_state(struct page *page, enum zone_stat_item item) { unsigned long flags; local_irq_save(flags); __dec_zone_page_state(page, item); local_irq_restore(flags); } EXPORT_SYMBOL(dec_zone_page_state); void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) { unsigned long flags; local_irq_save(flags); __inc_node_state(pgdat, item); local_irq_restore(flags); } EXPORT_SYMBOL(inc_node_state); void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, long delta) { unsigned long flags; local_irq_save(flags); __mod_node_page_state(pgdat, item, delta); local_irq_restore(flags); } EXPORT_SYMBOL(mod_node_page_state); void inc_node_page_state(struct page *page, enum node_stat_item item) { unsigned long flags; struct pglist_data *pgdat; pgdat = page_pgdat(page); local_irq_save(flags); __inc_node_state(pgdat, item); local_irq_restore(flags); } EXPORT_SYMBOL(inc_node_page_state); void dec_node_page_state(struct page *page, enum node_stat_item item) { unsigned long flags; local_irq_save(flags); __dec_node_page_state(page, item); local_irq_restore(flags); } EXPORT_SYMBOL(dec_node_page_state); #endif /* * Fold a differential into the global counters. * Returns the number of counters updated. */ static int fold_diff(int *zone_diff, int *node_diff) { int i; int changes = 0; for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) if (zone_diff[i]) { atomic_long_add(zone_diff[i], &vm_zone_stat[i]); changes++; } for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) if (node_diff[i]) { atomic_long_add(node_diff[i], &vm_node_stat[i]); changes++; } return changes; } /* * Update the zone counters for the current cpu. * * Note that refresh_cpu_vm_stats strives to only access * node local memory. The per cpu pagesets on remote zones are placed * in the memory local to the processor using that pageset. So the * loop over all zones will access a series of cachelines local to * the processor. * * The call to zone_page_state_add updates the cachelines with the * statistics in the remote zone struct as well as the global cachelines * with the global counters. These could cause remote node cache line * bouncing and will have to be only done when necessary. * * The function returns the number of global counters updated. */ static int refresh_cpu_vm_stats(bool do_pagesets) { struct pglist_data *pgdat; struct zone *zone; int i; int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; int changes = 0; for_each_populated_zone(zone) { struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset; for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { int v; v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0); if (v) { atomic_long_add(v, &zone->vm_stat[i]); global_zone_diff[i] += v; #ifdef CONFIG_NUMA /* 3 seconds idle till flush */ __this_cpu_write(pcp->expire, 3); #endif } } if (do_pagesets) { cond_resched(); changes += decay_pcp_high(zone, this_cpu_ptr(pcp)); #ifdef CONFIG_NUMA /* * Deal with draining the remote pageset of this * processor * * Check if there are pages remaining in this pageset * if not then there is nothing to expire. */ if (!__this_cpu_read(pcp->expire) || !__this_cpu_read(pcp->count)) continue; /* * We never drain zones local to this processor. */ if (zone_to_nid(zone) == numa_node_id()) { __this_cpu_write(pcp->expire, 0); continue; } if (__this_cpu_dec_return(pcp->expire)) { changes++; continue; } if (__this_cpu_read(pcp->count)) { drain_zone_pages(zone, this_cpu_ptr(pcp)); changes++; } #endif } } for_each_online_pgdat(pgdat) { struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { int v; v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); if (v) { atomic_long_add(v, &pgdat->vm_stat[i]); global_node_diff[i] += v; } } } changes += fold_diff(global_zone_diff, global_node_diff); return changes; } /* * Fold the data for an offline cpu into the global array. * There cannot be any access by the offline cpu and therefore * synchronization is simplified. */ void cpu_vm_stats_fold(int cpu) { struct pglist_data *pgdat; struct zone *zone; int i; int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; for_each_populated_zone(zone) { struct per_cpu_zonestat *pzstats; pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { if (pzstats->vm_stat_diff[i]) { int v; v = pzstats->vm_stat_diff[i]; pzstats->vm_stat_diff[i] = 0; atomic_long_add(v, &zone->vm_stat[i]); global_zone_diff[i] += v; } } #ifdef CONFIG_NUMA for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { if (pzstats->vm_numa_event[i]) { unsigned long v; v = pzstats->vm_numa_event[i]; pzstats->vm_numa_event[i] = 0; zone_numa_event_add(v, zone, i); } } #endif } for_each_online_pgdat(pgdat) { struct per_cpu_nodestat *p; p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) if (p->vm_node_stat_diff[i]) { int v; v = p->vm_node_stat_diff[i]; p->vm_node_stat_diff[i] = 0; atomic_long_add(v, &pgdat->vm_stat[i]); global_node_diff[i] += v; } } fold_diff(global_zone_diff, global_node_diff); } /* * this is only called if !populated_zone(zone), which implies no other users of * pset->vm_stat_diff[] exist. */ void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats) { unsigned long v; int i; for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { if (pzstats->vm_stat_diff[i]) { v = pzstats->vm_stat_diff[i]; pzstats->vm_stat_diff[i] = 0; zone_page_state_add(v, zone, i); } } #ifdef CONFIG_NUMA for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { if (pzstats->vm_numa_event[i]) { v = pzstats->vm_numa_event[i]; pzstats->vm_numa_event[i] = 0; zone_numa_event_add(v, zone, i); } } #endif } #endif #ifdef CONFIG_NUMA /* * Determine the per node value of a stat item. This function * is called frequently in a NUMA machine, so try to be as * frugal as possible. */ unsigned long sum_zone_node_page_state(int node, enum zone_stat_item item) { struct zone *zones = NODE_DATA(node)->node_zones; int i; unsigned long count = 0; for (i = 0; i < MAX_NR_ZONES; i++) count += zone_page_state(zones + i, item); return count; } /* Determine the per node value of a numa stat item. */ unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item) { struct zone *zones = NODE_DATA(node)->node_zones; unsigned long count = 0; int i; for (i = 0; i < MAX_NR_ZONES; i++) count += zone_numa_event_state(zones + i, item); return count; } /* * Determine the per node value of a stat item. */ unsigned long node_page_state_pages(struct pglist_data *pgdat, enum node_stat_item item) { long x = atomic_long_read(&pgdat->vm_stat[item]); #ifdef CONFIG_SMP if (x < 0) x = 0; #endif return x; } unsigned long node_page_state(struct pglist_data *pgdat, enum node_stat_item item) { VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); return node_page_state_pages(pgdat, item); } #endif /* * Count number of pages "struct page" and "struct page_ext" consume. * nr_memmap_boot_pages: # of pages allocated by boot allocator * nr_memmap_pages: # of pages that were allocated by buddy allocator */ static atomic_long_t nr_memmap_boot_pages = ATOMIC_LONG_INIT(0); static atomic_long_t nr_memmap_pages = ATOMIC_LONG_INIT(0); void memmap_boot_pages_add(long delta) { atomic_long_add(delta, &nr_memmap_boot_pages); } void memmap_pages_add(long delta) { atomic_long_add(delta, &nr_memmap_pages); } #ifdef CONFIG_COMPACTION struct contig_page_info { unsigned long free_pages; unsigned long free_blocks_total; unsigned long free_blocks_suitable; }; /* * Calculate the number of free pages in a zone, how many contiguous * pages are free and how many are large enough to satisfy an allocation of * the target size. Note that this function makes no attempt to estimate * how many suitable free blocks there *might* be if MOVABLE pages were * migrated. Calculating that is possible, but expensive and can be * figured out from userspace */ static void fill_contig_page_info(struct zone *zone, unsigned int suitable_order, struct contig_page_info *info) { unsigned int order; info->free_pages = 0; info->free_blocks_total = 0; info->free_blocks_suitable = 0; for (order = 0; order < NR_PAGE_ORDERS; order++) { unsigned long blocks; /* * Count number of free blocks. * * Access to nr_free is lockless as nr_free is used only for * diagnostic purposes. Use data_race to avoid KCSAN warning. */ blocks = data_race(zone->free_area[order].nr_free); info->free_blocks_total += blocks; /* Count free base pages */ info->free_pages += blocks << order; /* Count the suitable free blocks */ if (order >= suitable_order) info->free_blocks_suitable += blocks << (order - suitable_order); } } /* * A fragmentation index only makes sense if an allocation of a requested * size would fail. If that is true, the fragmentation index indicates * whether external fragmentation or a lack of memory was the problem. * The value can be used to determine if page reclaim or compaction * should be used */ static int __fragmentation_index(unsigned int order, struct contig_page_info *info) { unsigned long requested = 1UL << order; if (WARN_ON_ONCE(order > MAX_PAGE_ORDER)) return 0; if (!info->free_blocks_total) return 0; /* Fragmentation index only makes sense when a request would fail */ if (info->free_blocks_suitable) return -1000; /* * Index is between 0 and 1 so return within 3 decimal places * * 0 => allocation would fail due to lack of memory * 1 => allocation would fail due to fragmentation */ return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); } /* * Calculates external fragmentation within a zone wrt the given order. * It is defined as the percentage of pages found in blocks of size * less than 1 << order. It returns values in range [0, 100]. */ unsigned int extfrag_for_order(struct zone *zone, unsigned int order) { struct contig_page_info info; fill_contig_page_info(zone, order, &info); if (info.free_pages == 0) return 0; return div_u64((info.free_pages - (info.free_blocks_suitable << order)) * 100, info.free_pages); } /* Same as __fragmentation index but allocs contig_page_info on stack */ int fragmentation_index(struct zone *zone, unsigned int order) { struct contig_page_info info; fill_contig_page_info(zone, order, &info); return __fragmentation_index(order, &info); } #endif #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \ defined(CONFIG_NUMA) || defined(CONFIG_MEMCG) #ifdef CONFIG_ZONE_DMA #define TEXT_FOR_DMA(xx) xx "_dma", #else #define TEXT_FOR_DMA(xx) #endif #ifdef CONFIG_ZONE_DMA32 #define TEXT_FOR_DMA32(xx) xx "_dma32", #else #define TEXT_FOR_DMA32(xx) #endif #ifdef CONFIG_HIGHMEM #define TEXT_FOR_HIGHMEM(xx) xx "_high", #else #define TEXT_FOR_HIGHMEM(xx) #endif #ifdef CONFIG_ZONE_DEVICE #define TEXT_FOR_DEVICE(xx) xx "_device", #else #define TEXT_FOR_DEVICE(xx) #endif #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \ TEXT_FOR_HIGHMEM(xx) xx "_movable", \ TEXT_FOR_DEVICE(xx) const char * const vmstat_text[] = { /* enum zone_stat_item counters */ "nr_free_pages", "nr_zone_inactive_anon", "nr_zone_active_anon", "nr_zone_inactive_file", "nr_zone_active_file", "nr_zone_unevictable", "nr_zone_write_pending", "nr_mlock", "nr_bounce", #if IS_ENABLED(CONFIG_ZSMALLOC) "nr_zspages", #endif "nr_free_cma", #ifdef CONFIG_UNACCEPTED_MEMORY "nr_unaccepted", #endif /* enum numa_stat_item counters */ #ifdef CONFIG_NUMA "numa_hit", "numa_miss", "numa_foreign", "numa_interleave", "numa_local", "numa_other", #endif /* enum node_stat_item counters */ "nr_inactive_anon", "nr_active_anon", "nr_inactive_file", "nr_active_file", "nr_unevictable", "nr_slab_reclaimable", "nr_slab_unreclaimable", "nr_isolated_anon", "nr_isolated_file", "workingset_nodes", "workingset_refault_anon", "workingset_refault_file", "workingset_activate_anon", "workingset_activate_file", "workingset_restore_anon", "workingset_restore_file", "workingset_nodereclaim", "nr_anon_pages", "nr_mapped", "nr_file_pages", "nr_dirty", "nr_writeback", "nr_writeback_temp", "nr_shmem", "nr_shmem_hugepages", "nr_shmem_pmdmapped", "nr_file_hugepages", "nr_file_pmdmapped", "nr_anon_transparent_hugepages", "nr_vmscan_write", "nr_vmscan_immediate_reclaim", "nr_dirtied", "nr_written", "nr_throttled_written", "nr_kernel_misc_reclaimable", "nr_foll_pin_acquired", "nr_foll_pin_released", "nr_kernel_stack", #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) "nr_shadow_call_stack", #endif "nr_page_table_pages", "nr_sec_page_table_pages", #ifdef CONFIG_IOMMU_SUPPORT "nr_iommu_pages", #endif #ifdef CONFIG_SWAP "nr_swapcached", #endif #ifdef CONFIG_NUMA_BALANCING "pgpromote_success", "pgpromote_candidate", #endif "pgdemote_kswapd", "pgdemote_direct", "pgdemote_khugepaged", /* system-wide enum vm_stat_item counters */ "nr_dirty_threshold", "nr_dirty_background_threshold", "nr_memmap_pages", "nr_memmap_boot_pages", #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG) /* enum vm_event_item counters */ "pgpgin", "pgpgout", "pswpin", "pswpout", TEXTS_FOR_ZONES("pgalloc") TEXTS_FOR_ZONES("allocstall") TEXTS_FOR_ZONES("pgskip") "pgfree", "pgactivate", "pgdeactivate", "pglazyfree", "pgfault", "pgmajfault", "pglazyfreed", "pgrefill", "pgreuse", "pgsteal_kswapd", "pgsteal_direct", "pgsteal_khugepaged", "pgscan_kswapd", "pgscan_direct", "pgscan_khugepaged", "pgscan_direct_throttle", "pgscan_anon", "pgscan_file", "pgsteal_anon", "pgsteal_file", #ifdef CONFIG_NUMA "zone_reclaim_success", "zone_reclaim_failed", #endif "pginodesteal", "slabs_scanned", "kswapd_inodesteal", "kswapd_low_wmark_hit_quickly", "kswapd_high_wmark_hit_quickly", "pageoutrun", "pgrotated", "drop_pagecache", "drop_slab", "oom_kill", #ifdef CONFIG_NUMA_BALANCING "numa_pte_updates", "numa_huge_pte_updates", "numa_hint_faults", "numa_hint_faults_local", "numa_pages_migrated", #endif #ifdef CONFIG_MIGRATION "pgmigrate_success", "pgmigrate_fail", "thp_migration_success", "thp_migration_fail", "thp_migration_split", #endif #ifdef CONFIG_COMPACTION "compact_migrate_scanned", "compact_free_scanned", "compact_isolated", "compact_stall", "compact_fail", "compact_success", "compact_daemon_wake", "compact_daemon_migrate_scanned", "compact_daemon_free_scanned", #endif #ifdef CONFIG_HUGETLB_PAGE "htlb_buddy_alloc_success", "htlb_buddy_alloc_fail", #endif #ifdef CONFIG_CMA "cma_alloc_success", "cma_alloc_fail", #endif "unevictable_pgs_culled", "unevictable_pgs_scanned", "unevictable_pgs_rescued", "unevictable_pgs_mlocked", "unevictable_pgs_munlocked", "unevictable_pgs_cleared", "unevictable_pgs_stranded", #ifdef CONFIG_TRANSPARENT_HUGEPAGE "thp_fault_alloc", "thp_fault_fallback", "thp_fault_fallback_charge", "thp_collapse_alloc", "thp_collapse_alloc_failed", "thp_file_alloc", "thp_file_fallback", "thp_file_fallback_charge", "thp_file_mapped", "thp_split_page", "thp_split_page_failed", "thp_deferred_split_page", "thp_underused_split_page", "thp_split_pmd", "thp_scan_exceed_none_pte", "thp_scan_exceed_swap_pte", "thp_scan_exceed_share_pte", #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD "thp_split_pud", #endif "thp_zero_page_alloc", "thp_zero_page_alloc_failed", "thp_swpout", "thp_swpout_fallback", #endif #ifdef CONFIG_MEMORY_BALLOON "balloon_inflate", "balloon_deflate", #ifdef CONFIG_BALLOON_COMPACTION "balloon_migrate", #endif #endif /* CONFIG_MEMORY_BALLOON */ #ifdef CONFIG_DEBUG_TLBFLUSH "nr_tlb_remote_flush", "nr_tlb_remote_flush_received", "nr_tlb_local_flush_all", "nr_tlb_local_flush_one", #endif /* CONFIG_DEBUG_TLBFLUSH */ #ifdef CONFIG_SWAP "swap_ra", "swap_ra_hit", "swpin_zero", "swpout_zero", #ifdef CONFIG_KSM "ksm_swpin_copy", #endif #endif #ifdef CONFIG_KSM "cow_ksm", #endif #ifdef CONFIG_ZSWAP "zswpin", "zswpout", "zswpwb", #endif #ifdef CONFIG_X86 "direct_map_level2_splits", "direct_map_level3_splits", #endif #ifdef CONFIG_PER_VMA_LOCK_STATS "vma_lock_success", "vma_lock_abort", "vma_lock_retry", "vma_lock_miss", #endif #ifdef CONFIG_DEBUG_STACK_USAGE "kstack_1k", #if THREAD_SIZE > 1024 "kstack_2k", #endif #if THREAD_SIZE > 2048 "kstack_4k", #endif #if THREAD_SIZE > 4096 "kstack_8k", #endif #if THREAD_SIZE > 8192 "kstack_16k", #endif #if THREAD_SIZE > 16384 "kstack_32k", #endif #if THREAD_SIZE > 32768 "kstack_64k", #endif #if THREAD_SIZE > 65536 "kstack_rest", #endif #endif #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */ }; #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ defined(CONFIG_PROC_FS) static void *frag_start(struct seq_file *m, loff_t *pos) { pg_data_t *pgdat; loff_t node = *pos; for (pgdat = first_online_pgdat(); pgdat && node; pgdat = next_online_pgdat(pgdat)) --node; return pgdat; } static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) { pg_data_t *pgdat = (pg_data_t *)arg; (*pos)++; return next_online_pgdat(pgdat); } static void frag_stop(struct seq_file *m, void *arg) { } /* * Walk zones in a node and print using a callback. * If @assert_populated is true, only use callback for zones that are populated. */ static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, bool assert_populated, bool nolock, void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) { struct zone *zone; struct zone *node_zones = pgdat->node_zones; unsigned long flags; for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { if (assert_populated && !populated_zone(zone)) continue; if (!nolock) spin_lock_irqsave(&zone->lock, flags); print(m, pgdat, zone); if (!nolock) spin_unlock_irqrestore(&zone->lock, flags); } } #endif #ifdef CONFIG_PROC_FS static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { int order; seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); for (order = 0; order < NR_PAGE_ORDERS; ++order) /* * Access to nr_free is lockless as nr_free is used only for * printing purposes. Use data_race to avoid KCSAN warning. */ seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free)); seq_putc(m, '\n'); } /* * This walks the free areas for each zone. */ static int frag_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; walk_zones_in_node(m, pgdat, true, false, frag_show_print); return 0; } static void pagetypeinfo_showfree_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { int order, mtype; for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { seq_printf(m, "Node %4d, zone %8s, type %12s ", pgdat->node_id, zone->name, migratetype_names[mtype]); for (order = 0; order < NR_PAGE_ORDERS; ++order) { unsigned long freecount = 0; struct free_area *area; struct list_head *curr; bool overflow = false; area = &(zone->free_area[order]); list_for_each(curr, &area->free_list[mtype]) { /* * Cap the free_list iteration because it might * be really large and we are under a spinlock * so a long time spent here could trigger a * hard lockup detector. Anyway this is a * debugging tool so knowing there is a handful * of pages of this order should be more than * sufficient. */ if (++freecount >= 100000) { overflow = true; break; } } seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount); spin_unlock_irq(&zone->lock); cond_resched(); spin_lock_irq(&zone->lock); } seq_putc(m, '\n'); } } /* Print out the free pages at each order for each migatetype */ static void pagetypeinfo_showfree(struct seq_file *m, void *arg) { int order; pg_data_t *pgdat = (pg_data_t *)arg; /* Print header */ seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); for (order = 0; order < NR_PAGE_ORDERS; ++order) seq_printf(m, "%6d ", order); seq_putc(m, '\n'); walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print); } static void pagetypeinfo_showblockcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { int mtype; unsigned long pfn; unsigned long start_pfn = zone->zone_start_pfn; unsigned long end_pfn = zone_end_pfn(zone); unsigned long count[MIGRATE_TYPES] = { 0, }; for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { struct page *page; page = pfn_to_online_page(pfn); if (!page) continue; if (page_zone(page) != zone) continue; mtype = get_pageblock_migratetype(page); if (mtype < MIGRATE_TYPES) count[mtype]++; } /* Print counts */ seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) seq_printf(m, "%12lu ", count[mtype]); seq_putc(m, '\n'); } /* Print out the number of pageblocks for each migratetype */ static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg) { int mtype; pg_data_t *pgdat = (pg_data_t *)arg; seq_printf(m, "\n%-23s", "Number of blocks type "); for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) seq_printf(m, "%12s ", migratetype_names[mtype]); seq_putc(m, '\n'); walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showblockcount_print); } /* * Print out the number of pageblocks for each migratetype that contain pages * of other types. This gives an indication of how well fallbacks are being * contained by rmqueue_fallback(). It requires information from PAGE_OWNER * to determine what is going on */ static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) { #ifdef CONFIG_PAGE_OWNER int mtype; if (!static_branch_unlikely(&page_owner_inited)) return; drain_all_pages(NULL); seq_printf(m, "\n%-23s", "Number of mixed blocks "); for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) seq_printf(m, "%12s ", migratetype_names[mtype]); seq_putc(m, '\n'); walk_zones_in_node(m, pgdat, true, true, pagetypeinfo_showmixedcount_print); #endif /* CONFIG_PAGE_OWNER */ } /* * This prints out statistics in relation to grouping pages by mobility. * It is expensive to collect so do not constantly read the file. */ static int pagetypeinfo_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; /* check memoryless node */ if (!node_state(pgdat->node_id, N_MEMORY)) return 0; seq_printf(m, "Page block order: %d\n", pageblock_order); seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); seq_putc(m, '\n'); pagetypeinfo_showfree(m, pgdat); pagetypeinfo_showblockcount(m, pgdat); pagetypeinfo_showmixedcount(m, pgdat); return 0; } static const struct seq_operations fragmentation_op = { .start = frag_start, .next = frag_next, .stop = frag_stop, .show = frag_show, }; static const struct seq_operations pagetypeinfo_op = { .start = frag_start, .next = frag_next, .stop = frag_stop, .show = pagetypeinfo_show, }; static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) { int zid; for (zid = 0; zid < MAX_NR_ZONES; zid++) { struct zone *compare = &pgdat->node_zones[zid]; if (populated_zone(compare)) return zone == compare; } return false; } static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { int i; seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); if (is_zone_first_populated(pgdat, zone)) { seq_printf(m, "\n per-node stats"); for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { unsigned long pages = node_page_state_pages(pgdat, i); if (vmstat_item_print_in_thp(i)) pages /= HPAGE_PMD_NR; seq_printf(m, "\n %-12s %lu", node_stat_name(i), pages); } } seq_printf(m, "\n pages free %lu" "\n boost %lu" "\n min %lu" "\n low %lu" "\n high %lu" "\n promo %lu" "\n spanned %lu" "\n present %lu" "\n managed %lu" "\n cma %lu", zone_page_state(zone, NR_FREE_PAGES), zone->watermark_boost, min_wmark_pages(zone), low_wmark_pages(zone), high_wmark_pages(zone), promo_wmark_pages(zone), zone->spanned_pages, zone->present_pages, zone_managed_pages(zone), zone_cma_pages(zone)); seq_printf(m, "\n protection: (%ld", zone->lowmem_reserve[0]); for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) seq_printf(m, ", %ld", zone->lowmem_reserve[i]); seq_putc(m, ')'); /* If unpopulated, no other information is useful */ if (!populated_zone(zone)) { seq_putc(m, '\n'); return; } for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) seq_printf(m, "\n %-12s %lu", zone_stat_name(i), zone_page_state(zone, i)); #ifdef CONFIG_NUMA for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) seq_printf(m, "\n %-12s %lu", numa_stat_name(i), zone_numa_event_state(zone, i)); #endif seq_printf(m, "\n pagesets"); for_each_online_cpu(i) { struct per_cpu_pages *pcp; struct per_cpu_zonestat __maybe_unused *pzstats; pcp = per_cpu_ptr(zone->per_cpu_pageset, i); seq_printf(m, "\n cpu: %i" "\n count: %i" "\n high: %i" "\n batch: %i", i, pcp->count, pcp->high, pcp->batch); #ifdef CONFIG_SMP pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); seq_printf(m, "\n vm stats threshold: %d", pzstats->stat_threshold); #endif } seq_printf(m, "\n node_unreclaimable: %u" "\n start_pfn: %lu", pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES, zone->zone_start_pfn); seq_putc(m, '\n'); } /* * Output information about zones in @pgdat. All zones are printed regardless * of whether they are populated or not: lowmem_reserve_ratio operates on the * set of all zones and userspace would not be aware of such zones if they are * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio). */ static int zoneinfo_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); return 0; } static const struct seq_operations zoneinfo_op = { .start = frag_start, /* iterate over all zones. The same as in * fragmentation. */ .next = frag_next, .stop = frag_stop, .show = zoneinfo_show, }; #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \ NR_VM_NUMA_EVENT_ITEMS + \ NR_VM_NODE_STAT_ITEMS + \ NR_VM_STAT_ITEMS + \ (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \ NR_VM_EVENT_ITEMS : 0)) static void *vmstat_start(struct seq_file *m, loff_t *pos) { unsigned long *v; int i; if (*pos >= NR_VMSTAT_ITEMS) return NULL; BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) < NR_VMSTAT_ITEMS); fold_vm_numa_events(); v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL); m->private = v; if (!v) return ERR_PTR(-ENOMEM); for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) v[i] = global_zone_page_state(i); v += NR_VM_ZONE_STAT_ITEMS; #ifdef CONFIG_NUMA for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) v[i] = global_numa_event_state(i); v += NR_VM_NUMA_EVENT_ITEMS; #endif for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { v[i] = global_node_page_state_pages(i); if (vmstat_item_print_in_thp(i)) v[i] /= HPAGE_PMD_NR; } v += NR_VM_NODE_STAT_ITEMS; global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, v + NR_DIRTY_THRESHOLD); v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages); v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages); v += NR_VM_STAT_ITEMS; #ifdef CONFIG_VM_EVENT_COUNTERS all_vm_events(v); v[PGPGIN] /= 2; /* sectors -> kbytes */ v[PGPGOUT] /= 2; #endif return (unsigned long *)m->private + *pos; } static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) { (*pos)++; if (*pos >= NR_VMSTAT_ITEMS) return NULL; return (unsigned long *)m->private + *pos; } static int vmstat_show(struct seq_file *m, void *arg) { unsigned long *l = arg; unsigned long off = l - (unsigned long *)m->private; seq_puts(m, vmstat_text[off]); seq_put_decimal_ull(m, " ", *l); seq_putc(m, '\n'); if (off == NR_VMSTAT_ITEMS - 1) { /* * We've come to the end - add any deprecated counters to avoid * breaking userspace which might depend on them being present. */ seq_puts(m, "nr_unstable 0\n"); } return 0; } static void vmstat_stop(struct seq_file *m, void *arg) { kfree(m->private); m->private = NULL; } static const struct seq_operations vmstat_op = { .start = vmstat_start, .next = vmstat_next, .stop = vmstat_stop, .show = vmstat_show, }; #endif /* CONFIG_PROC_FS */ #ifdef CONFIG_SMP static DEFINE_PER_CPU(struct delayed_work, vmstat_work); int sysctl_stat_interval __read_mostly = HZ; #ifdef CONFIG_PROC_FS static void refresh_vm_stats(struct work_struct *work) { refresh_cpu_vm_stats(true); } int vmstat_refresh(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { long val; int err; int i; /* * The regular update, every sysctl_stat_interval, may come later * than expected: leaving a significant amount in per_cpu buckets. * This is particularly misleading when checking a quantity of HUGE * pages, immediately after running a test. /proc/sys/vm/stat_refresh, * which can equally be echo'ed to or cat'ted from (by root), * can be used to update the stats just before reading them. * * Oh, and since global_zone_page_state() etc. are so careful to hide * transiently negative values, report an error here if any of * the stats is negative, so we know to go looking for imbalance. */ err = schedule_on_each_cpu(refresh_vm_stats); if (err) return err; for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { /* * Skip checking stats known to go negative occasionally. */ switch (i) { case NR_ZONE_WRITE_PENDING: case NR_FREE_CMA_PAGES: continue; } val = atomic_long_read(&vm_zone_stat[i]); if (val < 0) { pr_warn("%s: %s %ld\n", __func__, zone_stat_name(i), val); } } for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { /* * Skip checking stats known to go negative occasionally. */ switch (i) { case NR_WRITEBACK: continue; } val = atomic_long_read(&vm_node_stat[i]); if (val < 0) { pr_warn("%s: %s %ld\n", __func__, node_stat_name(i), val); } } if (write) *ppos += *lenp; else *lenp = 0; return 0; } #endif /* CONFIG_PROC_FS */ static void vmstat_update(struct work_struct *w) { if (refresh_cpu_vm_stats(true)) { /* * Counters were updated so we expect more updates * to occur in the future. Keep on running the * update worker thread. */ queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, this_cpu_ptr(&vmstat_work), round_jiffies_relative(sysctl_stat_interval)); } } /* * Check if the diffs for a certain cpu indicate that * an update is needed. */ static bool need_update(int cpu) { pg_data_t *last_pgdat = NULL; struct zone *zone; for_each_populated_zone(zone) { struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); struct per_cpu_nodestat *n; /* * The fast way of checking if there are any vmstat diffs. */ if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff))) return true; if (last_pgdat == zone->zone_pgdat) continue; last_pgdat = zone->zone_pgdat; n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu); if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff))) return true; } return false; } /* * Switch off vmstat processing and then fold all the remaining differentials * until the diffs stay at zero. The function is used by NOHZ and can only be * invoked when tick processing is not active. */ void quiet_vmstat(void) { if (system_state != SYSTEM_RUNNING) return; if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) return; if (!need_update(smp_processor_id())) return; /* * Just refresh counters and do not care about the pending delayed * vmstat_update. It doesn't fire that often to matter and canceling * it would be too expensive from this path. * vmstat_shepherd will take care about that for us. */ refresh_cpu_vm_stats(false); } /* * Shepherd worker thread that checks the * differentials of processors that have their worker * threads for vm statistics updates disabled because of * inactivity. */ static void vmstat_shepherd(struct work_struct *w); static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); static void vmstat_shepherd(struct work_struct *w) { int cpu; cpus_read_lock(); /* Check processors whose vmstat worker threads have been disabled */ for_each_online_cpu(cpu) { struct delayed_work *dw = &per_cpu(vmstat_work, cpu); /* * In kernel users of vmstat counters either require the precise value and * they are using zone_page_state_snapshot interface or they can live with * an imprecision as the regular flushing can happen at arbitrary time and * cumulative error can grow (see calculate_normal_threshold). * * From that POV the regular flushing can be postponed for CPUs that have * been isolated from the kernel interference without critical * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd * for all isolated CPUs to avoid interference with the isolated workload. */ if (cpu_is_isolated(cpu)) continue; if (!delayed_work_pending(dw) && need_update(cpu)) queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); cond_resched(); } cpus_read_unlock(); schedule_delayed_work(&shepherd, round_jiffies_relative(sysctl_stat_interval)); } static void __init start_shepherd_timer(void) { int cpu; for_each_possible_cpu(cpu) INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), vmstat_update); schedule_delayed_work(&shepherd, round_jiffies_relative(sysctl_stat_interval)); } static void __init init_cpu_node_state(void) { int node; for_each_online_node(node) { if (!cpumask_empty(cpumask_of_node(node))) node_set_state(node, N_CPU); } } static int vmstat_cpu_online(unsigned int cpu) { refresh_zone_stat_thresholds(); if (!node_state(cpu_to_node(cpu), N_CPU)) { node_set_state(cpu_to_node(cpu), N_CPU); } return 0; } static int vmstat_cpu_down_prep(unsigned int cpu) { cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu)); return 0; } static int vmstat_cpu_dead(unsigned int cpu) { const struct cpumask *node_cpus; int node; node = cpu_to_node(cpu); refresh_zone_stat_thresholds(); node_cpus = cpumask_of_node(node); if (!cpumask_empty(node_cpus)) return 0; node_clear_state(node, N_CPU); return 0; } #endif struct workqueue_struct *mm_percpu_wq; void __init init_mm_internals(void) { int ret __maybe_unused; mm_percpu_wq = alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM, 0); #ifdef CONFIG_SMP ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", NULL, vmstat_cpu_dead); if (ret < 0) pr_err("vmstat: failed to register 'dead' hotplug state\n"); ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", vmstat_cpu_online, vmstat_cpu_down_prep); if (ret < 0) pr_err("vmstat: failed to register 'online' hotplug state\n"); cpus_read_lock(); init_cpu_node_state(); cpus_read_unlock(); start_shepherd_timer(); #endif #ifdef CONFIG_PROC_FS proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); proc_create_seq("vmstat", 0444, NULL, &vmstat_op); proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op); #endif } #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) /* * Return an index indicating how much of the available free memory is * unusable for an allocation of the requested size. */ static int unusable_free_index(unsigned int order, struct contig_page_info *info) { /* No free memory is interpreted as all free memory is unusable */ if (info->free_pages == 0) return 1000; /* * Index should be a value between 0 and 1. Return a value to 3 * decimal places. * * 0 => no fragmentation * 1 => high fragmentation */ return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); } static void unusable_show_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { unsigned int order; int index; struct contig_page_info info; seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); for (order = 0; order < NR_PAGE_ORDERS; ++order) { fill_contig_page_info(zone, order, &info); index = unusable_free_index(order, &info); seq_printf(m, "%d.%03d ", index / 1000, index % 1000); } seq_putc(m, '\n'); } /* * Display unusable free space index * * The unusable free space index measures how much of the available free * memory cannot be used to satisfy an allocation of a given size and is a * value between 0 and 1. The higher the value, the more of free memory is * unusable and by implication, the worse the external fragmentation is. This * can be expressed as a percentage by multiplying by 100. */ static int unusable_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; /* check memoryless node */ if (!node_state(pgdat->node_id, N_MEMORY)) return 0; walk_zones_in_node(m, pgdat, true, false, unusable_show_print); return 0; } static const struct seq_operations unusable_sops = { .start = frag_start, .next = frag_next, .stop = frag_stop, .show = unusable_show, }; DEFINE_SEQ_ATTRIBUTE(unusable); static void extfrag_show_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { unsigned int order; int index; /* Alloc on stack as interrupts are disabled for zone walk */ struct contig_page_info info; seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); for (order = 0; order < NR_PAGE_ORDERS; ++order) { fill_contig_page_info(zone, order, &info); index = __fragmentation_index(order, &info); seq_printf(m, "%2d.%03d ", index / 1000, index % 1000); } seq_putc(m, '\n'); } /* * Display fragmentation index for orders that allocations would fail for */ static int extfrag_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; walk_zones_in_node(m, pgdat, true, false, extfrag_show_print); return 0; } static const struct seq_operations extfrag_sops = { .start = frag_start, .next = frag_next, .stop = frag_stop, .show = extfrag_show, }; DEFINE_SEQ_ATTRIBUTE(extfrag); static int __init extfrag_debug_init(void) { struct dentry *extfrag_debug_root; extfrag_debug_root = debugfs_create_dir("extfrag", NULL); debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL, &unusable_fops); debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL, &extfrag_fops); return 0; } module_init(extfrag_debug_init); #endif
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1