| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Daniel Bristot de Oliveira | 9043 | 96.39% | 40 | 55.56% |
| Steven Rostedt | 217 | 2.31% | 14 | 19.44% |
| Li Wei | 42 | 0.45% | 3 | 4.17% |
| Nikita Yushchenko | 23 | 0.25% | 1 | 1.39% |
| Nicolas Saenz Julienne | 17 | 0.18% | 1 | 1.39% |
| Masami Hiramatsu | 7 | 0.07% | 1 | 1.39% |
| Nico Pache | 6 | 0.06% | 1 | 1.39% |
| Sebastian Andrzej Siewior | 5 | 0.05% | 1 | 1.39% |
| Tom Rix | 5 | 0.05% | 1 | 1.39% |
| Zhang Qiang | 3 | 0.03% | 1 | 1.39% |
| Frédéric Weisbecker | 3 | 0.03% | 1 | 1.39% |
| Chuang Wang | 2 | 0.02% | 1 | 1.39% |
| Delyan Kratunov | 2 | 0.02% | 1 | 1.39% |
| Valentin Schneider | 2 | 0.02% | 1 | 1.39% |
| Luis Claudio R. Goncalves | 2 | 0.02% | 1 | 1.39% |
| caihuoqing | 1 | 0.01% | 1 | 1.39% |
| Uladzislau Rezki | 1 | 0.01% | 1 | 1.39% |
| Davidlohr Bueso A | 1 | 0.01% | 1 | 1.39% |
| Total | 9382 | 72 |
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149
// SPDX-License-Identifier: GPL-2.0 /* * OS Noise Tracer: computes the OS Noise suffered by a running thread. * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread. * * Based on "hwlat_detector" tracer by: * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com> * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com> * With feedback from Clark Williams <williams@redhat.com> * * And also based on the rtsl tracer presented on: * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020. * * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com> */ #include <linux/kthread.h> #include <linux/tracefs.h> #include <linux/uaccess.h> #include <linux/cpumask.h> #include <linux/delay.h> #include <linux/sched/clock.h> #include <uapi/linux/sched/types.h> #include <linux/sched.h> #include "trace.h" #ifdef CONFIG_X86_LOCAL_APIC #include <asm/trace/irq_vectors.h> #undef TRACE_INCLUDE_PATH #undef TRACE_INCLUDE_FILE #endif /* CONFIG_X86_LOCAL_APIC */ #include <trace/events/irq.h> #include <trace/events/sched.h> #define CREATE_TRACE_POINTS #include <trace/events/osnoise.h> /* * Default values. */ #define BANNER "osnoise: " #define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */ #define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */ #define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */ #define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */ /* * osnoise/options entries. */ enum osnoise_options_index { OSN_DEFAULTS = 0, OSN_WORKLOAD, OSN_PANIC_ON_STOP, OSN_PREEMPT_DISABLE, OSN_IRQ_DISABLE, OSN_MAX }; static const char * const osnoise_options_str[OSN_MAX] = { "DEFAULTS", "OSNOISE_WORKLOAD", "PANIC_ON_STOP", "OSNOISE_PREEMPT_DISABLE", "OSNOISE_IRQ_DISABLE" }; #define OSN_DEFAULT_OPTIONS 0x2 static unsigned long osnoise_options = OSN_DEFAULT_OPTIONS; /* * trace_array of the enabled osnoise/timerlat instances. */ struct osnoise_instance { struct list_head list; struct trace_array *tr; }; static struct list_head osnoise_instances; static bool osnoise_has_registered_instances(void) { return !!list_first_or_null_rcu(&osnoise_instances, struct osnoise_instance, list); } /* * osnoise_instance_registered - check if a tr is already registered */ static int osnoise_instance_registered(struct trace_array *tr) { struct osnoise_instance *inst; int found = 0; rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { if (inst->tr == tr) found = 1; } rcu_read_unlock(); return found; } /* * osnoise_register_instance - register a new trace instance * * Register a trace_array *tr in the list of instances running * osnoise/timerlat tracers. */ static int osnoise_register_instance(struct trace_array *tr) { struct osnoise_instance *inst; /* * register/unregister serialization is provided by trace's * trace_types_lock. */ lockdep_assert_held(&trace_types_lock); inst = kmalloc(sizeof(*inst), GFP_KERNEL); if (!inst) return -ENOMEM; INIT_LIST_HEAD_RCU(&inst->list); inst->tr = tr; list_add_tail_rcu(&inst->list, &osnoise_instances); return 0; } /* * osnoise_unregister_instance - unregister a registered trace instance * * Remove the trace_array *tr from the list of instances running * osnoise/timerlat tracers. */ static void osnoise_unregister_instance(struct trace_array *tr) { struct osnoise_instance *inst; int found = 0; /* * register/unregister serialization is provided by trace's * trace_types_lock. */ list_for_each_entry_rcu(inst, &osnoise_instances, list, lockdep_is_held(&trace_types_lock)) { if (inst->tr == tr) { list_del_rcu(&inst->list); found = 1; break; } } if (!found) return; kvfree_rcu_mightsleep(inst); } /* * NMI runtime info. */ struct osn_nmi { u64 count; u64 delta_start; }; /* * IRQ runtime info. */ struct osn_irq { u64 count; u64 arrival_time; u64 delta_start; }; #define IRQ_CONTEXT 0 #define THREAD_CONTEXT 1 #define THREAD_URET 2 /* * sofirq runtime info. */ struct osn_softirq { u64 count; u64 arrival_time; u64 delta_start; }; /* * thread runtime info. */ struct osn_thread { u64 count; u64 arrival_time; u64 delta_start; }; /* * Runtime information: this structure saves the runtime information used by * one sampling thread. */ struct osnoise_variables { struct task_struct *kthread; bool sampling; pid_t pid; struct osn_nmi nmi; struct osn_irq irq; struct osn_softirq softirq; struct osn_thread thread; local_t int_counter; }; /* * Per-cpu runtime information. */ static DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var); /* * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU */ static inline struct osnoise_variables *this_cpu_osn_var(void) { return this_cpu_ptr(&per_cpu_osnoise_var); } /* * Protect the interface. */ static struct mutex interface_lock; #ifdef CONFIG_TIMERLAT_TRACER /* * Runtime information for the timer mode. */ struct timerlat_variables { struct task_struct *kthread; struct hrtimer timer; u64 rel_period; u64 abs_period; bool tracing_thread; u64 count; bool uthread_migrate; }; static DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var); /* * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU */ static inline struct timerlat_variables *this_cpu_tmr_var(void) { return this_cpu_ptr(&per_cpu_timerlat_var); } /* * tlat_var_reset - Reset the values of the given timerlat_variables */ static inline void tlat_var_reset(void) { struct timerlat_variables *tlat_var; int cpu; /* Synchronize with the timerlat interfaces */ mutex_lock(&interface_lock); /* * So far, all the values are initialized as 0, so * zeroing the structure is perfect. */ for_each_cpu(cpu, cpu_online_mask) { tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu); if (tlat_var->kthread) hrtimer_cancel(&tlat_var->timer); memset(tlat_var, 0, sizeof(*tlat_var)); } mutex_unlock(&interface_lock); } #else /* CONFIG_TIMERLAT_TRACER */ #define tlat_var_reset() do {} while (0) #endif /* CONFIG_TIMERLAT_TRACER */ /* * osn_var_reset - Reset the values of the given osnoise_variables */ static inline void osn_var_reset(void) { struct osnoise_variables *osn_var; int cpu; /* * So far, all the values are initialized as 0, so * zeroing the structure is perfect. */ for_each_cpu(cpu, cpu_online_mask) { osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu); memset(osn_var, 0, sizeof(*osn_var)); } } /* * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables */ static inline void osn_var_reset_all(void) { osn_var_reset(); tlat_var_reset(); } /* * Tells NMIs to call back to the osnoise tracer to record timestamps. */ bool trace_osnoise_callback_enabled; /* * osnoise sample structure definition. Used to store the statistics of a * sample run. */ struct osnoise_sample { u64 runtime; /* runtime */ u64 noise; /* noise */ u64 max_sample; /* max single noise sample */ int hw_count; /* # HW (incl. hypervisor) interference */ int nmi_count; /* # NMIs during this sample */ int irq_count; /* # IRQs during this sample */ int softirq_count; /* # softirqs during this sample */ int thread_count; /* # threads during this sample */ }; #ifdef CONFIG_TIMERLAT_TRACER /* * timerlat sample structure definition. Used to store the statistics of * a sample run. */ struct timerlat_sample { u64 timer_latency; /* timer_latency */ unsigned int seqnum; /* unique sequence */ int context; /* timer context */ }; #endif /* * Tracer data. */ static struct osnoise_data { u64 sample_period; /* total sampling period */ u64 sample_runtime; /* active sampling portion of period */ u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */ u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */ #ifdef CONFIG_TIMERLAT_TRACER u64 timerlat_period; /* timerlat period */ u64 print_stack; /* print IRQ stack if total > */ int timerlat_tracer; /* timerlat tracer */ #endif bool tainted; /* infor users and developers about a problem */ } osnoise_data = { .sample_period = DEFAULT_SAMPLE_PERIOD, .sample_runtime = DEFAULT_SAMPLE_RUNTIME, .stop_tracing = 0, .stop_tracing_total = 0, #ifdef CONFIG_TIMERLAT_TRACER .print_stack = 0, .timerlat_period = DEFAULT_TIMERLAT_PERIOD, .timerlat_tracer = 0, #endif }; #ifdef CONFIG_TIMERLAT_TRACER static inline bool timerlat_enabled(void) { return osnoise_data.timerlat_tracer; } static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var) { struct timerlat_variables *tlat_var = this_cpu_tmr_var(); /* * If the timerlat is enabled, but the irq handler did * not run yet enabling timerlat_tracer, do not trace. */ if (!tlat_var->tracing_thread) { osn_var->softirq.arrival_time = 0; osn_var->softirq.delta_start = 0; return 0; } return 1; } static inline int timerlat_thread_exit(struct osnoise_variables *osn_var) { struct timerlat_variables *tlat_var = this_cpu_tmr_var(); /* * If the timerlat is enabled, but the irq handler did * not run yet enabling timerlat_tracer, do not trace. */ if (!tlat_var->tracing_thread) { osn_var->thread.delta_start = 0; osn_var->thread.arrival_time = 0; return 0; } return 1; } #else /* CONFIG_TIMERLAT_TRACER */ static inline bool timerlat_enabled(void) { return false; } static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var) { return 1; } static inline int timerlat_thread_exit(struct osnoise_variables *osn_var) { return 1; } #endif #ifdef CONFIG_PREEMPT_RT /* * Print the osnoise header info. */ static void print_osnoise_headers(struct seq_file *s) { if (osnoise_data.tainted) seq_puts(s, "# osnoise is tainted!\n"); seq_puts(s, "# _-------=> irqs-off\n"); seq_puts(s, "# / _------=> need-resched\n"); seq_puts(s, "# | / _-----=> need-resched-lazy\n"); seq_puts(s, "# || / _----=> hardirq/softirq\n"); seq_puts(s, "# ||| / _---=> preempt-depth\n"); seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n"); seq_puts(s, "# ||||| / _-=> migrate-disable\n"); seq_puts(s, "# |||||| / "); seq_puts(s, " MAX\n"); seq_puts(s, "# ||||| / "); seq_puts(s, " SINGLE Interference counters:\n"); seq_puts(s, "# ||||||| RUNTIME "); seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n"); seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US "); seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n"); seq_puts(s, "# | | | ||||||| | | "); seq_puts(s, " | | | | | | | |\n"); } #else /* CONFIG_PREEMPT_RT */ static void print_osnoise_headers(struct seq_file *s) { if (osnoise_data.tainted) seq_puts(s, "# osnoise is tainted!\n"); seq_puts(s, "# _-----=> irqs-off\n"); seq_puts(s, "# / _----=> need-resched\n"); seq_puts(s, "# | / _---=> hardirq/softirq\n"); seq_puts(s, "# || / _--=> preempt-depth\n"); seq_puts(s, "# ||| / _-=> migrate-disable "); seq_puts(s, " MAX\n"); seq_puts(s, "# |||| / delay "); seq_puts(s, " SINGLE Interference counters:\n"); seq_puts(s, "# ||||| RUNTIME "); seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n"); seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US "); seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n"); seq_puts(s, "# | | | ||||| | | "); seq_puts(s, " | | | | | | | |\n"); } #endif /* CONFIG_PREEMPT_RT */ /* * osnoise_taint - report an osnoise error. */ #define osnoise_taint(msg) ({ \ struct osnoise_instance *inst; \ struct trace_buffer *buffer; \ \ rcu_read_lock(); \ list_for_each_entry_rcu(inst, &osnoise_instances, list) { \ buffer = inst->tr->array_buffer.buffer; \ trace_array_printk_buf(buffer, _THIS_IP_, msg); \ } \ rcu_read_unlock(); \ osnoise_data.tainted = true; \ }) /* * Record an osnoise_sample into the tracer buffer. */ static void __trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer) { struct ring_buffer_event *event; struct osnoise_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry), tracing_gen_ctx()); if (!event) return; entry = ring_buffer_event_data(event); entry->runtime = sample->runtime; entry->noise = sample->noise; entry->max_sample = sample->max_sample; entry->hw_count = sample->hw_count; entry->nmi_count = sample->nmi_count; entry->irq_count = sample->irq_count; entry->softirq_count = sample->softirq_count; entry->thread_count = sample->thread_count; trace_buffer_unlock_commit_nostack(buffer, event); } /* * Record an osnoise_sample on all osnoise instances. */ static void trace_osnoise_sample(struct osnoise_sample *sample) { struct osnoise_instance *inst; struct trace_buffer *buffer; rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { buffer = inst->tr->array_buffer.buffer; __trace_osnoise_sample(sample, buffer); } rcu_read_unlock(); } #ifdef CONFIG_TIMERLAT_TRACER /* * Print the timerlat header info. */ #ifdef CONFIG_PREEMPT_RT static void print_timerlat_headers(struct seq_file *s) { seq_puts(s, "# _-------=> irqs-off\n"); seq_puts(s, "# / _------=> need-resched\n"); seq_puts(s, "# | / _-----=> need-resched-lazy\n"); seq_puts(s, "# || / _----=> hardirq/softirq\n"); seq_puts(s, "# ||| / _---=> preempt-depth\n"); seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n"); seq_puts(s, "# ||||| / _-=> migrate-disable\n"); seq_puts(s, "# |||||| /\n"); seq_puts(s, "# ||||||| ACTIVATION\n"); seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID "); seq_puts(s, " CONTEXT LATENCY\n"); seq_puts(s, "# | | | ||||||| | | "); seq_puts(s, " | |\n"); } #else /* CONFIG_PREEMPT_RT */ static void print_timerlat_headers(struct seq_file *s) { seq_puts(s, "# _-----=> irqs-off\n"); seq_puts(s, "# / _----=> need-resched\n"); seq_puts(s, "# | / _---=> hardirq/softirq\n"); seq_puts(s, "# || / _--=> preempt-depth\n"); seq_puts(s, "# ||| / _-=> migrate-disable\n"); seq_puts(s, "# |||| / delay\n"); seq_puts(s, "# ||||| ACTIVATION\n"); seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID "); seq_puts(s, " CONTEXT LATENCY\n"); seq_puts(s, "# | | | ||||| | | "); seq_puts(s, " | |\n"); } #endif /* CONFIG_PREEMPT_RT */ static void __trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer) { struct ring_buffer_event *event; struct timerlat_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry), tracing_gen_ctx()); if (!event) return; entry = ring_buffer_event_data(event); entry->seqnum = sample->seqnum; entry->context = sample->context; entry->timer_latency = sample->timer_latency; trace_buffer_unlock_commit_nostack(buffer, event); } /* * Record an timerlat_sample into the tracer buffer. */ static void trace_timerlat_sample(struct timerlat_sample *sample) { struct osnoise_instance *inst; struct trace_buffer *buffer; rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { buffer = inst->tr->array_buffer.buffer; __trace_timerlat_sample(sample, buffer); } rcu_read_unlock(); } #ifdef CONFIG_STACKTRACE #define MAX_CALLS 256 /* * Stack trace will take place only at IRQ level, so, no need * to control nesting here. */ struct trace_stack { int stack_size; int nr_entries; unsigned long calls[MAX_CALLS]; }; static DEFINE_PER_CPU(struct trace_stack, trace_stack); /* * timerlat_save_stack - save a stack trace without printing * * Save the current stack trace without printing. The * stack will be printed later, after the end of the measurement. */ static void timerlat_save_stack(int skip) { unsigned int size, nr_entries; struct trace_stack *fstack; fstack = this_cpu_ptr(&trace_stack); size = ARRAY_SIZE(fstack->calls); nr_entries = stack_trace_save(fstack->calls, size, skip); fstack->stack_size = nr_entries * sizeof(unsigned long); fstack->nr_entries = nr_entries; return; } static void __timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size) { struct ring_buffer_event *event; struct stack_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size, tracing_gen_ctx()); if (!event) return; entry = ring_buffer_event_data(event); memcpy(&entry->caller, fstack->calls, size); entry->size = fstack->nr_entries; trace_buffer_unlock_commit_nostack(buffer, event); } /* * timerlat_dump_stack - dump a stack trace previously saved */ static void timerlat_dump_stack(u64 latency) { struct osnoise_instance *inst; struct trace_buffer *buffer; struct trace_stack *fstack; unsigned int size; /* * trace only if latency > print_stack config, if enabled. */ if (!osnoise_data.print_stack || osnoise_data.print_stack > latency) return; preempt_disable_notrace(); fstack = this_cpu_ptr(&trace_stack); size = fstack->stack_size; rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { buffer = inst->tr->array_buffer.buffer; __timerlat_dump_stack(buffer, fstack, size); } rcu_read_unlock(); preempt_enable_notrace(); } #else /* CONFIG_STACKTRACE */ #define timerlat_dump_stack(u64 latency) do {} while (0) #define timerlat_save_stack(a) do {} while (0) #endif /* CONFIG_STACKTRACE */ #endif /* CONFIG_TIMERLAT_TRACER */ /* * Macros to encapsulate the time capturing infrastructure. */ #define time_get() trace_clock_local() #define time_to_us(x) div_u64(x, 1000) #define time_sub(a, b) ((a) - (b)) /* * cond_move_irq_delta_start - Forward the delta_start of a running IRQ * * If an IRQ is preempted by an NMI, its delta_start is pushed forward * to discount the NMI interference. * * See get_int_safe_duration(). */ static inline void cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration) { if (osn_var->irq.delta_start) osn_var->irq.delta_start += duration; } #ifndef CONFIG_PREEMPT_RT /* * cond_move_softirq_delta_start - Forward the delta_start of a running softirq. * * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed * forward to discount the interference. * * See get_int_safe_duration(). */ static inline void cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration) { if (osn_var->softirq.delta_start) osn_var->softirq.delta_start += duration; } #else /* CONFIG_PREEMPT_RT */ #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0) #endif /* * cond_move_thread_delta_start - Forward the delta_start of a running thread * * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start * is pushed forward to discount the interference. * * See get_int_safe_duration(). */ static inline void cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration) { if (osn_var->thread.delta_start) osn_var->thread.delta_start += duration; } /* * get_int_safe_duration - Get the duration of a window * * The irq, softirq and thread varaibles need to have its duration without * the interference from higher priority interrupts. Instead of keeping a * variable to discount the interrupt interference from these variables, the * starting time of these variables are pushed forward with the interrupt's * duration. In this way, a single variable is used to: * * - Know if a given window is being measured. * - Account its duration. * - Discount the interference. * * To avoid getting inconsistent values, e.g.,: * * now = time_get() * ---> interrupt! * delta_start -= int duration; * <--- * duration = now - delta_start; * * result: negative duration if the variable duration before the * interrupt was smaller than the interrupt execution. * * A counter of interrupts is used. If the counter increased, try * to capture an interference safe duration. */ static inline s64 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start) { u64 int_counter, now; s64 duration; do { int_counter = local_read(&osn_var->int_counter); /* synchronize with interrupts */ barrier(); now = time_get(); duration = (now - *delta_start); /* synchronize with interrupts */ barrier(); } while (int_counter != local_read(&osn_var->int_counter)); /* * This is an evidence of race conditions that cause * a value to be "discounted" too much. */ if (duration < 0) osnoise_taint("Negative duration!\n"); *delta_start = 0; return duration; } /* * * set_int_safe_time - Save the current time on *time, aware of interference * * Get the time, taking into consideration a possible interference from * higher priority interrupts. * * See get_int_safe_duration() for an explanation. */ static u64 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time) { u64 int_counter; do { int_counter = local_read(&osn_var->int_counter); /* synchronize with interrupts */ barrier(); *time = time_get(); /* synchronize with interrupts */ barrier(); } while (int_counter != local_read(&osn_var->int_counter)); return int_counter; } #ifdef CONFIG_TIMERLAT_TRACER /* * copy_int_safe_time - Copy *src into *desc aware of interference */ static u64 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src) { u64 int_counter; do { int_counter = local_read(&osn_var->int_counter); /* synchronize with interrupts */ barrier(); *dst = *src; /* synchronize with interrupts */ barrier(); } while (int_counter != local_read(&osn_var->int_counter)); return int_counter; } #endif /* CONFIG_TIMERLAT_TRACER */ /* * trace_osnoise_callback - NMI entry/exit callback * * This function is called at the entry and exit NMI code. The bool enter * distinguishes between either case. This function is used to note a NMI * occurrence, compute the noise caused by the NMI, and to remove the noise * it is potentially causing on other interference variables. */ void trace_osnoise_callback(bool enter) { struct osnoise_variables *osn_var = this_cpu_osn_var(); u64 duration; if (!osn_var->sampling) return; /* * Currently trace_clock_local() calls sched_clock() and the * generic version is not NMI safe. */ if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) { if (enter) { osn_var->nmi.delta_start = time_get(); local_inc(&osn_var->int_counter); } else { duration = time_get() - osn_var->nmi.delta_start; trace_nmi_noise(osn_var->nmi.delta_start, duration); cond_move_irq_delta_start(osn_var, duration); cond_move_softirq_delta_start(osn_var, duration); cond_move_thread_delta_start(osn_var, duration); } } if (enter) osn_var->nmi.count++; } /* * osnoise_trace_irq_entry - Note the starting of an IRQ * * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs, * it is safe to use a single variable (ons_var->irq) to save the statistics. * The arrival_time is used to report... the arrival time. The delta_start * is used to compute the duration at the IRQ exit handler. See * cond_move_irq_delta_start(). */ void osnoise_trace_irq_entry(int id) { struct osnoise_variables *osn_var = this_cpu_osn_var(); if (!osn_var->sampling) return; /* * This value will be used in the report, but not to compute * the execution time, so it is safe to get it unsafe. */ osn_var->irq.arrival_time = time_get(); set_int_safe_time(osn_var, &osn_var->irq.delta_start); osn_var->irq.count++; local_inc(&osn_var->int_counter); } /* * osnoise_irq_exit - Note the end of an IRQ, sava data and trace * * Computes the duration of the IRQ noise, and trace it. Also discounts the * interference from other sources of noise could be currently being accounted. */ void osnoise_trace_irq_exit(int id, const char *desc) { struct osnoise_variables *osn_var = this_cpu_osn_var(); s64 duration; if (!osn_var->sampling) return; duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start); trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration); osn_var->irq.arrival_time = 0; cond_move_softirq_delta_start(osn_var, duration); cond_move_thread_delta_start(osn_var, duration); } /* * trace_irqentry_callback - Callback to the irq:irq_entry traceevent * * Used to note the starting of an IRQ occurece. */ static void trace_irqentry_callback(void *data, int irq, struct irqaction *action) { osnoise_trace_irq_entry(irq); } /* * trace_irqexit_callback - Callback to the irq:irq_exit traceevent * * Used to note the end of an IRQ occurece. */ static void trace_irqexit_callback(void *data, int irq, struct irqaction *action, int ret) { osnoise_trace_irq_exit(irq, action->name); } /* * arch specific register function. */ int __weak osnoise_arch_register(void) { return 0; } /* * arch specific unregister function. */ void __weak osnoise_arch_unregister(void) { return; } /* * hook_irq_events - Hook IRQ handling events * * This function hooks the IRQ related callbacks to the respective trace * events. */ static int hook_irq_events(void) { int ret; ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL); if (ret) goto out_err; ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL); if (ret) goto out_unregister_entry; ret = osnoise_arch_register(); if (ret) goto out_irq_exit; return 0; out_irq_exit: unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL); out_unregister_entry: unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL); out_err: return -EINVAL; } /* * unhook_irq_events - Unhook IRQ handling events * * This function unhooks the IRQ related callbacks to the respective trace * events. */ static void unhook_irq_events(void) { osnoise_arch_unregister(); unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL); unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL); } #ifndef CONFIG_PREEMPT_RT /* * trace_softirq_entry_callback - Note the starting of a softirq * * Save the starting time of a softirq. As softirqs are non-preemptive to * other softirqs, it is safe to use a single variable (ons_var->softirq) * to save the statistics. The arrival_time is used to report... the * arrival time. The delta_start is used to compute the duration at the * softirq exit handler. See cond_move_softirq_delta_start(). */ static void trace_softirq_entry_callback(void *data, unsigned int vec_nr) { struct osnoise_variables *osn_var = this_cpu_osn_var(); if (!osn_var->sampling) return; /* * This value will be used in the report, but not to compute * the execution time, so it is safe to get it unsafe. */ osn_var->softirq.arrival_time = time_get(); set_int_safe_time(osn_var, &osn_var->softirq.delta_start); osn_var->softirq.count++; local_inc(&osn_var->int_counter); } /* * trace_softirq_exit_callback - Note the end of an softirq * * Computes the duration of the softirq noise, and trace it. Also discounts the * interference from other sources of noise could be currently being accounted. */ static void trace_softirq_exit_callback(void *data, unsigned int vec_nr) { struct osnoise_variables *osn_var = this_cpu_osn_var(); s64 duration; if (!osn_var->sampling) return; if (unlikely(timerlat_enabled())) if (!timerlat_softirq_exit(osn_var)) return; duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start); trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration); cond_move_thread_delta_start(osn_var, duration); osn_var->softirq.arrival_time = 0; } /* * hook_softirq_events - Hook softirq handling events * * This function hooks the softirq related callbacks to the respective trace * events. */ static int hook_softirq_events(void) { int ret; ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL); if (ret) goto out_err; ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL); if (ret) goto out_unreg_entry; return 0; out_unreg_entry: unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL); out_err: return -EINVAL; } /* * unhook_softirq_events - Unhook softirq handling events * * This function hooks the softirq related callbacks to the respective trace * events. */ static void unhook_softirq_events(void) { unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL); unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL); } #else /* CONFIG_PREEMPT_RT */ /* * softirq are threads on the PREEMPT_RT mode. */ static int hook_softirq_events(void) { return 0; } static void unhook_softirq_events(void) { } #endif /* * thread_entry - Record the starting of a thread noise window * * It saves the context switch time for a noisy thread, and increments * the interference counters. */ static void thread_entry(struct osnoise_variables *osn_var, struct task_struct *t) { if (!osn_var->sampling) return; /* * The arrival time will be used in the report, but not to compute * the execution time, so it is safe to get it unsafe. */ osn_var->thread.arrival_time = time_get(); set_int_safe_time(osn_var, &osn_var->thread.delta_start); osn_var->thread.count++; local_inc(&osn_var->int_counter); } /* * thread_exit - Report the end of a thread noise window * * It computes the total noise from a thread, tracing if needed. */ static void thread_exit(struct osnoise_variables *osn_var, struct task_struct *t) { s64 duration; if (!osn_var->sampling) return; if (unlikely(timerlat_enabled())) if (!timerlat_thread_exit(osn_var)) return; duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start); trace_thread_noise(t, osn_var->thread.arrival_time, duration); osn_var->thread.arrival_time = 0; } #ifdef CONFIG_TIMERLAT_TRACER /* * osnoise_stop_exception - Stop tracing and the tracer. */ static __always_inline void osnoise_stop_exception(char *msg, int cpu) { struct osnoise_instance *inst; struct trace_array *tr; rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, "stop tracing hit on cpu %d due to exception: %s\n", smp_processor_id(), msg); if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options)) panic("tracer hit on cpu %d due to exception: %s\n", smp_processor_id(), msg); tracer_tracing_off(tr); } rcu_read_unlock(); } /* * trace_sched_migrate_callback - sched:sched_migrate_task trace event handler * * his function is hooked to the sched:sched_migrate_task trace event, and monitors * timerlat user-space thread migration. */ static void trace_sched_migrate_callback(void *data, struct task_struct *p, int dest_cpu) { struct osnoise_variables *osn_var; long cpu = task_cpu(p); osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu); if (osn_var->pid == p->pid && dest_cpu != cpu) { per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1; osnoise_taint("timerlat user-thread migrated\n"); osnoise_stop_exception("timerlat user-thread migrated", cpu); } } static int register_migration_monitor(void) { int ret = 0; /* * Timerlat thread migration check is only required when running timerlat in user-space. * Thus, enable callback only if timerlat is set with no workload. */ if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options)) ret = register_trace_sched_migrate_task(trace_sched_migrate_callback, NULL); return ret; } static void unregister_migration_monitor(void) { if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options)) unregister_trace_sched_migrate_task(trace_sched_migrate_callback, NULL); } #else static int register_migration_monitor(void) { return 0; } static void unregister_migration_monitor(void) {} #endif /* * trace_sched_switch - sched:sched_switch trace event handler * * This function is hooked to the sched:sched_switch trace event, and it is * used to record the beginning and to report the end of a thread noise window. */ static void trace_sched_switch_callback(void *data, bool preempt, struct task_struct *p, struct task_struct *n, unsigned int prev_state) { struct osnoise_variables *osn_var = this_cpu_osn_var(); int workload = test_bit(OSN_WORKLOAD, &osnoise_options); if ((p->pid != osn_var->pid) || !workload) thread_exit(osn_var, p); if ((n->pid != osn_var->pid) || !workload) thread_entry(osn_var, n); } /* * hook_thread_events - Hook the instrumentation for thread noise * * Hook the osnoise tracer callbacks to handle the noise from other * threads on the necessary kernel events. */ static int hook_thread_events(void) { int ret; ret = register_trace_sched_switch(trace_sched_switch_callback, NULL); if (ret) return -EINVAL; ret = register_migration_monitor(); if (ret) goto out_unreg; return 0; out_unreg: unregister_trace_sched_switch(trace_sched_switch_callback, NULL); return -EINVAL; } /* * unhook_thread_events - unhook the instrumentation for thread noise * * Unook the osnoise tracer callbacks to handle the noise from other * threads on the necessary kernel events. */ static void unhook_thread_events(void) { unregister_trace_sched_switch(trace_sched_switch_callback, NULL); unregister_migration_monitor(); } /* * save_osn_sample_stats - Save the osnoise_sample statistics * * Save the osnoise_sample statistics before the sampling phase. These * values will be used later to compute the diff betwneen the statistics * before and after the osnoise sampling. */ static void save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s) { s->nmi_count = osn_var->nmi.count; s->irq_count = osn_var->irq.count; s->softirq_count = osn_var->softirq.count; s->thread_count = osn_var->thread.count; } /* * diff_osn_sample_stats - Compute the osnoise_sample statistics * * After a sample period, compute the difference on the osnoise_sample * statistics. The struct osnoise_sample *s contains the statistics saved via * save_osn_sample_stats() before the osnoise sampling. */ static void diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s) { s->nmi_count = osn_var->nmi.count - s->nmi_count; s->irq_count = osn_var->irq.count - s->irq_count; s->softirq_count = osn_var->softirq.count - s->softirq_count; s->thread_count = osn_var->thread.count - s->thread_count; } /* * osnoise_stop_tracing - Stop tracing and the tracer. */ static __always_inline void osnoise_stop_tracing(void) { struct osnoise_instance *inst; struct trace_array *tr; rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, "stop tracing hit on cpu %d\n", smp_processor_id()); if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options)) panic("tracer hit stop condition on CPU %d\n", smp_processor_id()); tracer_tracing_off(tr); } rcu_read_unlock(); } /* * osnoise_has_tracing_on - Check if there is at least one instance on */ static __always_inline int osnoise_has_tracing_on(void) { struct osnoise_instance *inst; int trace_is_on = 0; rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) trace_is_on += tracer_tracing_is_on(inst->tr); rcu_read_unlock(); return trace_is_on; } /* * notify_new_max_latency - Notify a new max latency via fsnotify interface. */ static void notify_new_max_latency(u64 latency) { struct osnoise_instance *inst; struct trace_array *tr; rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; if (tracer_tracing_is_on(tr) && tr->max_latency < latency) { tr->max_latency = latency; latency_fsnotify(tr); } } rcu_read_unlock(); } /* * run_osnoise - Sample the time and look for osnoise * * Used to capture the time, looking for potential osnoise latency repeatedly. * Different from hwlat_detector, it is called with preemption and interrupts * enabled. This allows irqs, softirqs and threads to run, interfering on the * osnoise sampling thread, as they would do with a regular thread. */ static int run_osnoise(void) { bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options); struct osnoise_variables *osn_var = this_cpu_osn_var(); u64 start, sample, last_sample; u64 last_int_count, int_count; s64 noise = 0, max_noise = 0; s64 total, last_total = 0; struct osnoise_sample s; bool disable_preemption; unsigned int threshold; u64 runtime, stop_in; u64 sum_noise = 0; int hw_count = 0; int ret = -1; /* * Disabling preemption is only required if IRQs are enabled, * and the options is set on. */ disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options); /* * Considers the current thread as the workload. */ osn_var->pid = current->pid; /* * Save the current stats for the diff */ save_osn_sample_stats(osn_var, &s); /* * if threshold is 0, use the default value of 1 us. */ threshold = tracing_thresh ? : 1000; /* * Apply PREEMPT and IRQ disabled options. */ if (disable_irq) local_irq_disable(); if (disable_preemption) preempt_disable(); /* * Make sure NMIs see sampling first */ osn_var->sampling = true; barrier(); /* * Transform the *_us config to nanoseconds to avoid the * division on the main loop. */ runtime = osnoise_data.sample_runtime * NSEC_PER_USEC; stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC; /* * Start timestemp */ start = time_get(); /* * "previous" loop. */ last_int_count = set_int_safe_time(osn_var, &last_sample); do { /* * Get sample! */ int_count = set_int_safe_time(osn_var, &sample); noise = time_sub(sample, last_sample); /* * This shouldn't happen. */ if (noise < 0) { osnoise_taint("negative noise!"); goto out; } /* * Sample runtime. */ total = time_sub(sample, start); /* * Check for possible overflows. */ if (total < last_total) { osnoise_taint("total overflow!"); break; } last_total = total; if (noise >= threshold) { int interference = int_count - last_int_count; if (noise > max_noise) max_noise = noise; if (!interference) hw_count++; sum_noise += noise; trace_sample_threshold(last_sample, noise, interference); if (osnoise_data.stop_tracing) if (noise > stop_in) osnoise_stop_tracing(); } /* * In some cases, notably when running on a nohz_full CPU with * a stopped tick PREEMPT_RCU has no way to account for QSs. * This will eventually cause unwarranted noise as PREEMPT_RCU * will force preemption as the means of ending the current * grace period. We avoid this problem by calling * rcu_momentary_eqs(), which performs a zero duration * EQS allowing PREEMPT_RCU to end the current grace period. * This call shouldn't be wrapped inside an RCU critical * section. * * Note that in non PREEMPT_RCU kernels QSs are handled through * cond_resched() */ if (IS_ENABLED(CONFIG_PREEMPT_RCU)) { if (!disable_irq) local_irq_disable(); rcu_momentary_eqs(); if (!disable_irq) local_irq_enable(); } /* * For the non-preemptive kernel config: let threads runs, if * they so wish, unless set not do to so. */ if (!disable_irq && !disable_preemption) cond_resched(); last_sample = sample; last_int_count = int_count; } while (total < runtime && !kthread_should_stop()); /* * Finish the above in the view for interrupts. */ barrier(); osn_var->sampling = false; /* * Make sure sampling data is no longer updated. */ barrier(); /* * Return to the preemptive state. */ if (disable_preemption) preempt_enable(); if (disable_irq) local_irq_enable(); /* * Save noise info. */ s.noise = time_to_us(sum_noise); s.runtime = time_to_us(total); s.max_sample = time_to_us(max_noise); s.hw_count = hw_count; /* Save interference stats info */ diff_osn_sample_stats(osn_var, &s); trace_osnoise_sample(&s); notify_new_max_latency(max_noise); if (osnoise_data.stop_tracing_total) if (s.noise > osnoise_data.stop_tracing_total) osnoise_stop_tracing(); return 0; out: return ret; } static struct cpumask osnoise_cpumask; static struct cpumask save_cpumask; static struct cpumask kthread_cpumask; /* * osnoise_sleep - sleep until the next period */ static void osnoise_sleep(bool skip_period) { u64 interval; ktime_t wake_time; mutex_lock(&interface_lock); if (skip_period) interval = osnoise_data.sample_period; else interval = osnoise_data.sample_period - osnoise_data.sample_runtime; mutex_unlock(&interface_lock); /* * differently from hwlat_detector, the osnoise tracer can run * without a pause because preemption is on. */ if (!interval) { /* Let synchronize_rcu_tasks() make progress */ cond_resched_tasks_rcu_qs(); return; } wake_time = ktime_add_us(ktime_get(), interval); __set_current_state(TASK_INTERRUPTIBLE); while (schedule_hrtimeout(&wake_time, HRTIMER_MODE_ABS)) { if (kthread_should_stop()) break; } } /* * osnoise_migration_pending - checks if the task needs to migrate * * osnoise/timerlat threads are per-cpu. If there is a pending request to * migrate the thread away from the current CPU, something bad has happened. * Play the good citizen and leave. * * Returns 0 if it is safe to continue, 1 otherwise. */ static inline int osnoise_migration_pending(void) { if (!current->migration_pending) return 0; /* * If migration is pending, there is a task waiting for the * tracer to enable migration. The tracer does not allow migration, * thus: taint and leave to unblock the blocked thread. */ osnoise_taint("migration requested to osnoise threads, leaving."); /* * Unset this thread from the threads managed by the interface. * The tracers are responsible for cleaning their env before * exiting. */ mutex_lock(&interface_lock); this_cpu_osn_var()->kthread = NULL; cpumask_clear_cpu(smp_processor_id(), &kthread_cpumask); mutex_unlock(&interface_lock); return 1; } /* * osnoise_main - The osnoise detection kernel thread * * Calls run_osnoise() function to measure the osnoise for the configured runtime, * every period. */ static int osnoise_main(void *data) { unsigned long flags; /* * This thread was created pinned to the CPU using PF_NO_SETAFFINITY. * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread. * * To work around this limitation, disable migration and remove the * flag. */ migrate_disable(); raw_spin_lock_irqsave(¤t->pi_lock, flags); current->flags &= ~(PF_NO_SETAFFINITY); raw_spin_unlock_irqrestore(¤t->pi_lock, flags); while (!kthread_should_stop()) { if (osnoise_migration_pending()) break; /* skip a period if tracing is off on all instances */ if (!osnoise_has_tracing_on()) { osnoise_sleep(true); continue; } run_osnoise(); osnoise_sleep(false); } migrate_enable(); return 0; } #ifdef CONFIG_TIMERLAT_TRACER /* * timerlat_irq - hrtimer handler for timerlat. */ static enum hrtimer_restart timerlat_irq(struct hrtimer *timer) { struct osnoise_variables *osn_var = this_cpu_osn_var(); struct timerlat_variables *tlat; struct timerlat_sample s; u64 now; u64 diff; /* * I am not sure if the timer was armed for this CPU. So, get * the timerlat struct from the timer itself, not from this * CPU. */ tlat = container_of(timer, struct timerlat_variables, timer); now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer)); /* * Enable the osnoise: events for thread an softirq. */ tlat->tracing_thread = true; osn_var->thread.arrival_time = time_get(); /* * A hardirq is running: the timer IRQ. It is for sure preempting * a thread, and potentially preempting a softirq. * * At this point, it is not interesting to know the duration of the * preempted thread (and maybe softirq), but how much time they will * delay the beginning of the execution of the timer thread. * * To get the correct (net) delay added by the softirq, its delta_start * is set as the IRQ one. In this way, at the return of the IRQ, the delta * start of the sofitrq will be zeroed, accounting then only the time * after that. * * The thread follows the same principle. However, if a softirq is * running, the thread needs to receive the softirq delta_start. The * reason being is that the softirq will be the last to be unfolded, * resseting the thread delay to zero. * * The PREEMPT_RT is a special case, though. As softirqs run as threads * on RT, moving the thread is enough. */ if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) { copy_int_safe_time(osn_var, &osn_var->thread.delta_start, &osn_var->softirq.delta_start); copy_int_safe_time(osn_var, &osn_var->softirq.delta_start, &osn_var->irq.delta_start); } else { copy_int_safe_time(osn_var, &osn_var->thread.delta_start, &osn_var->irq.delta_start); } /* * Compute the current time with the expected time. */ diff = now - tlat->abs_period; tlat->count++; s.seqnum = tlat->count; s.timer_latency = diff; s.context = IRQ_CONTEXT; trace_timerlat_sample(&s); if (osnoise_data.stop_tracing) { if (time_to_us(diff) >= osnoise_data.stop_tracing) { /* * At this point, if stop_tracing is set and <= print_stack, * print_stack is set and would be printed in the thread handler. * * Thus, print the stack trace as it is helpful to define the * root cause of an IRQ latency. */ if (osnoise_data.stop_tracing <= osnoise_data.print_stack) { timerlat_save_stack(0); timerlat_dump_stack(time_to_us(diff)); } osnoise_stop_tracing(); notify_new_max_latency(diff); wake_up_process(tlat->kthread); return HRTIMER_NORESTART; } } wake_up_process(tlat->kthread); if (osnoise_data.print_stack) timerlat_save_stack(0); return HRTIMER_NORESTART; } /* * wait_next_period - Wait for the next period for timerlat */ static int wait_next_period(struct timerlat_variables *tlat) { ktime_t next_abs_period, now; u64 rel_period = osnoise_data.timerlat_period * 1000; now = hrtimer_cb_get_time(&tlat->timer); next_abs_period = ns_to_ktime(tlat->abs_period + rel_period); /* * Save the next abs_period. */ tlat->abs_period = (u64) ktime_to_ns(next_abs_period); /* * If the new abs_period is in the past, skip the activation. */ while (ktime_compare(now, next_abs_period) > 0) { next_abs_period = ns_to_ktime(tlat->abs_period + rel_period); tlat->abs_period = (u64) ktime_to_ns(next_abs_period); } set_current_state(TASK_INTERRUPTIBLE); hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD); schedule(); return 1; } /* * timerlat_main- Timerlat main */ static int timerlat_main(void *data) { struct osnoise_variables *osn_var = this_cpu_osn_var(); struct timerlat_variables *tlat = this_cpu_tmr_var(); struct timerlat_sample s; struct sched_param sp; unsigned long flags; u64 now, diff; /* * Make the thread RT, that is how cyclictest is usually used. */ sp.sched_priority = DEFAULT_TIMERLAT_PRIO; sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); /* * This thread was created pinned to the CPU using PF_NO_SETAFFINITY. * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread. * * To work around this limitation, disable migration and remove the * flag. */ migrate_disable(); raw_spin_lock_irqsave(¤t->pi_lock, flags); current->flags &= ~(PF_NO_SETAFFINITY); raw_spin_unlock_irqrestore(¤t->pi_lock, flags); tlat->count = 0; tlat->tracing_thread = false; hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD); tlat->timer.function = timerlat_irq; tlat->kthread = current; osn_var->pid = current->pid; /* * Anotate the arrival time. */ tlat->abs_period = hrtimer_cb_get_time(&tlat->timer); wait_next_period(tlat); osn_var->sampling = 1; while (!kthread_should_stop()) { now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer)); diff = now - tlat->abs_period; s.seqnum = tlat->count; s.timer_latency = diff; s.context = THREAD_CONTEXT; trace_timerlat_sample(&s); notify_new_max_latency(diff); timerlat_dump_stack(time_to_us(diff)); tlat->tracing_thread = false; if (osnoise_data.stop_tracing_total) if (time_to_us(diff) >= osnoise_data.stop_tracing_total) osnoise_stop_tracing(); if (osnoise_migration_pending()) break; wait_next_period(tlat); } hrtimer_cancel(&tlat->timer); migrate_enable(); return 0; } #else /* CONFIG_TIMERLAT_TRACER */ static int timerlat_main(void *data) { return 0; } #endif /* CONFIG_TIMERLAT_TRACER */ /* * stop_kthread - stop a workload thread */ static void stop_kthread(unsigned int cpu) { struct task_struct *kthread; kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL); if (kthread) { if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask) && !WARN_ON(!test_bit(OSN_WORKLOAD, &osnoise_options))) { kthread_stop(kthread); } else if (!WARN_ON(test_bit(OSN_WORKLOAD, &osnoise_options))) { /* * This is a user thread waiting on the timerlat_fd. We need * to close all users, and the best way to guarantee this is * by killing the thread. NOTE: this is a purpose specific file. */ kill_pid(kthread->thread_pid, SIGKILL, 1); put_task_struct(kthread); } } else { /* if no workload, just return */ if (!test_bit(OSN_WORKLOAD, &osnoise_options)) { /* * This is set in the osnoise tracer case. */ per_cpu(per_cpu_osnoise_var, cpu).sampling = false; barrier(); } } } /* * stop_per_cpu_kthread - Stop per-cpu threads * * Stop the osnoise sampling htread. Use this on unload and at system * shutdown. */ static void stop_per_cpu_kthreads(void) { int cpu; cpus_read_lock(); for_each_online_cpu(cpu) stop_kthread(cpu); cpus_read_unlock(); } /* * start_kthread - Start a workload tread */ static int start_kthread(unsigned int cpu) { struct task_struct *kthread; void *main = osnoise_main; char comm[24]; /* Do not start a new thread if it is already running */ if (per_cpu(per_cpu_osnoise_var, cpu).kthread) return 0; if (timerlat_enabled()) { snprintf(comm, 24, "timerlat/%d", cpu); main = timerlat_main; } else { /* if no workload, just return */ if (!test_bit(OSN_WORKLOAD, &osnoise_options)) { per_cpu(per_cpu_osnoise_var, cpu).sampling = true; barrier(); return 0; } snprintf(comm, 24, "osnoise/%d", cpu); } kthread = kthread_run_on_cpu(main, NULL, cpu, comm); if (IS_ERR(kthread)) { pr_err(BANNER "could not start sampling thread\n"); stop_per_cpu_kthreads(); return -ENOMEM; } per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread; cpumask_set_cpu(cpu, &kthread_cpumask); return 0; } /* * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads * * This starts the kernel thread that will look for osnoise on many * cpus. */ static int start_per_cpu_kthreads(void) { struct cpumask *current_mask = &save_cpumask; int retval = 0; int cpu; if (!test_bit(OSN_WORKLOAD, &osnoise_options)) { if (timerlat_enabled()) return 0; } cpus_read_lock(); /* * Run only on online CPUs in which osnoise is allowed to run. */ cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask); for_each_possible_cpu(cpu) { if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask)) { struct task_struct *kthread; kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL); if (!WARN_ON(!kthread)) kthread_stop(kthread); } } for_each_cpu(cpu, current_mask) { retval = start_kthread(cpu); if (retval) { cpus_read_unlock(); stop_per_cpu_kthreads(); return retval; } } cpus_read_unlock(); return retval; } #ifdef CONFIG_HOTPLUG_CPU static void osnoise_hotplug_workfn(struct work_struct *dummy) { unsigned int cpu = smp_processor_id(); mutex_lock(&trace_types_lock); if (!osnoise_has_registered_instances()) goto out_unlock_trace; mutex_lock(&interface_lock); cpus_read_lock(); if (!cpu_online(cpu)) goto out_unlock; if (!cpumask_test_cpu(cpu, &osnoise_cpumask)) goto out_unlock; start_kthread(cpu); out_unlock: cpus_read_unlock(); mutex_unlock(&interface_lock); out_unlock_trace: mutex_unlock(&trace_types_lock); } static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn); /* * osnoise_cpu_init - CPU hotplug online callback function */ static int osnoise_cpu_init(unsigned int cpu) { schedule_work_on(cpu, &osnoise_hotplug_work); return 0; } /* * osnoise_cpu_die - CPU hotplug offline callback function */ static int osnoise_cpu_die(unsigned int cpu) { stop_kthread(cpu); return 0; } static void osnoise_init_hotplug_support(void) { int ret; ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online", osnoise_cpu_init, osnoise_cpu_die); if (ret < 0) pr_warn(BANNER "Error to init cpu hotplug support\n"); return; } #else /* CONFIG_HOTPLUG_CPU */ static void osnoise_init_hotplug_support(void) { return; } #endif /* CONFIG_HOTPLUG_CPU */ /* * seq file functions for the osnoise/options file. */ static void *s_options_start(struct seq_file *s, loff_t *pos) { int option = *pos; mutex_lock(&interface_lock); if (option >= OSN_MAX) return NULL; return pos; } static void *s_options_next(struct seq_file *s, void *v, loff_t *pos) { int option = ++(*pos); if (option >= OSN_MAX) return NULL; return pos; } static int s_options_show(struct seq_file *s, void *v) { loff_t *pos = v; int option = *pos; if (option == OSN_DEFAULTS) { if (osnoise_options == OSN_DEFAULT_OPTIONS) seq_printf(s, "%s", osnoise_options_str[option]); else seq_printf(s, "NO_%s", osnoise_options_str[option]); goto out; } if (test_bit(option, &osnoise_options)) seq_printf(s, "%s", osnoise_options_str[option]); else seq_printf(s, "NO_%s", osnoise_options_str[option]); out: if (option != OSN_MAX) seq_puts(s, " "); return 0; } static void s_options_stop(struct seq_file *s, void *v) { seq_puts(s, "\n"); mutex_unlock(&interface_lock); } static const struct seq_operations osnoise_options_seq_ops = { .start = s_options_start, .next = s_options_next, .show = s_options_show, .stop = s_options_stop }; static int osnoise_options_open(struct inode *inode, struct file *file) { return seq_open(file, &osnoise_options_seq_ops); }; /** * osnoise_options_write - Write function for "options" entry * @filp: The active open file structure * @ubuf: The user buffer that contains the value to write * @cnt: The maximum number of bytes to write to "file" * @ppos: The current position in @file * * Writing the option name sets the option, writing the "NO_" * prefix in front of the option name disables it. * * Writing "DEFAULTS" resets the option values to the default ones. */ static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { int running, option, enable, retval; char buf[256], *option_str; if (cnt >= 256) return -EINVAL; if (copy_from_user(buf, ubuf, cnt)) return -EFAULT; buf[cnt] = 0; if (strncmp(buf, "NO_", 3)) { option_str = strstrip(buf); enable = true; } else { option_str = strstrip(&buf[3]); enable = false; } option = match_string(osnoise_options_str, OSN_MAX, option_str); if (option < 0) return -EINVAL; /* * trace_types_lock is taken to avoid concurrency on start/stop. */ mutex_lock(&trace_types_lock); running = osnoise_has_registered_instances(); if (running) stop_per_cpu_kthreads(); mutex_lock(&interface_lock); /* * avoid CPU hotplug operations that might read options. */ cpus_read_lock(); retval = cnt; if (enable) { if (option == OSN_DEFAULTS) osnoise_options = OSN_DEFAULT_OPTIONS; else set_bit(option, &osnoise_options); } else { if (option == OSN_DEFAULTS) retval = -EINVAL; else clear_bit(option, &osnoise_options); } cpus_read_unlock(); mutex_unlock(&interface_lock); if (running) start_per_cpu_kthreads(); mutex_unlock(&trace_types_lock); return retval; } /* * osnoise_cpus_read - Read function for reading the "cpus" file * @filp: The active open file structure * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * * Prints the "cpus" output into the user-provided buffer. */ static ssize_t osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count, loff_t *ppos) { char *mask_str; int len; mutex_lock(&interface_lock); len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1; mask_str = kmalloc(len, GFP_KERNEL); if (!mask_str) { count = -ENOMEM; goto out_unlock; } len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)); if (len >= count) { count = -EINVAL; goto out_free; } count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len); out_free: kfree(mask_str); out_unlock: mutex_unlock(&interface_lock); return count; } /* * osnoise_cpus_write - Write function for "cpus" entry * @filp: The active open file structure * @ubuf: The user buffer that contains the value to write * @cnt: The maximum number of bytes to write to "file" * @ppos: The current position in @file * * This function provides a write implementation for the "cpus" * interface to the osnoise trace. By default, it lists all CPUs, * in this way, allowing osnoise threads to run on any online CPU * of the system. It serves to restrict the execution of osnoise to the * set of CPUs writing via this interface. Why not use "tracing_cpumask"? * Because the user might be interested in tracing what is running on * other CPUs. For instance, one might run osnoise in one HT CPU * while observing what is running on the sibling HT CPU. */ static ssize_t osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count, loff_t *ppos) { cpumask_var_t osnoise_cpumask_new; int running, err; char buf[256]; if (count >= 256) return -EINVAL; if (copy_from_user(buf, ubuf, count)) return -EFAULT; if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL)) return -ENOMEM; err = cpulist_parse(buf, osnoise_cpumask_new); if (err) goto err_free; /* * trace_types_lock is taken to avoid concurrency on start/stop. */ mutex_lock(&trace_types_lock); running = osnoise_has_registered_instances(); if (running) stop_per_cpu_kthreads(); mutex_lock(&interface_lock); /* * osnoise_cpumask is read by CPU hotplug operations. */ cpus_read_lock(); cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new); cpus_read_unlock(); mutex_unlock(&interface_lock); if (running) start_per_cpu_kthreads(); mutex_unlock(&trace_types_lock); free_cpumask_var(osnoise_cpumask_new); return count; err_free: free_cpumask_var(osnoise_cpumask_new); return err; } #ifdef CONFIG_TIMERLAT_TRACER static int timerlat_fd_open(struct inode *inode, struct file *file) { struct osnoise_variables *osn_var; struct timerlat_variables *tlat; long cpu = (long) inode->i_cdev; mutex_lock(&interface_lock); /* * This file is accessible only if timerlat is enabled, and * NO_OSNOISE_WORKLOAD is set. */ if (!timerlat_enabled() || test_bit(OSN_WORKLOAD, &osnoise_options)) { mutex_unlock(&interface_lock); return -EINVAL; } migrate_disable(); osn_var = this_cpu_osn_var(); /* * The osn_var->pid holds the single access to this file. */ if (osn_var->pid) { mutex_unlock(&interface_lock); migrate_enable(); return -EBUSY; } /* * timerlat tracer is a per-cpu tracer. Check if the user-space too * is pinned to a single CPU. The tracer laters monitor if the task * migrates and then disables tracer if it does. However, it is * worth doing this basic acceptance test to avoid obviusly wrong * setup. */ if (current->nr_cpus_allowed > 1 || cpu != smp_processor_id()) { mutex_unlock(&interface_lock); migrate_enable(); return -EPERM; } /* * From now on, it is good to go. */ file->private_data = inode->i_cdev; get_task_struct(current); osn_var->kthread = current; osn_var->pid = current->pid; /* * Setup is done. */ mutex_unlock(&interface_lock); tlat = this_cpu_tmr_var(); tlat->count = 0; hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD); tlat->timer.function = timerlat_irq; migrate_enable(); return 0; }; /* * timerlat_fd_read - Read function for "timerlat_fd" file * @file: The active open file structure * @ubuf: The userspace provided buffer to read value into * @cnt: The maximum number of bytes to read * @ppos: The current "file" position * * Prints 1 on timerlat, the number of interferences on osnoise, -1 on error. */ static ssize_t timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { long cpu = (long) file->private_data; struct osnoise_variables *osn_var; struct timerlat_variables *tlat; struct timerlat_sample s; s64 diff; u64 now; migrate_disable(); tlat = this_cpu_tmr_var(); /* * While in user-space, the thread is migratable. There is nothing * we can do about it. * So, if the thread is running on another CPU, stop the machinery. */ if (cpu == smp_processor_id()) { if (tlat->uthread_migrate) { migrate_enable(); return -EINVAL; } } else { per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1; osnoise_taint("timerlat user thread migrate\n"); osnoise_stop_tracing(); migrate_enable(); return -EINVAL; } osn_var = this_cpu_osn_var(); /* * The timerlat in user-space runs in a different order: * the read() starts from the execution of the previous occurrence, * sleeping for the next occurrence. * * So, skip if we are entering on read() before the first wakeup * from timerlat IRQ: */ if (likely(osn_var->sampling)) { now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer)); diff = now - tlat->abs_period; /* * it was not a timer firing, but some other signal? */ if (diff < 0) goto out; s.seqnum = tlat->count; s.timer_latency = diff; s.context = THREAD_URET; trace_timerlat_sample(&s); notify_new_max_latency(diff); tlat->tracing_thread = false; if (osnoise_data.stop_tracing_total) if (time_to_us(diff) >= osnoise_data.stop_tracing_total) osnoise_stop_tracing(); } else { tlat->tracing_thread = false; tlat->kthread = current; /* Annotate now to drift new period */ tlat->abs_period = hrtimer_cb_get_time(&tlat->timer); osn_var->sampling = 1; } /* wait for the next period */ wait_next_period(tlat); /* This is the wakeup from this cycle */ now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer)); diff = now - tlat->abs_period; /* * it was not a timer firing, but some other signal? */ if (diff < 0) goto out; s.seqnum = tlat->count; s.timer_latency = diff; s.context = THREAD_CONTEXT; trace_timerlat_sample(&s); if (osnoise_data.stop_tracing_total) { if (time_to_us(diff) >= osnoise_data.stop_tracing_total) { timerlat_dump_stack(time_to_us(diff)); notify_new_max_latency(diff); osnoise_stop_tracing(); } } out: migrate_enable(); return 0; } static int timerlat_fd_release(struct inode *inode, struct file *file) { struct osnoise_variables *osn_var; struct timerlat_variables *tlat_var; long cpu = (long) file->private_data; migrate_disable(); mutex_lock(&interface_lock); osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu); tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu); if (tlat_var->kthread) hrtimer_cancel(&tlat_var->timer); memset(tlat_var, 0, sizeof(*tlat_var)); osn_var->sampling = 0; osn_var->pid = 0; /* * We are leaving, not being stopped... see stop_kthread(); */ if (osn_var->kthread) { put_task_struct(osn_var->kthread); osn_var->kthread = NULL; } mutex_unlock(&interface_lock); migrate_enable(); return 0; } #endif /* * osnoise/runtime_us: cannot be greater than the period. */ static struct trace_min_max_param osnoise_runtime = { .lock = &interface_lock, .val = &osnoise_data.sample_runtime, .max = &osnoise_data.sample_period, .min = NULL, }; /* * osnoise/period_us: cannot be smaller than the runtime. */ static struct trace_min_max_param osnoise_period = { .lock = &interface_lock, .val = &osnoise_data.sample_period, .max = NULL, .min = &osnoise_data.sample_runtime, }; /* * osnoise/stop_tracing_us: no limit. */ static struct trace_min_max_param osnoise_stop_tracing_in = { .lock = &interface_lock, .val = &osnoise_data.stop_tracing, .max = NULL, .min = NULL, }; /* * osnoise/stop_tracing_total_us: no limit. */ static struct trace_min_max_param osnoise_stop_tracing_total = { .lock = &interface_lock, .val = &osnoise_data.stop_tracing_total, .max = NULL, .min = NULL, }; #ifdef CONFIG_TIMERLAT_TRACER /* * osnoise/print_stack: print the stacktrace of the IRQ handler if the total * latency is higher than val. */ static struct trace_min_max_param osnoise_print_stack = { .lock = &interface_lock, .val = &osnoise_data.print_stack, .max = NULL, .min = NULL, }; /* * osnoise/timerlat_period: min 100 us, max 1 s */ static u64 timerlat_min_period = 100; static u64 timerlat_max_period = 1000000; static struct trace_min_max_param timerlat_period = { .lock = &interface_lock, .val = &osnoise_data.timerlat_period, .max = &timerlat_max_period, .min = &timerlat_min_period, }; static const struct file_operations timerlat_fd_fops = { .open = timerlat_fd_open, .read = timerlat_fd_read, .release = timerlat_fd_release, .llseek = generic_file_llseek, }; #endif static const struct file_operations cpus_fops = { .open = tracing_open_generic, .read = osnoise_cpus_read, .write = osnoise_cpus_write, .llseek = generic_file_llseek, }; static const struct file_operations osnoise_options_fops = { .open = osnoise_options_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release, .write = osnoise_options_write }; #ifdef CONFIG_TIMERLAT_TRACER #ifdef CONFIG_STACKTRACE static int init_timerlat_stack_tracefs(struct dentry *top_dir) { struct dentry *tmp; tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir, &osnoise_print_stack, &trace_min_max_fops); if (!tmp) return -ENOMEM; return 0; } #else /* CONFIG_STACKTRACE */ static int init_timerlat_stack_tracefs(struct dentry *top_dir) { return 0; } #endif /* CONFIG_STACKTRACE */ static int osnoise_create_cpu_timerlat_fd(struct dentry *top_dir) { struct dentry *timerlat_fd; struct dentry *per_cpu; struct dentry *cpu_dir; char cpu_str[30]; /* see trace.c: tracing_init_tracefs_percpu() */ long cpu; /* * Why not using tracing instance per_cpu/ dir? * * Because osnoise/timerlat have a single workload, having * multiple files like these are wast of memory. */ per_cpu = tracefs_create_dir("per_cpu", top_dir); if (!per_cpu) return -ENOMEM; for_each_possible_cpu(cpu) { snprintf(cpu_str, 30, "cpu%ld", cpu); cpu_dir = tracefs_create_dir(cpu_str, per_cpu); if (!cpu_dir) goto out_clean; timerlat_fd = trace_create_file("timerlat_fd", TRACE_MODE_READ, cpu_dir, NULL, &timerlat_fd_fops); if (!timerlat_fd) goto out_clean; /* Record the CPU */ d_inode(timerlat_fd)->i_cdev = (void *)(cpu); } return 0; out_clean: tracefs_remove(per_cpu); return -ENOMEM; } /* * init_timerlat_tracefs - A function to initialize the timerlat interface files */ static int init_timerlat_tracefs(struct dentry *top_dir) { struct dentry *tmp; int retval; tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir, &timerlat_period, &trace_min_max_fops); if (!tmp) return -ENOMEM; retval = osnoise_create_cpu_timerlat_fd(top_dir); if (retval) return retval; return init_timerlat_stack_tracefs(top_dir); } #else /* CONFIG_TIMERLAT_TRACER */ static int init_timerlat_tracefs(struct dentry *top_dir) { return 0; } #endif /* CONFIG_TIMERLAT_TRACER */ /* * init_tracefs - A function to initialize the tracefs interface files * * This function creates entries in tracefs for "osnoise" and "timerlat". * It creates these directories in the tracing directory, and within that * directory the use can change and view the configs. */ static int init_tracefs(void) { struct dentry *top_dir; struct dentry *tmp; int ret; ret = tracing_init_dentry(); if (ret) return -ENOMEM; top_dir = tracefs_create_dir("osnoise", NULL); if (!top_dir) return 0; tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir, &osnoise_period, &trace_min_max_fops); if (!tmp) goto err; tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir, &osnoise_runtime, &trace_min_max_fops); if (!tmp) goto err; tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir, &osnoise_stop_tracing_in, &trace_min_max_fops); if (!tmp) goto err; tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir, &osnoise_stop_tracing_total, &trace_min_max_fops); if (!tmp) goto err; tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops); if (!tmp) goto err; tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL, &osnoise_options_fops); if (!tmp) goto err; ret = init_timerlat_tracefs(top_dir); if (ret) goto err; return 0; err: tracefs_remove(top_dir); return -ENOMEM; } static int osnoise_hook_events(void) { int retval; /* * Trace is already hooked, we are re-enabling from * a stop_tracing_*. */ if (trace_osnoise_callback_enabled) return 0; retval = hook_irq_events(); if (retval) return -EINVAL; retval = hook_softirq_events(); if (retval) goto out_unhook_irq; retval = hook_thread_events(); /* * All fine! */ if (!retval) return 0; unhook_softirq_events(); out_unhook_irq: unhook_irq_events(); return -EINVAL; } static void osnoise_unhook_events(void) { unhook_thread_events(); unhook_softirq_events(); unhook_irq_events(); } /* * osnoise_workload_start - start the workload and hook to events */ static int osnoise_workload_start(void) { int retval; /* * Instances need to be registered after calling workload * start. Hence, if there is already an instance, the * workload was already registered. Otherwise, this * code is on the way to register the first instance, * and the workload will start. */ if (osnoise_has_registered_instances()) return 0; osn_var_reset_all(); retval = osnoise_hook_events(); if (retval) return retval; /* * Make sure that ftrace_nmi_enter/exit() see reset values * before enabling trace_osnoise_callback_enabled. */ barrier(); trace_osnoise_callback_enabled = true; retval = start_per_cpu_kthreads(); if (retval) { trace_osnoise_callback_enabled = false; /* * Make sure that ftrace_nmi_enter/exit() see * trace_osnoise_callback_enabled as false before continuing. */ barrier(); osnoise_unhook_events(); return retval; } return 0; } /* * osnoise_workload_stop - stop the workload and unhook the events */ static void osnoise_workload_stop(void) { /* * Instances need to be unregistered before calling * stop. Hence, if there is a registered instance, more * than one instance is running, and the workload will not * yet stop. Otherwise, this code is on the way to disable * the last instance, and the workload can stop. */ if (osnoise_has_registered_instances()) return; /* * If callbacks were already disabled in a previous stop * call, there is no need to disable then again. * * For instance, this happens when tracing is stopped via: * echo 0 > tracing_on * echo nop > current_tracer. */ if (!trace_osnoise_callback_enabled) return; trace_osnoise_callback_enabled = false; /* * Make sure that ftrace_nmi_enter/exit() see * trace_osnoise_callback_enabled as false before continuing. */ barrier(); stop_per_cpu_kthreads(); osnoise_unhook_events(); } static void osnoise_tracer_start(struct trace_array *tr) { int retval; /* * If the instance is already registered, there is no need to * register it again. */ if (osnoise_instance_registered(tr)) return; retval = osnoise_workload_start(); if (retval) pr_err(BANNER "Error starting osnoise tracer\n"); osnoise_register_instance(tr); } static void osnoise_tracer_stop(struct trace_array *tr) { osnoise_unregister_instance(tr); osnoise_workload_stop(); } static int osnoise_tracer_init(struct trace_array *tr) { /* * Only allow osnoise tracer if timerlat tracer is not running * already. */ if (timerlat_enabled()) return -EBUSY; tr->max_latency = 0; osnoise_tracer_start(tr); return 0; } static void osnoise_tracer_reset(struct trace_array *tr) { osnoise_tracer_stop(tr); } static struct tracer osnoise_tracer __read_mostly = { .name = "osnoise", .init = osnoise_tracer_init, .reset = osnoise_tracer_reset, .start = osnoise_tracer_start, .stop = osnoise_tracer_stop, .print_header = print_osnoise_headers, .allow_instances = true, }; #ifdef CONFIG_TIMERLAT_TRACER static void timerlat_tracer_start(struct trace_array *tr) { int retval; /* * If the instance is already registered, there is no need to * register it again. */ if (osnoise_instance_registered(tr)) return; retval = osnoise_workload_start(); if (retval) pr_err(BANNER "Error starting timerlat tracer\n"); osnoise_register_instance(tr); return; } static void timerlat_tracer_stop(struct trace_array *tr) { int cpu; osnoise_unregister_instance(tr); /* * Instruct the threads to stop only if this is the last instance. */ if (!osnoise_has_registered_instances()) { for_each_online_cpu(cpu) per_cpu(per_cpu_osnoise_var, cpu).sampling = 0; } osnoise_workload_stop(); } static int timerlat_tracer_init(struct trace_array *tr) { /* * Only allow timerlat tracer if osnoise tracer is not running already. */ if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer) return -EBUSY; /* * If this is the first instance, set timerlat_tracer to block * osnoise tracer start. */ if (!osnoise_has_registered_instances()) osnoise_data.timerlat_tracer = 1; tr->max_latency = 0; timerlat_tracer_start(tr); return 0; } static void timerlat_tracer_reset(struct trace_array *tr) { timerlat_tracer_stop(tr); /* * If this is the last instance, reset timerlat_tracer allowing * osnoise to be started. */ if (!osnoise_has_registered_instances()) osnoise_data.timerlat_tracer = 0; } static struct tracer timerlat_tracer __read_mostly = { .name = "timerlat", .init = timerlat_tracer_init, .reset = timerlat_tracer_reset, .start = timerlat_tracer_start, .stop = timerlat_tracer_stop, .print_header = print_timerlat_headers, .allow_instances = true, }; __init static int init_timerlat_tracer(void) { return register_tracer(&timerlat_tracer); } #else /* CONFIG_TIMERLAT_TRACER */ __init static int init_timerlat_tracer(void) { return 0; } #endif /* CONFIG_TIMERLAT_TRACER */ __init static int init_osnoise_tracer(void) { int ret; mutex_init(&interface_lock); cpumask_copy(&osnoise_cpumask, cpu_all_mask); ret = register_tracer(&osnoise_tracer); if (ret) { pr_err(BANNER "Error registering osnoise!\n"); return ret; } ret = init_timerlat_tracer(); if (ret) { pr_err(BANNER "Error registering timerlat!\n"); return ret; } osnoise_init_hotplug_support(); INIT_LIST_HEAD_RCU(&osnoise_instances); init_tracefs(); return 0; } late_initcall(init_osnoise_tracer);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1