| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Tejun Heo | 22540 | 88.17% | 103 | 51.24% |
| David Vernet | 1254 | 4.91% | 7 | 3.48% |
| Andrea Righi | 610 | 2.39% | 8 | 3.98% |
| Ingo Molnar | 318 | 1.24% | 9 | 4.48% |
| Peter Zijlstra | 240 | 0.94% | 18 | 8.96% |
| Mathieu Desnoyers | 219 | 0.86% | 4 | 1.99% |
| Linus Torvalds (pre-git) | 85 | 0.33% | 5 | 2.49% |
| Henry Huang | 37 | 0.14% | 2 | 1.00% |
| Hongyan Xia | 33 | 0.13% | 1 | 0.50% |
| Manu Bretelle | 30 | 0.12% | 1 | 0.50% |
| Changwoo Min | 29 | 0.11% | 3 | 1.49% |
| Phil Auld | 18 | 0.07% | 1 | 0.50% |
| Davidlohr Bueso A | 17 | 0.07% | 1 | 0.50% |
| Qian Cai | 16 | 0.06% | 1 | 0.50% |
| Gregory Haskins | 11 | 0.04% | 3 | 1.49% |
| Mel Gorman | 10 | 0.04% | 1 | 0.50% |
| Honglei Wang | 8 | 0.03% | 2 | 1.00% |
| Patrick Bellasi | 8 | 0.03% | 1 | 0.50% |
| Yipeng Zou | 7 | 0.03% | 1 | 0.50% |
| Bharata B Rao | 7 | 0.03% | 1 | 0.50% |
| Frédéric Weisbecker | 6 | 0.02% | 1 | 0.50% |
| Andrew Morton | 5 | 0.02% | 2 | 1.00% |
| Quentin Perret | 4 | 0.02% | 1 | 0.50% |
| Will Deacon | 4 | 0.02% | 1 | 0.50% |
| Tianchen Ding | 4 | 0.02% | 1 | 0.50% |
| Steven Rostedt | 4 | 0.02% | 1 | 0.50% |
| Hao Jia | 4 | 0.02% | 1 | 0.50% |
| Vincent Guittot | 4 | 0.02% | 1 | 0.50% |
| Zhang Qiao | 4 | 0.02% | 2 | 1.00% |
| Hariprasad Nellitheertha | 3 | 0.01% | 1 | 0.50% |
| Joel A Fernandes | 3 | 0.01% | 1 | 0.50% |
| Thomas Gleixner | 3 | 0.01% | 1 | 0.50% |
| Nico Pitre | 2 | 0.01% | 1 | 0.50% |
| Matthew Wilcox | 2 | 0.01% | 1 | 0.50% |
| Aboorva Devarajan | 2 | 0.01% | 1 | 0.50% |
| Zhao Mengmeng | 2 | 0.01% | 1 | 0.50% |
| Dominik Brodowski | 1 | 0.00% | 1 | 0.50% |
| Viresh Kumar | 1 | 0.00% | 1 | 0.50% |
| Emil Velikov | 1 | 0.00% | 1 | 0.50% |
| Venkatesh Pallipadi | 1 | 0.00% | 1 | 0.50% |
| Josh Don | 1 | 0.00% | 1 | 0.50% |
| Yisheng Xie | 1 | 0.00% | 1 | 0.50% |
| Greg Kroah-Hartman | 1 | 0.00% | 1 | 0.50% |
| Nicholas Piggin | 1 | 0.00% | 1 | 0.50% |
| Waiman Long | 1 | 0.00% | 1 | 0.50% |
| Pan Xinhui | 1 | 0.00% | 1 | 0.50% |
| Total | 25563 | 201 |
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694
/* SPDX-License-Identifier: GPL-2.0 */ /* * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst * * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. * Copyright (c) 2022 Tejun Heo <tj@kernel.org> * Copyright (c) 2022 David Vernet <dvernet@meta.com> */ #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) enum scx_consts { SCX_DSP_DFL_MAX_BATCH = 32, SCX_DSP_MAX_LOOPS = 32, SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, SCX_EXIT_BT_LEN = 64, SCX_EXIT_MSG_LEN = 1024, SCX_EXIT_DUMP_DFL_LEN = 32768, SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, /* * Iterating all tasks may take a while. Periodically drop * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. */ SCX_OPS_TASK_ITER_BATCH = 32, }; enum scx_exit_kind { SCX_EXIT_NONE, SCX_EXIT_DONE, SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ }; /* * An exit code can be specified when exiting with scx_bpf_exit() or * scx_ops_exit(), corresponding to exit_kind UNREG_BPF and UNREG_KERN * respectively. The codes are 64bit of the format: * * Bits: [63 .. 48 47 .. 32 31 .. 0] * [ SYS ACT ] [ SYS RSN ] [ USR ] * * SYS ACT: System-defined exit actions * SYS RSN: System-defined exit reasons * USR : User-defined exit codes and reasons * * Using the above, users may communicate intention and context by ORing system * actions and/or system reasons with a user-defined exit code. */ enum scx_exit_code { /* Reasons */ SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, /* Actions */ SCX_ECODE_ACT_RESTART = 1LLU << 48, }; /* * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is * being disabled. */ struct scx_exit_info { /* %SCX_EXIT_* - broad category of the exit reason */ enum scx_exit_kind kind; /* exit code if gracefully exiting */ s64 exit_code; /* textual representation of the above */ const char *reason; /* backtrace if exiting due to an error */ unsigned long *bt; u32 bt_len; /* informational message */ char *msg; /* debug dump */ char *dump; }; /* sched_ext_ops.flags */ enum scx_ops_flags { /* * Keep built-in idle tracking even if ops.update_idle() is implemented. */ SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, /* * By default, if there are no other task to run on the CPU, ext core * keeps running the current task even after its slice expires. If this * flag is specified, such tasks are passed to ops.enqueue() with * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. */ SCX_OPS_ENQ_LAST = 1LLU << 1, /* * An exiting task may schedule after PF_EXITING is set. In such cases, * bpf_task_from_pid() may not be able to find the task and if the BPF * scheduler depends on pid lookup for dispatching, the task will be * lost leading to various issues including RCU grace period stalls. * * To mask this problem, by default, unhashed tasks are automatically * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't * depend on pid lookups and wants to handle these tasks directly, the * following flag can be used. */ SCX_OPS_ENQ_EXITING = 1LLU << 2, /* * If set, only tasks with policy set to SCHED_EXT are attached to * sched_ext. If clear, SCHED_NORMAL tasks are also included. */ SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, /* * CPU cgroup support flags */ SCX_OPS_HAS_CGROUP_WEIGHT = 1LLU << 16, /* cpu.weight */ SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING | SCX_OPS_SWITCH_PARTIAL | SCX_OPS_HAS_CGROUP_WEIGHT, }; /* argument container for ops.init_task() */ struct scx_init_task_args { /* * Set if ops.init_task() is being invoked on the fork path, as opposed * to the scheduler transition path. */ bool fork; #ifdef CONFIG_EXT_GROUP_SCHED /* the cgroup the task is joining */ struct cgroup *cgroup; #endif }; /* argument container for ops.exit_task() */ struct scx_exit_task_args { /* Whether the task exited before running on sched_ext. */ bool cancelled; }; /* argument container for ops->cgroup_init() */ struct scx_cgroup_init_args { /* the weight of the cgroup [1..10000] */ u32 weight; }; enum scx_cpu_preempt_reason { /* next task is being scheduled by &sched_class_rt */ SCX_CPU_PREEMPT_RT, /* next task is being scheduled by &sched_class_dl */ SCX_CPU_PREEMPT_DL, /* next task is being scheduled by &sched_class_stop */ SCX_CPU_PREEMPT_STOP, /* unknown reason for SCX being preempted */ SCX_CPU_PREEMPT_UNKNOWN, }; /* * Argument container for ops->cpu_acquire(). Currently empty, but may be * expanded in the future. */ struct scx_cpu_acquire_args {}; /* argument container for ops->cpu_release() */ struct scx_cpu_release_args { /* the reason the CPU was preempted */ enum scx_cpu_preempt_reason reason; /* the task that's going to be scheduled on the CPU */ struct task_struct *task; }; /* * Informational context provided to dump operations. */ struct scx_dump_ctx { enum scx_exit_kind kind; s64 exit_code; const char *reason; u64 at_ns; u64 at_jiffies; }; /** * struct sched_ext_ops - Operation table for BPF scheduler implementation * * A BPF scheduler can implement an arbitrary scheduling policy by * implementing and loading operations in this table. Note that a userland * scheduling policy can also be implemented using the BPF scheduler * as a shim layer. */ struct sched_ext_ops { /** * select_cpu - Pick the target CPU for a task which is being woken up * @p: task being woken up * @prev_cpu: the cpu @p was on before sleeping * @wake_flags: SCX_WAKE_* * * Decision made here isn't final. @p may be moved to any CPU while it * is getting dispatched for execution later. However, as @p is not on * the rq at this point, getting the eventual execution CPU right here * saves a small bit of overhead down the line. * * If an idle CPU is returned, the CPU is kicked and will try to * dispatch. While an explicit custom mechanism can be added, * select_cpu() serves as the default way to wake up idle CPUs. * * @p may be inserted into a DSQ directly by calling * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ * of the CPU returned by this operation. * * Note that select_cpu() is never called for tasks that can only run * on a single CPU or tasks with migration disabled, as they don't have * the option to select a different CPU. See select_task_rq() for * details. */ s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); /** * enqueue - Enqueue a task on the BPF scheduler * @p: task being enqueued * @enq_flags: %SCX_ENQ_* * * @p is ready to run. Insert directly into a DSQ by calling * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, * the task will stall. * * If @p was inserted into a DSQ from ops.select_cpu(), this callback is * skipped. */ void (*enqueue)(struct task_struct *p, u64 enq_flags); /** * dequeue - Remove a task from the BPF scheduler * @p: task being dequeued * @deq_flags: %SCX_DEQ_* * * Remove @p from the BPF scheduler. This is usually called to isolate * the task while updating its scheduling properties (e.g. priority). * * The ext core keeps track of whether the BPF side owns a given task or * not and can gracefully ignore spurious dispatches from BPF side, * which makes it safe to not implement this method. However, depending * on the scheduling logic, this can lead to confusing behaviors - e.g. * scheduling position not being updated across a priority change. */ void (*dequeue)(struct task_struct *p, u64 deq_flags); /** * dispatch - Dispatch tasks from the BPF scheduler and/or user DSQs * @cpu: CPU to dispatch tasks for * @prev: previous task being switched out * * Called when a CPU's local dsq is empty. The operation should dispatch * one or more tasks from the BPF scheduler into the DSQs using * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ * using scx_bpf_dsq_move_to_local(). * * The maximum number of times scx_bpf_dsq_insert() can be called * without an intervening scx_bpf_dsq_move_to_local() is specified by * ops.dispatch_max_batch. See the comments on top of the two functions * for more details. * * When not %NULL, @prev is an SCX task with its slice depleted. If * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in * @prev->scx.flags, it is not enqueued yet and will be enqueued after * ops.dispatch() returns. To keep executing @prev, return without * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. */ void (*dispatch)(s32 cpu, struct task_struct *prev); /** * tick - Periodic tick * @p: task running currently * * This operation is called every 1/HZ seconds on CPUs which are * executing an SCX task. Setting @p->scx.slice to 0 will trigger an * immediate dispatch cycle on the CPU. */ void (*tick)(struct task_struct *p); /** * runnable - A task is becoming runnable on its associated CPU * @p: task becoming runnable * @enq_flags: %SCX_ENQ_* * * This and the following three functions can be used to track a task's * execution state transitions. A task becomes ->runnable() on a CPU, * and then goes through one or more ->running() and ->stopping() pairs * as it runs on the CPU, and eventually becomes ->quiescent() when it's * done running on the CPU. * * @p is becoming runnable on the CPU because it's * * - waking up (%SCX_ENQ_WAKEUP) * - being moved from another CPU * - being restored after temporarily taken off the queue for an * attribute change. * * This and ->enqueue() are related but not coupled. This operation * notifies @p's state transition and may not be followed by ->enqueue() * e.g. when @p is being dispatched to a remote CPU, or when @p is * being enqueued on a CPU experiencing a hotplug event. Likewise, a * task may be ->enqueue()'d without being preceded by this operation * e.g. after exhausting its slice. */ void (*runnable)(struct task_struct *p, u64 enq_flags); /** * running - A task is starting to run on its associated CPU * @p: task starting to run * * See ->runnable() for explanation on the task state notifiers. */ void (*running)(struct task_struct *p); /** * stopping - A task is stopping execution * @p: task stopping to run * @runnable: is task @p still runnable? * * See ->runnable() for explanation on the task state notifiers. If * !@runnable, ->quiescent() will be invoked after this operation * returns. */ void (*stopping)(struct task_struct *p, bool runnable); /** * quiescent - A task is becoming not runnable on its associated CPU * @p: task becoming not runnable * @deq_flags: %SCX_DEQ_* * * See ->runnable() for explanation on the task state notifiers. * * @p is becoming quiescent on the CPU because it's * * - sleeping (%SCX_DEQ_SLEEP) * - being moved to another CPU * - being temporarily taken off the queue for an attribute change * (%SCX_DEQ_SAVE) * * This and ->dequeue() are related but not coupled. This operation * notifies @p's state transition and may not be preceded by ->dequeue() * e.g. when @p is being dispatched to a remote CPU. */ void (*quiescent)(struct task_struct *p, u64 deq_flags); /** * yield - Yield CPU * @from: yielding task * @to: optional yield target task * * If @to is NULL, @from is yielding the CPU to other runnable tasks. * The BPF scheduler should ensure that other available tasks are * dispatched before the yielding task. Return value is ignored in this * case. * * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf * scheduler can implement the request, return %true; otherwise, %false. */ bool (*yield)(struct task_struct *from, struct task_struct *to); /** * core_sched_before - Task ordering for core-sched * @a: task A * @b: task B * * Used by core-sched to determine the ordering between two tasks. See * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on * core-sched. * * Both @a and @b are runnable and may or may not currently be queued on * the BPF scheduler. Should return %true if @a should run before @b. * %false if there's no required ordering or @b should run before @a. * * If not specified, the default is ordering them according to when they * became runnable. */ bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); /** * set_weight - Set task weight * @p: task to set weight for * @weight: new weight [1..10000] * * Update @p's weight to @weight. */ void (*set_weight)(struct task_struct *p, u32 weight); /** * set_cpumask - Set CPU affinity * @p: task to set CPU affinity for * @cpumask: cpumask of cpus that @p can run on * * Update @p's CPU affinity to @cpumask. */ void (*set_cpumask)(struct task_struct *p, const struct cpumask *cpumask); /** * update_idle - Update the idle state of a CPU * @cpu: CPU to udpate the idle state for * @idle: whether entering or exiting the idle state * * This operation is called when @rq's CPU goes or leaves the idle * state. By default, implementing this operation disables the built-in * idle CPU tracking and the following helpers become unavailable: * * - scx_bpf_select_cpu_dfl() * - scx_bpf_test_and_clear_cpu_idle() * - scx_bpf_pick_idle_cpu() * * The user also must implement ops.select_cpu() as the default * implementation relies on scx_bpf_select_cpu_dfl(). * * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle * tracking. */ void (*update_idle)(s32 cpu, bool idle); /** * cpu_acquire - A CPU is becoming available to the BPF scheduler * @cpu: The CPU being acquired by the BPF scheduler. * @args: Acquire arguments, see the struct definition. * * A CPU that was previously released from the BPF scheduler is now once * again under its control. */ void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); /** * cpu_release - A CPU is taken away from the BPF scheduler * @cpu: The CPU being released by the BPF scheduler. * @args: Release arguments, see the struct definition. * * The specified CPU is no longer under the control of the BPF * scheduler. This could be because it was preempted by a higher * priority sched_class, though there may be other reasons as well. The * caller should consult @args->reason to determine the cause. */ void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); /** * init_task - Initialize a task to run in a BPF scheduler * @p: task to initialize for BPF scheduling * @args: init arguments, see the struct definition * * Either we're loading a BPF scheduler or a new task is being forked. * Initialize @p for BPF scheduling. This operation may block and can * be used for allocations, and is called exactly once for a task. * * Return 0 for success, -errno for failure. An error return while * loading will abort loading of the BPF scheduler. During a fork, it * will abort that specific fork. */ s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); /** * exit_task - Exit a previously-running task from the system * @p: task to exit * * @p is exiting or the BPF scheduler is being unloaded. Perform any * necessary cleanup for @p. */ void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); /** * enable - Enable BPF scheduling for a task * @p: task to enable BPF scheduling for * * Enable @p for BPF scheduling. enable() is called on @p any time it * enters SCX, and is always paired with a matching disable(). */ void (*enable)(struct task_struct *p); /** * disable - Disable BPF scheduling for a task * @p: task to disable BPF scheduling for * * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. * Disable BPF scheduling for @p. A disable() call is always matched * with a prior enable() call. */ void (*disable)(struct task_struct *p); /** * dump - Dump BPF scheduler state on error * @ctx: debug dump context * * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. */ void (*dump)(struct scx_dump_ctx *ctx); /** * dump_cpu - Dump BPF scheduler state for a CPU on error * @ctx: debug dump context * @cpu: CPU to generate debug dump for * @idle: @cpu is currently idle without any runnable tasks * * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for * @cpu. If @idle is %true and this operation doesn't produce any * output, @cpu is skipped for dump. */ void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); /** * dump_task - Dump BPF scheduler state for a runnable task on error * @ctx: debug dump context * @p: runnable task to generate debug dump for * * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for * @p. */ void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); #ifdef CONFIG_EXT_GROUP_SCHED /** * cgroup_init - Initialize a cgroup * @cgrp: cgroup being initialized * @args: init arguments, see the struct definition * * Either the BPF scheduler is being loaded or @cgrp created, initialize * @cgrp for sched_ext. This operation may block. * * Return 0 for success, -errno for failure. An error return while * loading will abort loading of the BPF scheduler. During cgroup * creation, it will abort the specific cgroup creation. */ s32 (*cgroup_init)(struct cgroup *cgrp, struct scx_cgroup_init_args *args); /** * cgroup_exit - Exit a cgroup * @cgrp: cgroup being exited * * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit * @cgrp for sched_ext. This operation my block. */ void (*cgroup_exit)(struct cgroup *cgrp); /** * cgroup_prep_move - Prepare a task to be moved to a different cgroup * @p: task being moved * @from: cgroup @p is being moved from * @to: cgroup @p is being moved to * * Prepare @p for move from cgroup @from to @to. This operation may * block and can be used for allocations. * * Return 0 for success, -errno for failure. An error return aborts the * migration. */ s32 (*cgroup_prep_move)(struct task_struct *p, struct cgroup *from, struct cgroup *to); /** * cgroup_move - Commit cgroup move * @p: task being moved * @from: cgroup @p is being moved from * @to: cgroup @p is being moved to * * Commit the move. @p is dequeued during this operation. */ void (*cgroup_move)(struct task_struct *p, struct cgroup *from, struct cgroup *to); /** * cgroup_cancel_move - Cancel cgroup move * @p: task whose cgroup move is being canceled * @from: cgroup @p was being moved from * @to: cgroup @p was being moved to * * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). * Undo the preparation. */ void (*cgroup_cancel_move)(struct task_struct *p, struct cgroup *from, struct cgroup *to); /** * cgroup_set_weight - A cgroup's weight is being changed * @cgrp: cgroup whose weight is being updated * @weight: new weight [1..10000] * * Update @tg's weight to @weight. */ void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); #endif /* CONFIG_EXT_GROUP_SCHED */ /* * All online ops must come before ops.cpu_online(). */ /** * cpu_online - A CPU became online * @cpu: CPU which just came up * * @cpu just came online. @cpu will not call ops.enqueue() or * ops.dispatch(), nor run tasks associated with other CPUs beforehand. */ void (*cpu_online)(s32 cpu); /** * cpu_offline - A CPU is going offline * @cpu: CPU which is going offline * * @cpu is going offline. @cpu will not call ops.enqueue() or * ops.dispatch(), nor run tasks associated with other CPUs afterwards. */ void (*cpu_offline)(s32 cpu); /* * All CPU hotplug ops must come before ops.init(). */ /** * init - Initialize the BPF scheduler */ s32 (*init)(void); /** * exit - Clean up after the BPF scheduler * @info: Exit info * * ops.exit() is also called on ops.init() failure, which is a bit * unusual. This is to allow rich reporting through @info on how * ops.init() failed. */ void (*exit)(struct scx_exit_info *info); /** * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch */ u32 dispatch_max_batch; /** * flags - %SCX_OPS_* flags */ u64 flags; /** * timeout_ms - The maximum amount of time, in milliseconds, that a * runnable task should be able to wait before being scheduled. The * maximum timeout may not exceed the default timeout of 30 seconds. * * Defaults to the maximum allowed timeout value of 30 seconds. */ u32 timeout_ms; /** * exit_dump_len - scx_exit_info.dump buffer length. If 0, the default * value of 32768 is used. */ u32 exit_dump_len; /** * hotplug_seq - A sequence number that may be set by the scheduler to * detect when a hotplug event has occurred during the loading process. * If 0, no detection occurs. Otherwise, the scheduler will fail to * load if the sequence number does not match @scx_hotplug_seq on the * enable path. */ u64 hotplug_seq; /** * name - BPF scheduler's name * * Must be a non-zero valid BPF object name including only isalnum(), * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the * BPF scheduler is enabled. */ char name[SCX_OPS_NAME_LEN]; }; enum scx_opi { SCX_OPI_BEGIN = 0, SCX_OPI_NORMAL_BEGIN = 0, SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), SCX_OPI_END = SCX_OP_IDX(init), }; enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, SCX_WAKE_TTWU = WF_TTWU, SCX_WAKE_SYNC = WF_SYNC, }; enum scx_enq_flags { /* expose select ENQUEUE_* flags as enums */ SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, SCX_ENQ_HEAD = ENQUEUE_HEAD, SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, /* high 32bits are SCX specific */ /* * Set the following to trigger preemption when calling * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the * current task is cleared to zero and the CPU is kicked into the * scheduling path. Implies %SCX_ENQ_HEAD. */ SCX_ENQ_PREEMPT = 1LLU << 32, /* * The task being enqueued was previously enqueued on the current CPU's * %SCX_DSQ_LOCAL, but was removed from it in a call to the * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was * invoked in a ->cpu_release() callback, and the task is again * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the * task will not be scheduled on the CPU until at least the next invocation * of the ->cpu_acquire() callback. */ SCX_ENQ_REENQ = 1LLU << 40, /* * The task being enqueued is the only task available for the cpu. By * default, ext core keeps executing such tasks but when * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the * %SCX_ENQ_LAST flag set. * * The BPF scheduler is responsible for triggering a follow-up * scheduling event. Otherwise, Execution may stall. */ SCX_ENQ_LAST = 1LLU << 41, /* high 8 bits are internal */ __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, SCX_ENQ_CLEAR_OPSS = 1LLU << 56, SCX_ENQ_DSQ_PRIQ = 1LLU << 57, }; enum scx_deq_flags { /* expose select DEQUEUE_* flags as enums */ SCX_DEQ_SLEEP = DEQUEUE_SLEEP, /* high 32bits are SCX specific */ /* * The generic core-sched layer decided to execute the task even though * it hasn't been dispatched yet. Dequeue from the BPF side. */ SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, }; enum scx_pick_idle_cpu_flags { SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ }; enum scx_kick_flags { /* * Kick the target CPU if idle. Guarantees that the target CPU goes * through at least one full scheduling cycle before going idle. If the * target CPU can be determined to be currently not idle and going to go * through a scheduling cycle before going idle, noop. */ SCX_KICK_IDLE = 1LLU << 0, /* * Preempt the current task and execute the dispatch path. If the * current task of the target CPU is an SCX task, its ->scx.slice is * cleared to zero before the scheduling path is invoked so that the * task expires and the dispatch path is invoked. */ SCX_KICK_PREEMPT = 1LLU << 1, /* * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will * return after the target CPU finishes picking the next task. */ SCX_KICK_WAIT = 1LLU << 2, }; enum scx_tg_flags { SCX_TG_ONLINE = 1U << 0, SCX_TG_INITED = 1U << 1, }; enum scx_ops_enable_state { SCX_OPS_ENABLING, SCX_OPS_ENABLED, SCX_OPS_DISABLING, SCX_OPS_DISABLED, }; static const char *scx_ops_enable_state_str[] = { [SCX_OPS_ENABLING] = "enabling", [SCX_OPS_ENABLED] = "enabled", [SCX_OPS_DISABLING] = "disabling", [SCX_OPS_DISABLED] = "disabled", }; /* * sched_ext_entity->ops_state * * Used to track the task ownership between the SCX core and the BPF scheduler. * State transitions look as follows: * * NONE -> QUEUEING -> QUEUED -> DISPATCHING * ^ | | * | v v * \-------------------------------/ * * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call * sites for explanations on the conditions being waited upon and why they are * safe. Transitions out of them into NONE or QUEUED must store_release and the * waiters should load_acquire. * * Tracking scx_ops_state enables sched_ext core to reliably determine whether * any given task can be dispatched by the BPF scheduler at all times and thus * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler * to try to dispatch any task anytime regardless of its state as the SCX core * can safely reject invalid dispatches. */ enum scx_ops_state { SCX_OPSS_NONE, /* owned by the SCX core */ SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ /* * QSEQ brands each QUEUED instance so that, when dispatch races * dequeue/requeue, the dispatcher can tell whether it still has a claim * on the task being dispatched. * * As some 32bit archs can't do 64bit store_release/load_acquire, * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on * 32bit machines. The dispatch race window QSEQ protects is very narrow * and runs with IRQ disabled. 30 bits should be sufficient. */ SCX_OPSS_QSEQ_SHIFT = 2, }; /* Use macros to ensure that the type is unsigned long for the masks */ #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) /* * During exit, a task may schedule after losing its PIDs. When disabling the * BPF scheduler, we need to be able to iterate tasks in every state to * guarantee system safety. Maintain a dedicated task list which contains every * task between its fork and eventual free. */ static DEFINE_SPINLOCK(scx_tasks_lock); static LIST_HEAD(scx_tasks); /* ops enable/disable */ static struct kthread_worker *scx_ops_helper; static DEFINE_MUTEX(scx_ops_enable_mutex); DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); static unsigned long scx_in_softlockup; static atomic_t scx_ops_breather_depth = ATOMIC_INIT(0); static int scx_ops_bypass_depth; static bool scx_ops_init_task_enabled; static bool scx_switching_all; DEFINE_STATIC_KEY_FALSE(__scx_switched_all); static struct sched_ext_ops scx_ops; static bool scx_warned_zero_slice; static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); static DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); #ifdef CONFIG_SMP static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_llc); static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_numa); #endif static struct static_key_false scx_has_op[SCX_OPI_END] = { [0 ... SCX_OPI_END-1] = STATIC_KEY_FALSE_INIT }; static atomic_t scx_exit_kind = ATOMIC_INIT(SCX_EXIT_DONE); static struct scx_exit_info *scx_exit_info; static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); /* * A monotically increasing sequence number that is incremented every time a * scheduler is enabled. This can be used by to check if any custom sched_ext * scheduler has ever been used in the system. */ static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); /* * The maximum amount of time in jiffies that a task may be runnable without * being scheduled on a CPU. If this timeout is exceeded, it will trigger * scx_ops_error(). */ static unsigned long scx_watchdog_timeout; /* * The last time the delayed work was run. This delayed work relies on * ksoftirqd being able to run to service timer interrupts, so it's possible * that this work itself could get wedged. To account for this, we check that * it's not stalled in the timer tick, and trigger an error if it is. */ static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; static struct delayed_work scx_watchdog_work; /* idle tracking */ #ifdef CONFIG_SMP #ifdef CONFIG_CPUMASK_OFFSTACK #define CL_ALIGNED_IF_ONSTACK #else #define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp #endif static struct { cpumask_var_t cpu; cpumask_var_t smt; } idle_masks CL_ALIGNED_IF_ONSTACK; #endif /* CONFIG_SMP */ /* for %SCX_KICK_WAIT */ static unsigned long __percpu *scx_kick_cpus_pnt_seqs; /* * Direct dispatch marker. * * Non-NULL values are used for direct dispatch from enqueue path. A valid * pointer points to the task currently being enqueued. An ERR_PTR value is used * to indicate that direct dispatch has already happened. */ static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); /* * Dispatch queues. * * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. This is * to avoid live-locking in bypass mode where all tasks are dispatched to * %SCX_DSQ_GLOBAL and all CPUs consume from it. If per-node split isn't * sufficient, it can be further split. */ static struct scx_dispatch_q **global_dsqs; static const struct rhashtable_params dsq_hash_params = { .key_len = 8, .key_offset = offsetof(struct scx_dispatch_q, id), .head_offset = offsetof(struct scx_dispatch_q, hash_node), }; static struct rhashtable dsq_hash; static LLIST_HEAD(dsqs_to_free); /* dispatch buf */ struct scx_dsp_buf_ent { struct task_struct *task; unsigned long qseq; u64 dsq_id; u64 enq_flags; }; static u32 scx_dsp_max_batch; struct scx_dsp_ctx { struct rq *rq; u32 cursor; u32 nr_tasks; struct scx_dsp_buf_ent buf[]; }; static struct scx_dsp_ctx __percpu *scx_dsp_ctx; /* string formatting from BPF */ struct scx_bstr_buf { u64 data[MAX_BPRINTF_VARARGS]; char line[SCX_EXIT_MSG_LEN]; }; static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); static struct scx_bstr_buf scx_exit_bstr_buf; /* ops debug dump */ struct scx_dump_data { s32 cpu; bool first; s32 cursor; struct seq_buf *s; const char *prefix; struct scx_bstr_buf buf; }; static struct scx_dump_data scx_dump_data = { .cpu = -1, }; /* /sys/kernel/sched_ext interface */ static struct kset *scx_kset; static struct kobject *scx_root_kobj; #define CREATE_TRACE_POINTS #include <trace/events/sched_ext.h> static void process_ddsp_deferred_locals(struct rq *rq); static void scx_bpf_kick_cpu(s32 cpu, u64 flags); static __printf(3, 4) void scx_ops_exit_kind(enum scx_exit_kind kind, s64 exit_code, const char *fmt, ...); #define scx_ops_error_kind(err, fmt, args...) \ scx_ops_exit_kind((err), 0, fmt, ##args) #define scx_ops_exit(code, fmt, args...) \ scx_ops_exit_kind(SCX_EXIT_UNREG_KERN, (code), fmt, ##args) #define scx_ops_error(fmt, args...) \ scx_ops_error_kind(SCX_EXIT_ERROR, fmt, ##args) #define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) static long jiffies_delta_msecs(unsigned long at, unsigned long now) { if (time_after(at, now)) return jiffies_to_msecs(at - now); else return -(long)jiffies_to_msecs(now - at); } /* if the highest set bit is N, return a mask with bits [N+1, 31] set */ static u32 higher_bits(u32 flags) { return ~((1 << fls(flags)) - 1); } /* return the mask with only the highest bit set */ static u32 highest_bit(u32 flags) { int bit = fls(flags); return ((u64)1 << bit) >> 1; } static bool u32_before(u32 a, u32 b) { return (s32)(a - b) < 0; } static struct scx_dispatch_q *find_global_dsq(struct task_struct *p) { return global_dsqs[cpu_to_node(task_cpu(p))]; } static struct scx_dispatch_q *find_user_dsq(u64 dsq_id) { return rhashtable_lookup_fast(&dsq_hash, &dsq_id, dsq_hash_params); } /* * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check * whether it's running from an allowed context. * * @mask is constant, always inline to cull the mask calculations. */ static __always_inline void scx_kf_allow(u32 mask) { /* nesting is allowed only in increasing scx_kf_mask order */ WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask, "invalid nesting current->scx.kf_mask=0x%x mask=0x%x\n", current->scx.kf_mask, mask); current->scx.kf_mask |= mask; barrier(); } static void scx_kf_disallow(u32 mask) { barrier(); current->scx.kf_mask &= ~mask; } #define SCX_CALL_OP(mask, op, args...) \ do { \ if (mask) { \ scx_kf_allow(mask); \ scx_ops.op(args); \ scx_kf_disallow(mask); \ } else { \ scx_ops.op(args); \ } \ } while (0) #define SCX_CALL_OP_RET(mask, op, args...) \ ({ \ __typeof__(scx_ops.op(args)) __ret; \ if (mask) { \ scx_kf_allow(mask); \ __ret = scx_ops.op(args); \ scx_kf_disallow(mask); \ } else { \ __ret = scx_ops.op(args); \ } \ __ret; \ }) /* * Some kfuncs are allowed only on the tasks that are subjects of the * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such * restrictions, the following SCX_CALL_OP_*() variants should be used when * invoking scx_ops operations that take task arguments. These can only be used * for non-nesting operations due to the way the tasks are tracked. * * kfuncs which can only operate on such tasks can in turn use * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on * the specific task. */ #define SCX_CALL_OP_TASK(mask, op, task, args...) \ do { \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task; \ SCX_CALL_OP(mask, op, task, ##args); \ current->scx.kf_tasks[0] = NULL; \ } while (0) #define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ ({ \ __typeof__(scx_ops.op(task, ##args)) __ret; \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task; \ __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ current->scx.kf_tasks[0] = NULL; \ __ret; \ }) #define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ ({ \ __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task0; \ current->scx.kf_tasks[1] = task1; \ __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ current->scx.kf_tasks[0] = NULL; \ current->scx.kf_tasks[1] = NULL; \ __ret; \ }) /* @mask is constant, always inline to cull unnecessary branches */ static __always_inline bool scx_kf_allowed(u32 mask) { if (unlikely(!(current->scx.kf_mask & mask))) { scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", mask, current->scx.kf_mask); return false; } /* * Enforce nesting boundaries. e.g. A kfunc which can be called from * DISPATCH must not be called if we're running DEQUEUE which is nested * inside ops.dispatch(). We don't need to check boundaries for any * blocking kfuncs as the verifier ensures they're only called from * sleepable progs. */ if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { scx_ops_error("cpu_release kfunc called from a nested operation"); return false; } if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) { scx_ops_error("dispatch kfunc called from a nested operation"); return false; } return true; } /* see SCX_CALL_OP_TASK() */ static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, struct task_struct *p) { if (!scx_kf_allowed(mask)) return false; if (unlikely((p != current->scx.kf_tasks[0] && p != current->scx.kf_tasks[1]))) { scx_ops_error("called on a task not being operated on"); return false; } return true; } static bool scx_kf_allowed_if_unlocked(void) { return !current->scx.kf_mask; } /** * nldsq_next_task - Iterate to the next task in a non-local DSQ * @dsq: user dsq being interated * @cur: current position, %NULL to start iteration * @rev: walk backwards * * Returns %NULL when iteration is finished. */ static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, struct task_struct *cur, bool rev) { struct list_head *list_node; struct scx_dsq_list_node *dsq_lnode; lockdep_assert_held(&dsq->lock); if (cur) list_node = &cur->scx.dsq_list.node; else list_node = &dsq->list; /* find the next task, need to skip BPF iteration cursors */ do { if (rev) list_node = list_node->prev; else list_node = list_node->next; if (list_node == &dsq->list) return NULL; dsq_lnode = container_of(list_node, struct scx_dsq_list_node, node); } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); return container_of(dsq_lnode, struct task_struct, scx.dsq_list); } #define nldsq_for_each_task(p, dsq) \ for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ (p) = nldsq_next_task((dsq), (p), false)) /* * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] * dispatch order. BPF-visible iterator is opaque and larger to allow future * changes without breaking backward compatibility. Can be used with * bpf_for_each(). See bpf_iter_scx_dsq_*(). */ enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | __SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME, }; struct bpf_iter_scx_dsq_kern { struct scx_dsq_list_node cursor; struct scx_dispatch_q *dsq; u64 slice; u64 vtime; } __attribute__((aligned(8))); struct bpf_iter_scx_dsq { u64 __opaque[6]; } __attribute__((aligned(8))); /* * SCX task iterator. */ struct scx_task_iter { struct sched_ext_entity cursor; struct task_struct *locked; struct rq *rq; struct rq_flags rf; u32 cnt; }; /** * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration * @iter: iterator to init * * Initialize @iter and return with scx_tasks_lock held. Once initialized, @iter * must eventually be stopped with scx_task_iter_stop(). * * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() * between this and the first next() call or between any two next() calls. If * the locks are released between two next() calls, the caller is responsible * for ensuring that the task being iterated remains accessible either through * RCU read lock or obtaining a reference count. * * All tasks which existed when the iteration started are guaranteed to be * visited as long as they still exist. */ static void scx_task_iter_start(struct scx_task_iter *iter) { BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); spin_lock_irq(&scx_tasks_lock); iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; list_add(&iter->cursor.tasks_node, &scx_tasks); iter->locked = NULL; iter->cnt = 0; } static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) { if (iter->locked) { task_rq_unlock(iter->rq, iter->locked, &iter->rf); iter->locked = NULL; } } /** * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator * @iter: iterator to unlock * * If @iter is in the middle of a locked iteration, it may be locking the rq of * the task currently being visited in addition to scx_tasks_lock. Unlock both. * This function can be safely called anytime during an iteration. */ static void scx_task_iter_unlock(struct scx_task_iter *iter) { __scx_task_iter_rq_unlock(iter); spin_unlock_irq(&scx_tasks_lock); } /** * scx_task_iter_relock - Lock scx_tasks_lock released by scx_task_iter_unlock() * @iter: iterator to re-lock * * Re-lock scx_tasks_lock unlocked by scx_task_iter_unlock(). Note that it * doesn't re-lock the rq lock. Must be called before other iterator operations. */ static void scx_task_iter_relock(struct scx_task_iter *iter) { spin_lock_irq(&scx_tasks_lock); } /** * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock * @iter: iterator to exit * * Exit a previously initialized @iter. Must be called with scx_tasks_lock held * which is released on return. If the iterator holds a task's rq lock, that rq * lock is also released. See scx_task_iter_start() for details. */ static void scx_task_iter_stop(struct scx_task_iter *iter) { list_del_init(&iter->cursor.tasks_node); scx_task_iter_unlock(iter); } /** * scx_task_iter_next - Next task * @iter: iterator to walk * * Visit the next task. See scx_task_iter_start() for details. Locks are dropped * and re-acquired every %SCX_OPS_TASK_ITER_BATCH iterations to avoid causing * stalls by holding scx_tasks_lock for too long. */ static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) { struct list_head *cursor = &iter->cursor.tasks_node; struct sched_ext_entity *pos; if (!(++iter->cnt % SCX_OPS_TASK_ITER_BATCH)) { scx_task_iter_unlock(iter); cond_resched(); scx_task_iter_relock(iter); } list_for_each_entry(pos, cursor, tasks_node) { if (&pos->tasks_node == &scx_tasks) return NULL; if (!(pos->flags & SCX_TASK_CURSOR)) { list_move(cursor, &pos->tasks_node); return container_of(pos, struct task_struct, scx); } } /* can't happen, should always terminate at scx_tasks above */ BUG(); } /** * scx_task_iter_next_locked - Next non-idle task with its rq locked * @iter: iterator to walk * @include_dead: Whether we should include dead tasks in the iteration * * Visit the non-idle task with its rq lock held. Allows callers to specify * whether they would like to filter out dead tasks. See scx_task_iter_start() * for details. */ static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) { struct task_struct *p; __scx_task_iter_rq_unlock(iter); while ((p = scx_task_iter_next(iter))) { /* * scx_task_iter is used to prepare and move tasks into SCX * while loading the BPF scheduler and vice-versa while * unloading. The init_tasks ("swappers") should be excluded * from the iteration because: * * - It's unsafe to use __setschduler_prio() on an init_task to * determine the sched_class to use as it won't preserve its * idle_sched_class. * * - ops.init/exit_task() can easily be confused if called with * init_tasks as they, e.g., share PID 0. * * As init_tasks are never scheduled through SCX, they can be * skipped safely. Note that is_idle_task() which tests %PF_IDLE * doesn't work here: * * - %PF_IDLE may not be set for an init_task whose CPU hasn't * yet been onlined. * * - %PF_IDLE can be set on tasks that are not init_tasks. See * play_idle_precise() used by CONFIG_IDLE_INJECT. * * Test for idle_sched_class as only init_tasks are on it. */ if (p->sched_class != &idle_sched_class) break; } if (!p) return NULL; iter->rq = task_rq_lock(p, &iter->rf); iter->locked = p; return p; } static enum scx_ops_enable_state scx_ops_enable_state(void) { return atomic_read(&scx_ops_enable_state_var); } static enum scx_ops_enable_state scx_ops_set_enable_state(enum scx_ops_enable_state to) { return atomic_xchg(&scx_ops_enable_state_var, to); } static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, enum scx_ops_enable_state from) { int from_v = from; return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); } static bool scx_rq_bypassing(struct rq *rq) { return unlikely(rq->scx.flags & SCX_RQ_BYPASSING); } /** * wait_ops_state - Busy-wait the specified ops state to end * @p: target task * @opss: state to wait the end of * * Busy-wait for @p to transition out of @opss. This can only be used when the * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also * has load_acquire semantics to ensure that the caller can see the updates made * in the enqueueing and dispatching paths. */ static void wait_ops_state(struct task_struct *p, unsigned long opss) { do { cpu_relax(); } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); } /** * ops_cpu_valid - Verify a cpu number * @cpu: cpu number which came from a BPF ops * @where: extra information reported on error * * @cpu is a cpu number which came from the BPF scheduler and can be any value. * Verify that it is in range and one of the possible cpus. If invalid, trigger * an ops error. */ static bool ops_cpu_valid(s32 cpu, const char *where) { if (likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu))) { return true; } else { scx_ops_error("invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); return false; } } /** * ops_sanitize_err - Sanitize a -errno value * @ops_name: operation to blame on failure * @err: -errno value to sanitize * * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return * -%EPROTO. This is necessary because returning a rogue -errno up the chain can * cause misbehaviors. For an example, a large negative return from * ops.init_task() triggers an oops when passed up the call chain because the * value fails IS_ERR() test after being encoded with ERR_PTR() and then is * handled as a pointer. */ static int ops_sanitize_err(const char *ops_name, s32 err) { if (err < 0 && err >= -MAX_ERRNO) return err; scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); return -EPROTO; } static void run_deferred(struct rq *rq) { process_ddsp_deferred_locals(rq); } #ifdef CONFIG_SMP static void deferred_bal_cb_workfn(struct rq *rq) { run_deferred(rq); } #endif static void deferred_irq_workfn(struct irq_work *irq_work) { struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); raw_spin_rq_lock(rq); run_deferred(rq); raw_spin_rq_unlock(rq); } /** * schedule_deferred - Schedule execution of deferred actions on an rq * @rq: target rq * * Schedule execution of deferred actions on @rq. Must be called with @rq * locked. Deferred actions are executed with @rq locked but unpinned, and thus * can unlock @rq to e.g. migrate tasks to other rqs. */ static void schedule_deferred(struct rq *rq) { lockdep_assert_rq_held(rq); #ifdef CONFIG_SMP /* * If in the middle of waking up a task, task_woken_scx() will be called * afterwards which will then run the deferred actions, no need to * schedule anything. */ if (rq->scx.flags & SCX_RQ_IN_WAKEUP) return; /* * If in balance, the balance callbacks will be called before rq lock is * released. Schedule one. */ if (rq->scx.flags & SCX_RQ_IN_BALANCE) { queue_balance_callback(rq, &rq->scx.deferred_bal_cb, deferred_bal_cb_workfn); return; } #endif /* * No scheduler hooks available. Queue an irq work. They are executed on * IRQ re-enable which may take a bit longer than the scheduler hooks. * The above WAKEUP and BALANCE paths should cover most of the cases and * the time to IRQ re-enable shouldn't be long. */ irq_work_queue(&rq->scx.deferred_irq_work); } /** * touch_core_sched - Update timestamp used for core-sched task ordering * @rq: rq to read clock from, must be locked * @p: task to update the timestamp for * * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to * implement global or local-DSQ FIFO ordering for core-sched. Should be called * when a task becomes runnable and its turn on the CPU ends (e.g. slice * exhaustion). */ static void touch_core_sched(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); #ifdef CONFIG_SCHED_CORE /* * It's okay to update the timestamp spuriously. Use * sched_core_disabled() which is cheaper than enabled(). * * As this is used to determine ordering between tasks of sibling CPUs, * it may be better to use per-core dispatch sequence instead. */ if (!sched_core_disabled()) p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); #endif } /** * touch_core_sched_dispatch - Update core-sched timestamp on dispatch * @rq: rq to read clock from, must be locked * @p: task being dispatched * * If the BPF scheduler implements custom core-sched ordering via * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO * ordering within each local DSQ. This function is called from dispatch paths * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. */ static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); #ifdef CONFIG_SCHED_CORE if (SCX_HAS_OP(core_sched_before)) touch_core_sched(rq, p); #endif } static void update_curr_scx(struct rq *rq) { struct task_struct *curr = rq->curr; s64 delta_exec; delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return; if (curr->scx.slice != SCX_SLICE_INF) { curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); if (!curr->scx.slice) touch_core_sched(rq, curr); } } static bool scx_dsq_priq_less(struct rb_node *node_a, const struct rb_node *node_b) { const struct task_struct *a = container_of(node_a, struct task_struct, scx.dsq_priq); const struct task_struct *b = container_of(node_b, struct task_struct, scx.dsq_priq); return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); } static void dsq_mod_nr(struct scx_dispatch_q *dsq, s32 delta) { /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ WRITE_ONCE(dsq->nr, dsq->nr + delta); } static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { bool is_local = dsq->id == SCX_DSQ_LOCAL; WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || !RB_EMPTY_NODE(&p->scx.dsq_priq)); if (!is_local) { raw_spin_lock(&dsq->lock); if (unlikely(dsq->id == SCX_DSQ_INVALID)) { scx_ops_error("attempting to dispatch to a destroyed dsq"); /* fall back to the global dsq */ raw_spin_unlock(&dsq->lock); dsq = find_global_dsq(p); raw_spin_lock(&dsq->lock); } } if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && (enq_flags & SCX_ENQ_DSQ_PRIQ))) { /* * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from * their FIFO queues. To avoid confusion and accidentally * starving vtime-dispatched tasks by FIFO-dispatched tasks, we * disallow any internal DSQ from doing vtime ordering of * tasks. */ scx_ops_error("cannot use vtime ordering for built-in DSQs"); enq_flags &= ~SCX_ENQ_DSQ_PRIQ; } if (enq_flags & SCX_ENQ_DSQ_PRIQ) { struct rb_node *rbp; /* * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are * linked to both the rbtree and list on PRIQs, this can only be * tested easily when adding the first task. */ if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && nldsq_next_task(dsq, NULL, false))) scx_ops_error("DSQ ID 0x%016llx already had FIFO-enqueued tasks", dsq->id); p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); /* * Find the previous task and insert after it on the list so * that @dsq->list is vtime ordered. */ rbp = rb_prev(&p->scx.dsq_priq); if (rbp) { struct task_struct *prev = container_of(rbp, struct task_struct, scx.dsq_priq); list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); } else { list_add(&p->scx.dsq_list.node, &dsq->list); } } else { /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) scx_ops_error("DSQ ID 0x%016llx already had PRIQ-enqueued tasks", dsq->id); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) list_add(&p->scx.dsq_list.node, &dsq->list); else list_add_tail(&p->scx.dsq_list.node, &dsq->list); } /* seq records the order tasks are queued, used by BPF DSQ iterator */ dsq->seq++; p->scx.dsq_seq = dsq->seq; dsq_mod_nr(dsq, 1); p->scx.dsq = dsq; /* * scx.ddsp_dsq_id and scx.ddsp_enq_flags are only relevant on the * direct dispatch path, but we clear them here because the direct * dispatch verdict may be overridden on the enqueue path during e.g. * bypass. */ p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; p->scx.ddsp_enq_flags = 0; /* * We're transitioning out of QUEUEING or DISPATCHING. store_release to * match waiters' load_acquire. */ if (enq_flags & SCX_ENQ_CLEAR_OPSS) atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); if (is_local) { struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); bool preempt = false; if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { rq->curr->scx.slice = 0; preempt = true; } if (preempt || sched_class_above(&ext_sched_class, rq->curr->sched_class)) resched_curr(rq); } else { raw_spin_unlock(&dsq->lock); } } static void task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) { WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { rb_erase(&p->scx.dsq_priq, &dsq->priq); RB_CLEAR_NODE(&p->scx.dsq_priq); p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; } list_del_init(&p->scx.dsq_list.node); dsq_mod_nr(dsq, -1); } static void dispatch_dequeue(struct rq *rq, struct task_struct *p) { struct scx_dispatch_q *dsq = p->scx.dsq; bool is_local = dsq == &rq->scx.local_dsq; if (!dsq) { /* * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. * Unlinking is all that's needed to cancel. */ if (unlikely(!list_empty(&p->scx.dsq_list.node))) list_del_init(&p->scx.dsq_list.node); /* * When dispatching directly from the BPF scheduler to a local * DSQ, the task isn't associated with any DSQ but * @p->scx.holding_cpu may be set under the protection of * %SCX_OPSS_DISPATCHING. */ if (p->scx.holding_cpu >= 0) p->scx.holding_cpu = -1; return; } if (!is_local) raw_spin_lock(&dsq->lock); /* * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't * change underneath us. */ if (p->scx.holding_cpu < 0) { /* @p must still be on @dsq, dequeue */ task_unlink_from_dsq(p, dsq); } else { /* * We're racing against dispatch_to_local_dsq() which already * removed @p from @dsq and set @p->scx.holding_cpu. Clear the * holding_cpu which tells dispatch_to_local_dsq() that it lost * the race. */ WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); p->scx.holding_cpu = -1; } p->scx.dsq = NULL; if (!is_local) raw_spin_unlock(&dsq->lock); } static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, struct task_struct *p) { struct scx_dispatch_q *dsq; if (dsq_id == SCX_DSQ_LOCAL) return &rq->scx.local_dsq; if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; if (!ops_cpu_valid(cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) return find_global_dsq(p); return &cpu_rq(cpu)->scx.local_dsq; } if (dsq_id == SCX_DSQ_GLOBAL) dsq = find_global_dsq(p); else dsq = find_user_dsq(dsq_id); if (unlikely(!dsq)) { scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", dsq_id, p->comm, p->pid); return find_global_dsq(p); } return dsq; } static void mark_direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p, u64 dsq_id, u64 enq_flags) { /* * Mark that dispatch already happened from ops.select_cpu() or * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value * which can never match a valid task pointer. */ __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); /* @p must match the task on the enqueue path */ if (unlikely(p != ddsp_task)) { if (IS_ERR(ddsp_task)) scx_ops_error("%s[%d] already direct-dispatched", p->comm, p->pid); else scx_ops_error("scheduling for %s[%d] but trying to direct-dispatch %s[%d]", ddsp_task->comm, ddsp_task->pid, p->comm, p->pid); return; } WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); WARN_ON_ONCE(p->scx.ddsp_enq_flags); p->scx.ddsp_dsq_id = dsq_id; p->scx.ddsp_enq_flags = enq_flags; } static void direct_dispatch(struct task_struct *p, u64 enq_flags) { struct rq *rq = task_rq(p); struct scx_dispatch_q *dsq = find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p); touch_core_sched_dispatch(rq, p); p->scx.ddsp_enq_flags |= enq_flags; /* * We are in the enqueue path with @rq locked and pinned, and thus can't * double lock a remote rq and enqueue to its local DSQ. For * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer * the enqueue so that it's executed when @rq can be unlocked. */ if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { unsigned long opss; opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; switch (opss & SCX_OPSS_STATE_MASK) { case SCX_OPSS_NONE: break; case SCX_OPSS_QUEUEING: /* * As @p was never passed to the BPF side, _release is * not strictly necessary. Still do it for consistency. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); break; default: WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", p->comm, p->pid, opss); atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); break; } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); list_add_tail(&p->scx.dsq_list.node, &rq->scx.ddsp_deferred_locals); schedule_deferred(rq); return; } dispatch_enqueue(dsq, p, p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); } static bool scx_rq_online(struct rq *rq) { /* * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates * the online state as seen from the BPF scheduler. cpu_active() test * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will * stay set until the current scheduling operation is complete even if * we aren't locking @rq. */ return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); } static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu) { struct task_struct **ddsp_taskp; unsigned long qseq; WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); /* rq migration */ if (sticky_cpu == cpu_of(rq)) goto local_norefill; /* * If !scx_rq_online(), we already told the BPF scheduler that the CPU * is offline and are just running the hotplug path. Don't bother the * BPF scheduler. */ if (!scx_rq_online(rq)) goto local; if (scx_rq_bypassing(rq)) goto global; if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct; /* see %SCX_OPS_ENQ_EXITING */ if (!static_branch_unlikely(&scx_ops_enq_exiting) && unlikely(p->flags & PF_EXITING)) goto local; if (!SCX_HAS_OP(enqueue)) goto global; /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); WARN_ON_ONCE(*ddsp_taskp); *ddsp_taskp = p; SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); *ddsp_taskp = NULL; if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct; /* * If not directly dispatched, QUEUEING isn't clear yet and dispatch or * dequeue may be waiting. The store_release matches their load_acquire. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); return; direct: direct_dispatch(p, enq_flags); return; local: /* * For task-ordering, slice refill must be treated as implying the end * of the current slice. Otherwise, the longer @p stays on the CPU, the * higher priority it becomes from scx_prio_less()'s POV. */ touch_core_sched(rq, p); p->scx.slice = SCX_SLICE_DFL; local_norefill: dispatch_enqueue(&rq->scx.local_dsq, p, enq_flags); return; global: touch_core_sched(rq, p); /* see the comment in local: */ p->scx.slice = SCX_SLICE_DFL; dispatch_enqueue(find_global_dsq(p), p, enq_flags); } static bool task_runnable(const struct task_struct *p) { return !list_empty(&p->scx.runnable_node); } static void set_task_runnable(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { p->scx.runnable_at = jiffies; p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; } /* * list_add_tail() must be used. scx_ops_bypass() depends on tasks being * appened to the runnable_list. */ list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); } static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) { list_del_init(&p->scx.runnable_node); if (reset_runnable_at) p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; } static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) { int sticky_cpu = p->scx.sticky_cpu; if (enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; enq_flags |= rq->scx.extra_enq_flags; if (sticky_cpu >= 0) p->scx.sticky_cpu = -1; /* * Restoring a running task will be immediately followed by * set_next_task_scx() which expects the task to not be on the BPF * scheduler as tasks can only start running through local DSQs. Force * direct-dispatch into the local DSQ by setting the sticky_cpu. */ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) sticky_cpu = cpu_of(rq); if (p->scx.flags & SCX_TASK_QUEUED) { WARN_ON_ONCE(!task_runnable(p)); goto out; } set_task_runnable(rq, p); p->scx.flags |= SCX_TASK_QUEUED; rq->scx.nr_running++; add_nr_running(rq, 1); if (SCX_HAS_OP(runnable) && !task_on_rq_migrating(p)) SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); if (enq_flags & SCX_ENQ_WAKEUP) touch_core_sched(rq, p); do_enqueue_task(rq, p, enq_flags, sticky_cpu); out: rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; } static void ops_dequeue(struct task_struct *p, u64 deq_flags) { unsigned long opss; /* dequeue is always temporary, don't reset runnable_at */ clr_task_runnable(p, false); /* acquire ensures that we see the preceding updates on QUEUED */ opss = atomic_long_read_acquire(&p->scx.ops_state); switch (opss & SCX_OPSS_STATE_MASK) { case SCX_OPSS_NONE: break; case SCX_OPSS_QUEUEING: /* * QUEUEING is started and finished while holding @p's rq lock. * As we're holding the rq lock now, we shouldn't see QUEUEING. */ BUG(); case SCX_OPSS_QUEUED: if (SCX_HAS_OP(dequeue)) SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, SCX_OPSS_NONE)) break; fallthrough; case SCX_OPSS_DISPATCHING: /* * If @p is being dispatched from the BPF scheduler to a DSQ, * wait for the transfer to complete so that @p doesn't get * added to its DSQ after dequeueing is complete. * * As we're waiting on DISPATCHING with the rq locked, the * dispatching side shouldn't try to lock the rq while * DISPATCHING is set. See dispatch_to_local_dsq(). * * DISPATCHING shouldn't have qseq set and control can reach * here with NONE @opss from the above QUEUED case block. * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. */ wait_ops_state(p, SCX_OPSS_DISPATCHING); BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); break; } } static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) { if (!(p->scx.flags & SCX_TASK_QUEUED)) { WARN_ON_ONCE(task_runnable(p)); return true; } ops_dequeue(p, deq_flags); /* * A currently running task which is going off @rq first gets dequeued * and then stops running. As we want running <-> stopping transitions * to be contained within runnable <-> quiescent transitions, trigger * ->stopping() early here instead of in put_prev_task_scx(). * * @p may go through multiple stopping <-> running transitions between * here and put_prev_task_scx() if task attribute changes occur while * balance_scx() leaves @rq unlocked. However, they don't contain any * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ if (SCX_HAS_OP(stopping) && task_current(rq, p)) { update_curr_scx(rq); SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); } if (SCX_HAS_OP(quiescent) && !task_on_rq_migrating(p)) SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); if (deq_flags & SCX_DEQ_SLEEP) p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; else p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; p->scx.flags &= ~SCX_TASK_QUEUED; rq->scx.nr_running--; sub_nr_running(rq, 1); dispatch_dequeue(rq, p); return true; } static void yield_task_scx(struct rq *rq) { struct task_struct *p = rq->curr; if (SCX_HAS_OP(yield)) SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); else p->scx.slice = 0; } static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) { struct task_struct *from = rq->curr; if (SCX_HAS_OP(yield)) return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); else return false; } static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *src_dsq, struct rq *dst_rq) { struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; /* @dsq is locked and @p is on @dst_rq */ lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) list_add(&p->scx.dsq_list.node, &dst_dsq->list); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); dsq_mod_nr(dst_dsq, 1); p->scx.dsq = dst_dsq; } #ifdef CONFIG_SMP /** * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ * @p: task to move * @enq_flags: %SCX_ENQ_* * @src_rq: rq to move the task from, locked on entry, released on return * @dst_rq: rq to move the task into, locked on return * * Move @p which is currently on @src_rq to @dst_rq's local DSQ. */ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { lockdep_assert_rq_held(src_rq); /* the following marks @p MIGRATING which excludes dequeue */ deactivate_task(src_rq, p, 0); set_task_cpu(p, cpu_of(dst_rq)); p->scx.sticky_cpu = cpu_of(dst_rq); raw_spin_rq_unlock(src_rq); raw_spin_rq_lock(dst_rq); /* * We want to pass scx-specific enq_flags but activate_task() will * truncate the upper 32 bit. As we own @rq, we can pass them through * @rq->scx.extra_enq_flags instead. */ WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); dst_rq->scx.extra_enq_flags = enq_flags; activate_task(dst_rq, p, 0); dst_rq->scx.extra_enq_flags = 0; } /* * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two * differences: * * - is_cpu_allowed() asks "Can this task run on this CPU?" while * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to * this CPU?". * * While migration is disabled, is_cpu_allowed() has to say "yes" as the task * must be allowed to finish on the CPU that it's currently on regardless of * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the * BPF scheduler shouldn't attempt to migrate a task which has migration * disabled. * * - The BPF scheduler is bypassed while the rq is offline and we can always say * no to the BPF scheduler initiated migrations while offline. */ static bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq, bool trigger_error) { int cpu = cpu_of(rq); /* * We don't require the BPF scheduler to avoid dispatching to offline * CPUs mostly for convenience but also because CPUs can go offline * between scx_bpf_dsq_insert() calls and here. Trigger error iff the * picked CPU is outside the allowed mask. */ if (!task_allowed_on_cpu(p, cpu)) { if (trigger_error) scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", cpu_of(rq), p->comm, p->pid); return false; } if (unlikely(is_migration_disabled(p))) return false; if (!scx_rq_online(rq)) return false; return true; } /** * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq * @p: target task * @dsq: locked DSQ @p is currently on * @src_rq: rq @p is currently on, stable with @dsq locked * * Called with @dsq locked but no rq's locked. We want to move @p to a different * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is * required when transferring into a local DSQ. Even when transferring into a * non-local DSQ, it's better to use the same mechanism to protect against * dequeues and maintain the invariant that @p->scx.dsq can only change while * @src_rq is locked, which e.g. scx_dump_task() depends on. * * We want to grab @src_rq but that can deadlock if we try while locking @dsq, * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As * this may race with dequeue, which can't drop the rq lock or fail, do a little * dancing from our side. * * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu * would be cleared to -1. While other cpus may have updated it to different * values afterwards, as this operation can't be preempted or recurse, the * holding_cpu can never become this CPU again before we're done. Thus, we can * tell whether we lost to dequeue by testing whether the holding_cpu still * points to this CPU. See dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is * still valid. %false if lost to dequeue. */ static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *src_rq) { s32 cpu = raw_smp_processor_id(); lockdep_assert_held(&dsq->lock); WARN_ON_ONCE(p->scx.holding_cpu >= 0); task_unlink_from_dsq(p, dsq); p->scx.holding_cpu = cpu; raw_spin_unlock(&dsq->lock); raw_spin_rq_lock(src_rq); /* task_rq couldn't have changed if we're still the holding cpu */ return likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p)); } static bool consume_remote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *src_rq) { raw_spin_rq_unlock(this_rq); if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { move_remote_task_to_local_dsq(p, 0, src_rq, this_rq); return true; } else { raw_spin_rq_unlock(src_rq); raw_spin_rq_lock(this_rq); return false; } } #else /* CONFIG_SMP */ static inline void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { WARN_ON_ONCE(1); } static inline bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq, bool trigger_error) { return false; } static inline bool consume_remote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *task_rq) { return false; } #endif /* CONFIG_SMP */ /** * move_task_between_dsqs() - Move a task from one DSQ to another * @p: target task * @enq_flags: %SCX_ENQ_* * @src_dsq: DSQ @p is currently on, must not be a local DSQ * @dst_dsq: DSQ @p is being moved to, can be any DSQ * * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq * will change. As @p's task_rq is locked, this function doesn't need to use the * holding_cpu mechanism. * * On return, @src_dsq is unlocked and only @p's new task_rq, which is the * return value, is locked. */ static struct rq *move_task_between_dsqs(struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *src_dsq, struct scx_dispatch_q *dst_dsq) { struct rq *src_rq = task_rq(p), *dst_rq; BUG_ON(src_dsq->id == SCX_DSQ_LOCAL); lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(src_rq); if (dst_dsq->id == SCX_DSQ_LOCAL) { dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); if (!task_can_run_on_remote_rq(p, dst_rq, true)) { dst_dsq = find_global_dsq(p); dst_rq = src_rq; } } else { /* no need to migrate if destination is a non-local DSQ */ dst_rq = src_rq; } /* * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different * CPU, @p will be migrated. */ if (dst_dsq->id == SCX_DSQ_LOCAL) { /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { task_unlink_from_dsq(p, src_dsq); move_local_task_to_local_dsq(p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); } else { raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(p, enq_flags, src_rq, dst_rq); } } else { /* * @p is going from a non-local DSQ to a non-local DSQ. As * $src_dsq is already locked, do an abbreviated dequeue. */ task_unlink_from_dsq(p, src_dsq); p->scx.dsq = NULL; raw_spin_unlock(&src_dsq->lock); dispatch_enqueue(dst_dsq, p, enq_flags); } return dst_rq; } /* * A poorly behaving BPF scheduler can live-lock the system by e.g. incessantly * banging on the same DSQ on a large NUMA system to the point where switching * to the bypass mode can take a long time. Inject artifical delays while the * bypass mode is switching to guarantee timely completion. */ static void scx_ops_breather(struct rq *rq) { u64 until; lockdep_assert_rq_held(rq); if (likely(!atomic_read(&scx_ops_breather_depth))) return; raw_spin_rq_unlock(rq); until = ktime_get_ns() + NSEC_PER_MSEC; do { int cnt = 1024; while (atomic_read(&scx_ops_breather_depth) && --cnt) cpu_relax(); } while (atomic_read(&scx_ops_breather_depth) && time_before64(ktime_get_ns(), until)); raw_spin_rq_lock(rq); } static bool consume_dispatch_q(struct rq *rq, struct scx_dispatch_q *dsq) { struct task_struct *p; retry: /* * This retry loop can repeatedly race against scx_ops_bypass() * dequeueing tasks from @dsq trying to put the system into the bypass * mode. On some multi-socket machines (e.g. 2x Intel 8480c), this can * live-lock the machine into soft lockups. Give a breather. */ scx_ops_breather(rq); /* * The caller can't expect to successfully consume a task if the task's * addition to @dsq isn't guaranteed to be visible somehow. Test * @dsq->list without locking and skip if it seems empty. */ if (list_empty(&dsq->list)) return false; raw_spin_lock(&dsq->lock); nldsq_for_each_task(p, dsq) { struct rq *task_rq = task_rq(p); if (rq == task_rq) { task_unlink_from_dsq(p, dsq); move_local_task_to_local_dsq(p, 0, dsq, rq); raw_spin_unlock(&dsq->lock); return true; } if (task_can_run_on_remote_rq(p, rq, false)) { if (likely(consume_remote_task(rq, p, dsq, task_rq))) return true; goto retry; } } raw_spin_unlock(&dsq->lock); return false; } static bool consume_global_dsq(struct rq *rq) { int node = cpu_to_node(cpu_of(rq)); return consume_dispatch_q(rq, global_dsqs[node]); } /** * dispatch_to_local_dsq - Dispatch a task to a local dsq * @rq: current rq which is locked * @dst_dsq: destination DSQ * @p: task to dispatch * @enq_flags: %SCX_ENQ_* * * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local * DSQ. This function performs all the synchronization dancing needed because * local DSQs are protected with rq locks. * * The caller must have exclusive ownership of @p (e.g. through * %SCX_OPSS_DISPATCHING). */ static void dispatch_to_local_dsq(struct rq *rq, struct scx_dispatch_q *dst_dsq, struct task_struct *p, u64 enq_flags) { struct rq *src_rq = task_rq(p); struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); /* * We're synchronized against dequeue through DISPATCHING. As @p can't * be dequeued, its task_rq and cpus_allowed are stable too. * * If dispatching to @rq that @p is already on, no lock dancing needed. */ if (rq == src_rq && rq == dst_rq) { dispatch_enqueue(dst_dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); return; } #ifdef CONFIG_SMP if (unlikely(!task_can_run_on_remote_rq(p, dst_rq, true))) { dispatch_enqueue(find_global_dsq(p), p, enq_flags | SCX_ENQ_CLEAR_OPSS); return; } /* * @p is on a possibly remote @src_rq which we need to lock to move the * task. If dequeue is in progress, it'd be locking @src_rq and waiting * on DISPATCHING, so we can't grab @src_rq lock while holding * DISPATCHING. * * As DISPATCHING guarantees that @p is wholly ours, we can pretend that * we're moving from a DSQ and use the same mechanism - mark the task * under transfer with holding_cpu, release DISPATCHING and then follow * the same protocol. See unlink_dsq_and_lock_src_rq(). */ p->scx.holding_cpu = raw_smp_processor_id(); /* store_release ensures that dequeue sees the above */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); /* switch to @src_rq lock */ if (rq != src_rq) { raw_spin_rq_unlock(rq); raw_spin_rq_lock(src_rq); } /* task_rq couldn't have changed if we're still the holding cpu */ if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && !WARN_ON_ONCE(src_rq != task_rq(p))) { /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by * abbreviating move_remote_task_to_local_dsq(). */ if (src_rq == dst_rq) { p->scx.holding_cpu = -1; dispatch_enqueue(&dst_rq->scx.local_dsq, p, enq_flags); } else { move_remote_task_to_local_dsq(p, enq_flags, src_rq, dst_rq); } /* if the destination CPU is idle, wake it up */ if (sched_class_above(p->sched_class, dst_rq->curr->sched_class)) resched_curr(dst_rq); } /* switch back to @rq lock */ if (rq != dst_rq) { raw_spin_rq_unlock(dst_rq); raw_spin_rq_lock(rq); } #else /* CONFIG_SMP */ BUG(); /* control can not reach here on UP */ #endif /* CONFIG_SMP */ } /** * finish_dispatch - Asynchronously finish dispatching a task * @rq: current rq which is locked * @p: task to finish dispatching * @qseq_at_dispatch: qseq when @p started getting dispatched * @dsq_id: destination DSQ ID * @enq_flags: %SCX_ENQ_* * * Dispatching to local DSQs may need to wait for queueing to complete or * require rq lock dancing. As we don't wanna do either while inside * ops.dispatch() to avoid locking order inversion, we split dispatching into * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the * task and its qseq. Once ops.dispatch() returns, this function is called to * finish up. * * There is no guarantee that @p is still valid for dispatching or even that it * was valid in the first place. Make sure that the task is still owned by the * BPF scheduler and claim the ownership before dispatching. */ static void finish_dispatch(struct rq *rq, struct task_struct *p, unsigned long qseq_at_dispatch, u64 dsq_id, u64 enq_flags) { struct scx_dispatch_q *dsq; unsigned long opss; touch_core_sched_dispatch(rq, p); retry: /* * No need for _acquire here. @p is accessed only after a successful * try_cmpxchg to DISPATCHING. */ opss = atomic_long_read(&p->scx.ops_state); switch (opss & SCX_OPSS_STATE_MASK) { case SCX_OPSS_DISPATCHING: case SCX_OPSS_NONE: /* someone else already got to it */ return; case SCX_OPSS_QUEUED: /* * If qseq doesn't match, @p has gone through at least one * dispatch/dequeue and re-enqueue cycle between * scx_bpf_dsq_insert() and here and we have no claim on it. */ if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) return; /* * While we know @p is accessible, we don't yet have a claim on * it - the BPF scheduler is allowed to dispatch tasks * spuriously and there can be a racing dequeue attempt. Let's * claim @p by atomically transitioning it from QUEUED to * DISPATCHING. */ if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, SCX_OPSS_DISPATCHING))) break; goto retry; case SCX_OPSS_QUEUEING: /* * do_enqueue_task() is in the process of transferring the task * to the BPF scheduler while holding @p's rq lock. As we aren't * holding any kernel or BPF resource that the enqueue path may * depend upon, it's safe to wait. */ wait_ops_state(p, opss); goto retry; } BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); dsq = find_dsq_for_dispatch(this_rq(), dsq_id, p); if (dsq->id == SCX_DSQ_LOCAL) dispatch_to_local_dsq(rq, dsq, p, enq_flags); else dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); } static void flush_dispatch_buf(struct rq *rq) { struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); u32 u; for (u = 0; u < dspc->cursor; u++) { struct scx_dsp_buf_ent *ent = &dspc->buf[u]; finish_dispatch(rq, ent->task, ent->qseq, ent->dsq_id, ent->enq_flags); } dspc->nr_tasks += dspc->cursor; dspc->cursor = 0; } static int balance_one(struct rq *rq, struct task_struct *prev) { struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); bool prev_on_scx = prev->sched_class == &ext_sched_class; bool prev_on_rq = prev->scx.flags & SCX_TASK_QUEUED; int nr_loops = SCX_DSP_MAX_LOOPS; lockdep_assert_rq_held(rq); rq->scx.flags |= SCX_RQ_IN_BALANCE; rq->scx.flags &= ~(SCX_RQ_BAL_PENDING | SCX_RQ_BAL_KEEP); if (static_branch_unlikely(&scx_ops_cpu_preempt) && unlikely(rq->scx.cpu_released)) { /* * If the previous sched_class for the current CPU was not SCX, * notify the BPF scheduler that it again has control of the * core. This callback complements ->cpu_release(), which is * emitted in switch_class(). */ if (SCX_HAS_OP(cpu_acquire)) SCX_CALL_OP(SCX_KF_REST, cpu_acquire, cpu_of(rq), NULL); rq->scx.cpu_released = false; } if (prev_on_scx) { update_curr_scx(rq); /* * If @prev is runnable & has slice left, it has priority and * fetching more just increases latency for the fetched tasks. * Tell pick_task_scx() to keep running @prev. If the BPF * scheduler wants to handle this explicitly, it should * implement ->cpu_release(). * * See scx_ops_disable_workfn() for the explanation on the * bypassing test. */ if (prev_on_rq && prev->scx.slice && !scx_rq_bypassing(rq)) { rq->scx.flags |= SCX_RQ_BAL_KEEP; goto has_tasks; } } /* if there already are tasks to run, nothing to do */ if (rq->scx.local_dsq.nr) goto has_tasks; if (consume_global_dsq(rq)) goto has_tasks; if (!SCX_HAS_OP(dispatch) || scx_rq_bypassing(rq) || !scx_rq_online(rq)) goto no_tasks; dspc->rq = rq; /* * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, * the local DSQ might still end up empty after a successful * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() * produced some tasks, retry. The BPF scheduler may depend on this * looping behavior to simplify its implementation. */ do { dspc->nr_tasks = 0; SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), prev_on_scx ? prev : NULL); flush_dispatch_buf(rq); if (prev_on_rq && prev->scx.slice) { rq->scx.flags |= SCX_RQ_BAL_KEEP; goto has_tasks; } if (rq->scx.local_dsq.nr) goto has_tasks; if (consume_global_dsq(rq)) goto has_tasks; /* * ops.dispatch() can trap us in this loop by repeatedly * dispatching ineligible tasks. Break out once in a while to * allow the watchdog to run. As IRQ can't be enabled in * balance(), we want to complete this scheduling cycle and then * start a new one. IOW, we want to call resched_curr() on the * next, most likely idle, task, not the current one. Use * scx_bpf_kick_cpu() for deferred kicking. */ if (unlikely(!--nr_loops)) { scx_bpf_kick_cpu(cpu_of(rq), 0); break; } } while (dspc->nr_tasks); no_tasks: /* * Didn't find another task to run. Keep running @prev unless * %SCX_OPS_ENQ_LAST is in effect. */ if (prev_on_rq && (!static_branch_unlikely(&scx_ops_enq_last) || scx_rq_bypassing(rq))) { rq->scx.flags |= SCX_RQ_BAL_KEEP; goto has_tasks; } rq->scx.flags &= ~SCX_RQ_IN_BALANCE; return false; has_tasks: rq->scx.flags &= ~SCX_RQ_IN_BALANCE; return true; } static int balance_scx(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) { int ret; rq_unpin_lock(rq, rf); ret = balance_one(rq, prev); #ifdef CONFIG_SCHED_SMT /* * When core-sched is enabled, this ops.balance() call will be followed * by pick_task_scx() on this CPU and the SMT siblings. Balance the * siblings too. */ if (sched_core_enabled(rq)) { const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); int scpu; for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { struct rq *srq = cpu_rq(scpu); struct task_struct *sprev = srq->curr; WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); update_rq_clock(srq); balance_one(srq, sprev); } } #endif rq_repin_lock(rq, rf); return ret; } static void process_ddsp_deferred_locals(struct rq *rq) { struct task_struct *p; lockdep_assert_rq_held(rq); /* * Now that @rq can be unlocked, execute the deferred enqueueing of * tasks directly dispatched to the local DSQs of other CPUs. See * direct_dispatch(). Keep popping from the head instead of using * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq * temporarily. */ while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, struct task_struct, scx.dsq_list.node))) { struct scx_dispatch_q *dsq; list_del_init(&p->scx.dsq_list.node); dsq = find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p); if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) dispatch_to_local_dsq(rq, dsq, p, p->scx.ddsp_enq_flags); } } static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) { if (p->scx.flags & SCX_TASK_QUEUED) { /* * Core-sched might decide to execute @p before it is * dispatched. Call ops_dequeue() to notify the BPF scheduler. */ ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); dispatch_dequeue(rq, p); } p->se.exec_start = rq_clock_task(rq); /* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(running) && (p->scx.flags & SCX_TASK_QUEUED)) SCX_CALL_OP_TASK(SCX_KF_REST, running, p); clr_task_runnable(p, true); /* * @p is getting newly scheduled or got kicked after someone updated its * slice. Refresh whether tick can be stopped. See scx_can_stop_tick(). */ if ((p->scx.slice == SCX_SLICE_INF) != (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { if (p->scx.slice == SCX_SLICE_INF) rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; else rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; sched_update_tick_dependency(rq); /* * For now, let's refresh the load_avgs just when transitioning * in and out of nohz. In the future, we might want to add a * mechanism which calls the following periodically on * tick-stopped CPUs. */ update_other_load_avgs(rq); } } static enum scx_cpu_preempt_reason preempt_reason_from_class(const struct sched_class *class) { #ifdef CONFIG_SMP if (class == &stop_sched_class) return SCX_CPU_PREEMPT_STOP; #endif if (class == &dl_sched_class) return SCX_CPU_PREEMPT_DL; if (class == &rt_sched_class) return SCX_CPU_PREEMPT_RT; return SCX_CPU_PREEMPT_UNKNOWN; } static void switch_class(struct rq *rq, struct task_struct *next) { const struct sched_class *next_class = next->sched_class; #ifdef CONFIG_SMP /* * Pairs with the smp_load_acquire() issued by a CPU in * kick_cpus_irq_workfn() who is waiting for this CPU to perform a * resched. */ smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1); #endif if (!static_branch_unlikely(&scx_ops_cpu_preempt)) return; /* * The callback is conceptually meant to convey that the CPU is no * longer under the control of SCX. Therefore, don't invoke the callback * if the next class is below SCX (in which case the BPF scheduler has * actively decided not to schedule any tasks on the CPU). */ if (sched_class_above(&ext_sched_class, next_class)) return; /* * At this point we know that SCX was preempted by a higher priority * sched_class, so invoke the ->cpu_release() callback if we have not * done so already. We only send the callback once between SCX being * preempted, and it regaining control of the CPU. * * ->cpu_release() complements ->cpu_acquire(), which is emitted the * next time that balance_scx() is invoked. */ if (!rq->scx.cpu_released) { if (SCX_HAS_OP(cpu_release)) { struct scx_cpu_release_args args = { .reason = preempt_reason_from_class(next_class), .task = next, }; SCX_CALL_OP(SCX_KF_CPU_RELEASE, cpu_release, cpu_of(rq), &args); } rq->scx.cpu_released = true; } } static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { update_curr_scx(rq); /* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(stopping) && (p->scx.flags & SCX_TASK_QUEUED)) SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); if (p->scx.flags & SCX_TASK_QUEUED) { set_task_runnable(rq, p); /* * If @p has slice left and is being put, @p is getting * preempted by a higher priority scheduler class or core-sched * forcing a different task. Leave it at the head of the local * DSQ. */ if (p->scx.slice && !scx_rq_bypassing(rq)) { dispatch_enqueue(&rq->scx.local_dsq, p, SCX_ENQ_HEAD); goto switch_class; } /* * If @p is runnable but we're about to enter a lower * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. */ if (sched_class_above(&ext_sched_class, next->sched_class)) { WARN_ON_ONCE(!static_branch_unlikely(&scx_ops_enq_last)); do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); } else { do_enqueue_task(rq, p, 0, -1); } } switch_class: if (next && next->sched_class != &ext_sched_class) switch_class(rq, next); } static struct task_struct *first_local_task(struct rq *rq) { return list_first_entry_or_null(&rq->scx.local_dsq.list, struct task_struct, scx.dsq_list.node); } static struct task_struct *pick_task_scx(struct rq *rq) { struct task_struct *prev = rq->curr; struct task_struct *p; bool prev_on_scx = prev->sched_class == &ext_sched_class; bool keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; bool kick_idle = false; /* * WORKAROUND: * * %SCX_RQ_BAL_KEEP should be set iff $prev is on SCX as it must just * have gone through balance_scx(). Unfortunately, there currently is a * bug where fair could say yes on balance() but no on pick_task(), * which then ends up calling pick_task_scx() without preceding * balance_scx(). * * Keep running @prev if possible and avoid stalling from entering idle * without balancing. * * Once fair is fixed, remove the workaround and trigger WARN_ON_ONCE() * if pick_task_scx() is called without preceding balance_scx(). */ if (unlikely(rq->scx.flags & SCX_RQ_BAL_PENDING)) { if (prev_on_scx) { keep_prev = true; } else { keep_prev = false; kick_idle = true; } } else if (unlikely(keep_prev && !prev_on_scx)) { /* only allowed during transitions */ WARN_ON_ONCE(scx_ops_enable_state() == SCX_OPS_ENABLED); keep_prev = false; } /* * If balance_scx() is telling us to keep running @prev, replenish slice * if necessary and keep running @prev. Otherwise, pop the first one * from the local DSQ. */ if (keep_prev) { p = prev; if (!p->scx.slice) p->scx.slice = SCX_SLICE_DFL; } else { p = first_local_task(rq); if (!p) { if (kick_idle) scx_bpf_kick_cpu(cpu_of(rq), SCX_KICK_IDLE); return NULL; } if (unlikely(!p->scx.slice)) { if (!scx_rq_bypassing(rq) && !scx_warned_zero_slice) { printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", p->comm, p->pid, __func__); scx_warned_zero_slice = true; } p->scx.slice = SCX_SLICE_DFL; } } return p; } #ifdef CONFIG_SCHED_CORE /** * scx_prio_less - Task ordering for core-sched * @a: task A * @b: task B * * Core-sched is implemented as an additional scheduling layer on top of the * usual sched_class'es and needs to find out the expected task ordering. For * SCX, core-sched calls this function to interrogate the task ordering. * * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used * to implement the default task ordering. The older the timestamp, the higher * prority the task - the global FIFO ordering matching the default scheduling * behavior. * * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to * implement FIFO ordering within each local DSQ. See pick_task_scx(). */ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, bool in_fi) { /* * The const qualifiers are dropped from task_struct pointers when * calling ops.core_sched_before(). Accesses are controlled by the * verifier. */ if (SCX_HAS_OP(core_sched_before) && !scx_rq_bypassing(task_rq(a))) return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, (struct task_struct *)a, (struct task_struct *)b); else return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); } #endif /* CONFIG_SCHED_CORE */ #ifdef CONFIG_SMP static bool test_and_clear_cpu_idle(int cpu) { #ifdef CONFIG_SCHED_SMT /* * SMT mask should be cleared whether we can claim @cpu or not. The SMT * cluster is not wholly idle either way. This also prevents * scx_pick_idle_cpu() from getting caught in an infinite loop. */ if (sched_smt_active()) { const struct cpumask *smt = cpu_smt_mask(cpu); /* * If offline, @cpu is not its own sibling and * scx_pick_idle_cpu() can get caught in an infinite loop as * @cpu is never cleared from idle_masks.smt. Ensure that @cpu * is eventually cleared. */ if (cpumask_intersects(smt, idle_masks.smt)) cpumask_andnot(idle_masks.smt, idle_masks.smt, smt); else if (cpumask_test_cpu(cpu, idle_masks.smt)) __cpumask_clear_cpu(cpu, idle_masks.smt); } #endif return cpumask_test_and_clear_cpu(cpu, idle_masks.cpu); } static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags) { int cpu; retry: if (sched_smt_active()) { cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); if (cpu < nr_cpu_ids) goto found; if (flags & SCX_PICK_IDLE_CORE) return -EBUSY; } cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); if (cpu >= nr_cpu_ids) return -EBUSY; found: if (test_and_clear_cpu_idle(cpu)) return cpu; else goto retry; } /* * Return true if the LLC domains do not perfectly overlap with the NUMA * domains, false otherwise. */ static bool llc_numa_mismatch(void) { int cpu; /* * We need to scan all online CPUs to verify whether their scheduling * domains overlap. * * While it is rare to encounter architectures with asymmetric NUMA * topologies, CPU hotplugging or virtualized environments can result * in asymmetric configurations. * * For example: * * NUMA 0: * - LLC 0: cpu0..cpu7 * - LLC 1: cpu8..cpu15 [offline] * * NUMA 1: * - LLC 0: cpu16..cpu23 * - LLC 1: cpu24..cpu31 * * In this case, if we only check the first online CPU (cpu0), we might * incorrectly assume that the LLC and NUMA domains are fully * overlapping, which is incorrect (as NUMA 1 has two distinct LLC * domains). */ for_each_online_cpu(cpu) { const struct cpumask *numa_cpus; struct sched_domain *sd; sd = rcu_dereference(per_cpu(sd_llc, cpu)); if (!sd) return true; numa_cpus = cpumask_of_node(cpu_to_node(cpu)); if (sd->span_weight != cpumask_weight(numa_cpus)) return true; } return false; } /* * Initialize topology-aware scheduling. * * Detect if the system has multiple LLC or multiple NUMA domains and enable * cache-aware / NUMA-aware scheduling optimizations in the default CPU idle * selection policy. * * Assumption: the kernel's internal topology representation assumes that each * CPU belongs to a single LLC domain, and that each LLC domain is entirely * contained within a single NUMA node. */ static void update_selcpu_topology(void) { bool enable_llc = false, enable_numa = false; struct sched_domain *sd; const struct cpumask *cpus; s32 cpu = cpumask_first(cpu_online_mask); /* * Enable LLC domain optimization only when there are multiple LLC * domains among the online CPUs. If all online CPUs are part of a * single LLC domain, the idle CPU selection logic can choose any * online CPU without bias. * * Note that it is sufficient to check the LLC domain of the first * online CPU to determine whether a single LLC domain includes all * CPUs. */ rcu_read_lock(); sd = rcu_dereference(per_cpu(sd_llc, cpu)); if (sd) { if (sd->span_weight < num_online_cpus()) enable_llc = true; } /* * Enable NUMA optimization only when there are multiple NUMA domains * among the online CPUs and the NUMA domains don't perfectly overlaps * with the LLC domains. * * If all CPUs belong to the same NUMA node and the same LLC domain, * enabling both NUMA and LLC optimizations is unnecessary, as checking * for an idle CPU in the same domain twice is redundant. */ cpus = cpumask_of_node(cpu_to_node(cpu)); if ((cpumask_weight(cpus) < num_online_cpus()) && llc_numa_mismatch()) enable_numa = true; rcu_read_unlock(); pr_debug("sched_ext: LLC idle selection %s\n", enable_llc ? "enabled" : "disabled"); pr_debug("sched_ext: NUMA idle selection %s\n", enable_numa ? "enabled" : "disabled"); if (enable_llc) static_branch_enable_cpuslocked(&scx_selcpu_topo_llc); else static_branch_disable_cpuslocked(&scx_selcpu_topo_llc); if (enable_numa) static_branch_enable_cpuslocked(&scx_selcpu_topo_numa); else static_branch_disable_cpuslocked(&scx_selcpu_topo_numa); } /* * Built-in CPU idle selection policy: * * 1. Prioritize full-idle cores: * - always prioritize CPUs from fully idle cores (both logical CPUs are * idle) to avoid interference caused by SMT. * * 2. Reuse the same CPU: * - prefer the last used CPU to take advantage of cached data (L1, L2) and * branch prediction optimizations. * * 3. Pick a CPU within the same LLC (Last-Level Cache): * - if the above conditions aren't met, pick a CPU that shares the same LLC * to maintain cache locality. * * 4. Pick a CPU within the same NUMA node, if enabled: * - choose a CPU from the same NUMA node to reduce memory access latency. * * Step 3 and 4 are performed only if the system has, respectively, multiple * LLC domains / multiple NUMA nodes (see scx_selcpu_topo_llc and * scx_selcpu_topo_numa). * * NOTE: tasks that can only run on 1 CPU are excluded by this logic, because * we never call ops.select_cpu() for them, see select_task_rq(). */ static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *found) { const struct cpumask *llc_cpus = NULL; const struct cpumask *numa_cpus = NULL; s32 cpu; *found = false; /* * This is necessary to protect llc_cpus. */ rcu_read_lock(); /* * Determine the scheduling domain only if the task is allowed to run * on all CPUs. * * This is done primarily for efficiency, as it avoids the overhead of * updating a cpumask every time we need to select an idle CPU (which * can be costly in large SMP systems), but it also aligns logically: * if a task's scheduling domain is restricted by user-space (through * CPU affinity), the task will simply use the flat scheduling domain * defined by user-space. */ if (p->nr_cpus_allowed >= num_possible_cpus()) { if (static_branch_maybe(CONFIG_NUMA, &scx_selcpu_topo_numa)) numa_cpus = cpumask_of_node(cpu_to_node(prev_cpu)); if (static_branch_maybe(CONFIG_SCHED_MC, &scx_selcpu_topo_llc)) { struct sched_domain *sd; sd = rcu_dereference(per_cpu(sd_llc, prev_cpu)); if (sd) llc_cpus = sched_domain_span(sd); } } /* * If WAKE_SYNC, try to migrate the wakee to the waker's CPU. */ if (wake_flags & SCX_WAKE_SYNC) { cpu = smp_processor_id(); /* * If the waker's CPU is cache affine and prev_cpu is idle, * then avoid a migration. */ if (cpus_share_cache(cpu, prev_cpu) && test_and_clear_cpu_idle(prev_cpu)) { cpu = prev_cpu; goto cpu_found; } /* * If the waker's local DSQ is empty, and the system is under * utilized, try to wake up @p to the local DSQ of the waker. * * Checking only for an empty local DSQ is insufficient as it * could give the wakee an unfair advantage when the system is * oversaturated. * * Checking only for the presence of idle CPUs is also * insufficient as the local DSQ of the waker could have tasks * piled up on it even if there is an idle core elsewhere on * the system. */ if (!cpumask_empty(idle_masks.cpu) && !(current->flags & PF_EXITING) && cpu_rq(cpu)->scx.local_dsq.nr == 0) { if (cpumask_test_cpu(cpu, p->cpus_ptr)) goto cpu_found; } } /* * If CPU has SMT, any wholly idle CPU is likely a better pick than * partially idle @prev_cpu. */ if (sched_smt_active()) { /* * Keep using @prev_cpu if it's part of a fully idle core. */ if (cpumask_test_cpu(prev_cpu, idle_masks.smt) && test_and_clear_cpu_idle(prev_cpu)) { cpu = prev_cpu; goto cpu_found; } /* * Search for any fully idle core in the same LLC domain. */ if (llc_cpus) { cpu = scx_pick_idle_cpu(llc_cpus, SCX_PICK_IDLE_CORE); if (cpu >= 0) goto cpu_found; } /* * Search for any fully idle core in the same NUMA node. */ if (numa_cpus) { cpu = scx_pick_idle_cpu(numa_cpus, SCX_PICK_IDLE_CORE); if (cpu >= 0) goto cpu_found; } /* * Search for any full idle core usable by the task. */ cpu = scx_pick_idle_cpu(p->cpus_ptr, SCX_PICK_IDLE_CORE); if (cpu >= 0) goto cpu_found; } /* * Use @prev_cpu if it's idle. */ if (test_and_clear_cpu_idle(prev_cpu)) { cpu = prev_cpu; goto cpu_found; } /* * Search for any idle CPU in the same LLC domain. */ if (llc_cpus) { cpu = scx_pick_idle_cpu(llc_cpus, 0); if (cpu >= 0) goto cpu_found; } /* * Search for any idle CPU in the same NUMA node. */ if (numa_cpus) { cpu = scx_pick_idle_cpu(numa_cpus, 0); if (cpu >= 0) goto cpu_found; } /* * Search for any idle CPU usable by the task. */ cpu = scx_pick_idle_cpu(p->cpus_ptr, 0); if (cpu >= 0) goto cpu_found; rcu_read_unlock(); return prev_cpu; cpu_found: rcu_read_unlock(); *found = true; return cpu; } static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) { /* * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it * can be a good migration opportunity with low cache and memory * footprint. Returning a CPU different than @prev_cpu triggers * immediate rq migration. However, for SCX, as the current rq * association doesn't dictate where the task is going to run, this * doesn't fit well. If necessary, we can later add a dedicated method * which can decide to preempt self to force it through the regular * scheduling path. */ if (unlikely(wake_flags & WF_EXEC)) return prev_cpu; if (SCX_HAS_OP(select_cpu) && !scx_rq_bypassing(task_rq(p))) { s32 cpu; struct task_struct **ddsp_taskp; ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); WARN_ON_ONCE(*ddsp_taskp); *ddsp_taskp = p; cpu = SCX_CALL_OP_TASK_RET(SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU, select_cpu, p, prev_cpu, wake_flags); *ddsp_taskp = NULL; if (ops_cpu_valid(cpu, "from ops.select_cpu()")) return cpu; else return prev_cpu; } else { bool found; s32 cpu; cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, &found); if (found) { p->scx.slice = SCX_SLICE_DFL; p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; } return cpu; } } static void task_woken_scx(struct rq *rq, struct task_struct *p) { run_deferred(rq); } static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ac) { set_cpus_allowed_common(p, ac); /* * The effective cpumask is stored in @p->cpus_ptr which may temporarily * differ from the configured one in @p->cpus_mask. Always tell the bpf * scheduler the effective one. * * Fine-grained memory write control is enforced by BPF making the const * designation pointless. Cast it away when calling the operation. */ if (SCX_HAS_OP(set_cpumask)) SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, (struct cpumask *)p->cpus_ptr); } static void reset_idle_masks(void) { /* * Consider all online cpus idle. Should converge to the actual state * quickly. */ cpumask_copy(idle_masks.cpu, cpu_online_mask); cpumask_copy(idle_masks.smt, cpu_online_mask); } static void update_builtin_idle(int cpu, bool idle) { if (idle) cpumask_set_cpu(cpu, idle_masks.cpu); else cpumask_clear_cpu(cpu, idle_masks.cpu); #ifdef CONFIG_SCHED_SMT if (sched_smt_active()) { const struct cpumask *smt = cpu_smt_mask(cpu); if (idle) { /* * idle_masks.smt handling is racy but that's fine as * it's only for optimization and self-correcting. */ for_each_cpu(cpu, smt) { if (!cpumask_test_cpu(cpu, idle_masks.cpu)) return; } cpumask_or(idle_masks.smt, idle_masks.smt, smt); } else { cpumask_andnot(idle_masks.smt, idle_masks.smt, smt); } } #endif } /* * Update the idle state of a CPU to @idle. * * If @do_notify is true, ops.update_idle() is invoked to notify the scx * scheduler of an actual idle state transition (idle to busy or vice * versa). If @do_notify is false, only the idle state in the idle masks is * refreshed without invoking ops.update_idle(). * * This distinction is necessary, because an idle CPU can be "reserved" and * awakened via scx_bpf_pick_idle_cpu() + scx_bpf_kick_cpu(), marking it as * busy even if no tasks are dispatched. In this case, the CPU may return * to idle without a true state transition. Refreshing the idle masks * without invoking ops.update_idle() ensures accurate idle state tracking * while avoiding unnecessary updates and maintaining balanced state * transitions. */ void __scx_update_idle(struct rq *rq, bool idle, bool do_notify) { int cpu = cpu_of(rq); lockdep_assert_rq_held(rq); /* * Trigger ops.update_idle() only when transitioning from a task to * the idle thread and vice versa. * * Idle transitions are indicated by do_notify being set to true, * managed by put_prev_task_idle()/set_next_task_idle(). */ if (SCX_HAS_OP(update_idle) && do_notify && !scx_rq_bypassing(rq)) SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); /* * Update the idle masks: * - for real idle transitions (do_notify == true) * - for idle-to-idle transitions (indicated by the previous task * being the idle thread, managed by pick_task_idle()) * * Skip updating idle masks if the previous task is not the idle * thread, since set_next_task_idle() has already handled it when * transitioning from a task to the idle thread (calling this * function with do_notify == true). * * In this way we can avoid updating the idle masks twice, * unnecessarily. */ if (static_branch_likely(&scx_builtin_idle_enabled)) if (do_notify || is_idle_task(rq->curr)) update_builtin_idle(cpu, idle); } static void handle_hotplug(struct rq *rq, bool online) { int cpu = cpu_of(rq); atomic_long_inc(&scx_hotplug_seq); if (scx_enabled()) update_selcpu_topology(); if (online && SCX_HAS_OP(cpu_online)) SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_online, cpu); else if (!online && SCX_HAS_OP(cpu_offline)) SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_offline, cpu); else scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, "cpu %d going %s, exiting scheduler", cpu, online ? "online" : "offline"); } void scx_rq_activate(struct rq *rq) { handle_hotplug(rq, true); } void scx_rq_deactivate(struct rq *rq) { handle_hotplug(rq, false); } static void rq_online_scx(struct rq *rq) { rq->scx.flags |= SCX_RQ_ONLINE; } static void rq_offline_scx(struct rq *rq) { rq->scx.flags &= ~SCX_RQ_ONLINE; } #else /* CONFIG_SMP */ static bool test_and_clear_cpu_idle(int cpu) { return false; } static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags) { return -EBUSY; } static void reset_idle_masks(void) {} #endif /* CONFIG_SMP */ static bool check_rq_for_timeouts(struct rq *rq) { struct task_struct *p; struct rq_flags rf; bool timed_out = false; rq_lock_irqsave(rq, &rf); list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { unsigned long last_runnable = p->scx.runnable_at; if (unlikely(time_after(jiffies, last_runnable + scx_watchdog_timeout))) { u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); scx_ops_error_kind(SCX_EXIT_ERROR_STALL, "%s[%d] failed to run for %u.%03us", p->comm, p->pid, dur_ms / 1000, dur_ms % 1000); timed_out = true; break; } } rq_unlock_irqrestore(rq, &rf); return timed_out; } static void scx_watchdog_workfn(struct work_struct *work) { int cpu; WRITE_ONCE(scx_watchdog_timestamp, jiffies); for_each_online_cpu(cpu) { if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) break; cond_resched(); } queue_delayed_work(system_unbound_wq, to_delayed_work(work), scx_watchdog_timeout / 2); } void scx_tick(struct rq *rq) { unsigned long last_check; if (!scx_enabled()) return; last_check = READ_ONCE(scx_watchdog_timestamp); if (unlikely(time_after(jiffies, last_check + READ_ONCE(scx_watchdog_timeout)))) { u32 dur_ms = jiffies_to_msecs(jiffies - last_check); scx_ops_error_kind(SCX_EXIT_ERROR_STALL, "watchdog failed to check in for %u.%03us", dur_ms / 1000, dur_ms % 1000); } update_other_load_avgs(rq); } static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) { update_curr_scx(rq); /* * While disabling, always resched and refresh core-sched timestamp as * we can't trust the slice management or ops.core_sched_before(). */ if (scx_rq_bypassing(rq)) { curr->scx.slice = 0; touch_core_sched(rq, curr); } else if (SCX_HAS_OP(tick)) { SCX_CALL_OP(SCX_KF_REST, tick, curr); } if (!curr->scx.slice) resched_curr(rq); } #ifdef CONFIG_EXT_GROUP_SCHED static struct cgroup *tg_cgrp(struct task_group *tg) { /* * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the * root cgroup. */ if (tg && tg->css.cgroup) return tg->css.cgroup; else return &cgrp_dfl_root.cgrp; } #define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), #else /* CONFIG_EXT_GROUP_SCHED */ #define SCX_INIT_TASK_ARGS_CGROUP(tg) #endif /* CONFIG_EXT_GROUP_SCHED */ static enum scx_task_state scx_get_task_state(const struct task_struct *p) { return (p->scx.flags & SCX_TASK_STATE_MASK) >> SCX_TASK_STATE_SHIFT; } static void scx_set_task_state(struct task_struct *p, enum scx_task_state state) { enum scx_task_state prev_state = scx_get_task_state(p); bool warn = false; BUILD_BUG_ON(SCX_TASK_NR_STATES > (1 << SCX_TASK_STATE_BITS)); switch (state) { case SCX_TASK_NONE: break; case SCX_TASK_INIT: warn = prev_state != SCX_TASK_NONE; break; case SCX_TASK_READY: warn = prev_state == SCX_TASK_NONE; break; case SCX_TASK_ENABLED: warn = prev_state != SCX_TASK_READY; break; default: warn = true; return; } WARN_ONCE(warn, "sched_ext: Invalid task state transition %d -> %d for %s[%d]", prev_state, state, p->comm, p->pid); p->scx.flags &= ~SCX_TASK_STATE_MASK; p->scx.flags |= state << SCX_TASK_STATE_SHIFT; } static int scx_ops_init_task(struct task_struct *p, struct task_group *tg, bool fork) { int ret; p->scx.disallow = false; if (SCX_HAS_OP(init_task)) { struct scx_init_task_args args = { SCX_INIT_TASK_ARGS_CGROUP(tg) .fork = fork, }; ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init_task, p, &args); if (unlikely(ret)) { ret = ops_sanitize_err("init_task", ret); return ret; } } scx_set_task_state(p, SCX_TASK_INIT); if (p->scx.disallow) { if (!fork) { struct rq *rq; struct rq_flags rf; rq = task_rq_lock(p, &rf); /* * We're in the load path and @p->policy will be applied * right after. Reverting @p->policy here and rejecting * %SCHED_EXT transitions from scx_check_setscheduler() * guarantees that if ops.init_task() sets @p->disallow, * @p can never be in SCX. */ if (p->policy == SCHED_EXT) { p->policy = SCHED_NORMAL; atomic_long_inc(&scx_nr_rejected); } task_rq_unlock(rq, p, &rf); } else if (p->policy == SCHED_EXT) { scx_ops_error("ops.init_task() set task->scx.disallow for %s[%d] during fork", p->comm, p->pid); } } p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; return 0; } static void scx_ops_enable_task(struct task_struct *p) { u32 weight; lockdep_assert_rq_held(task_rq(p)); /* * Set the weight before calling ops.enable() so that the scheduler * doesn't see a stale value if they inspect the task struct. */ if (task_has_idle_policy(p)) weight = WEIGHT_IDLEPRIO; else weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; p->scx.weight = sched_weight_to_cgroup(weight); if (SCX_HAS_OP(enable)) SCX_CALL_OP_TASK(SCX_KF_REST, enable, p); scx_set_task_state(p, SCX_TASK_ENABLED); if (SCX_HAS_OP(set_weight)) SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx.weight); } static void scx_ops_disable_task(struct task_struct *p) { lockdep_assert_rq_held(task_rq(p)); WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); if (SCX_HAS_OP(disable)) SCX_CALL_OP(SCX_KF_REST, disable, p); scx_set_task_state(p, SCX_TASK_READY); } static void scx_ops_exit_task(struct task_struct *p) { struct scx_exit_task_args args = { .cancelled = false, }; lockdep_assert_rq_held(task_rq(p)); switch (scx_get_task_state(p)) { case SCX_TASK_NONE: return; case SCX_TASK_INIT: args.cancelled = true; break; case SCX_TASK_READY: break; case SCX_TASK_ENABLED: scx_ops_disable_task(p); break; default: WARN_ON_ONCE(true); return; } if (SCX_HAS_OP(exit_task)) SCX_CALL_OP(SCX_KF_REST, exit_task, p, &args); scx_set_task_state(p, SCX_TASK_NONE); } void init_scx_entity(struct sched_ext_entity *scx) { memset(scx, 0, sizeof(*scx)); INIT_LIST_HEAD(&scx->dsq_list.node); RB_CLEAR_NODE(&scx->dsq_priq); scx->sticky_cpu = -1; scx->holding_cpu = -1; INIT_LIST_HEAD(&scx->runnable_node); scx->runnable_at = jiffies; scx->ddsp_dsq_id = SCX_DSQ_INVALID; scx->slice = SCX_SLICE_DFL; } void scx_pre_fork(struct task_struct *p) { /* * BPF scheduler enable/disable paths want to be able to iterate and * update all tasks which can become complex when racing forks. As * enable/disable are very cold paths, let's use a percpu_rwsem to * exclude forks. */ percpu_down_read(&scx_fork_rwsem); } int scx_fork(struct task_struct *p) { percpu_rwsem_assert_held(&scx_fork_rwsem); if (scx_ops_init_task_enabled) return scx_ops_init_task(p, task_group(p), true); else return 0; } void scx_post_fork(struct task_struct *p) { if (scx_ops_init_task_enabled) { scx_set_task_state(p, SCX_TASK_READY); /* * Enable the task immediately if it's running on sched_ext. * Otherwise, it'll be enabled in switching_to_scx() if and * when it's ever configured to run with a SCHED_EXT policy. */ if (p->sched_class == &ext_sched_class) { struct rq_flags rf; struct rq *rq; rq = task_rq_lock(p, &rf); scx_ops_enable_task(p); task_rq_unlock(rq, p, &rf); } } spin_lock_irq(&scx_tasks_lock); list_add_tail(&p->scx.tasks_node, &scx_tasks); spin_unlock_irq(&scx_tasks_lock); percpu_up_read(&scx_fork_rwsem); } void scx_cancel_fork(struct task_struct *p) { if (scx_enabled()) { struct rq *rq; struct rq_flags rf; rq = task_rq_lock(p, &rf); WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); scx_ops_exit_task(p); task_rq_unlock(rq, p, &rf); } percpu_up_read(&scx_fork_rwsem); } void sched_ext_free(struct task_struct *p) { unsigned long flags; spin_lock_irqsave(&scx_tasks_lock, flags); list_del_init(&p->scx.tasks_node); spin_unlock_irqrestore(&scx_tasks_lock, flags); /* * @p is off scx_tasks and wholly ours. scx_ops_enable()'s READY -> * ENABLED transitions can't race us. Disable ops for @p. */ if (scx_get_task_state(p) != SCX_TASK_NONE) { struct rq_flags rf; struct rq *rq; rq = task_rq_lock(p, &rf); scx_ops_exit_task(p); task_rq_unlock(rq, p, &rf); } } static void reweight_task_scx(struct rq *rq, struct task_struct *p, const struct load_weight *lw) { lockdep_assert_rq_held(task_rq(p)); p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); if (SCX_HAS_OP(set_weight)) SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx.weight); } static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) { } static void switching_to_scx(struct rq *rq, struct task_struct *p) { scx_ops_enable_task(p); /* * set_cpus_allowed_scx() is not called while @p is associated with a * different scheduler class. Keep the BPF scheduler up-to-date. */ if (SCX_HAS_OP(set_cpumask)) SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, (struct cpumask *)p->cpus_ptr); } static void switched_from_scx(struct rq *rq, struct task_struct *p) { scx_ops_disable_task(p); } static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} static void switched_to_scx(struct rq *rq, struct task_struct *p) {} int scx_check_setscheduler(struct task_struct *p, int policy) { lockdep_assert_rq_held(task_rq(p)); /* if disallow, reject transitioning into SCX */ if (scx_enabled() && READ_ONCE(p->scx.disallow) && p->policy != policy && policy == SCHED_EXT) return -EACCES; return 0; } #ifdef CONFIG_NO_HZ_FULL bool scx_can_stop_tick(struct rq *rq) { struct task_struct *p = rq->curr; if (scx_rq_bypassing(rq)) return false; if (p->sched_class != &ext_sched_class) return true; /* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks * iff the BPF scheduler indicated so. See set_next_task_scx(). */ return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; } #endif #ifdef CONFIG_EXT_GROUP_SCHED DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_rwsem); static bool scx_cgroup_enabled; static bool cgroup_warned_missing_weight; static bool cgroup_warned_missing_idle; static void scx_cgroup_warn_missing_weight(struct task_group *tg) { if (scx_ops_enable_state() == SCX_OPS_DISABLED || cgroup_warned_missing_weight) return; if ((scx_ops.flags & SCX_OPS_HAS_CGROUP_WEIGHT) || !tg->css.parent) return; pr_warn("sched_ext: \"%s\" does not implement cgroup cpu.weight\n", scx_ops.name); cgroup_warned_missing_weight = true; } static void scx_cgroup_warn_missing_idle(struct task_group *tg) { if (!scx_cgroup_enabled || cgroup_warned_missing_idle) return; if (!tg->idle) return; pr_warn("sched_ext: \"%s\" does not implement cgroup cpu.idle\n", scx_ops.name); cgroup_warned_missing_idle = true; } int scx_tg_online(struct task_group *tg) { int ret = 0; WARN_ON_ONCE(tg->scx_flags & (SCX_TG_ONLINE | SCX_TG_INITED)); percpu_down_read(&scx_cgroup_rwsem); scx_cgroup_warn_missing_weight(tg); if (scx_cgroup_enabled) { if (SCX_HAS_OP(cgroup_init)) { struct scx_cgroup_init_args args = { .weight = tg->scx_weight }; ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_init, tg->css.cgroup, &args); if (ret) ret = ops_sanitize_err("cgroup_init", ret); } if (ret == 0) tg->scx_flags |= SCX_TG_ONLINE | SCX_TG_INITED; } else { tg->scx_flags |= SCX_TG_ONLINE; } percpu_up_read(&scx_cgroup_rwsem); return ret; } void scx_tg_offline(struct task_group *tg) { WARN_ON_ONCE(!(tg->scx_flags & SCX_TG_ONLINE)); percpu_down_read(&scx_cgroup_rwsem); if (SCX_HAS_OP(cgroup_exit) && (tg->scx_flags & SCX_TG_INITED)) SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_exit, tg->css.cgroup); tg->scx_flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); percpu_up_read(&scx_cgroup_rwsem); } int scx_cgroup_can_attach(struct cgroup_taskset *tset) { struct cgroup_subsys_state *css; struct task_struct *p; int ret; /* released in scx_finish/cancel_attach() */ percpu_down_read(&scx_cgroup_rwsem); if (!scx_cgroup_enabled) return 0; cgroup_taskset_for_each(p, css, tset) { struct cgroup *from = tg_cgrp(task_group(p)); struct cgroup *to = tg_cgrp(css_tg(css)); WARN_ON_ONCE(p->scx.cgrp_moving_from); /* * sched_move_task() omits identity migrations. Let's match the * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() * always match one-to-one. */ if (from == to) continue; if (SCX_HAS_OP(cgroup_prep_move)) { ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_prep_move, p, from, css->cgroup); if (ret) goto err; } p->scx.cgrp_moving_from = from; } return 0; err: cgroup_taskset_for_each(p, css, tset) { if (SCX_HAS_OP(cgroup_cancel_move) && p->scx.cgrp_moving_from) SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_cancel_move, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; } percpu_up_read(&scx_cgroup_rwsem); return ops_sanitize_err("cgroup_prep_move", ret); } void scx_move_task(struct task_struct *p) { if (!scx_cgroup_enabled) return; /* * We're called from sched_move_task() which handles both cgroup and * autogroup moves. Ignore the latter. * * Also ignore exiting tasks, because in the exit path tasks transition * from the autogroup to the root group, so task_group_is_autogroup() * alone isn't able to catch exiting autogroup tasks. This is safe for * cgroup_move(), because cgroup migrations never happen for PF_EXITING * tasks. */ if (task_group_is_autogroup(task_group(p)) || (p->flags & PF_EXITING)) return; /* * @p must have ops.cgroup_prep_move() called on it and thus * cgrp_moving_from set. */ if (SCX_HAS_OP(cgroup_move) && !WARN_ON_ONCE(!p->scx.cgrp_moving_from)) SCX_CALL_OP_TASK(SCX_KF_UNLOCKED, cgroup_move, p, p->scx.cgrp_moving_from, tg_cgrp(task_group(p))); p->scx.cgrp_moving_from = NULL; } void scx_cgroup_finish_attach(void) { percpu_up_read(&scx_cgroup_rwsem); } void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) { struct cgroup_subsys_state *css; struct task_struct *p; if (!scx_cgroup_enabled) goto out_unlock; cgroup_taskset_for_each(p, css, tset) { if (SCX_HAS_OP(cgroup_cancel_move) && p->scx.cgrp_moving_from) SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_cancel_move, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; } out_unlock: percpu_up_read(&scx_cgroup_rwsem); } void scx_group_set_weight(struct task_group *tg, unsigned long weight) { percpu_down_read(&scx_cgroup_rwsem); if (scx_cgroup_enabled && tg->scx_weight != weight) { if (SCX_HAS_OP(cgroup_set_weight)) SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_set_weight, tg_cgrp(tg), weight); tg->scx_weight = weight; } percpu_up_read(&scx_cgroup_rwsem); } void scx_group_set_idle(struct task_group *tg, bool idle) { percpu_down_read(&scx_cgroup_rwsem); scx_cgroup_warn_missing_idle(tg); percpu_up_read(&scx_cgroup_rwsem); } static void scx_cgroup_lock(void) { percpu_down_write(&scx_cgroup_rwsem); } static void scx_cgroup_unlock(void) { percpu_up_write(&scx_cgroup_rwsem); } #else /* CONFIG_EXT_GROUP_SCHED */ static inline void scx_cgroup_lock(void) {} static inline void scx_cgroup_unlock(void) {} #endif /* CONFIG_EXT_GROUP_SCHED */ /* * Omitted operations: * * - wakeup_preempt: NOOP as it isn't useful in the wakeup path because the task * isn't tied to the CPU at that point. Preemption is implemented by resetting * the victim task's slice to 0 and triggering reschedule on the target CPU. * * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. * * - task_fork/dead: We need fork/dead notifications for all tasks regardless of * their current sched_class. Call them directly from sched core instead. */ DEFINE_SCHED_CLASS(ext) = { .enqueue_task = enqueue_task_scx, .dequeue_task = dequeue_task_scx, .yield_task = yield_task_scx, .yield_to_task = yield_to_task_scx, .wakeup_preempt = wakeup_preempt_scx, .balance = balance_scx, .pick_task = pick_task_scx, .put_prev_task = put_prev_task_scx, .set_next_task = set_next_task_scx, #ifdef CONFIG_SMP .select_task_rq = select_task_rq_scx, .task_woken = task_woken_scx, .set_cpus_allowed = set_cpus_allowed_scx, .rq_online = rq_online_scx, .rq_offline = rq_offline_scx, #endif .task_tick = task_tick_scx, .switching_to = switching_to_scx, .switched_from = switched_from_scx, .switched_to = switched_to_scx, .reweight_task = reweight_task_scx, .prio_changed = prio_changed_scx, .update_curr = update_curr_scx, #ifdef CONFIG_UCLAMP_TASK .uclamp_enabled = 1, #endif }; static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) { memset(dsq, 0, sizeof(*dsq)); raw_spin_lock_init(&dsq->lock); INIT_LIST_HEAD(&dsq->list); dsq->id = dsq_id; } static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) { struct scx_dispatch_q *dsq; int ret; if (dsq_id & SCX_DSQ_FLAG_BUILTIN) return ERR_PTR(-EINVAL); dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); if (!dsq) return ERR_PTR(-ENOMEM); init_dsq(dsq, dsq_id); ret = rhashtable_insert_fast(&dsq_hash, &dsq->hash_node, dsq_hash_params); if (ret) { kfree(dsq); return ERR_PTR(ret); } return dsq; } static void free_dsq_irq_workfn(struct irq_work *irq_work) { struct llist_node *to_free = llist_del_all(&dsqs_to_free); struct scx_dispatch_q *dsq, *tmp_dsq; llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) kfree_rcu(dsq, rcu); } static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); static void destroy_dsq(u64 dsq_id) { struct scx_dispatch_q *dsq; unsigned long flags; rcu_read_lock(); dsq = find_user_dsq(dsq_id); if (!dsq) goto out_unlock_rcu; raw_spin_lock_irqsave(&dsq->lock, flags); if (dsq->nr) { scx_ops_error("attempting to destroy in-use dsq 0x%016llx (nr=%u)", dsq->id, dsq->nr); goto out_unlock_dsq; } if (rhashtable_remove_fast(&dsq_hash, &dsq->hash_node, dsq_hash_params)) goto out_unlock_dsq; /* * Mark dead by invalidating ->id to prevent dispatch_enqueue() from * queueing more tasks. As this function can be called from anywhere, * freeing is bounced through an irq work to avoid nesting RCU * operations inside scheduler locks. */ dsq->id = SCX_DSQ_INVALID; llist_add(&dsq->free_node, &dsqs_to_free); irq_work_queue(&free_dsq_irq_work); out_unlock_dsq: raw_spin_unlock_irqrestore(&dsq->lock, flags); out_unlock_rcu: rcu_read_unlock(); } #ifdef CONFIG_EXT_GROUP_SCHED static void scx_cgroup_exit(void) { struct cgroup_subsys_state *css; percpu_rwsem_assert_held(&scx_cgroup_rwsem); scx_cgroup_enabled = false; /* * scx_tg_on/offline() are excluded through scx_cgroup_rwsem. If we walk * cgroups and exit all the inited ones, all online cgroups are exited. */ rcu_read_lock(); css_for_each_descendant_post(css, &root_task_group.css) { struct task_group *tg = css_tg(css); if (!(tg->scx_flags & SCX_TG_INITED)) continue; tg->scx_flags &= ~SCX_TG_INITED; if (!scx_ops.cgroup_exit) continue; if (WARN_ON_ONCE(!css_tryget(css))) continue; rcu_read_unlock(); SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_exit, css->cgroup); rcu_read_lock(); css_put(css); } rcu_read_unlock(); } static int scx_cgroup_init(void) { struct cgroup_subsys_state *css; int ret; percpu_rwsem_assert_held(&scx_cgroup_rwsem); cgroup_warned_missing_weight = false; cgroup_warned_missing_idle = false; /* * scx_tg_on/offline() are excluded thorugh scx_cgroup_rwsem. If we walk * cgroups and init, all online cgroups are initialized. */ rcu_read_lock(); css_for_each_descendant_pre(css, &root_task_group.css) { struct task_group *tg = css_tg(css); struct scx_cgroup_init_args args = { .weight = tg->scx_weight }; scx_cgroup_warn_missing_weight(tg); scx_cgroup_warn_missing_idle(tg); if ((tg->scx_flags & (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) continue; if (!scx_ops.cgroup_init) { tg->scx_flags |= SCX_TG_INITED; continue; } if (WARN_ON_ONCE(!css_tryget(css))) continue; rcu_read_unlock(); ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_init, css->cgroup, &args); if (ret) { css_put(css); scx_ops_error("ops.cgroup_init() failed (%d)", ret); return ret; } tg->scx_flags |= SCX_TG_INITED; rcu_read_lock(); css_put(css); } rcu_read_unlock(); WARN_ON_ONCE(scx_cgroup_enabled); scx_cgroup_enabled = true; return 0; } #else static void scx_cgroup_exit(void) {} static int scx_cgroup_init(void) { return 0; } #endif /******************************************************************************** * Sysfs interface and ops enable/disable. */ #define SCX_ATTR(_name) \ static struct kobj_attribute scx_attr_##_name = { \ .attr = { .name = __stringify(_name), .mode = 0444 }, \ .show = scx_attr_##_name##_show, \ } static ssize_t scx_attr_state_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf) { return sysfs_emit(buf, "%s\n", scx_ops_enable_state_str[scx_ops_enable_state()]); } SCX_ATTR(state); static ssize_t scx_attr_switch_all_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf) { return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all)); } SCX_ATTR(switch_all); static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf) { return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected)); } SCX_ATTR(nr_rejected); static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf) { return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq)); } SCX_ATTR(hotplug_seq); static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf) { return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); } SCX_ATTR(enable_seq); static struct attribute *scx_global_attrs[] = { &scx_attr_state.attr, &scx_attr_switch_all.attr, &scx_attr_nr_rejected.attr, &scx_attr_hotplug_seq.attr, &scx_attr_enable_seq.attr, NULL, }; static const struct attribute_group scx_global_attr_group = { .attrs = scx_global_attrs, }; static void scx_kobj_release(struct kobject *kobj) { kfree(kobj); } static ssize_t scx_attr_ops_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf) { return sysfs_emit(buf, "%s\n", scx_ops.name); } SCX_ATTR(ops); static struct attribute *scx_sched_attrs[] = { &scx_attr_ops.attr, NULL, }; ATTRIBUTE_GROUPS(scx_sched); static const struct kobj_type scx_ktype = { .release = scx_kobj_release, .sysfs_ops = &kobj_sysfs_ops, .default_groups = scx_sched_groups, }; static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) { return add_uevent_var(env, "SCXOPS=%s", scx_ops.name); } static const struct kset_uevent_ops scx_uevent_ops = { .uevent = scx_uevent, }; /* * Used by sched_fork() and __setscheduler_prio() to pick the matching * sched_class. dl/rt are already handled. */ bool task_should_scx(int policy) { if (!scx_enabled() || unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING)) return false; if (READ_ONCE(scx_switching_all)) return true; return policy == SCHED_EXT; } /** * scx_softlockup - sched_ext softlockup handler * * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can * live-lock the system by making many CPUs target the same DSQ to the point * where soft-lockup detection triggers. This function is called from * soft-lockup watchdog when the triggering point is close and tries to unjam * the system by enabling the breather and aborting the BPF scheduler. */ void scx_softlockup(u32 dur_s) { switch (scx_ops_enable_state()) { case SCX_OPS_ENABLING: case SCX_OPS_ENABLED: break; default: return; } /* allow only one instance, cleared at the end of scx_ops_bypass() */ if (test_and_set_bit(0, &scx_in_softlockup)) return; printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU%d stuck for %us, disabling \"%s\"\n", smp_processor_id(), dur_s, scx_ops.name); /* * Some CPUs may be trapped in the dispatch paths. Enable breather * immediately; otherwise, we might even be able to get to * scx_ops_bypass(). */ atomic_inc(&scx_ops_breather_depth); scx_ops_error("soft lockup - CPU#%d stuck for %us", smp_processor_id(), dur_s); } static void scx_clear_softlockup(void) { if (test_and_clear_bit(0, &scx_in_softlockup)) atomic_dec(&scx_ops_breather_depth); } /** * scx_ops_bypass - [Un]bypass scx_ops and guarantee forward progress * * Bypassing guarantees that all runnable tasks make forward progress without * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might * be held by tasks that the BPF scheduler is forgetting to run, which * unfortunately also excludes toggling the static branches. * * Let's work around by overriding a couple ops and modifying behaviors based on * the DISABLING state and then cycling the queued tasks through dequeue/enqueue * to force global FIFO scheduling. * * - ops.select_cpu() is ignored and the default select_cpu() is used. * * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. * %SCX_OPS_ENQ_LAST is also ignored. * * - ops.dispatch() is ignored. * * - balance_scx() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to * the tail of the queue with core_sched_at touched. * * - pick_next_task() suppresses zero slice warning. * * - scx_bpf_kick_cpu() is disabled to avoid irq_work malfunction during PM * operations. * * - scx_prio_less() reverts to the default core_sched_at order. */ static void scx_ops_bypass(bool bypass) { static DEFINE_RAW_SPINLOCK(bypass_lock); int cpu; unsigned long flags; raw_spin_lock_irqsave(&bypass_lock, flags); if (bypass) { scx_ops_bypass_depth++; WARN_ON_ONCE(scx_ops_bypass_depth <= 0); if (scx_ops_bypass_depth != 1) goto unlock; } else { scx_ops_bypass_depth--; WARN_ON_ONCE(scx_ops_bypass_depth < 0); if (scx_ops_bypass_depth != 0) goto unlock; } atomic_inc(&scx_ops_breather_depth); /* * No task property is changing. We just need to make sure all currently * queued tasks are re-queued according to the new scx_rq_bypassing() * state. As an optimization, walk each rq's runnable_list instead of * the scx_tasks list. * * This function can't trust the scheduler and thus can't use * cpus_read_lock(). Walk all possible CPUs instead of online. */ for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); struct task_struct *p, *n; raw_spin_rq_lock(rq); if (bypass) { WARN_ON_ONCE(rq->scx.flags & SCX_RQ_BYPASSING); rq->scx.flags |= SCX_RQ_BYPASSING; } else { WARN_ON_ONCE(!(rq->scx.flags & SCX_RQ_BYPASSING)); rq->scx.flags &= ~SCX_RQ_BYPASSING; } /* * We need to guarantee that no tasks are on the BPF scheduler * while bypassing. Either we see enabled or the enable path * sees scx_rq_bypassing() before moving tasks to SCX. */ if (!scx_enabled()) { raw_spin_rq_unlock(rq); continue; } /* * The use of list_for_each_entry_safe_reverse() is required * because each task is going to be removed from and added back * to the runnable_list during iteration. Because they're added * to the tail of the list, safe reverse iteration can still * visit all nodes. */ list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) { struct sched_enq_and_set_ctx ctx; /* cycling deq/enq is enough, see the function comment */ sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx); sched_enq_and_set_task(&ctx); } /* resched to restore ticks and idle state */ if (cpu_online(cpu) || cpu == smp_processor_id()) resched_curr(rq); raw_spin_rq_unlock(rq); } atomic_dec(&scx_ops_breather_depth); unlock: raw_spin_unlock_irqrestore(&bypass_lock, flags); scx_clear_softlockup(); } static void free_exit_info(struct scx_exit_info *ei) { kfree(ei->dump); kfree(ei->msg); kfree(ei->bt); kfree(ei); } static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) { struct scx_exit_info *ei; ei = kzalloc(sizeof(*ei), GFP_KERNEL); if (!ei) return NULL; ei->bt = kcalloc(SCX_EXIT_BT_LEN, sizeof(ei->bt[0]), GFP_KERNEL); ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); ei->dump = kzalloc(exit_dump_len, GFP_KERNEL); if (!ei->bt || !ei->msg || !ei->dump) { free_exit_info(ei); return NULL; } return ei; } static const char *scx_exit_reason(enum scx_exit_kind kind) { switch (kind) { case SCX_EXIT_UNREG: return "unregistered from user space"; case SCX_EXIT_UNREG_BPF: return "unregistered from BPF"; case SCX_EXIT_UNREG_KERN: return "unregistered from the main kernel"; case SCX_EXIT_SYSRQ: return "disabled by sysrq-S"; case SCX_EXIT_ERROR: return "runtime error"; case SCX_EXIT_ERROR_BPF: return "scx_bpf_error"; case SCX_EXIT_ERROR_STALL: return "runnable task stall"; default: return "<UNKNOWN>"; } } static void scx_ops_disable_workfn(struct kthread_work *work) { struct scx_exit_info *ei = scx_exit_info; struct scx_task_iter sti; struct task_struct *p; struct rhashtable_iter rht_iter; struct scx_dispatch_q *dsq; int i, kind; kind = atomic_read(&scx_exit_kind); while (true) { /* * NONE indicates that a new scx_ops has been registered since * disable was scheduled - don't kill the new ops. DONE * indicates that the ops has already been disabled. */ if (kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE) return; if (atomic_try_cmpxchg(&scx_exit_kind, &kind, SCX_EXIT_DONE)) break; } ei->kind = kind; ei->reason = scx_exit_reason(ei->kind); /* guarantee forward progress by bypassing scx_ops */ scx_ops_bypass(true); switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { case SCX_OPS_DISABLING: WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); break; case SCX_OPS_DISABLED: pr_warn("sched_ext: ops error detected without ops (%s)\n", scx_exit_info->msg); WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != SCX_OPS_DISABLING); goto done; default: break; } /* * Here, every runnable task is guaranteed to make forward progress and * we can safely use blocking synchronization constructs. Actually * disable ops. */ mutex_lock(&scx_ops_enable_mutex); static_branch_disable(&__scx_switched_all); WRITE_ONCE(scx_switching_all, false); /* * Shut down cgroup support before tasks so that the cgroup attach path * doesn't race against scx_ops_exit_task(). */ scx_cgroup_lock(); scx_cgroup_exit(); scx_cgroup_unlock(); /* * The BPF scheduler is going away. All tasks including %TASK_DEAD ones * must be switched out and exited synchronously. */ percpu_down_write(&scx_fork_rwsem); scx_ops_init_task_enabled = false; scx_task_iter_start(&sti); while ((p = scx_task_iter_next_locked(&sti))) { const struct sched_class *old_class = p->sched_class; const struct sched_class *new_class = __setscheduler_class(p->policy, p->prio); struct sched_enq_and_set_ctx ctx; if (old_class != new_class && p->se.sched_delayed) dequeue_task(task_rq(p), p, DEQUEUE_SLEEP | DEQUEUE_DELAYED); sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx); p->sched_class = new_class; check_class_changing(task_rq(p), p, old_class); sched_enq_and_set_task(&ctx); check_class_changed(task_rq(p), p, old_class, p->prio); scx_ops_exit_task(p); } scx_task_iter_stop(&sti); percpu_up_write(&scx_fork_rwsem); /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_ops_enabled); for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) static_branch_disable(&scx_has_op[i]); static_branch_disable(&scx_ops_enq_last); static_branch_disable(&scx_ops_enq_exiting); static_branch_disable(&scx_ops_cpu_preempt); static_branch_disable(&scx_builtin_idle_enabled); synchronize_rcu(); if (ei->kind >= SCX_EXIT_ERROR) { pr_err("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", scx_ops.name, ei->reason); if (ei->msg[0] != '\0') pr_err("sched_ext: %s: %s\n", scx_ops.name, ei->msg); #ifdef CONFIG_STACKTRACE stack_trace_print(ei->bt, ei->bt_len, 2); #endif } else { pr_info("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", scx_ops.name, ei->reason); } if (scx_ops.exit) SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); cancel_delayed_work_sync(&scx_watchdog_work); /* * Delete the kobject from the hierarchy eagerly in addition to just * dropping a reference. Otherwise, if the object is deleted * asynchronously, sysfs could observe an object of the same name still * in the hierarchy when another scheduler is loaded. */ kobject_del(scx_root_kobj); kobject_put(scx_root_kobj); scx_root_kobj = NULL; memset(&scx_ops, 0, sizeof(scx_ops)); rhashtable_walk_enter(&dsq_hash, &rht_iter); do { rhashtable_walk_start(&rht_iter); while ((dsq = rhashtable_walk_next(&rht_iter)) && !IS_ERR(dsq)) destroy_dsq(dsq->id); rhashtable_walk_stop(&rht_iter); } while (dsq == ERR_PTR(-EAGAIN)); rhashtable_walk_exit(&rht_iter); free_percpu(scx_dsp_ctx); scx_dsp_ctx = NULL; scx_dsp_max_batch = 0; free_exit_info(scx_exit_info); scx_exit_info = NULL; mutex_unlock(&scx_ops_enable_mutex); WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != SCX_OPS_DISABLING); done: scx_ops_bypass(false); } static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); static void schedule_scx_ops_disable_work(void) { struct kthread_worker *helper = READ_ONCE(scx_ops_helper); /* * We may be called spuriously before the first bpf_sched_ext_reg(). If * scx_ops_helper isn't set up yet, there's nothing to do. */ if (helper) kthread_queue_work(helper, &scx_ops_disable_work); } static void scx_ops_disable(enum scx_exit_kind kind) { int none = SCX_EXIT_NONE; if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) kind = SCX_EXIT_ERROR; atomic_try_cmpxchg(&scx_exit_kind, &none, kind); schedule_scx_ops_disable_work(); } static void dump_newline(struct seq_buf *s) { trace_sched_ext_dump(""); /* @s may be zero sized and seq_buf triggers WARN if so */ if (s->size) seq_buf_putc(s, '\n'); } static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) { va_list args; #ifdef CONFIG_TRACEPOINTS if (trace_sched_ext_dump_enabled()) { /* protected by scx_dump_state()::dump_lock */ static char line_buf[SCX_EXIT_MSG_LEN]; va_start(args, fmt); vscnprintf(line_buf, sizeof(line_buf), fmt, args); va_end(args); trace_sched_ext_dump(line_buf); } #endif /* @s may be zero sized and seq_buf triggers WARN if so */ if (s->size) { va_start(args, fmt); seq_buf_vprintf(s, fmt, args); va_end(args); seq_buf_putc(s, '\n'); } } static void dump_stack_trace(struct seq_buf *s, const char *prefix, const unsigned long *bt, unsigned int len) { unsigned int i; for (i = 0; i < len; i++) dump_line(s, "%s%pS", prefix, (void *)bt[i]); } static void ops_dump_init(struct seq_buf *s, const char *prefix) { struct scx_dump_data *dd = &scx_dump_data; lockdep_assert_irqs_disabled(); dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ dd->first = true; dd->cursor = 0; dd->s = s; dd->prefix = prefix; } static void ops_dump_flush(void) { struct scx_dump_data *dd = &scx_dump_data; char *line = dd->buf.line; if (!dd->cursor) return; /* * There's something to flush and this is the first line. Insert a blank * line to distinguish ops dump. */ if (dd->first) { dump_newline(dd->s); dd->first = false; } /* * There may be multiple lines in $line. Scan and emit each line * separately. */ while (true) { char *end = line; char c; while (*end != '\n' && *end != '\0') end++; /* * If $line overflowed, it may not have newline at the end. * Always emit with a newline. */ c = *end; *end = '\0'; dump_line(dd->s, "%s%s", dd->prefix, line); if (c == '\0') break; /* move to the next line */ end++; if (*end == '\0') break; line = end; } dd->cursor = 0; } static void ops_dump_exit(void) { ops_dump_flush(); scx_dump_data.cpu = -1; } static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx, struct task_struct *p, char marker) { static unsigned long bt[SCX_EXIT_BT_LEN]; char dsq_id_buf[19] = "(n/a)"; unsigned long ops_state = atomic_long_read(&p->scx.ops_state); unsigned int bt_len = 0; if (p->scx.dsq) scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", (unsigned long long)p->scx.dsq->id); dump_newline(s); dump_line(s, " %c%c %s[%d] %+ldms", marker, task_state_to_char(p), p->comm, p->pid, jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", scx_get_task_state(p), p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT); dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s dsq_vtime=%llu", p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf, p->scx.dsq_vtime); dump_line(s, " cpus=%*pb", cpumask_pr_args(p->cpus_ptr)); if (SCX_HAS_OP(dump_task)) { ops_dump_init(s, " "); SCX_CALL_OP(SCX_KF_REST, dump_task, dctx, p); ops_dump_exit(); } #ifdef CONFIG_STACKTRACE bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1); #endif if (bt_len) { dump_newline(s); dump_stack_trace(s, " ", bt, bt_len); } } static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len) { static DEFINE_SPINLOCK(dump_lock); static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; struct scx_dump_ctx dctx = { .kind = ei->kind, .exit_code = ei->exit_code, .reason = ei->reason, .at_ns = ktime_get_ns(), .at_jiffies = jiffies, }; struct seq_buf s; unsigned long flags; char *buf; int cpu; spin_lock_irqsave(&dump_lock, flags); seq_buf_init(&s, ei->dump, dump_len); if (ei->kind == SCX_EXIT_NONE) { dump_line(&s, "Debug dump triggered by %s", ei->reason); } else { dump_line(&s, "%s[%d] triggered exit kind %d:", current->comm, current->pid, ei->kind); dump_line(&s, " %s (%s)", ei->reason, ei->msg); dump_newline(&s); dump_line(&s, "Backtrace:"); dump_stack_trace(&s, " ", ei->bt, ei->bt_len); } if (SCX_HAS_OP(dump)) { ops_dump_init(&s, ""); SCX_CALL_OP(SCX_KF_UNLOCKED, dump, &dctx); ops_dump_exit(); } dump_newline(&s); dump_line(&s, "CPU states"); dump_line(&s, "----------"); for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); struct rq_flags rf; struct task_struct *p; struct seq_buf ns; size_t avail, used; bool idle; rq_lock(rq, &rf); idle = list_empty(&rq->scx.runnable_list) && rq->curr->sched_class == &idle_sched_class; if (idle && !SCX_HAS_OP(dump_cpu)) goto next; /* * We don't yet know whether ops.dump_cpu() will produce output * and we may want to skip the default CPU dump if it doesn't. * Use a nested seq_buf to generate the standard dump so that we * can decide whether to commit later. */ avail = seq_buf_get_buf(&s, &buf); seq_buf_init(&ns, buf, avail); dump_newline(&ns); dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu pnt_seq=%lu", cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, rq->scx.ops_qseq, rq->scx.pnt_seq); dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); if (!cpumask_empty(rq->scx.cpus_to_kick)) dump_line(&ns, " cpus_to_kick : %*pb", cpumask_pr_args(rq->scx.cpus_to_kick)); if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) dump_line(&ns, " idle_to_kick : %*pb", cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); if (!cpumask_empty(rq->scx.cpus_to_preempt)) dump_line(&ns, " cpus_to_preempt: %*pb", cpumask_pr_args(rq->scx.cpus_to_preempt)); if (!cpumask_empty(rq->scx.cpus_to_wait)) dump_line(&ns, " cpus_to_wait : %*pb", cpumask_pr_args(rq->scx.cpus_to_wait)); used = seq_buf_used(&ns); if (SCX_HAS_OP(dump_cpu)) { ops_dump_init(&ns, " "); SCX_CALL_OP(SCX_KF_REST, dump_cpu, &dctx, cpu, idle); ops_dump_exit(); } /* * If idle && nothing generated by ops.dump_cpu(), there's * nothing interesting. Skip. */ if (idle && used == seq_buf_used(&ns)) goto next; /* * $s may already have overflowed when $ns was created. If so, * calling commit on it will trigger BUG. */ if (avail) { seq_buf_commit(&s, seq_buf_used(&ns)); if (seq_buf_has_overflowed(&ns)) seq_buf_set_overflow(&s); } if (rq->curr->sched_class == &ext_sched_class) scx_dump_task(&s, &dctx, rq->curr, '*'); list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) scx_dump_task(&s, &dctx, p, ' '); next: rq_unlock(rq, &rf); } if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) memcpy(ei->dump + dump_len - sizeof(trunc_marker), trunc_marker, sizeof(trunc_marker)); spin_unlock_irqrestore(&dump_lock, flags); } static void scx_ops_error_irq_workfn(struct irq_work *irq_work) { struct scx_exit_info *ei = scx_exit_info; if (ei->kind >= SCX_EXIT_ERROR) scx_dump_state(ei, scx_ops.exit_dump_len); schedule_scx_ops_disable_work(); } static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); static __printf(3, 4) void scx_ops_exit_kind(enum scx_exit_kind kind, s64 exit_code, const char *fmt, ...) { struct scx_exit_info *ei = scx_exit_info; int none = SCX_EXIT_NONE; va_list args; if (!atomic_try_cmpxchg(&scx_exit_kind, &none, kind)) return; ei->exit_code = exit_code; #ifdef CONFIG_STACKTRACE if (kind >= SCX_EXIT_ERROR) ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); #endif va_start(args, fmt); vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); va_end(args); /* * Set ei->kind and ->reason for scx_dump_state(). They'll be set again * in scx_ops_disable_workfn(). */ ei->kind = kind; ei->reason = scx_exit_reason(ei->kind); irq_work_queue(&scx_ops_error_irq_work); } static struct kthread_worker *scx_create_rt_helper(const char *name) { struct kthread_worker *helper; helper = kthread_create_worker(0, name); if (helper) sched_set_fifo(helper->task); return helper; } static void check_hotplug_seq(const struct sched_ext_ops *ops) { unsigned long long global_hotplug_seq; /* * If a hotplug event has occurred between when a scheduler was * initialized, and when we were able to attach, exit and notify user * space about it. */ if (ops->hotplug_seq) { global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); if (ops->hotplug_seq != global_hotplug_seq) { scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, "expected hotplug seq %llu did not match actual %llu", ops->hotplug_seq, global_hotplug_seq); } } } static int validate_ops(const struct sched_ext_ops *ops) { /* * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the * ops.enqueue() callback isn't implemented. */ if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { scx_ops_error("SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); return -EINVAL; } return 0; } static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) { struct scx_task_iter sti; struct task_struct *p; unsigned long timeout; int i, cpu, node, ret; if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), cpu_possible_mask)) { pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); return -EINVAL; } mutex_lock(&scx_ops_enable_mutex); if (!scx_ops_helper) { WRITE_ONCE(scx_ops_helper, scx_create_rt_helper("sched_ext_ops_helper")); if (!scx_ops_helper) { ret = -ENOMEM; goto err_unlock; } } if (!global_dsqs) { struct scx_dispatch_q **dsqs; dsqs = kcalloc(nr_node_ids, sizeof(dsqs[0]), GFP_KERNEL); if (!dsqs) { ret = -ENOMEM; goto err_unlock; } for_each_node_state(node, N_POSSIBLE) { struct scx_dispatch_q *dsq; dsq = kzalloc_node(sizeof(*dsq), GFP_KERNEL, node); if (!dsq) { for_each_node_state(node, N_POSSIBLE) kfree(dsqs[node]); kfree(dsqs); ret = -ENOMEM; goto err_unlock; } init_dsq(dsq, SCX_DSQ_GLOBAL); dsqs[node] = dsq; } global_dsqs = dsqs; } if (scx_ops_enable_state() != SCX_OPS_DISABLED) { ret = -EBUSY; goto err_unlock; } scx_root_kobj = kzalloc(sizeof(*scx_root_kobj), GFP_KERNEL); if (!scx_root_kobj) { ret = -ENOMEM; goto err_unlock; } scx_root_kobj->kset = scx_kset; ret = kobject_init_and_add(scx_root_kobj, &scx_ktype, NULL, "root"); if (ret < 0) goto err; scx_exit_info = alloc_exit_info(ops->exit_dump_len); if (!scx_exit_info) { ret = -ENOMEM; goto err_del; } /* * Set scx_ops, transition to ENABLING and clear exit info to arm the * disable path. Failure triggers full disabling from here on. */ scx_ops = *ops; WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_ENABLING) != SCX_OPS_DISABLED); atomic_set(&scx_exit_kind, SCX_EXIT_NONE); scx_warned_zero_slice = false; atomic_long_set(&scx_nr_rejected, 0); for_each_possible_cpu(cpu) cpu_rq(cpu)->scx.cpuperf_target = SCX_CPUPERF_ONE; /* * Keep CPUs stable during enable so that the BPF scheduler can track * online CPUs by watching ->on/offline_cpu() after ->init(). */ cpus_read_lock(); if (scx_ops.init) { ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init); if (ret) { ret = ops_sanitize_err("init", ret); cpus_read_unlock(); scx_ops_error("ops.init() failed (%d)", ret); goto err_disable; } } for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) if (((void (**)(void))ops)[i]) static_branch_enable_cpuslocked(&scx_has_op[i]); check_hotplug_seq(ops); #ifdef CONFIG_SMP update_selcpu_topology(); #endif cpus_read_unlock(); ret = validate_ops(ops); if (ret) goto err_disable; WARN_ON_ONCE(scx_dsp_ctx); scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; scx_dsp_ctx = __alloc_percpu(struct_size_t(struct scx_dsp_ctx, buf, scx_dsp_max_batch), __alignof__(struct scx_dsp_ctx)); if (!scx_dsp_ctx) { ret = -ENOMEM; goto err_disable; } if (ops->timeout_ms) timeout = msecs_to_jiffies(ops->timeout_ms); else timeout = SCX_WATCHDOG_MAX_TIMEOUT; WRITE_ONCE(scx_watchdog_timeout, timeout); WRITE_ONCE(scx_watchdog_timestamp, jiffies); queue_delayed_work(system_unbound_wq, &scx_watchdog_work, scx_watchdog_timeout / 2); /* * Once __scx_ops_enabled is set, %current can be switched to SCX * anytime. This can lead to stalls as some BPF schedulers (e.g. * userspace scheduling) may not function correctly before all tasks are * switched. Init in bypass mode to guarantee forward progress. */ scx_ops_bypass(true); for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) if (((void (**)(void))ops)[i]) static_branch_enable(&scx_has_op[i]); if (ops->flags & SCX_OPS_ENQ_LAST) static_branch_enable(&scx_ops_enq_last); if (ops->flags & SCX_OPS_ENQ_EXITING) static_branch_enable(&scx_ops_enq_exiting); if (scx_ops.cpu_acquire || scx_ops.cpu_release) static_branch_enable(&scx_ops_cpu_preempt); if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { reset_idle_masks(); static_branch_enable(&scx_builtin_idle_enabled); } else { static_branch_disable(&scx_builtin_idle_enabled); } /* * Lock out forks, cgroup on/offlining and moves before opening the * floodgate so that they don't wander into the operations prematurely. */ percpu_down_write(&scx_fork_rwsem); WARN_ON_ONCE(scx_ops_init_task_enabled); scx_ops_init_task_enabled = true; /* * Enable ops for every task. Fork is excluded by scx_fork_rwsem * preventing new tasks from being added. No need to exclude tasks * leaving as sched_ext_free() can handle both prepped and enabled * tasks. Prep all tasks first and then enable them with preemption * disabled. * * All cgroups should be initialized before scx_ops_init_task() so that * the BPF scheduler can reliably track each task's cgroup membership * from scx_ops_init_task(). Lock out cgroup on/offlining and task * migrations while tasks are being initialized so that * scx_cgroup_can_attach() never sees uninitialized tasks. */ scx_cgroup_lock(); ret = scx_cgroup_init(); if (ret) goto err_disable_unlock_all; scx_task_iter_start(&sti); while ((p = scx_task_iter_next_locked(&sti))) { /* * @p may already be dead, have lost all its usages counts and * be waiting for RCU grace period before being freed. @p can't * be initialized for SCX in such cases and should be ignored. */ if (!tryget_task_struct(p)) continue; scx_task_iter_unlock(&sti); ret = scx_ops_init_task(p, task_group(p), false); if (ret) { put_task_struct(p); scx_task_iter_relock(&sti); scx_task_iter_stop(&sti); scx_ops_error("ops.init_task() failed (%d) for %s[%d]", ret, p->comm, p->pid); goto err_disable_unlock_all; } scx_set_task_state(p, SCX_TASK_READY); put_task_struct(p); scx_task_iter_relock(&sti); } scx_task_iter_stop(&sti); scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); /* * All tasks are READY. It's safe to turn on scx_enabled() and switch * all eligible tasks. */ WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); static_branch_enable(&__scx_ops_enabled); /* * We're fully committed and can't fail. The task READY -> ENABLED * transitions here are synchronized against sched_ext_free() through * scx_tasks_lock. */ percpu_down_write(&scx_fork_rwsem); scx_task_iter_start(&sti); while ((p = scx_task_iter_next_locked(&sti))) { const struct sched_class *old_class = p->sched_class; const struct sched_class *new_class = __setscheduler_class(p->policy, p->prio); struct sched_enq_and_set_ctx ctx; if (old_class != new_class && p->se.sched_delayed) dequeue_task(task_rq(p), p, DEQUEUE_SLEEP | DEQUEUE_DELAYED); sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx); p->scx.slice = SCX_SLICE_DFL; p->sched_class = new_class; check_class_changing(task_rq(p), p, old_class); sched_enq_and_set_task(&ctx); check_class_changed(task_rq(p), p, old_class, p->prio); } scx_task_iter_stop(&sti); percpu_up_write(&scx_fork_rwsem); scx_ops_bypass(false); if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE); goto err_disable; } if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) static_branch_enable(&__scx_switched_all); pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", scx_ops.name, scx_switched_all() ? "" : " (partial)"); kobject_uevent(scx_root_kobj, KOBJ_ADD); mutex_unlock(&scx_ops_enable_mutex); atomic_long_inc(&scx_enable_seq); return 0; err_del: kobject_del(scx_root_kobj); err: kobject_put(scx_root_kobj); scx_root_kobj = NULL; if (scx_exit_info) { free_exit_info(scx_exit_info); scx_exit_info = NULL; } err_unlock: mutex_unlock(&scx_ops_enable_mutex); return ret; err_disable_unlock_all: scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); scx_ops_bypass(false); err_disable: mutex_unlock(&scx_ops_enable_mutex); /* * Returning an error code here would not pass all the error information * to userspace. Record errno using scx_ops_error() for cases * scx_ops_error() wasn't already invoked and exit indicating success so * that the error is notified through ops.exit() with all the details. * * Flush scx_ops_disable_work to ensure that error is reported before * init completion. */ scx_ops_error("scx_ops_enable() failed (%d)", ret); kthread_flush_work(&scx_ops_disable_work); return 0; } /******************************************************************************** * bpf_struct_ops plumbing. */ #include <linux/bpf_verifier.h> #include <linux/bpf.h> #include <linux/btf.h> static const struct btf_type *task_struct_type; static bool bpf_scx_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) { if (type != BPF_READ) return false; if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) return false; if (off % size != 0) return false; return btf_ctx_access(off, size, type, prog, info); } static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, int off, int size) { const struct btf_type *t; t = btf_type_by_id(reg->btf, reg->btf_id); if (t == task_struct_type) { if (off >= offsetof(struct task_struct, scx.slice) && off + size <= offsetofend(struct task_struct, scx.slice)) return SCALAR_VALUE; if (off >= offsetof(struct task_struct, scx.dsq_vtime) && off + size <= offsetofend(struct task_struct, scx.dsq_vtime)) return SCALAR_VALUE; if (off >= offsetof(struct task_struct, scx.disallow) && off + size <= offsetofend(struct task_struct, scx.disallow)) return SCALAR_VALUE; } return -EACCES; } static const struct bpf_func_proto * bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_task_storage_get: return &bpf_task_storage_get_proto; case BPF_FUNC_task_storage_delete: return &bpf_task_storage_delete_proto; default: return bpf_base_func_proto(func_id, prog); } } static const struct bpf_verifier_ops bpf_scx_verifier_ops = { .get_func_proto = bpf_scx_get_func_proto, .is_valid_access = bpf_scx_is_valid_access, .btf_struct_access = bpf_scx_btf_struct_access, }; static int bpf_scx_init_member(const struct btf_type *t, const struct btf_member *member, void *kdata, const void *udata) { const struct sched_ext_ops *uops = udata; struct sched_ext_ops *ops = kdata; u32 moff = __btf_member_bit_offset(t, member) / 8; int ret; switch (moff) { case offsetof(struct sched_ext_ops, dispatch_max_batch): if (*(u32 *)(udata + moff) > INT_MAX) return -E2BIG; ops->dispatch_max_batch = *(u32 *)(udata + moff); return 1; case offsetof(struct sched_ext_ops, flags): if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) return -EINVAL; ops->flags = *(u64 *)(udata + moff); return 1; case offsetof(struct sched_ext_ops, name): ret = bpf_obj_name_cpy(ops->name, uops->name, sizeof(ops->name)); if (ret < 0) return ret; if (ret == 0) return -EINVAL; return 1; case offsetof(struct sched_ext_ops, timeout_ms): if (msecs_to_jiffies(*(u32 *)(udata + moff)) > SCX_WATCHDOG_MAX_TIMEOUT) return -E2BIG; ops->timeout_ms = *(u32 *)(udata + moff); return 1; case offsetof(struct sched_ext_ops, exit_dump_len): ops->exit_dump_len = *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; return 1; case offsetof(struct sched_ext_ops, hotplug_seq): ops->hotplug_seq = *(u64 *)(udata + moff); return 1; } return 0; } static int bpf_scx_check_member(const struct btf_type *t, const struct btf_member *member, const struct bpf_prog *prog) { u32 moff = __btf_member_bit_offset(t, member) / 8; switch (moff) { case offsetof(struct sched_ext_ops, init_task): #ifdef CONFIG_EXT_GROUP_SCHED case offsetof(struct sched_ext_ops, cgroup_init): case offsetof(struct sched_ext_ops, cgroup_exit): case offsetof(struct sched_ext_ops, cgroup_prep_move): #endif case offsetof(struct sched_ext_ops, cpu_online): case offsetof(struct sched_ext_ops, cpu_offline): case offsetof(struct sched_ext_ops, init): case offsetof(struct sched_ext_ops, exit): break; default: if (prog->sleepable) return -EINVAL; } return 0; } static int bpf_scx_reg(void *kdata, struct bpf_link *link) { return scx_ops_enable(kdata, link); } static void bpf_scx_unreg(void *kdata, struct bpf_link *link) { scx_ops_disable(SCX_EXIT_UNREG); kthread_flush_work(&scx_ops_disable_work); } static int bpf_scx_init(struct btf *btf) { task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]); return 0; } static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link) { /* * sched_ext does not support updating the actively-loaded BPF * scheduler, as registering a BPF scheduler can always fail if the * scheduler returns an error code for e.g. ops.init(), ops.init_task(), * etc. Similarly, we can always race with unregistration happening * elsewhere, such as with sysrq. */ return -EOPNOTSUPP; } static int bpf_scx_validate(void *kdata) { return 0; } static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; } static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {} static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {} static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {} static void sched_ext_ops__tick(struct task_struct *p) {} static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {} static void sched_ext_ops__running(struct task_struct *p) {} static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {} static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {} static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; } static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; } static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {} static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {} static void sched_ext_ops__update_idle(s32 cpu, bool idle) {} static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {} static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {} static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; } static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {} static void sched_ext_ops__enable(struct task_struct *p) {} static void sched_ext_ops__disable(struct task_struct *p) {} #ifdef CONFIG_EXT_GROUP_SCHED static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; } static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {} static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; } static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {} #endif static void sched_ext_ops__cpu_online(s32 cpu) {} static void sched_ext_ops__cpu_offline(s32 cpu) {} static s32 sched_ext_ops__init(void) { return -EINVAL; } static void sched_ext_ops__exit(struct scx_exit_info *info) {} static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {} static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {} static struct sched_ext_ops __bpf_ops_sched_ext_ops = { .select_cpu = sched_ext_ops__select_cpu, .enqueue = sched_ext_ops__enqueue, .dequeue = sched_ext_ops__dequeue, .dispatch = sched_ext_ops__dispatch, .tick = sched_ext_ops__tick, .runnable = sched_ext_ops__runnable, .running = sched_ext_ops__running, .stopping = sched_ext_ops__stopping, .quiescent = sched_ext_ops__quiescent, .yield = sched_ext_ops__yield, .core_sched_before = sched_ext_ops__core_sched_before, .set_weight = sched_ext_ops__set_weight, .set_cpumask = sched_ext_ops__set_cpumask, .update_idle = sched_ext_ops__update_idle, .cpu_acquire = sched_ext_ops__cpu_acquire, .cpu_release = sched_ext_ops__cpu_release, .init_task = sched_ext_ops__init_task, .exit_task = sched_ext_ops__exit_task, .enable = sched_ext_ops__enable, .disable = sched_ext_ops__disable, #ifdef CONFIG_EXT_GROUP_SCHED .cgroup_init = sched_ext_ops__cgroup_init, .cgroup_exit = sched_ext_ops__cgroup_exit, .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, .cgroup_move = sched_ext_ops__cgroup_move, .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, #endif .cpu_online = sched_ext_ops__cpu_online, .cpu_offline = sched_ext_ops__cpu_offline, .init = sched_ext_ops__init, .exit = sched_ext_ops__exit, .dump = sched_ext_ops__dump, .dump_cpu = sched_ext_ops__dump_cpu, .dump_task = sched_ext_ops__dump_task, }; static struct bpf_struct_ops bpf_sched_ext_ops = { .verifier_ops = &bpf_scx_verifier_ops, .reg = bpf_scx_reg, .unreg = bpf_scx_unreg, .check_member = bpf_scx_check_member, .init_member = bpf_scx_init_member, .init = bpf_scx_init, .update = bpf_scx_update, .validate = bpf_scx_validate, .name = "sched_ext_ops", .owner = THIS_MODULE, .cfi_stubs = &__bpf_ops_sched_ext_ops }; /******************************************************************************** * System integration and init. */ static void sysrq_handle_sched_ext_reset(u8 key) { if (scx_ops_helper) scx_ops_disable(SCX_EXIT_SYSRQ); else pr_info("sched_ext: BPF scheduler not yet used\n"); } static const struct sysrq_key_op sysrq_sched_ext_reset_op = { .handler = sysrq_handle_sched_ext_reset, .help_msg = "reset-sched-ext(S)", .action_msg = "Disable sched_ext and revert all tasks to CFS", .enable_mask = SYSRQ_ENABLE_RTNICE, }; static void sysrq_handle_sched_ext_dump(u8 key) { struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .reason = "SysRq-D" }; if (scx_enabled()) scx_dump_state(&ei, 0); } static const struct sysrq_key_op sysrq_sched_ext_dump_op = { .handler = sysrq_handle_sched_ext_dump, .help_msg = "dump-sched-ext(D)", .action_msg = "Trigger sched_ext debug dump", .enable_mask = SYSRQ_ENABLE_RTNICE, }; static bool can_skip_idle_kick(struct rq *rq) { lockdep_assert_rq_held(rq); /* * We can skip idle kicking if @rq is going to go through at least one * full SCX scheduling cycle before going idle. Just checking whether * curr is not idle is insufficient because we could be racing * balance_one() trying to pull the next task from a remote rq, which * may fail, and @rq may become idle afterwards. * * The race window is small and we don't and can't guarantee that @rq is * only kicked while idle anyway. Skip only when sure. */ return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); } static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *pseqs) { struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; bool should_wait = false; unsigned long flags; raw_spin_rq_lock_irqsave(rq, flags); /* * During CPU hotplug, a CPU may depend on kicking itself to make * forward progress. Allow kicking self regardless of online state. */ if (cpu_online(cpu) || cpu == cpu_of(this_rq)) { if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { if (rq->curr->sched_class == &ext_sched_class) rq->curr->scx.slice = 0; cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); } if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { pseqs[cpu] = rq->scx.pnt_seq; should_wait = true; } resched_curr(rq); } else { cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); } raw_spin_rq_unlock_irqrestore(rq, flags); return should_wait; } static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) { struct rq *rq = cpu_rq(cpu); unsigned long flags; raw_spin_rq_lock_irqsave(rq, flags); if (!can_skip_idle_kick(rq) && (cpu_online(cpu) || cpu == cpu_of(this_rq))) resched_curr(rq); raw_spin_rq_unlock_irqrestore(rq, flags); } static void kick_cpus_irq_workfn(struct irq_work *irq_work) { struct rq *this_rq = this_rq(); struct scx_rq *this_scx = &this_rq->scx; unsigned long *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); bool should_wait = false; s32 cpu; for_each_cpu(cpu, this_scx->cpus_to_kick) { should_wait |= kick_one_cpu(cpu, this_rq, pseqs); cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); } for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { kick_one_cpu_if_idle(cpu, this_rq); cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); } if (!should_wait) return; for_each_cpu(cpu, this_scx->cpus_to_wait) { unsigned long *wait_pnt_seq = &cpu_rq(cpu)->scx.pnt_seq; if (cpu != cpu_of(this_rq)) { /* * Pairs with smp_store_release() issued by this CPU in * switch_class() on the resched path. * * We busy-wait here to guarantee that no other task can * be scheduled on our core before the target CPU has * entered the resched path. */ while (smp_load_acquire(wait_pnt_seq) == pseqs[cpu]) cpu_relax(); } cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); } } /** * print_scx_info - print out sched_ext scheduler state * @log_lvl: the log level to use when printing * @p: target task * * If a sched_ext scheduler is enabled, print the name and state of the * scheduler. If @p is on sched_ext, print further information about the task. * * This function can be safely called on any task as long as the task_struct * itself is accessible. While safe, this function isn't synchronized and may * print out mixups or garbages of limited length. */ void print_scx_info(const char *log_lvl, struct task_struct *p) { enum scx_ops_enable_state state = scx_ops_enable_state(); const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; char runnable_at_buf[22] = "?"; struct sched_class *class; unsigned long runnable_at; if (state == SCX_OPS_DISABLED) return; /* * Carefully check if the task was running on sched_ext, and then * carefully copy the time it's been runnable, and its state. */ if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || class != &ext_sched_class) { printk("%sSched_ext: %s (%s%s)", log_lvl, scx_ops.name, scx_ops_enable_state_str[state], all); return; } if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, sizeof(runnable_at))) scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", jiffies_delta_msecs(runnable_at, jiffies)); /* print everything onto one line to conserve console space */ printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", log_lvl, scx_ops.name, scx_ops_enable_state_str[state], all, runnable_at_buf); } static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr) { /* * SCX schedulers often have userspace components which are sometimes * involved in critial scheduling paths. PM operations involve freezing * userspace which can lead to scheduling misbehaviors including stalls. * Let's bypass while PM operations are in progress. */ switch (event) { case PM_HIBERNATION_PREPARE: case PM_SUSPEND_PREPARE: case PM_RESTORE_PREPARE: scx_ops_bypass(true); break; case PM_POST_HIBERNATION: case PM_POST_SUSPEND: case PM_POST_RESTORE: scx_ops_bypass(false); break; } return NOTIFY_OK; } static struct notifier_block scx_pm_notifier = { .notifier_call = scx_pm_handler, }; void __init init_sched_ext_class(void) { s32 cpu, v; /* * The following is to prevent the compiler from optimizing out the enum * definitions so that BPF scheduler implementations can use them * through the generated vmlinux.h. */ WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | SCX_TG_ONLINE); BUG_ON(rhashtable_init(&dsq_hash, &dsq_hash_params)); #ifdef CONFIG_SMP BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); #endif scx_kick_cpus_pnt_seqs = __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * nr_cpu_ids, __alignof__(scx_kick_cpus_pnt_seqs[0])); BUG_ON(!scx_kick_cpus_pnt_seqs); for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL); INIT_LIST_HEAD(&rq->scx.runnable_list); INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_kick, GFP_KERNEL)); BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL)); BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_preempt, GFP_KERNEL)); BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_wait, GFP_KERNEL)); init_irq_work(&rq->scx.deferred_irq_work, deferred_irq_workfn); init_irq_work(&rq->scx.kick_cpus_irq_work, kick_cpus_irq_workfn); if (cpu_online(cpu)) cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; } register_sysrq_key('S', &sysrq_sched_ext_reset_op); register_sysrq_key('D', &sysrq_sched_ext_dump_op); INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); } /******************************************************************************** * Helpers that can be called from the BPF scheduler. */ #include <linux/btf_ids.h> __bpf_kfunc_start_defs(); /** * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu() * @p: task_struct to select a CPU for * @prev_cpu: CPU @p was on previously * @wake_flags: %SCX_WAKE_* flags * @is_idle: out parameter indicating whether the returned CPU is idle * * Can only be called from ops.select_cpu() if the built-in CPU selection is * enabled - ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE is set. * @p, @prev_cpu and @wake_flags match ops.select_cpu(). * * Returns the picked CPU with *@is_idle indicating whether the picked CPU is * currently idle and thus a good candidate for direct dispatching. */ __bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) { if (!static_branch_likely(&scx_builtin_idle_enabled)) { scx_ops_error("built-in idle tracking is disabled"); goto prev_cpu; } if (!scx_kf_allowed(SCX_KF_SELECT_CPU)) goto prev_cpu; #ifdef CONFIG_SMP return scx_select_cpu_dfl(p, prev_cpu, wake_flags, is_idle); #endif prev_cpu: *is_idle = false; return prev_cpu; } __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_select_cpu) BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_RCU) BTF_KFUNCS_END(scx_kfunc_ids_select_cpu) static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_select_cpu, }; static bool scx_dsq_insert_preamble(struct task_struct *p, u64 enq_flags) { if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) return false; lockdep_assert_irqs_disabled(); if (unlikely(!p)) { scx_ops_error("called with NULL task"); return false; } if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { scx_ops_error("invalid enq_flags 0x%llx", enq_flags); return false; } return true; } static void scx_dsq_insert_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) { struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); struct task_struct *ddsp_task; ddsp_task = __this_cpu_read(direct_dispatch_task); if (ddsp_task) { mark_direct_dispatch(ddsp_task, p, dsq_id, enq_flags); return; } if (unlikely(dspc->cursor >= scx_dsp_max_batch)) { scx_ops_error("dispatch buffer overflow"); return; } dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ .task = p, .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, .dsq_id = dsq_id, .enq_flags = enq_flags, }; } __bpf_kfunc_start_defs(); /** * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ * @p: task_struct to insert * @dsq_id: DSQ to insert into * @slice: duration @p can run for in nsecs, 0 to keep the current value * @enq_flags: SCX_ENQ_* * * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to * call this function spuriously. Can be called from ops.enqueue(), * ops.select_cpu(), and ops.dispatch(). * * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch * and @p must match the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be * used to target the local DSQ of a CPU other than the enqueueing one. Use * ops.select_cpu() to be on the target CPU in the first place. * * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p * will be directly inserted into the corresponding dispatch queue after * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be * inserted into the local DSQ of the CPU returned by ops.select_cpu(). * @enq_flags are OR'd with the enqueue flags on the enqueue path before the * task is inserted. * * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id * and this function can be called upto ops.dispatch_max_batch times to insert * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. * * This function doesn't have any locking restrictions and may be called under * BPF locks (in the future when BPF introduces more flexible locking). * * @p is allowed to run for @slice. The scheduling path is triggered on slice * exhaustion. If zero, the current residual slice is maintained. If * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with * scx_bpf_kick_cpu() to trigger scheduling. */ __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) { if (!scx_dsq_insert_preamble(p, enq_flags)) return; if (slice) p->scx.slice = slice; else p->scx.slice = p->scx.slice ?: 1; scx_dsq_insert_commit(p, dsq_id, enq_flags); } /* for backward compatibility, will be removed in v6.15 */ __bpf_kfunc void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) { printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch() renamed to scx_bpf_dsq_insert()"); scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags); } /** * scx_bpf_dsq_insert_vtime - Insert a task into the vtime priority queue of a DSQ * @p: task_struct to insert * @dsq_id: DSQ to insert into * @slice: duration @p can run for in nsecs, 0 to keep the current value * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ * @enq_flags: SCX_ENQ_* * * Insert @p into the vtime priority queue of the DSQ identified by @dsq_id. * Tasks queued into the priority queue are ordered by @vtime. All other aspects * are identical to scx_bpf_dsq_insert(). * * @vtime ordering is according to time_before64() which considers wrapping. A * numerically larger vtime may indicate an earlier position in the ordering and * vice-versa. * * A DSQ can only be used as a FIFO or priority queue at any given time and this * function must not be called on a DSQ which already has one or more FIFO tasks * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and * SCX_DSQ_GLOBAL) cannot be used as priority queues. */ __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) { if (!scx_dsq_insert_preamble(p, enq_flags)) return; if (slice) p->scx.slice = slice; else p->scx.slice = p->scx.slice ?: 1; p->scx.dsq_vtime = vtime; scx_dsq_insert_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); } /* for backward compatibility, will be removed in v6.15 */ __bpf_kfunc void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) { printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_vtime() renamed to scx_bpf_dsq_insert_vtime()"); scx_bpf_dsq_insert_vtime(p, dsq_id, slice, vtime, enq_flags); } __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch) BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dispatch, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime, KF_RCU) BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch) static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_enqueue_dispatch, }; static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, struct task_struct *p, u64 dsq_id, u64 enq_flags) { struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; struct rq *this_rq, *src_rq, *locked_rq; bool dispatched = false; bool in_balance; unsigned long flags; if (!scx_kf_allowed_if_unlocked() && !scx_kf_allowed(SCX_KF_DISPATCH)) return false; /* * Can be called from either ops.dispatch() locking this_rq() or any * context where no rq lock is held. If latter, lock @p's task_rq which * we'll likely need anyway. */ src_rq = task_rq(p); local_irq_save(flags); this_rq = this_rq(); in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; if (in_balance) { if (this_rq != src_rq) { raw_spin_rq_unlock(this_rq); raw_spin_rq_lock(src_rq); } } else { raw_spin_rq_lock(src_rq); } /* * If the BPF scheduler keeps calling this function repeatedly, it can * cause similar live-lock conditions as consume_dispatch_q(). Insert a * breather if necessary. */ scx_ops_breather(src_rq); locked_rq = src_rq; raw_spin_lock(&src_dsq->lock); /* * Did someone else get to it? @p could have already left $src_dsq, got * re-enqueud, or be in the process of being consumed by someone else. */ if (unlikely(p->scx.dsq != src_dsq || u32_before(kit->cursor.priv, p->scx.dsq_seq) || p->scx.holding_cpu >= 0) || WARN_ON_ONCE(src_rq != task_rq(p))) { raw_spin_unlock(&src_dsq->lock); goto out; } /* @p is still on $src_dsq and stable, determine the destination */ dst_dsq = find_dsq_for_dispatch(this_rq, dsq_id, p); /* * Apply vtime and slice updates before moving so that the new time is * visible before inserting into $dst_dsq. @p is still on $src_dsq but * this is safe as we're locking it. */ if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) p->scx.dsq_vtime = kit->vtime; if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) p->scx.slice = kit->slice; /* execute move */ locked_rq = move_task_between_dsqs(p, enq_flags, src_dsq, dst_dsq); dispatched = true; out: if (in_balance) { if (this_rq != locked_rq) { raw_spin_rq_unlock(locked_rq); raw_spin_rq_lock(this_rq); } } else { raw_spin_rq_unlock_irqrestore(locked_rq, flags); } kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME); return dispatched; } __bpf_kfunc_start_defs(); /** * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots * * Can only be called from ops.dispatch(). */ __bpf_kfunc u32 scx_bpf_dispatch_nr_slots(void) { if (!scx_kf_allowed(SCX_KF_DISPATCH)) return 0; return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx->cursor); } /** * scx_bpf_dispatch_cancel - Cancel the latest dispatch * * Cancel the latest dispatch. Can be called multiple times to cancel further * dispatches. Can only be called from ops.dispatch(). */ __bpf_kfunc void scx_bpf_dispatch_cancel(void) { struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); if (!scx_kf_allowed(SCX_KF_DISPATCH)) return; if (dspc->cursor > 0) dspc->cursor--; else scx_ops_error("dispatch buffer underflow"); } /** * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ * @dsq_id: DSQ to move task from * * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's * local DSQ for execution. Can only be called from ops.dispatch(). * * This function flushes the in-flight dispatches from scx_bpf_dsq_insert() * before trying to move from the specified DSQ. It may also grab rq locks and * thus can't be called under any BPF locks. * * Returns %true if a task has been moved, %false if there isn't any task to * move. */ __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id) { struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); struct scx_dispatch_q *dsq; if (!scx_kf_allowed(SCX_KF_DISPATCH)) return false; flush_dispatch_buf(dspc->rq); dsq = find_user_dsq(dsq_id); if (unlikely(!dsq)) { scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); return false; } if (consume_dispatch_q(dspc->rq, dsq)) { /* * A successfully consumed task can be dequeued before it starts * running while the CPU is trying to migrate other dispatched * tasks. Bump nr_tasks to tell balance_scx() to retry on empty * local DSQ. */ dspc->nr_tasks++; return true; } else { return false; } } /* for backward compatibility, will be removed in v6.15 */ __bpf_kfunc bool scx_bpf_consume(u64 dsq_id) { printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_consume() renamed to scx_bpf_dsq_move_to_local()"); return scx_bpf_dsq_move_to_local(dsq_id); } /** * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs * @it__iter: DSQ iterator in progress * @slice: duration the moved task can run for in nsecs * * Override the slice of the next task that will be moved from @it__iter using * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous * slice duration is kept. */ __bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, u64 slice) { struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; kit->slice = slice; kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; } /* for backward compatibility, will be removed in v6.15 */ __bpf_kfunc void scx_bpf_dispatch_from_dsq_set_slice( struct bpf_iter_scx_dsq *it__iter, u64 slice) { printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_from_dsq_set_slice() renamed to scx_bpf_dsq_move_set_slice()"); scx_bpf_dsq_move_set_slice(it__iter, slice); } /** * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs * @it__iter: DSQ iterator in progress * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ * * Override the vtime of the next task that will be moved from @it__iter using * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the * override is ignored and cleared. */ __bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, u64 vtime) { struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; kit->vtime = vtime; kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; } /* for backward compatibility, will be removed in v6.15 */ __bpf_kfunc void scx_bpf_dispatch_from_dsq_set_vtime( struct bpf_iter_scx_dsq *it__iter, u64 vtime) { printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_from_dsq_set_vtime() renamed to scx_bpf_dsq_move_set_vtime()"); scx_bpf_dsq_move_set_vtime(it__iter, vtime); } /** * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ * @it__iter: DSQ iterator in progress * @p: task to transfer * @dsq_id: DSQ to move @p to * @enq_flags: SCX_ENQ_* * * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can * be the destination. * * For the transfer to be successful, @p must still be on the DSQ and have been * queued before the DSQ iteration started. This function doesn't care whether * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have * been queued before the iteration started. * * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update. * * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq * lock (e.g. BPF timers or SYSCALL programs). * * Returns %true if @p has been consumed, %false if @p had already been consumed * or dequeued. */ __bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) { return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, p, dsq_id, enq_flags); } /* for backward compatibility, will be removed in v6.15 */ __bpf_kfunc bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) { printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_from_dsq() renamed to scx_bpf_dsq_move()"); return scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags); } /** * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ * @it__iter: DSQ iterator in progress * @p: task to transfer * @dsq_id: DSQ to move @p to * @enq_flags: SCX_ENQ_* * * Transfer @p which is on the DSQ currently iterated by @it__iter to the * priority queue of the DSQ specified by @dsq_id. The destination must be a * user DSQ as only user DSQs support priority queue. * * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice() * and scx_bpf_dsq_move_set_vtime() to update. * * All other aspects are identical to scx_bpf_dsq_move(). See * scx_bpf_dsq_insert_vtime() for more information on @vtime. */ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) { return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); } /* for backward compatibility, will be removed in v6.15 */ __bpf_kfunc bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) { printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_from_dsq_vtime() renamed to scx_bpf_dsq_move_vtime()"); return scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags); } __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_dispatch) BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel) BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local) BTF_ID_FLAGS(func, scx_bpf_consume) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_slice) BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_vtime) BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime_from_dsq, KF_RCU) BTF_KFUNCS_END(scx_kfunc_ids_dispatch) static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_dispatch, }; __bpf_kfunc_start_defs(); /** * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ * * Iterate over all of the tasks currently enqueued on the local DSQ of the * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of * processed tasks. Can only be called from ops.cpu_release(). */ __bpf_kfunc u32 scx_bpf_reenqueue_local(void) { LIST_HEAD(tasks); u32 nr_enqueued = 0; struct rq *rq; struct task_struct *p, *n; if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) return 0; rq = cpu_rq(smp_processor_id()); lockdep_assert_rq_held(rq); /* * The BPF scheduler may choose to dispatch tasks back to * @rq->scx.local_dsq. Move all candidate tasks off to a private list * first to avoid processing the same tasks repeatedly. */ list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, scx.dsq_list.node) { /* * If @p is being migrated, @p's current CPU may not agree with * its allowed CPUs and the migration_cpu_stop is about to * deactivate and re-activate @p anyway. Skip re-enqueueing. * * While racing sched property changes may also dequeue and * re-enqueue a migrating task while its current CPU and allowed * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to * the current local DSQ for running tasks and thus are not * visible to the BPF scheduler. */ if (p->migration_pending) continue; dispatch_dequeue(rq, p); list_add_tail(&p->scx.dsq_list.node, &tasks); } list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { list_del_init(&p->scx.dsq_list.node); do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); nr_enqueued++; } return nr_enqueued; } __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_cpu_release) BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) BTF_KFUNCS_END(scx_kfunc_ids_cpu_release) static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_cpu_release, }; __bpf_kfunc_start_defs(); /** * scx_bpf_create_dsq - Create a custom DSQ * @dsq_id: DSQ to create * @node: NUMA node to allocate from * * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. */ __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) { if (unlikely(node >= (int)nr_node_ids || (node < 0 && node != NUMA_NO_NODE))) return -EINVAL; return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); } __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_unlocked) BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_SLEEPABLE) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_slice) BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_vtime) BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime_from_dsq, KF_RCU) BTF_KFUNCS_END(scx_kfunc_ids_unlocked) static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_unlocked, }; __bpf_kfunc_start_defs(); /** * scx_bpf_kick_cpu - Trigger reschedule on a CPU * @cpu: cpu to kick * @flags: %SCX_KICK_* flags * * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or * trigger rescheduling on a busy CPU. This can be called from any online * scx_ops operation and the actual kicking is performed asynchronously through * an irq work. */ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags) { struct rq *this_rq; unsigned long irq_flags; if (!ops_cpu_valid(cpu, NULL)) return; local_irq_save(irq_flags); this_rq = this_rq(); /* * While bypassing for PM ops, IRQ handling may not be online which can * lead to irq_work_queue() malfunction such as infinite busy wait for * IRQ status update. Suppress kicking. */ if (scx_rq_bypassing(this_rq)) goto out; /* * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting * rq locks. We can probably be smarter and avoid bouncing if called * from ops which don't hold a rq lock. */ if (flags & SCX_KICK_IDLE) { struct rq *target_rq = cpu_rq(cpu); if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) scx_ops_error("PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); if (raw_spin_rq_trylock(target_rq)) { if (can_skip_idle_kick(target_rq)) { raw_spin_rq_unlock(target_rq); goto out; } raw_spin_rq_unlock(target_rq); } cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); } else { cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); if (flags & SCX_KICK_PREEMPT) cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); if (flags & SCX_KICK_WAIT) cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); } irq_work_queue(&this_rq->scx.kick_cpus_irq_work); out: local_irq_restore(irq_flags); } /** * scx_bpf_dsq_nr_queued - Return the number of queued tasks * @dsq_id: id of the DSQ * * Return the number of tasks in the DSQ matching @dsq_id. If not found, * -%ENOENT is returned. */ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id) { struct scx_dispatch_q *dsq; s32 ret; preempt_disable(); if (dsq_id == SCX_DSQ_LOCAL) { ret = READ_ONCE(this_rq()->scx.local_dsq.nr); goto out; } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; if (ops_cpu_valid(cpu, NULL)) { ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); goto out; } } else { dsq = find_user_dsq(dsq_id); if (dsq) { ret = READ_ONCE(dsq->nr); goto out; } } ret = -ENOENT; out: preempt_enable(); return ret; } /** * scx_bpf_destroy_dsq - Destroy a custom DSQ * @dsq_id: DSQ to destroy * * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is * empty and no further tasks are dispatched to it. Ignored if called on a DSQ * which doesn't exist. Can be called from any online scx_ops operations. */ __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id) { destroy_dsq(dsq_id); } /** * bpf_iter_scx_dsq_new - Create a DSQ iterator * @it: iterator to initialize * @dsq_id: DSQ to iterate * @flags: %SCX_DSQ_ITER_* * * Initialize BPF iterator @it which can be used with bpf_for_each() to walk * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes * tasks which are already queued when this function is invoked. */ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) { struct bpf_iter_scx_dsq_kern *kit = (void *)it; BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > sizeof(struct bpf_iter_scx_dsq)); BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != __alignof__(struct bpf_iter_scx_dsq)); if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) return -EINVAL; kit->dsq = find_user_dsq(dsq_id); if (!kit->dsq) return -ENOENT; INIT_LIST_HEAD(&kit->cursor.node); kit->cursor.flags = SCX_DSQ_LNODE_ITER_CURSOR | flags; kit->cursor.priv = READ_ONCE(kit->dsq->seq); return 0; } /** * bpf_iter_scx_dsq_next - Progress a DSQ iterator * @it: iterator to progress * * Return the next task. See bpf_iter_scx_dsq_new(). */ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) { struct bpf_iter_scx_dsq_kern *kit = (void *)it; bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV; struct task_struct *p; unsigned long flags; if (!kit->dsq) return NULL; raw_spin_lock_irqsave(&kit->dsq->lock, flags); if (list_empty(&kit->cursor.node)) p = NULL; else p = container_of(&kit->cursor, struct task_struct, scx.dsq_list); /* * Only tasks which were queued before the iteration started are * visible. This bounds BPF iterations and guarantees that vtime never * jumps in the other direction while iterating. */ do { p = nldsq_next_task(kit->dsq, p, rev); } while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq))); if (p) { if (rev) list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node); else list_move(&kit->cursor.node, &p->scx.dsq_list.node); } else { list_del_init(&kit->cursor.node); } raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); return p; } /** * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * * Undo scx_iter_scx_dsq_new(). */ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) { struct bpf_iter_scx_dsq_kern *kit = (void *)it; if (!kit->dsq) return; if (!list_empty(&kit->cursor.node)) { unsigned long flags; raw_spin_lock_irqsave(&kit->dsq->lock, flags); list_del_init(&kit->cursor.node); raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); } kit->dsq = NULL; } __bpf_kfunc_end_defs(); static s32 __bstr_format(u64 *data_buf, char *line_buf, size_t line_size, char *fmt, unsigned long long *data, u32 data__sz) { struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; s32 ret; if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || (data__sz && !data)) { scx_ops_error("invalid data=%p and data__sz=%u", (void *)data, data__sz); return -EINVAL; } ret = copy_from_kernel_nofault(data_buf, data, data__sz); if (ret < 0) { scx_ops_error("failed to read data fields (%d)", ret); return ret; } ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, &bprintf_data); if (ret < 0) { scx_ops_error("format preparation failed (%d)", ret); return ret; } ret = bstr_printf(line_buf, line_size, fmt, bprintf_data.bin_args); bpf_bprintf_cleanup(&bprintf_data); if (ret < 0) { scx_ops_error("(\"%s\", %p, %u) failed to format", fmt, data, data__sz); return ret; } return ret; } static s32 bstr_format(struct scx_bstr_buf *buf, char *fmt, unsigned long long *data, u32 data__sz) { return __bstr_format(buf->data, buf->line, sizeof(buf->line), fmt, data, data__sz); } __bpf_kfunc_start_defs(); /** * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. * @exit_code: Exit value to pass to user space via struct scx_exit_info. * @fmt: error message format string * @data: format string parameters packaged using ___bpf_fill() macro * @data__sz: @data len, must end in '__sz' for the verifier * * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops * disabling. */ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) { unsigned long flags; raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); if (bstr_format(&scx_exit_bstr_buf, fmt, data, data__sz) >= 0) scx_ops_exit_kind(SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); } /** * scx_bpf_error_bstr - Indicate fatal error * @fmt: error message format string * @data: format string parameters packaged using ___bpf_fill() macro * @data__sz: @data len, must end in '__sz' for the verifier * * Indicate that the BPF scheduler encountered a fatal error and initiate ops * disabling. */ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz) { unsigned long flags; raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); if (bstr_format(&scx_exit_bstr_buf, fmt, data, data__sz) >= 0) scx_ops_exit_kind(SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); } /** * scx_bpf_dump - Generate extra debug dump specific to the BPF scheduler * @fmt: format string * @data: format string parameters packaged using ___bpf_fill() macro * @data__sz: @data len, must end in '__sz' for the verifier * * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and * dump_task() to generate extra debug dump specific to the BPF scheduler. * * The extra dump may be multiple lines. A single line may be split over * multiple calls. The last line is automatically terminated. */ __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data__sz) { struct scx_dump_data *dd = &scx_dump_data; struct scx_bstr_buf *buf = &dd->buf; s32 ret; if (raw_smp_processor_id() != dd->cpu) { scx_ops_error("scx_bpf_dump() must only be called from ops.dump() and friends"); return; } /* append the formatted string to the line buf */ ret = __bstr_format(buf->data, buf->line + dd->cursor, sizeof(buf->line) - dd->cursor, fmt, data, data__sz); if (ret < 0) { dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", dd->prefix, fmt, data, data__sz, ret); return; } dd->cursor += ret; dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line)); if (!dd->cursor) return; /* * If the line buf overflowed or ends in a newline, flush it into the * dump. This is to allow the caller to generate a single line over * multiple calls. As ops_dump_flush() can also handle multiple lines in * the line buf, the only case which can lead to an unexpected * truncation is when the caller keeps generating newlines in the middle * instead of the end consecutively. Don't do that. */ if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') ops_dump_flush(); } /** * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU * @cpu: CPU of interest * * Return the maximum relative capacity of @cpu in relation to the most * performant CPU in the system. The return value is in the range [1, * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). */ __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu) { if (ops_cpu_valid(cpu, NULL)) return arch_scale_cpu_capacity(cpu); else return SCX_CPUPERF_ONE; } /** * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU * @cpu: CPU of interest * * Return the current relative performance of @cpu in relation to its maximum. * The return value is in the range [1, %SCX_CPUPERF_ONE]. * * The current performance level of a CPU in relation to the maximum performance * available in the system can be calculated as follows: * * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE * * The result is in the range [1, %SCX_CPUPERF_ONE]. */ __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu) { if (ops_cpu_valid(cpu, NULL)) return arch_scale_freq_capacity(cpu); else return SCX_CPUPERF_ONE; } /** * scx_bpf_cpuperf_set - Set the relative performance target of a CPU * @cpu: CPU of interest * @perf: target performance level [0, %SCX_CPUPERF_ONE] * @flags: %SCX_CPUPERF_* flags * * Set the target performance level of @cpu to @perf. @perf is in linear * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the * schedutil cpufreq governor chooses the target frequency. * * The actual performance level chosen, CPU grouping, and the overhead and * latency of the operations are dependent on the hardware and cpufreq driver in * use. Consult hardware and cpufreq documentation for more information. The * current performance level can be monitored using scx_bpf_cpuperf_cur(). */ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf) { if (unlikely(perf > SCX_CPUPERF_ONE)) { scx_ops_error("Invalid cpuperf target %u for CPU %d", perf, cpu); return; } if (ops_cpu_valid(cpu, NULL)) { struct rq *rq = cpu_rq(cpu); rq->scx.cpuperf_target = perf; rcu_read_lock_sched_notrace(); cpufreq_update_util(cpu_rq(cpu), 0); rcu_read_unlock_sched_notrace(); } } /** * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs * * All valid CPU IDs in the system are smaller than the returned value. */ __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) { return nr_cpu_ids; } /** * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask */ __bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void) { return cpu_possible_mask; } /** * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask */ __bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void) { return cpu_online_mask; } /** * scx_bpf_put_cpumask - Release a possible/online cpumask * @cpumask: cpumask to release */ __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) { /* * Empty function body because we aren't actually acquiring or releasing * a reference to a global cpumask, which is read-only in the caller and * is never released. The acquire / release semantics here are just used * to make the cpumask is a trusted pointer in the caller. */ } /** * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking * per-CPU cpumask. * * Returns NULL if idle tracking is not enabled, or running on a UP kernel. */ __bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(void) { if (!static_branch_likely(&scx_builtin_idle_enabled)) { scx_ops_error("built-in idle tracking is disabled"); return cpu_none_mask; } #ifdef CONFIG_SMP return idle_masks.cpu; #else return cpu_none_mask; #endif } /** * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, * per-physical-core cpumask. Can be used to determine if an entire physical * core is free. * * Returns NULL if idle tracking is not enabled, or running on a UP kernel. */ __bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(void) { if (!static_branch_likely(&scx_builtin_idle_enabled)) { scx_ops_error("built-in idle tracking is disabled"); return cpu_none_mask; } #ifdef CONFIG_SMP if (sched_smt_active()) return idle_masks.smt; else return idle_masks.cpu; #else return cpu_none_mask; #endif } /** * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to * either the percpu, or SMT idle-tracking cpumask. */ __bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) { /* * Empty function body because we aren't actually acquiring or releasing * a reference to a global idle cpumask, which is read-only in the * caller and is never released. The acquire / release semantics here * are just used to make the cpumask a trusted pointer in the caller. */ } /** * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state * @cpu: cpu to test and clear idle for * * Returns %true if @cpu was idle and its idle state was successfully cleared. * %false otherwise. * * Unavailable if ops.update_idle() is implemented and * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. */ __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) { if (!static_branch_likely(&scx_builtin_idle_enabled)) { scx_ops_error("built-in idle tracking is disabled"); return false; } if (ops_cpu_valid(cpu, NULL)) return test_and_clear_cpu_idle(cpu); else return false; } /** * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu * @cpus_allowed: Allowed cpumask * @flags: %SCX_PICK_IDLE_CPU_* flags * * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu * number on success. -%EBUSY if no matching cpu was found. * * Idle CPU tracking may race against CPU scheduling state transitions. For * example, this function may return -%EBUSY as CPUs are transitioning into the * idle state. If the caller then assumes that there will be dispatch events on * the CPUs as they were all busy, the scheduler may end up stalling with CPUs * idling while there are pending tasks. Use scx_bpf_pick_any_cpu() and * scx_bpf_kick_cpu() to guarantee that there will be at least one dispatch * event in the near future. * * Unavailable if ops.update_idle() is implemented and * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. */ __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags) { if (!static_branch_likely(&scx_builtin_idle_enabled)) { scx_ops_error("built-in idle tracking is disabled"); return -EBUSY; } return scx_pick_idle_cpu(cpus_allowed, flags); } /** * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU * @cpus_allowed: Allowed cpumask * @flags: %SCX_PICK_IDLE_CPU_* flags * * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is * empty. * * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not * set, this function can't tell which CPUs are idle and will always pick any * CPU. */ __bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed, u64 flags) { s32 cpu; if (static_branch_likely(&scx_builtin_idle_enabled)) { cpu = scx_pick_idle_cpu(cpus_allowed, flags); if (cpu >= 0) return cpu; } cpu = cpumask_any_distribute(cpus_allowed); if (cpu < nr_cpu_ids) return cpu; else return -EBUSY; } /** * scx_bpf_task_running - Is task currently running? * @p: task of interest */ __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) { return task_rq(p)->curr == p; } /** * scx_bpf_task_cpu - CPU a task is currently associated with * @p: task of interest */ __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) { return task_cpu(p); } /** * scx_bpf_cpu_rq - Fetch the rq of a CPU * @cpu: CPU of the rq */ __bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu) { if (!ops_cpu_valid(cpu, NULL)) return NULL; return cpu_rq(cpu); } /** * scx_bpf_task_cgroup - Return the sched cgroup of a task * @p: task of interest * * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with * from the scheduler's POV. SCX operations should use this function to * determine @p's current cgroup as, unlike following @p->cgroups, * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all * rq-locked operations. Can be called on the parameter tasks of rq-locked * operations. The restriction guarantees that @p's rq is locked by the caller. */ #ifdef CONFIG_CGROUP_SCHED __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) { struct task_group *tg = p->sched_task_group; struct cgroup *cgrp = &cgrp_dfl_root.cgrp; if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) goto out; cgrp = tg_cgrp(tg); out: cgroup_get(cgrp); return cgrp; } #endif __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_any) BTF_ID_FLAGS(func, scx_bpf_kick_cpu) BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_ITER_NEW | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_TRUSTED_ARGS) BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_TRUSTED_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur) BTF_ID_FLAGS(func, scx_bpf_cpuperf_set) BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_cpu_rq) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_RCU | KF_ACQUIRE) #endif BTF_KFUNCS_END(scx_kfunc_ids_any) static const struct btf_kfunc_id_set scx_kfunc_set_any = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_any, }; static int __init scx_init(void) { int ret; /* * kfunc registration can't be done from init_sched_ext_class() as * register_btf_kfunc_id_set() needs most of the system to be up. * * Some kfuncs are context-sensitive and can only be called from * specific SCX ops. They are grouped into BTF sets accordingly. * Unfortunately, BPF currently doesn't have a way of enforcing such * restrictions. Eventually, the verifier should be able to enforce * them. For now, register them the same and make each kfunc explicitly * check using scx_kf_allowed(). */ if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_select_cpu)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_enqueue_dispatch)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_dispatch)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cpu_release)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_unlocked)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_unlocked)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_any)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_any)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_any))) { pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); return ret; } ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); if (ret) { pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); return ret; } ret = register_pm_notifier(&scx_pm_notifier); if (ret) { pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); return ret; } scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); if (!scx_kset) { pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); return -ENOMEM; } ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); if (ret < 0) { pr_err("sched_ext: Failed to add global attributes\n"); return ret; } return 0; } __initcall(scx_init);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1