ftrace.c 167 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Infrastructure for profiling code inserted by 'gcc -pg'.
  4. *
  5. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  6. * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
  7. *
  8. * Originally ported from the -rt patch by:
  9. * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
  10. *
  11. * Based on code in the latency_tracer, that is:
  12. *
  13. * Copyright (C) 2004-2006 Ingo Molnar
  14. * Copyright (C) 2004 Nadia Yvette Chambers
  15. */
  16. #include <linux/stop_machine.h>
  17. #include <linux/clocksource.h>
  18. #include <linux/sched/task.h>
  19. #include <linux/kallsyms.h>
  20. #include <linux/seq_file.h>
  21. #include <linux/suspend.h>
  22. #include <linux/tracefs.h>
  23. #include <linux/hardirq.h>
  24. #include <linux/kthread.h>
  25. #include <linux/uaccess.h>
  26. #include <linux/bsearch.h>
  27. #include <linux/module.h>
  28. #include <linux/ftrace.h>
  29. #include <linux/sysctl.h>
  30. #include <linux/slab.h>
  31. #include <linux/ctype.h>
  32. #include <linux/sort.h>
  33. #include <linux/list.h>
  34. #include <linux/hash.h>
  35. #include <linux/rcupdate.h>
  36. #include <linux/kprobes.h>
  37. #include <trace/events/sched.h>
  38. #include <asm/sections.h>
  39. #include <asm/setup.h>
  40. #include "trace_output.h"
  41. #include "trace_stat.h"
  42. #define FTRACE_WARN_ON(cond) \
  43. ({ \
  44. int ___r = cond; \
  45. if (WARN_ON(___r)) \
  46. ftrace_kill(); \
  47. ___r; \
  48. })
  49. #define FTRACE_WARN_ON_ONCE(cond) \
  50. ({ \
  51. int ___r = cond; \
  52. if (WARN_ON_ONCE(___r)) \
  53. ftrace_kill(); \
  54. ___r; \
  55. })
  56. /* hash bits for specific function selection */
  57. #define FTRACE_HASH_BITS 7
  58. #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  59. #define FTRACE_HASH_DEFAULT_BITS 10
  60. #define FTRACE_HASH_MAX_BITS 12
  61. #ifdef CONFIG_DYNAMIC_FTRACE
  62. #define INIT_OPS_HASH(opsname) \
  63. .func_hash = &opsname.local_hash, \
  64. .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
  65. #define ASSIGN_OPS_HASH(opsname, val) \
  66. .func_hash = val, \
  67. .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
  68. #else
  69. #define INIT_OPS_HASH(opsname)
  70. #define ASSIGN_OPS_HASH(opsname, val)
  71. #endif
  72. static struct ftrace_ops ftrace_list_end __read_mostly = {
  73. .func = ftrace_stub,
  74. .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
  75. INIT_OPS_HASH(ftrace_list_end)
  76. };
  77. /* ftrace_enabled is a method to turn ftrace on or off */
  78. int ftrace_enabled __read_mostly;
  79. static int last_ftrace_enabled;
  80. /* Current function tracing op */
  81. struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
  82. /* What to set function_trace_op to */
  83. static struct ftrace_ops *set_function_trace_op;
  84. static bool ftrace_pids_enabled(struct ftrace_ops *ops)
  85. {
  86. struct trace_array *tr;
  87. if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
  88. return false;
  89. tr = ops->private;
  90. return tr->function_pids != NULL;
  91. }
  92. static void ftrace_update_trampoline(struct ftrace_ops *ops);
  93. /*
  94. * ftrace_disabled is set when an anomaly is discovered.
  95. * ftrace_disabled is much stronger than ftrace_enabled.
  96. */
  97. static int ftrace_disabled __read_mostly;
  98. static DEFINE_MUTEX(ftrace_lock);
  99. static struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
  100. ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  101. static struct ftrace_ops global_ops;
  102. #if ARCH_SUPPORTS_FTRACE_OPS
  103. static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
  104. struct ftrace_ops *op, struct pt_regs *regs);
  105. #else
  106. /* See comment below, where ftrace_ops_list_func is defined */
  107. static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
  108. #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
  109. #endif
  110. /*
  111. * Traverse the ftrace_global_list, invoking all entries. The reason that we
  112. * can use rcu_dereference_raw_notrace() is that elements removed from this list
  113. * are simply leaked, so there is no need to interact with a grace-period
  114. * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle
  115. * concurrent insertions into the ftrace_global_list.
  116. *
  117. * Silly Alpha and silly pointer-speculation compiler optimizations!
  118. */
  119. #define do_for_each_ftrace_op(op, list) \
  120. op = rcu_dereference_raw_notrace(list); \
  121. do
  122. /*
  123. * Optimized for just a single item in the list (as that is the normal case).
  124. */
  125. #define while_for_each_ftrace_op(op) \
  126. while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \
  127. unlikely((op) != &ftrace_list_end))
  128. static inline void ftrace_ops_init(struct ftrace_ops *ops)
  129. {
  130. #ifdef CONFIG_DYNAMIC_FTRACE
  131. if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
  132. mutex_init(&ops->local_hash.regex_lock);
  133. ops->func_hash = &ops->local_hash;
  134. ops->flags |= FTRACE_OPS_FL_INITIALIZED;
  135. }
  136. #endif
  137. }
  138. static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
  139. struct ftrace_ops *op, struct pt_regs *regs)
  140. {
  141. struct trace_array *tr = op->private;
  142. if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
  143. return;
  144. op->saved_func(ip, parent_ip, op, regs);
  145. }
  146. static void ftrace_sync(struct work_struct *work)
  147. {
  148. /*
  149. * This function is just a stub to implement a hard force
  150. * of synchronize_sched(). This requires synchronizing
  151. * tasks even in userspace and idle.
  152. *
  153. * Yes, function tracing is rude.
  154. */
  155. }
  156. static void ftrace_sync_ipi(void *data)
  157. {
  158. /* Probably not needed, but do it anyway */
  159. smp_rmb();
  160. }
  161. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  162. static void update_function_graph_func(void);
  163. /* Both enabled by default (can be cleared by function_graph tracer flags */
  164. static bool fgraph_sleep_time = true;
  165. static bool fgraph_graph_time = true;
  166. #else
  167. static inline void update_function_graph_func(void) { }
  168. #endif
  169. static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
  170. {
  171. /*
  172. * If this is a dynamic, RCU, or per CPU ops, or we force list func,
  173. * then it needs to call the list anyway.
  174. */
  175. if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
  176. FTRACE_FORCE_LIST_FUNC)
  177. return ftrace_ops_list_func;
  178. return ftrace_ops_get_func(ops);
  179. }
  180. static void update_ftrace_function(void)
  181. {
  182. ftrace_func_t func;
  183. /*
  184. * Prepare the ftrace_ops that the arch callback will use.
  185. * If there's only one ftrace_ops registered, the ftrace_ops_list
  186. * will point to the ops we want.
  187. */
  188. set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
  189. lockdep_is_held(&ftrace_lock));
  190. /* If there's no ftrace_ops registered, just call the stub function */
  191. if (set_function_trace_op == &ftrace_list_end) {
  192. func = ftrace_stub;
  193. /*
  194. * If we are at the end of the list and this ops is
  195. * recursion safe and not dynamic and the arch supports passing ops,
  196. * then have the mcount trampoline call the function directly.
  197. */
  198. } else if (rcu_dereference_protected(ftrace_ops_list->next,
  199. lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
  200. func = ftrace_ops_get_list_func(ftrace_ops_list);
  201. } else {
  202. /* Just use the default ftrace_ops */
  203. set_function_trace_op = &ftrace_list_end;
  204. func = ftrace_ops_list_func;
  205. }
  206. update_function_graph_func();
  207. /* If there's no change, then do nothing more here */
  208. if (ftrace_trace_function == func)
  209. return;
  210. /*
  211. * If we are using the list function, it doesn't care
  212. * about the function_trace_ops.
  213. */
  214. if (func == ftrace_ops_list_func) {
  215. ftrace_trace_function = func;
  216. /*
  217. * Don't even bother setting function_trace_ops,
  218. * it would be racy to do so anyway.
  219. */
  220. return;
  221. }
  222. #ifndef CONFIG_DYNAMIC_FTRACE
  223. /*
  224. * For static tracing, we need to be a bit more careful.
  225. * The function change takes affect immediately. Thus,
  226. * we need to coorditate the setting of the function_trace_ops
  227. * with the setting of the ftrace_trace_function.
  228. *
  229. * Set the function to the list ops, which will call the
  230. * function we want, albeit indirectly, but it handles the
  231. * ftrace_ops and doesn't depend on function_trace_op.
  232. */
  233. ftrace_trace_function = ftrace_ops_list_func;
  234. /*
  235. * Make sure all CPUs see this. Yes this is slow, but static
  236. * tracing is slow and nasty to have enabled.
  237. */
  238. schedule_on_each_cpu(ftrace_sync);
  239. /* Now all cpus are using the list ops. */
  240. function_trace_op = set_function_trace_op;
  241. /* Make sure the function_trace_op is visible on all CPUs */
  242. smp_wmb();
  243. /* Nasty way to force a rmb on all cpus */
  244. smp_call_function(ftrace_sync_ipi, NULL, 1);
  245. /* OK, we are all set to update the ftrace_trace_function now! */
  246. #endif /* !CONFIG_DYNAMIC_FTRACE */
  247. ftrace_trace_function = func;
  248. }
  249. static void add_ftrace_ops(struct ftrace_ops __rcu **list,
  250. struct ftrace_ops *ops)
  251. {
  252. rcu_assign_pointer(ops->next, *list);
  253. /*
  254. * We are entering ops into the list but another
  255. * CPU might be walking that list. We need to make sure
  256. * the ops->next pointer is valid before another CPU sees
  257. * the ops pointer included into the list.
  258. */
  259. rcu_assign_pointer(*list, ops);
  260. }
  261. static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
  262. struct ftrace_ops *ops)
  263. {
  264. struct ftrace_ops **p;
  265. /*
  266. * If we are removing the last function, then simply point
  267. * to the ftrace_stub.
  268. */
  269. if (rcu_dereference_protected(*list,
  270. lockdep_is_held(&ftrace_lock)) == ops &&
  271. rcu_dereference_protected(ops->next,
  272. lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
  273. *list = &ftrace_list_end;
  274. return 0;
  275. }
  276. for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
  277. if (*p == ops)
  278. break;
  279. if (*p != ops)
  280. return -1;
  281. *p = (*p)->next;
  282. return 0;
  283. }
  284. static void ftrace_update_trampoline(struct ftrace_ops *ops);
  285. static int __register_ftrace_function(struct ftrace_ops *ops)
  286. {
  287. if (ops->flags & FTRACE_OPS_FL_DELETED)
  288. return -EINVAL;
  289. if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
  290. return -EBUSY;
  291. #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  292. /*
  293. * If the ftrace_ops specifies SAVE_REGS, then it only can be used
  294. * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
  295. * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
  296. */
  297. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
  298. !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
  299. return -EINVAL;
  300. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
  301. ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
  302. #endif
  303. if (!core_kernel_data((unsigned long)ops))
  304. ops->flags |= FTRACE_OPS_FL_DYNAMIC;
  305. add_ftrace_ops(&ftrace_ops_list, ops);
  306. /* Always save the function, and reset at unregistering */
  307. ops->saved_func = ops->func;
  308. if (ftrace_pids_enabled(ops))
  309. ops->func = ftrace_pid_func;
  310. ftrace_update_trampoline(ops);
  311. if (ftrace_enabled)
  312. update_ftrace_function();
  313. return 0;
  314. }
  315. static int __unregister_ftrace_function(struct ftrace_ops *ops)
  316. {
  317. int ret;
  318. if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
  319. return -EBUSY;
  320. ret = remove_ftrace_ops(&ftrace_ops_list, ops);
  321. if (ret < 0)
  322. return ret;
  323. if (ftrace_enabled)
  324. update_ftrace_function();
  325. ops->func = ops->saved_func;
  326. return 0;
  327. }
  328. static void ftrace_update_pid_func(void)
  329. {
  330. struct ftrace_ops *op;
  331. /* Only do something if we are tracing something */
  332. if (ftrace_trace_function == ftrace_stub)
  333. return;
  334. do_for_each_ftrace_op(op, ftrace_ops_list) {
  335. if (op->flags & FTRACE_OPS_FL_PID) {
  336. op->func = ftrace_pids_enabled(op) ?
  337. ftrace_pid_func : op->saved_func;
  338. ftrace_update_trampoline(op);
  339. }
  340. } while_for_each_ftrace_op(op);
  341. update_ftrace_function();
  342. }
  343. #ifdef CONFIG_FUNCTION_PROFILER
  344. struct ftrace_profile {
  345. struct hlist_node node;
  346. unsigned long ip;
  347. unsigned long counter;
  348. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  349. unsigned long long time;
  350. unsigned long long time_squared;
  351. #endif
  352. };
  353. struct ftrace_profile_page {
  354. struct ftrace_profile_page *next;
  355. unsigned long index;
  356. struct ftrace_profile records[];
  357. };
  358. struct ftrace_profile_stat {
  359. atomic_t disabled;
  360. struct hlist_head *hash;
  361. struct ftrace_profile_page *pages;
  362. struct ftrace_profile_page *start;
  363. struct tracer_stat stat;
  364. };
  365. #define PROFILE_RECORDS_SIZE \
  366. (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
  367. #define PROFILES_PER_PAGE \
  368. (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
  369. static int ftrace_profile_enabled __read_mostly;
  370. /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
  371. static DEFINE_MUTEX(ftrace_profile_lock);
  372. static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
  373. #define FTRACE_PROFILE_HASH_BITS 10
  374. #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
  375. static void *
  376. function_stat_next(void *v, int idx)
  377. {
  378. struct ftrace_profile *rec = v;
  379. struct ftrace_profile_page *pg;
  380. pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
  381. again:
  382. if (idx != 0)
  383. rec++;
  384. if ((void *)rec >= (void *)&pg->records[pg->index]) {
  385. pg = pg->next;
  386. if (!pg)
  387. return NULL;
  388. rec = &pg->records[0];
  389. if (!rec->counter)
  390. goto again;
  391. }
  392. return rec;
  393. }
  394. static void *function_stat_start(struct tracer_stat *trace)
  395. {
  396. struct ftrace_profile_stat *stat =
  397. container_of(trace, struct ftrace_profile_stat, stat);
  398. if (!stat || !stat->start)
  399. return NULL;
  400. return function_stat_next(&stat->start->records[0], 0);
  401. }
  402. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  403. /* function graph compares on total time */
  404. static int function_stat_cmp(void *p1, void *p2)
  405. {
  406. struct ftrace_profile *a = p1;
  407. struct ftrace_profile *b = p2;
  408. if (a->time < b->time)
  409. return -1;
  410. if (a->time > b->time)
  411. return 1;
  412. else
  413. return 0;
  414. }
  415. #else
  416. /* not function graph compares against hits */
  417. static int function_stat_cmp(void *p1, void *p2)
  418. {
  419. struct ftrace_profile *a = p1;
  420. struct ftrace_profile *b = p2;
  421. if (a->counter < b->counter)
  422. return -1;
  423. if (a->counter > b->counter)
  424. return 1;
  425. else
  426. return 0;
  427. }
  428. #endif
  429. static int function_stat_headers(struct seq_file *m)
  430. {
  431. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  432. seq_puts(m, " Function "
  433. "Hit Time Avg s^2\n"
  434. " -------- "
  435. "--- ---- --- ---\n");
  436. #else
  437. seq_puts(m, " Function Hit\n"
  438. " -------- ---\n");
  439. #endif
  440. return 0;
  441. }
  442. static int function_stat_show(struct seq_file *m, void *v)
  443. {
  444. struct ftrace_profile *rec = v;
  445. char str[KSYM_SYMBOL_LEN];
  446. int ret = 0;
  447. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  448. static struct trace_seq s;
  449. unsigned long long avg;
  450. unsigned long long stddev;
  451. #endif
  452. mutex_lock(&ftrace_profile_lock);
  453. /* we raced with function_profile_reset() */
  454. if (unlikely(rec->counter == 0)) {
  455. ret = -EBUSY;
  456. goto out;
  457. }
  458. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  459. avg = div64_ul(rec->time, rec->counter);
  460. if (tracing_thresh && (avg < tracing_thresh))
  461. goto out;
  462. #endif
  463. kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
  464. seq_printf(m, " %-30.30s %10lu", str, rec->counter);
  465. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  466. seq_puts(m, " ");
  467. /* Sample standard deviation (s^2) */
  468. if (rec->counter <= 1)
  469. stddev = 0;
  470. else {
  471. /*
  472. * Apply Welford's method:
  473. * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
  474. */
  475. stddev = rec->counter * rec->time_squared -
  476. rec->time * rec->time;
  477. /*
  478. * Divide only 1000 for ns^2 -> us^2 conversion.
  479. * trace_print_graph_duration will divide 1000 again.
  480. */
  481. stddev = div64_ul(stddev,
  482. rec->counter * (rec->counter - 1) * 1000);
  483. }
  484. trace_seq_init(&s);
  485. trace_print_graph_duration(rec->time, &s);
  486. trace_seq_puts(&s, " ");
  487. trace_print_graph_duration(avg, &s);
  488. trace_seq_puts(&s, " ");
  489. trace_print_graph_duration(stddev, &s);
  490. trace_print_seq(m, &s);
  491. #endif
  492. seq_putc(m, '\n');
  493. out:
  494. mutex_unlock(&ftrace_profile_lock);
  495. return ret;
  496. }
  497. static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
  498. {
  499. struct ftrace_profile_page *pg;
  500. pg = stat->pages = stat->start;
  501. while (pg) {
  502. memset(pg->records, 0, PROFILE_RECORDS_SIZE);
  503. pg->index = 0;
  504. pg = pg->next;
  505. }
  506. memset(stat->hash, 0,
  507. FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
  508. }
  509. int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
  510. {
  511. struct ftrace_profile_page *pg;
  512. int functions;
  513. int pages;
  514. int i;
  515. /* If we already allocated, do nothing */
  516. if (stat->pages)
  517. return 0;
  518. stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
  519. if (!stat->pages)
  520. return -ENOMEM;
  521. #ifdef CONFIG_DYNAMIC_FTRACE
  522. functions = ftrace_update_tot_cnt;
  523. #else
  524. /*
  525. * We do not know the number of functions that exist because
  526. * dynamic tracing is what counts them. With past experience
  527. * we have around 20K functions. That should be more than enough.
  528. * It is highly unlikely we will execute every function in
  529. * the kernel.
  530. */
  531. functions = 20000;
  532. #endif
  533. pg = stat->start = stat->pages;
  534. pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
  535. for (i = 1; i < pages; i++) {
  536. pg->next = (void *)get_zeroed_page(GFP_KERNEL);
  537. if (!pg->next)
  538. goto out_free;
  539. pg = pg->next;
  540. }
  541. return 0;
  542. out_free:
  543. pg = stat->start;
  544. while (pg) {
  545. unsigned long tmp = (unsigned long)pg;
  546. pg = pg->next;
  547. free_page(tmp);
  548. }
  549. stat->pages = NULL;
  550. stat->start = NULL;
  551. return -ENOMEM;
  552. }
  553. static int ftrace_profile_init_cpu(int cpu)
  554. {
  555. struct ftrace_profile_stat *stat;
  556. int size;
  557. stat = &per_cpu(ftrace_profile_stats, cpu);
  558. if (stat->hash) {
  559. /* If the profile is already created, simply reset it */
  560. ftrace_profile_reset(stat);
  561. return 0;
  562. }
  563. /*
  564. * We are profiling all functions, but usually only a few thousand
  565. * functions are hit. We'll make a hash of 1024 items.
  566. */
  567. size = FTRACE_PROFILE_HASH_SIZE;
  568. stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL);
  569. if (!stat->hash)
  570. return -ENOMEM;
  571. /* Preallocate the function profiling pages */
  572. if (ftrace_profile_pages_init(stat) < 0) {
  573. kfree(stat->hash);
  574. stat->hash = NULL;
  575. return -ENOMEM;
  576. }
  577. return 0;
  578. }
  579. static int ftrace_profile_init(void)
  580. {
  581. int cpu;
  582. int ret = 0;
  583. for_each_possible_cpu(cpu) {
  584. ret = ftrace_profile_init_cpu(cpu);
  585. if (ret)
  586. break;
  587. }
  588. return ret;
  589. }
  590. /* interrupts must be disabled */
  591. static struct ftrace_profile *
  592. ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
  593. {
  594. struct ftrace_profile *rec;
  595. struct hlist_head *hhd;
  596. unsigned long key;
  597. key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
  598. hhd = &stat->hash[key];
  599. if (hlist_empty(hhd))
  600. return NULL;
  601. hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
  602. if (rec->ip == ip)
  603. return rec;
  604. }
  605. return NULL;
  606. }
  607. static void ftrace_add_profile(struct ftrace_profile_stat *stat,
  608. struct ftrace_profile *rec)
  609. {
  610. unsigned long key;
  611. key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
  612. hlist_add_head_rcu(&rec->node, &stat->hash[key]);
  613. }
  614. /*
  615. * The memory is already allocated, this simply finds a new record to use.
  616. */
  617. static struct ftrace_profile *
  618. ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
  619. {
  620. struct ftrace_profile *rec = NULL;
  621. /* prevent recursion (from NMIs) */
  622. if (atomic_inc_return(&stat->disabled) != 1)
  623. goto out;
  624. /*
  625. * Try to find the function again since an NMI
  626. * could have added it
  627. */
  628. rec = ftrace_find_profiled_func(stat, ip);
  629. if (rec)
  630. goto out;
  631. if (stat->pages->index == PROFILES_PER_PAGE) {
  632. if (!stat->pages->next)
  633. goto out;
  634. stat->pages = stat->pages->next;
  635. }
  636. rec = &stat->pages->records[stat->pages->index++];
  637. rec->ip = ip;
  638. ftrace_add_profile(stat, rec);
  639. out:
  640. atomic_dec(&stat->disabled);
  641. return rec;
  642. }
  643. static void
  644. function_profile_call(unsigned long ip, unsigned long parent_ip,
  645. struct ftrace_ops *ops, struct pt_regs *regs)
  646. {
  647. struct ftrace_profile_stat *stat;
  648. struct ftrace_profile *rec;
  649. unsigned long flags;
  650. if (!ftrace_profile_enabled)
  651. return;
  652. local_irq_save(flags);
  653. stat = this_cpu_ptr(&ftrace_profile_stats);
  654. if (!stat->hash || !ftrace_profile_enabled)
  655. goto out;
  656. rec = ftrace_find_profiled_func(stat, ip);
  657. if (!rec) {
  658. rec = ftrace_profile_alloc(stat, ip);
  659. if (!rec)
  660. goto out;
  661. }
  662. rec->counter++;
  663. out:
  664. local_irq_restore(flags);
  665. }
  666. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  667. static int profile_graph_entry(struct ftrace_graph_ent *trace)
  668. {
  669. int index = current->curr_ret_stack;
  670. function_profile_call(trace->func, 0, NULL, NULL);
  671. /* If function graph is shutting down, ret_stack can be NULL */
  672. if (!current->ret_stack)
  673. return 0;
  674. if (index >= 0 && index < FTRACE_RETFUNC_DEPTH)
  675. current->ret_stack[index].subtime = 0;
  676. return 1;
  677. }
  678. static void profile_graph_return(struct ftrace_graph_ret *trace)
  679. {
  680. struct ftrace_profile_stat *stat;
  681. unsigned long long calltime;
  682. struct ftrace_profile *rec;
  683. unsigned long flags;
  684. local_irq_save(flags);
  685. stat = this_cpu_ptr(&ftrace_profile_stats);
  686. if (!stat->hash || !ftrace_profile_enabled)
  687. goto out;
  688. /* If the calltime was zero'd ignore it */
  689. if (!trace->calltime)
  690. goto out;
  691. calltime = trace->rettime - trace->calltime;
  692. if (!fgraph_graph_time) {
  693. int index;
  694. index = current->curr_ret_stack;
  695. /* Append this call time to the parent time to subtract */
  696. if (index)
  697. current->ret_stack[index - 1].subtime += calltime;
  698. if (current->ret_stack[index].subtime < calltime)
  699. calltime -= current->ret_stack[index].subtime;
  700. else
  701. calltime = 0;
  702. }
  703. rec = ftrace_find_profiled_func(stat, trace->func);
  704. if (rec) {
  705. rec->time += calltime;
  706. rec->time_squared += calltime * calltime;
  707. }
  708. out:
  709. local_irq_restore(flags);
  710. }
  711. static int register_ftrace_profiler(void)
  712. {
  713. return register_ftrace_graph(&profile_graph_return,
  714. &profile_graph_entry);
  715. }
  716. static void unregister_ftrace_profiler(void)
  717. {
  718. unregister_ftrace_graph();
  719. }
  720. #else
  721. static struct ftrace_ops ftrace_profile_ops __read_mostly = {
  722. .func = function_profile_call,
  723. .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
  724. INIT_OPS_HASH(ftrace_profile_ops)
  725. };
  726. static int register_ftrace_profiler(void)
  727. {
  728. return register_ftrace_function(&ftrace_profile_ops);
  729. }
  730. static void unregister_ftrace_profiler(void)
  731. {
  732. unregister_ftrace_function(&ftrace_profile_ops);
  733. }
  734. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  735. static ssize_t
  736. ftrace_profile_write(struct file *filp, const char __user *ubuf,
  737. size_t cnt, loff_t *ppos)
  738. {
  739. unsigned long val;
  740. int ret;
  741. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  742. if (ret)
  743. return ret;
  744. val = !!val;
  745. mutex_lock(&ftrace_profile_lock);
  746. if (ftrace_profile_enabled ^ val) {
  747. if (val) {
  748. ret = ftrace_profile_init();
  749. if (ret < 0) {
  750. cnt = ret;
  751. goto out;
  752. }
  753. ret = register_ftrace_profiler();
  754. if (ret < 0) {
  755. cnt = ret;
  756. goto out;
  757. }
  758. ftrace_profile_enabled = 1;
  759. } else {
  760. ftrace_profile_enabled = 0;
  761. /*
  762. * unregister_ftrace_profiler calls stop_machine
  763. * so this acts like an synchronize_sched.
  764. */
  765. unregister_ftrace_profiler();
  766. }
  767. }
  768. out:
  769. mutex_unlock(&ftrace_profile_lock);
  770. *ppos += cnt;
  771. return cnt;
  772. }
  773. static ssize_t
  774. ftrace_profile_read(struct file *filp, char __user *ubuf,
  775. size_t cnt, loff_t *ppos)
  776. {
  777. char buf[64]; /* big enough to hold a number */
  778. int r;
  779. r = sprintf(buf, "%u\n", ftrace_profile_enabled);
  780. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  781. }
  782. static const struct file_operations ftrace_profile_fops = {
  783. .open = tracing_open_generic,
  784. .read = ftrace_profile_read,
  785. .write = ftrace_profile_write,
  786. .llseek = default_llseek,
  787. };
  788. /* used to initialize the real stat files */
  789. static struct tracer_stat function_stats __initdata = {
  790. .name = "functions",
  791. .stat_start = function_stat_start,
  792. .stat_next = function_stat_next,
  793. .stat_cmp = function_stat_cmp,
  794. .stat_headers = function_stat_headers,
  795. .stat_show = function_stat_show
  796. };
  797. static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
  798. {
  799. struct ftrace_profile_stat *stat;
  800. struct dentry *entry;
  801. char *name;
  802. int ret;
  803. int cpu;
  804. for_each_possible_cpu(cpu) {
  805. stat = &per_cpu(ftrace_profile_stats, cpu);
  806. name = kasprintf(GFP_KERNEL, "function%d", cpu);
  807. if (!name) {
  808. /*
  809. * The files created are permanent, if something happens
  810. * we still do not free memory.
  811. */
  812. WARN(1,
  813. "Could not allocate stat file for cpu %d\n",
  814. cpu);
  815. return;
  816. }
  817. stat->stat = function_stats;
  818. stat->stat.name = name;
  819. ret = register_stat_tracer(&stat->stat);
  820. if (ret) {
  821. WARN(1,
  822. "Could not register function stat for cpu %d\n",
  823. cpu);
  824. kfree(name);
  825. return;
  826. }
  827. }
  828. entry = tracefs_create_file("function_profile_enabled", 0644,
  829. d_tracer, NULL, &ftrace_profile_fops);
  830. if (!entry)
  831. pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
  832. }
  833. #else /* CONFIG_FUNCTION_PROFILER */
  834. static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
  835. {
  836. }
  837. #endif /* CONFIG_FUNCTION_PROFILER */
  838. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  839. static int ftrace_graph_active;
  840. #else
  841. # define ftrace_graph_active 0
  842. #endif
  843. #ifdef CONFIG_DYNAMIC_FTRACE
  844. static struct ftrace_ops *removed_ops;
  845. /*
  846. * Set when doing a global update, like enabling all recs or disabling them.
  847. * It is not set when just updating a single ftrace_ops.
  848. */
  849. static bool update_all_ops;
  850. #ifndef CONFIG_FTRACE_MCOUNT_RECORD
  851. # error Dynamic ftrace depends on MCOUNT_RECORD
  852. #endif
  853. struct ftrace_func_entry {
  854. struct hlist_node hlist;
  855. unsigned long ip;
  856. };
  857. struct ftrace_func_probe {
  858. struct ftrace_probe_ops *probe_ops;
  859. struct ftrace_ops ops;
  860. struct trace_array *tr;
  861. struct list_head list;
  862. void *data;
  863. int ref;
  864. };
  865. /*
  866. * We make these constant because no one should touch them,
  867. * but they are used as the default "empty hash", to avoid allocating
  868. * it all the time. These are in a read only section such that if
  869. * anyone does try to modify it, it will cause an exception.
  870. */
  871. static const struct hlist_head empty_buckets[1];
  872. static const struct ftrace_hash empty_hash = {
  873. .buckets = (struct hlist_head *)empty_buckets,
  874. };
  875. #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
  876. static struct ftrace_ops global_ops = {
  877. .func = ftrace_stub,
  878. .local_hash.notrace_hash = EMPTY_HASH,
  879. .local_hash.filter_hash = EMPTY_HASH,
  880. INIT_OPS_HASH(global_ops)
  881. .flags = FTRACE_OPS_FL_RECURSION_SAFE |
  882. FTRACE_OPS_FL_INITIALIZED |
  883. FTRACE_OPS_FL_PID,
  884. };
  885. /*
  886. * Used by the stack undwinder to know about dynamic ftrace trampolines.
  887. */
  888. struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
  889. {
  890. struct ftrace_ops *op = NULL;
  891. /*
  892. * Some of the ops may be dynamically allocated,
  893. * they are freed after a synchronize_sched().
  894. */
  895. preempt_disable_notrace();
  896. do_for_each_ftrace_op(op, ftrace_ops_list) {
  897. /*
  898. * This is to check for dynamically allocated trampolines.
  899. * Trampolines that are in kernel text will have
  900. * core_kernel_text() return true.
  901. */
  902. if (op->trampoline && op->trampoline_size)
  903. if (addr >= op->trampoline &&
  904. addr < op->trampoline + op->trampoline_size) {
  905. preempt_enable_notrace();
  906. return op;
  907. }
  908. } while_for_each_ftrace_op(op);
  909. preempt_enable_notrace();
  910. return NULL;
  911. }
  912. /*
  913. * This is used by __kernel_text_address() to return true if the
  914. * address is on a dynamically allocated trampoline that would
  915. * not return true for either core_kernel_text() or
  916. * is_module_text_address().
  917. */
  918. bool is_ftrace_trampoline(unsigned long addr)
  919. {
  920. return ftrace_ops_trampoline(addr) != NULL;
  921. }
  922. struct ftrace_page {
  923. struct ftrace_page *next;
  924. struct dyn_ftrace *records;
  925. int index;
  926. int size;
  927. };
  928. #define ENTRY_SIZE sizeof(struct dyn_ftrace)
  929. #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
  930. /* estimate from running different kernels */
  931. #define NR_TO_INIT 10000
  932. static struct ftrace_page *ftrace_pages_start;
  933. static struct ftrace_page *ftrace_pages;
  934. static __always_inline unsigned long
  935. ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
  936. {
  937. if (hash->size_bits > 0)
  938. return hash_long(ip, hash->size_bits);
  939. return 0;
  940. }
  941. /* Only use this function if ftrace_hash_empty() has already been tested */
  942. static __always_inline struct ftrace_func_entry *
  943. __ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
  944. {
  945. unsigned long key;
  946. struct ftrace_func_entry *entry;
  947. struct hlist_head *hhd;
  948. key = ftrace_hash_key(hash, ip);
  949. hhd = &hash->buckets[key];
  950. hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
  951. if (entry->ip == ip)
  952. return entry;
  953. }
  954. return NULL;
  955. }
  956. /**
  957. * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
  958. * @hash: The hash to look at
  959. * @ip: The instruction pointer to test
  960. *
  961. * Search a given @hash to see if a given instruction pointer (@ip)
  962. * exists in it.
  963. *
  964. * Returns the entry that holds the @ip if found. NULL otherwise.
  965. */
  966. struct ftrace_func_entry *
  967. ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
  968. {
  969. if (ftrace_hash_empty(hash))
  970. return NULL;
  971. return __ftrace_lookup_ip(hash, ip);
  972. }
  973. static void __add_hash_entry(struct ftrace_hash *hash,
  974. struct ftrace_func_entry *entry)
  975. {
  976. struct hlist_head *hhd;
  977. unsigned long key;
  978. key = ftrace_hash_key(hash, entry->ip);
  979. hhd = &hash->buckets[key];
  980. hlist_add_head(&entry->hlist, hhd);
  981. hash->count++;
  982. }
  983. static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
  984. {
  985. struct ftrace_func_entry *entry;
  986. entry = kmalloc(sizeof(*entry), GFP_KERNEL);
  987. if (!entry)
  988. return -ENOMEM;
  989. entry->ip = ip;
  990. __add_hash_entry(hash, entry);
  991. return 0;
  992. }
  993. static void
  994. free_hash_entry(struct ftrace_hash *hash,
  995. struct ftrace_func_entry *entry)
  996. {
  997. hlist_del(&entry->hlist);
  998. kfree(entry);
  999. hash->count--;
  1000. }
  1001. static void
  1002. remove_hash_entry(struct ftrace_hash *hash,
  1003. struct ftrace_func_entry *entry)
  1004. {
  1005. hlist_del_rcu(&entry->hlist);
  1006. hash->count--;
  1007. }
  1008. static void ftrace_hash_clear(struct ftrace_hash *hash)
  1009. {
  1010. struct hlist_head *hhd;
  1011. struct hlist_node *tn;
  1012. struct ftrace_func_entry *entry;
  1013. int size = 1 << hash->size_bits;
  1014. int i;
  1015. if (!hash->count)
  1016. return;
  1017. for (i = 0; i < size; i++) {
  1018. hhd = &hash->buckets[i];
  1019. hlist_for_each_entry_safe(entry, tn, hhd, hlist)
  1020. free_hash_entry(hash, entry);
  1021. }
  1022. FTRACE_WARN_ON(hash->count);
  1023. }
  1024. static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
  1025. {
  1026. list_del(&ftrace_mod->list);
  1027. kfree(ftrace_mod->module);
  1028. kfree(ftrace_mod->func);
  1029. kfree(ftrace_mod);
  1030. }
  1031. static void clear_ftrace_mod_list(struct list_head *head)
  1032. {
  1033. struct ftrace_mod_load *p, *n;
  1034. /* stack tracer isn't supported yet */
  1035. if (!head)
  1036. return;
  1037. mutex_lock(&ftrace_lock);
  1038. list_for_each_entry_safe(p, n, head, list)
  1039. free_ftrace_mod(p);
  1040. mutex_unlock(&ftrace_lock);
  1041. }
  1042. static void free_ftrace_hash(struct ftrace_hash *hash)
  1043. {
  1044. if (!hash || hash == EMPTY_HASH)
  1045. return;
  1046. ftrace_hash_clear(hash);
  1047. kfree(hash->buckets);
  1048. kfree(hash);
  1049. }
  1050. static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
  1051. {
  1052. struct ftrace_hash *hash;
  1053. hash = container_of(rcu, struct ftrace_hash, rcu);
  1054. free_ftrace_hash(hash);
  1055. }
  1056. static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
  1057. {
  1058. if (!hash || hash == EMPTY_HASH)
  1059. return;
  1060. call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
  1061. }
  1062. void ftrace_free_filter(struct ftrace_ops *ops)
  1063. {
  1064. ftrace_ops_init(ops);
  1065. free_ftrace_hash(ops->func_hash->filter_hash);
  1066. free_ftrace_hash(ops->func_hash->notrace_hash);
  1067. }
  1068. static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
  1069. {
  1070. struct ftrace_hash *hash;
  1071. int size;
  1072. hash = kzalloc(sizeof(*hash), GFP_KERNEL);
  1073. if (!hash)
  1074. return NULL;
  1075. size = 1 << size_bits;
  1076. hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
  1077. if (!hash->buckets) {
  1078. kfree(hash);
  1079. return NULL;
  1080. }
  1081. hash->size_bits = size_bits;
  1082. return hash;
  1083. }
  1084. static int ftrace_add_mod(struct trace_array *tr,
  1085. const char *func, const char *module,
  1086. int enable)
  1087. {
  1088. struct ftrace_mod_load *ftrace_mod;
  1089. struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;
  1090. ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
  1091. if (!ftrace_mod)
  1092. return -ENOMEM;
  1093. ftrace_mod->func = kstrdup(func, GFP_KERNEL);
  1094. ftrace_mod->module = kstrdup(module, GFP_KERNEL);
  1095. ftrace_mod->enable = enable;
  1096. if (!ftrace_mod->func || !ftrace_mod->module)
  1097. goto out_free;
  1098. list_add(&ftrace_mod->list, mod_head);
  1099. return 0;
  1100. out_free:
  1101. free_ftrace_mod(ftrace_mod);
  1102. return -ENOMEM;
  1103. }
  1104. static struct ftrace_hash *
  1105. alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
  1106. {
  1107. struct ftrace_func_entry *entry;
  1108. struct ftrace_hash *new_hash;
  1109. int size;
  1110. int ret;
  1111. int i;
  1112. new_hash = alloc_ftrace_hash(size_bits);
  1113. if (!new_hash)
  1114. return NULL;
  1115. if (hash)
  1116. new_hash->flags = hash->flags;
  1117. /* Empty hash? */
  1118. if (ftrace_hash_empty(hash))
  1119. return new_hash;
  1120. size = 1 << hash->size_bits;
  1121. for (i = 0; i < size; i++) {
  1122. hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
  1123. ret = add_hash_entry(new_hash, entry->ip);
  1124. if (ret < 0)
  1125. goto free_hash;
  1126. }
  1127. }
  1128. FTRACE_WARN_ON(new_hash->count != hash->count);
  1129. return new_hash;
  1130. free_hash:
  1131. free_ftrace_hash(new_hash);
  1132. return NULL;
  1133. }
  1134. static void
  1135. ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
  1136. static void
  1137. ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
  1138. static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
  1139. struct ftrace_hash *new_hash);
  1140. static struct ftrace_hash *
  1141. __ftrace_hash_move(struct ftrace_hash *src)
  1142. {
  1143. struct ftrace_func_entry *entry;
  1144. struct hlist_node *tn;
  1145. struct hlist_head *hhd;
  1146. struct ftrace_hash *new_hash;
  1147. int size = src->count;
  1148. int bits = 0;
  1149. int i;
  1150. /*
  1151. * If the new source is empty, just return the empty_hash.
  1152. */
  1153. if (ftrace_hash_empty(src))
  1154. return EMPTY_HASH;
  1155. /*
  1156. * Make the hash size about 1/2 the # found
  1157. */
  1158. for (size /= 2; size; size >>= 1)
  1159. bits++;
  1160. /* Don't allocate too much */
  1161. if (bits > FTRACE_HASH_MAX_BITS)
  1162. bits = FTRACE_HASH_MAX_BITS;
  1163. new_hash = alloc_ftrace_hash(bits);
  1164. if (!new_hash)
  1165. return NULL;
  1166. new_hash->flags = src->flags;
  1167. size = 1 << src->size_bits;
  1168. for (i = 0; i < size; i++) {
  1169. hhd = &src->buckets[i];
  1170. hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
  1171. remove_hash_entry(src, entry);
  1172. __add_hash_entry(new_hash, entry);
  1173. }
  1174. }
  1175. return new_hash;
  1176. }
  1177. static int
  1178. ftrace_hash_move(struct ftrace_ops *ops, int enable,
  1179. struct ftrace_hash **dst, struct ftrace_hash *src)
  1180. {
  1181. struct ftrace_hash *new_hash;
  1182. int ret;
  1183. /* Reject setting notrace hash on IPMODIFY ftrace_ops */
  1184. if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
  1185. return -EINVAL;
  1186. new_hash = __ftrace_hash_move(src);
  1187. if (!new_hash)
  1188. return -ENOMEM;
  1189. /* Make sure this can be applied if it is IPMODIFY ftrace_ops */
  1190. if (enable) {
  1191. /* IPMODIFY should be updated only when filter_hash updating */
  1192. ret = ftrace_hash_ipmodify_update(ops, new_hash);
  1193. if (ret < 0) {
  1194. free_ftrace_hash(new_hash);
  1195. return ret;
  1196. }
  1197. }
  1198. /*
  1199. * Remove the current set, update the hash and add
  1200. * them back.
  1201. */
  1202. ftrace_hash_rec_disable_modify(ops, enable);
  1203. rcu_assign_pointer(*dst, new_hash);
  1204. ftrace_hash_rec_enable_modify(ops, enable);
  1205. return 0;
  1206. }
  1207. static bool hash_contains_ip(unsigned long ip,
  1208. struct ftrace_ops_hash *hash)
  1209. {
  1210. /*
  1211. * The function record is a match if it exists in the filter
  1212. * hash and not in the notrace hash. Note, an emty hash is
  1213. * considered a match for the filter hash, but an empty
  1214. * notrace hash is considered not in the notrace hash.
  1215. */
  1216. return (ftrace_hash_empty(hash->filter_hash) ||
  1217. __ftrace_lookup_ip(hash->filter_hash, ip)) &&
  1218. (ftrace_hash_empty(hash->notrace_hash) ||
  1219. !__ftrace_lookup_ip(hash->notrace_hash, ip));
  1220. }
  1221. /*
  1222. * Test the hashes for this ops to see if we want to call
  1223. * the ops->func or not.
  1224. *
  1225. * It's a match if the ip is in the ops->filter_hash or
  1226. * the filter_hash does not exist or is empty,
  1227. * AND
  1228. * the ip is not in the ops->notrace_hash.
  1229. *
  1230. * This needs to be called with preemption disabled as
  1231. * the hashes are freed with call_rcu_sched().
  1232. */
  1233. static int
  1234. ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
  1235. {
  1236. struct ftrace_ops_hash hash;
  1237. int ret;
  1238. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  1239. /*
  1240. * There's a small race when adding ops that the ftrace handler
  1241. * that wants regs, may be called without them. We can not
  1242. * allow that handler to be called if regs is NULL.
  1243. */
  1244. if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
  1245. return 0;
  1246. #endif
  1247. rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
  1248. rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
  1249. if (hash_contains_ip(ip, &hash))
  1250. ret = 1;
  1251. else
  1252. ret = 0;
  1253. return ret;
  1254. }
  1255. /*
  1256. * This is a double for. Do not use 'break' to break out of the loop,
  1257. * you must use a goto.
  1258. */
  1259. #define do_for_each_ftrace_rec(pg, rec) \
  1260. for (pg = ftrace_pages_start; pg; pg = pg->next) { \
  1261. int _____i; \
  1262. for (_____i = 0; _____i < pg->index; _____i++) { \
  1263. rec = &pg->records[_____i];
  1264. #define while_for_each_ftrace_rec() \
  1265. } \
  1266. }
  1267. static int ftrace_cmp_recs(const void *a, const void *b)
  1268. {
  1269. const struct dyn_ftrace *key = a;
  1270. const struct dyn_ftrace *rec = b;
  1271. if (key->flags < rec->ip)
  1272. return -1;
  1273. if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
  1274. return 1;
  1275. return 0;
  1276. }
  1277. /**
  1278. * ftrace_location_range - return the first address of a traced location
  1279. * if it touches the given ip range
  1280. * @start: start of range to search.
  1281. * @end: end of range to search (inclusive). @end points to the last byte
  1282. * to check.
  1283. *
  1284. * Returns rec->ip if the related ftrace location is a least partly within
  1285. * the given address range. That is, the first address of the instruction
  1286. * that is either a NOP or call to the function tracer. It checks the ftrace
  1287. * internal tables to determine if the address belongs or not.
  1288. */
  1289. unsigned long ftrace_location_range(unsigned long start, unsigned long end)
  1290. {
  1291. struct ftrace_page *pg;
  1292. struct dyn_ftrace *rec;
  1293. struct dyn_ftrace key;
  1294. key.ip = start;
  1295. key.flags = end; /* overload flags, as it is unsigned long */
  1296. for (pg = ftrace_pages_start; pg; pg = pg->next) {
  1297. if (end < pg->records[0].ip ||
  1298. start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
  1299. continue;
  1300. rec = bsearch(&key, pg->records, pg->index,
  1301. sizeof(struct dyn_ftrace),
  1302. ftrace_cmp_recs);
  1303. if (rec)
  1304. return rec->ip;
  1305. }
  1306. return 0;
  1307. }
  1308. /**
  1309. * ftrace_location - return true if the ip giving is a traced location
  1310. * @ip: the instruction pointer to check
  1311. *
  1312. * Returns rec->ip if @ip given is a pointer to a ftrace location.
  1313. * That is, the instruction that is either a NOP or call to
  1314. * the function tracer. It checks the ftrace internal tables to
  1315. * determine if the address belongs or not.
  1316. */
  1317. unsigned long ftrace_location(unsigned long ip)
  1318. {
  1319. return ftrace_location_range(ip, ip);
  1320. }
  1321. /**
  1322. * ftrace_text_reserved - return true if range contains an ftrace location
  1323. * @start: start of range to search
  1324. * @end: end of range to search (inclusive). @end points to the last byte to check.
  1325. *
  1326. * Returns 1 if @start and @end contains a ftrace location.
  1327. * That is, the instruction that is either a NOP or call to
  1328. * the function tracer. It checks the ftrace internal tables to
  1329. * determine if the address belongs or not.
  1330. */
  1331. int ftrace_text_reserved(const void *start, const void *end)
  1332. {
  1333. unsigned long ret;
  1334. ret = ftrace_location_range((unsigned long)start,
  1335. (unsigned long)end);
  1336. return (int)!!ret;
  1337. }
  1338. /* Test if ops registered to this rec needs regs */
  1339. static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
  1340. {
  1341. struct ftrace_ops *ops;
  1342. bool keep_regs = false;
  1343. for (ops = ftrace_ops_list;
  1344. ops != &ftrace_list_end; ops = ops->next) {
  1345. /* pass rec in as regs to have non-NULL val */
  1346. if (ftrace_ops_test(ops, rec->ip, rec)) {
  1347. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
  1348. keep_regs = true;
  1349. break;
  1350. }
  1351. }
  1352. }
  1353. return keep_regs;
  1354. }
  1355. static struct ftrace_ops *
  1356. ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
  1357. static struct ftrace_ops *
  1358. ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude);
  1359. static struct ftrace_ops *
  1360. ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
  1361. static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
  1362. int filter_hash,
  1363. bool inc)
  1364. {
  1365. struct ftrace_hash *hash;
  1366. struct ftrace_hash *other_hash;
  1367. struct ftrace_page *pg;
  1368. struct dyn_ftrace *rec;
  1369. bool update = false;
  1370. int count = 0;
  1371. int all = false;
  1372. /* Only update if the ops has been registered */
  1373. if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
  1374. return false;
  1375. /*
  1376. * In the filter_hash case:
  1377. * If the count is zero, we update all records.
  1378. * Otherwise we just update the items in the hash.
  1379. *
  1380. * In the notrace_hash case:
  1381. * We enable the update in the hash.
  1382. * As disabling notrace means enabling the tracing,
  1383. * and enabling notrace means disabling, the inc variable
  1384. * gets inversed.
  1385. */
  1386. if (filter_hash) {
  1387. hash = ops->func_hash->filter_hash;
  1388. other_hash = ops->func_hash->notrace_hash;
  1389. if (ftrace_hash_empty(hash))
  1390. all = true;
  1391. } else {
  1392. inc = !inc;
  1393. hash = ops->func_hash->notrace_hash;
  1394. other_hash = ops->func_hash->filter_hash;
  1395. /*
  1396. * If the notrace hash has no items,
  1397. * then there's nothing to do.
  1398. */
  1399. if (ftrace_hash_empty(hash))
  1400. return false;
  1401. }
  1402. do_for_each_ftrace_rec(pg, rec) {
  1403. int in_other_hash = 0;
  1404. int in_hash = 0;
  1405. int match = 0;
  1406. if (rec->flags & FTRACE_FL_DISABLED)
  1407. continue;
  1408. if (all) {
  1409. /*
  1410. * Only the filter_hash affects all records.
  1411. * Update if the record is not in the notrace hash.
  1412. */
  1413. if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
  1414. match = 1;
  1415. } else {
  1416. in_hash = !!ftrace_lookup_ip(hash, rec->ip);
  1417. in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
  1418. /*
  1419. * If filter_hash is set, we want to match all functions
  1420. * that are in the hash but not in the other hash.
  1421. *
  1422. * If filter_hash is not set, then we are decrementing.
  1423. * That means we match anything that is in the hash
  1424. * and also in the other_hash. That is, we need to turn
  1425. * off functions in the other hash because they are disabled
  1426. * by this hash.
  1427. */
  1428. if (filter_hash && in_hash && !in_other_hash)
  1429. match = 1;
  1430. else if (!filter_hash && in_hash &&
  1431. (in_other_hash || ftrace_hash_empty(other_hash)))
  1432. match = 1;
  1433. }
  1434. if (!match)
  1435. continue;
  1436. if (inc) {
  1437. rec->flags++;
  1438. if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
  1439. return false;
  1440. /*
  1441. * If there's only a single callback registered to a
  1442. * function, and the ops has a trampoline registered
  1443. * for it, then we can call it directly.
  1444. */
  1445. if (ftrace_rec_count(rec) == 1 && ops->trampoline)
  1446. rec->flags |= FTRACE_FL_TRAMP;
  1447. else
  1448. /*
  1449. * If we are adding another function callback
  1450. * to this function, and the previous had a
  1451. * custom trampoline in use, then we need to go
  1452. * back to the default trampoline.
  1453. */
  1454. rec->flags &= ~FTRACE_FL_TRAMP;
  1455. /*
  1456. * If any ops wants regs saved for this function
  1457. * then all ops will get saved regs.
  1458. */
  1459. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
  1460. rec->flags |= FTRACE_FL_REGS;
  1461. } else {
  1462. if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
  1463. return false;
  1464. rec->flags--;
  1465. /*
  1466. * If the rec had REGS enabled and the ops that is
  1467. * being removed had REGS set, then see if there is
  1468. * still any ops for this record that wants regs.
  1469. * If not, we can stop recording them.
  1470. */
  1471. if (ftrace_rec_count(rec) > 0 &&
  1472. rec->flags & FTRACE_FL_REGS &&
  1473. ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
  1474. if (!test_rec_ops_needs_regs(rec))
  1475. rec->flags &= ~FTRACE_FL_REGS;
  1476. }
  1477. /*
  1478. * The TRAMP needs to be set only if rec count
  1479. * is decremented to one, and the ops that is
  1480. * left has a trampoline. As TRAMP can only be
  1481. * enabled if there is only a single ops attached
  1482. * to it.
  1483. */
  1484. if (ftrace_rec_count(rec) == 1 &&
  1485. ftrace_find_tramp_ops_any_other(rec, ops))
  1486. rec->flags |= FTRACE_FL_TRAMP;
  1487. else
  1488. rec->flags &= ~FTRACE_FL_TRAMP;
  1489. /*
  1490. * flags will be cleared in ftrace_check_record()
  1491. * if rec count is zero.
  1492. */
  1493. }
  1494. count++;
  1495. /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
  1496. update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;
  1497. /* Shortcut, if we handled all records, we are done. */
  1498. if (!all && count == hash->count)
  1499. return update;
  1500. } while_for_each_ftrace_rec();
  1501. return update;
  1502. }
  1503. static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
  1504. int filter_hash)
  1505. {
  1506. return __ftrace_hash_rec_update(ops, filter_hash, 0);
  1507. }
  1508. static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
  1509. int filter_hash)
  1510. {
  1511. return __ftrace_hash_rec_update(ops, filter_hash, 1);
  1512. }
  1513. static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
  1514. int filter_hash, int inc)
  1515. {
  1516. struct ftrace_ops *op;
  1517. __ftrace_hash_rec_update(ops, filter_hash, inc);
  1518. if (ops->func_hash != &global_ops.local_hash)
  1519. return;
  1520. /*
  1521. * If the ops shares the global_ops hash, then we need to update
  1522. * all ops that are enabled and use this hash.
  1523. */
  1524. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1525. /* Already done */
  1526. if (op == ops)
  1527. continue;
  1528. if (op->func_hash == &global_ops.local_hash)
  1529. __ftrace_hash_rec_update(op, filter_hash, inc);
  1530. } while_for_each_ftrace_op(op);
  1531. }
  1532. static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
  1533. int filter_hash)
  1534. {
  1535. ftrace_hash_rec_update_modify(ops, filter_hash, 0);
  1536. }
  1537. static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
  1538. int filter_hash)
  1539. {
  1540. ftrace_hash_rec_update_modify(ops, filter_hash, 1);
  1541. }
  1542. /*
  1543. * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
  1544. * or no-needed to update, -EBUSY if it detects a conflict of the flag
  1545. * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
  1546. * Note that old_hash and new_hash has below meanings
  1547. * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
  1548. * - If the hash is EMPTY_HASH, it hits nothing
  1549. * - Anything else hits the recs which match the hash entries.
  1550. */
  1551. static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
  1552. struct ftrace_hash *old_hash,
  1553. struct ftrace_hash *new_hash)
  1554. {
  1555. struct ftrace_page *pg;
  1556. struct dyn_ftrace *rec, *end = NULL;
  1557. int in_old, in_new;
  1558. /* Only update if the ops has been registered */
  1559. if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
  1560. return 0;
  1561. if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
  1562. return 0;
  1563. /*
  1564. * Since the IPMODIFY is a very address sensitive action, we do not
  1565. * allow ftrace_ops to set all functions to new hash.
  1566. */
  1567. if (!new_hash || !old_hash)
  1568. return -EINVAL;
  1569. /* Update rec->flags */
  1570. do_for_each_ftrace_rec(pg, rec) {
  1571. if (rec->flags & FTRACE_FL_DISABLED)
  1572. continue;
  1573. /* We need to update only differences of filter_hash */
  1574. in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
  1575. in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
  1576. if (in_old == in_new)
  1577. continue;
  1578. if (in_new) {
  1579. /* New entries must ensure no others are using it */
  1580. if (rec->flags & FTRACE_FL_IPMODIFY)
  1581. goto rollback;
  1582. rec->flags |= FTRACE_FL_IPMODIFY;
  1583. } else /* Removed entry */
  1584. rec->flags &= ~FTRACE_FL_IPMODIFY;
  1585. } while_for_each_ftrace_rec();
  1586. return 0;
  1587. rollback:
  1588. end = rec;
  1589. /* Roll back what we did above */
  1590. do_for_each_ftrace_rec(pg, rec) {
  1591. if (rec->flags & FTRACE_FL_DISABLED)
  1592. continue;
  1593. if (rec == end)
  1594. goto err_out;
  1595. in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
  1596. in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
  1597. if (in_old == in_new)
  1598. continue;
  1599. if (in_new)
  1600. rec->flags &= ~FTRACE_FL_IPMODIFY;
  1601. else
  1602. rec->flags |= FTRACE_FL_IPMODIFY;
  1603. } while_for_each_ftrace_rec();
  1604. err_out:
  1605. return -EBUSY;
  1606. }
  1607. static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
  1608. {
  1609. struct ftrace_hash *hash = ops->func_hash->filter_hash;
  1610. if (ftrace_hash_empty(hash))
  1611. hash = NULL;
  1612. return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
  1613. }
  1614. /* Disabling always succeeds */
  1615. static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
  1616. {
  1617. struct ftrace_hash *hash = ops->func_hash->filter_hash;
  1618. if (ftrace_hash_empty(hash))
  1619. hash = NULL;
  1620. __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
  1621. }
  1622. static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
  1623. struct ftrace_hash *new_hash)
  1624. {
  1625. struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
  1626. if (ftrace_hash_empty(old_hash))
  1627. old_hash = NULL;
  1628. if (ftrace_hash_empty(new_hash))
  1629. new_hash = NULL;
  1630. return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
  1631. }
  1632. static void print_ip_ins(const char *fmt, const unsigned char *p)
  1633. {
  1634. int i;
  1635. printk(KERN_CONT "%s", fmt);
  1636. for (i = 0; i < MCOUNT_INSN_SIZE; i++)
  1637. printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
  1638. }
  1639. enum ftrace_bug_type ftrace_bug_type;
  1640. const void *ftrace_expected;
  1641. static void print_bug_type(void)
  1642. {
  1643. switch (ftrace_bug_type) {
  1644. case FTRACE_BUG_UNKNOWN:
  1645. break;
  1646. case FTRACE_BUG_INIT:
  1647. pr_info("Initializing ftrace call sites\n");
  1648. break;
  1649. case FTRACE_BUG_NOP:
  1650. pr_info("Setting ftrace call site to NOP\n");
  1651. break;
  1652. case FTRACE_BUG_CALL:
  1653. pr_info("Setting ftrace call site to call ftrace function\n");
  1654. break;
  1655. case FTRACE_BUG_UPDATE:
  1656. pr_info("Updating ftrace call site to call a different ftrace function\n");
  1657. break;
  1658. }
  1659. }
  1660. /**
  1661. * ftrace_bug - report and shutdown function tracer
  1662. * @failed: The failed type (EFAULT, EINVAL, EPERM)
  1663. * @rec: The record that failed
  1664. *
  1665. * The arch code that enables or disables the function tracing
  1666. * can call ftrace_bug() when it has detected a problem in
  1667. * modifying the code. @failed should be one of either:
  1668. * EFAULT - if the problem happens on reading the @ip address
  1669. * EINVAL - if what is read at @ip is not what was expected
  1670. * EPERM - if the problem happens on writting to the @ip address
  1671. */
  1672. void ftrace_bug(int failed, struct dyn_ftrace *rec)
  1673. {
  1674. unsigned long ip = rec ? rec->ip : 0;
  1675. switch (failed) {
  1676. case -EFAULT:
  1677. FTRACE_WARN_ON_ONCE(1);
  1678. pr_info("ftrace faulted on modifying ");
  1679. print_ip_sym(ip);
  1680. break;
  1681. case -EINVAL:
  1682. FTRACE_WARN_ON_ONCE(1);
  1683. pr_info("ftrace failed to modify ");
  1684. print_ip_sym(ip);
  1685. print_ip_ins(" actual: ", (unsigned char *)ip);
  1686. pr_cont("\n");
  1687. if (ftrace_expected) {
  1688. print_ip_ins(" expected: ", ftrace_expected);
  1689. pr_cont("\n");
  1690. }
  1691. break;
  1692. case -EPERM:
  1693. FTRACE_WARN_ON_ONCE(1);
  1694. pr_info("ftrace faulted on writing ");
  1695. print_ip_sym(ip);
  1696. break;
  1697. default:
  1698. FTRACE_WARN_ON_ONCE(1);
  1699. pr_info("ftrace faulted on unknown error ");
  1700. print_ip_sym(ip);
  1701. }
  1702. print_bug_type();
  1703. if (rec) {
  1704. struct ftrace_ops *ops = NULL;
  1705. pr_info("ftrace record flags: %lx\n", rec->flags);
  1706. pr_cont(" (%ld)%s", ftrace_rec_count(rec),
  1707. rec->flags & FTRACE_FL_REGS ? " R" : " ");
  1708. if (rec->flags & FTRACE_FL_TRAMP_EN) {
  1709. ops = ftrace_find_tramp_ops_any(rec);
  1710. if (ops) {
  1711. do {
  1712. pr_cont("\ttramp: %pS (%pS)",
  1713. (void *)ops->trampoline,
  1714. (void *)ops->func);
  1715. ops = ftrace_find_tramp_ops_next(rec, ops);
  1716. } while (ops);
  1717. } else
  1718. pr_cont("\ttramp: ERROR!");
  1719. }
  1720. ip = ftrace_get_addr_curr(rec);
  1721. pr_cont("\n expected tramp: %lx\n", ip);
  1722. }
  1723. }
  1724. static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
  1725. {
  1726. unsigned long flag = 0UL;
  1727. ftrace_bug_type = FTRACE_BUG_UNKNOWN;
  1728. if (rec->flags & FTRACE_FL_DISABLED)
  1729. return FTRACE_UPDATE_IGNORE;
  1730. /*
  1731. * If we are updating calls:
  1732. *
  1733. * If the record has a ref count, then we need to enable it
  1734. * because someone is using it.
  1735. *
  1736. * Otherwise we make sure its disabled.
  1737. *
  1738. * If we are disabling calls, then disable all records that
  1739. * are enabled.
  1740. */
  1741. if (enable && ftrace_rec_count(rec))
  1742. flag = FTRACE_FL_ENABLED;
  1743. /*
  1744. * If enabling and the REGS flag does not match the REGS_EN, or
  1745. * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
  1746. * this record. Set flags to fail the compare against ENABLED.
  1747. */
  1748. if (flag) {
  1749. if (!(rec->flags & FTRACE_FL_REGS) !=
  1750. !(rec->flags & FTRACE_FL_REGS_EN))
  1751. flag |= FTRACE_FL_REGS;
  1752. if (!(rec->flags & FTRACE_FL_TRAMP) !=
  1753. !(rec->flags & FTRACE_FL_TRAMP_EN))
  1754. flag |= FTRACE_FL_TRAMP;
  1755. }
  1756. /* If the state of this record hasn't changed, then do nothing */
  1757. if ((rec->flags & FTRACE_FL_ENABLED) == flag)
  1758. return FTRACE_UPDATE_IGNORE;
  1759. if (flag) {
  1760. /* Save off if rec is being enabled (for return value) */
  1761. flag ^= rec->flags & FTRACE_FL_ENABLED;
  1762. if (update) {
  1763. rec->flags |= FTRACE_FL_ENABLED;
  1764. if (flag & FTRACE_FL_REGS) {
  1765. if (rec->flags & FTRACE_FL_REGS)
  1766. rec->flags |= FTRACE_FL_REGS_EN;
  1767. else
  1768. rec->flags &= ~FTRACE_FL_REGS_EN;
  1769. }
  1770. if (flag & FTRACE_FL_TRAMP) {
  1771. if (rec->flags & FTRACE_FL_TRAMP)
  1772. rec->flags |= FTRACE_FL_TRAMP_EN;
  1773. else
  1774. rec->flags &= ~FTRACE_FL_TRAMP_EN;
  1775. }
  1776. }
  1777. /*
  1778. * If this record is being updated from a nop, then
  1779. * return UPDATE_MAKE_CALL.
  1780. * Otherwise,
  1781. * return UPDATE_MODIFY_CALL to tell the caller to convert
  1782. * from the save regs, to a non-save regs function or
  1783. * vice versa, or from a trampoline call.
  1784. */
  1785. if (flag & FTRACE_FL_ENABLED) {
  1786. ftrace_bug_type = FTRACE_BUG_CALL;
  1787. return FTRACE_UPDATE_MAKE_CALL;
  1788. }
  1789. ftrace_bug_type = FTRACE_BUG_UPDATE;
  1790. return FTRACE_UPDATE_MODIFY_CALL;
  1791. }
  1792. if (update) {
  1793. /* If there's no more users, clear all flags */
  1794. if (!ftrace_rec_count(rec))
  1795. rec->flags = 0;
  1796. else
  1797. /*
  1798. * Just disable the record, but keep the ops TRAMP
  1799. * and REGS states. The _EN flags must be disabled though.
  1800. */
  1801. rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
  1802. FTRACE_FL_REGS_EN);
  1803. }
  1804. ftrace_bug_type = FTRACE_BUG_NOP;
  1805. return FTRACE_UPDATE_MAKE_NOP;
  1806. }
  1807. /**
  1808. * ftrace_update_record, set a record that now is tracing or not
  1809. * @rec: the record to update
  1810. * @enable: set to 1 if the record is tracing, zero to force disable
  1811. *
  1812. * The records that represent all functions that can be traced need
  1813. * to be updated when tracing has been enabled.
  1814. */
  1815. int ftrace_update_record(struct dyn_ftrace *rec, int enable)
  1816. {
  1817. return ftrace_check_record(rec, enable, 1);
  1818. }
  1819. /**
  1820. * ftrace_test_record, check if the record has been enabled or not
  1821. * @rec: the record to test
  1822. * @enable: set to 1 to check if enabled, 0 if it is disabled
  1823. *
  1824. * The arch code may need to test if a record is already set to
  1825. * tracing to determine how to modify the function code that it
  1826. * represents.
  1827. */
  1828. int ftrace_test_record(struct dyn_ftrace *rec, int enable)
  1829. {
  1830. return ftrace_check_record(rec, enable, 0);
  1831. }
  1832. static struct ftrace_ops *
  1833. ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
  1834. {
  1835. struct ftrace_ops *op;
  1836. unsigned long ip = rec->ip;
  1837. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1838. if (!op->trampoline)
  1839. continue;
  1840. if (hash_contains_ip(ip, op->func_hash))
  1841. return op;
  1842. } while_for_each_ftrace_op(op);
  1843. return NULL;
  1844. }
  1845. static struct ftrace_ops *
  1846. ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude)
  1847. {
  1848. struct ftrace_ops *op;
  1849. unsigned long ip = rec->ip;
  1850. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1851. if (op == op_exclude || !op->trampoline)
  1852. continue;
  1853. if (hash_contains_ip(ip, op->func_hash))
  1854. return op;
  1855. } while_for_each_ftrace_op(op);
  1856. return NULL;
  1857. }
  1858. static struct ftrace_ops *
  1859. ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
  1860. struct ftrace_ops *op)
  1861. {
  1862. unsigned long ip = rec->ip;
  1863. while_for_each_ftrace_op(op) {
  1864. if (!op->trampoline)
  1865. continue;
  1866. if (hash_contains_ip(ip, op->func_hash))
  1867. return op;
  1868. }
  1869. return NULL;
  1870. }
  1871. static struct ftrace_ops *
  1872. ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
  1873. {
  1874. struct ftrace_ops *op;
  1875. unsigned long ip = rec->ip;
  1876. /*
  1877. * Need to check removed ops first.
  1878. * If they are being removed, and this rec has a tramp,
  1879. * and this rec is in the ops list, then it would be the
  1880. * one with the tramp.
  1881. */
  1882. if (removed_ops) {
  1883. if (hash_contains_ip(ip, &removed_ops->old_hash))
  1884. return removed_ops;
  1885. }
  1886. /*
  1887. * Need to find the current trampoline for a rec.
  1888. * Now, a trampoline is only attached to a rec if there
  1889. * was a single 'ops' attached to it. But this can be called
  1890. * when we are adding another op to the rec or removing the
  1891. * current one. Thus, if the op is being added, we can
  1892. * ignore it because it hasn't attached itself to the rec
  1893. * yet.
  1894. *
  1895. * If an ops is being modified (hooking to different functions)
  1896. * then we don't care about the new functions that are being
  1897. * added, just the old ones (that are probably being removed).
  1898. *
  1899. * If we are adding an ops to a function that already is using
  1900. * a trampoline, it needs to be removed (trampolines are only
  1901. * for single ops connected), then an ops that is not being
  1902. * modified also needs to be checked.
  1903. */
  1904. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1905. if (!op->trampoline)
  1906. continue;
  1907. /*
  1908. * If the ops is being added, it hasn't gotten to
  1909. * the point to be removed from this tree yet.
  1910. */
  1911. if (op->flags & FTRACE_OPS_FL_ADDING)
  1912. continue;
  1913. /*
  1914. * If the ops is being modified and is in the old
  1915. * hash, then it is probably being removed from this
  1916. * function.
  1917. */
  1918. if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
  1919. hash_contains_ip(ip, &op->old_hash))
  1920. return op;
  1921. /*
  1922. * If the ops is not being added or modified, and it's
  1923. * in its normal filter hash, then this must be the one
  1924. * we want!
  1925. */
  1926. if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
  1927. hash_contains_ip(ip, op->func_hash))
  1928. return op;
  1929. } while_for_each_ftrace_op(op);
  1930. return NULL;
  1931. }
  1932. static struct ftrace_ops *
  1933. ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
  1934. {
  1935. struct ftrace_ops *op;
  1936. unsigned long ip = rec->ip;
  1937. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1938. /* pass rec in as regs to have non-NULL val */
  1939. if (hash_contains_ip(ip, op->func_hash))
  1940. return op;
  1941. } while_for_each_ftrace_op(op);
  1942. return NULL;
  1943. }
  1944. /**
  1945. * ftrace_get_addr_new - Get the call address to set to
  1946. * @rec: The ftrace record descriptor
  1947. *
  1948. * If the record has the FTRACE_FL_REGS set, that means that it
  1949. * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
  1950. * is not not set, then it wants to convert to the normal callback.
  1951. *
  1952. * Returns the address of the trampoline to set to
  1953. */
  1954. unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
  1955. {
  1956. struct ftrace_ops *ops;
  1957. /* Trampolines take precedence over regs */
  1958. if (rec->flags & FTRACE_FL_TRAMP) {
  1959. ops = ftrace_find_tramp_ops_new(rec);
  1960. if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
  1961. pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
  1962. (void *)rec->ip, (void *)rec->ip, rec->flags);
  1963. /* Ftrace is shutting down, return anything */
  1964. return (unsigned long)FTRACE_ADDR;
  1965. }
  1966. return ops->trampoline;
  1967. }
  1968. if (rec->flags & FTRACE_FL_REGS)
  1969. return (unsigned long)FTRACE_REGS_ADDR;
  1970. else
  1971. return (unsigned long)FTRACE_ADDR;
  1972. }
  1973. /**
  1974. * ftrace_get_addr_curr - Get the call address that is already there
  1975. * @rec: The ftrace record descriptor
  1976. *
  1977. * The FTRACE_FL_REGS_EN is set when the record already points to
  1978. * a function that saves all the regs. Basically the '_EN' version
  1979. * represents the current state of the function.
  1980. *
  1981. * Returns the address of the trampoline that is currently being called
  1982. */
  1983. unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
  1984. {
  1985. struct ftrace_ops *ops;
  1986. /* Trampolines take precedence over regs */
  1987. if (rec->flags & FTRACE_FL_TRAMP_EN) {
  1988. ops = ftrace_find_tramp_ops_curr(rec);
  1989. if (FTRACE_WARN_ON(!ops)) {
  1990. pr_warn("Bad trampoline accounting at: %p (%pS)\n",
  1991. (void *)rec->ip, (void *)rec->ip);
  1992. /* Ftrace is shutting down, return anything */
  1993. return (unsigned long)FTRACE_ADDR;
  1994. }
  1995. return ops->trampoline;
  1996. }
  1997. if (rec->flags & FTRACE_FL_REGS_EN)
  1998. return (unsigned long)FTRACE_REGS_ADDR;
  1999. else
  2000. return (unsigned long)FTRACE_ADDR;
  2001. }
  2002. static int
  2003. __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
  2004. {
  2005. unsigned long ftrace_old_addr;
  2006. unsigned long ftrace_addr;
  2007. int ret;
  2008. ftrace_addr = ftrace_get_addr_new(rec);
  2009. /* This needs to be done before we call ftrace_update_record */
  2010. ftrace_old_addr = ftrace_get_addr_curr(rec);
  2011. ret = ftrace_update_record(rec, enable);
  2012. ftrace_bug_type = FTRACE_BUG_UNKNOWN;
  2013. switch (ret) {
  2014. case FTRACE_UPDATE_IGNORE:
  2015. return 0;
  2016. case FTRACE_UPDATE_MAKE_CALL:
  2017. ftrace_bug_type = FTRACE_BUG_CALL;
  2018. return ftrace_make_call(rec, ftrace_addr);
  2019. case FTRACE_UPDATE_MAKE_NOP:
  2020. ftrace_bug_type = FTRACE_BUG_NOP;
  2021. return ftrace_make_nop(NULL, rec, ftrace_old_addr);
  2022. case FTRACE_UPDATE_MODIFY_CALL:
  2023. ftrace_bug_type = FTRACE_BUG_UPDATE;
  2024. return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
  2025. }
  2026. return -1; /* unknow ftrace bug */
  2027. }
  2028. void __weak ftrace_replace_code(int enable)
  2029. {
  2030. struct dyn_ftrace *rec;
  2031. struct ftrace_page *pg;
  2032. int failed;
  2033. if (unlikely(ftrace_disabled))
  2034. return;
  2035. do_for_each_ftrace_rec(pg, rec) {
  2036. if (rec->flags & FTRACE_FL_DISABLED)
  2037. continue;
  2038. failed = __ftrace_replace_code(rec, enable);
  2039. if (failed) {
  2040. ftrace_bug(failed, rec);
  2041. /* Stop processing */
  2042. return;
  2043. }
  2044. } while_for_each_ftrace_rec();
  2045. }
  2046. struct ftrace_rec_iter {
  2047. struct ftrace_page *pg;
  2048. int index;
  2049. };
  2050. /**
  2051. * ftrace_rec_iter_start, start up iterating over traced functions
  2052. *
  2053. * Returns an iterator handle that is used to iterate over all
  2054. * the records that represent address locations where functions
  2055. * are traced.
  2056. *
  2057. * May return NULL if no records are available.
  2058. */
  2059. struct ftrace_rec_iter *ftrace_rec_iter_start(void)
  2060. {
  2061. /*
  2062. * We only use a single iterator.
  2063. * Protected by the ftrace_lock mutex.
  2064. */
  2065. static struct ftrace_rec_iter ftrace_rec_iter;
  2066. struct ftrace_rec_iter *iter = &ftrace_rec_iter;
  2067. iter->pg = ftrace_pages_start;
  2068. iter->index = 0;
  2069. /* Could have empty pages */
  2070. while (iter->pg && !iter->pg->index)
  2071. iter->pg = iter->pg->next;
  2072. if (!iter->pg)
  2073. return NULL;
  2074. return iter;
  2075. }
  2076. /**
  2077. * ftrace_rec_iter_next, get the next record to process.
  2078. * @iter: The handle to the iterator.
  2079. *
  2080. * Returns the next iterator after the given iterator @iter.
  2081. */
  2082. struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
  2083. {
  2084. iter->index++;
  2085. if (iter->index >= iter->pg->index) {
  2086. iter->pg = iter->pg->next;
  2087. iter->index = 0;
  2088. /* Could have empty pages */
  2089. while (iter->pg && !iter->pg->index)
  2090. iter->pg = iter->pg->next;
  2091. }
  2092. if (!iter->pg)
  2093. return NULL;
  2094. return iter;
  2095. }
  2096. /**
  2097. * ftrace_rec_iter_record, get the record at the iterator location
  2098. * @iter: The current iterator location
  2099. *
  2100. * Returns the record that the current @iter is at.
  2101. */
  2102. struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
  2103. {
  2104. return &iter->pg->records[iter->index];
  2105. }
  2106. static int
  2107. ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
  2108. {
  2109. int ret;
  2110. if (unlikely(ftrace_disabled))
  2111. return 0;
  2112. ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
  2113. if (ret) {
  2114. ftrace_bug_type = FTRACE_BUG_INIT;
  2115. ftrace_bug(ret, rec);
  2116. return 0;
  2117. }
  2118. return 1;
  2119. }
  2120. /*
  2121. * archs can override this function if they must do something
  2122. * before the modifying code is performed.
  2123. */
  2124. int __weak ftrace_arch_code_modify_prepare(void)
  2125. {
  2126. return 0;
  2127. }
  2128. /*
  2129. * archs can override this function if they must do something
  2130. * after the modifying code is performed.
  2131. */
  2132. int __weak ftrace_arch_code_modify_post_process(void)
  2133. {
  2134. return 0;
  2135. }
  2136. void ftrace_modify_all_code(int command)
  2137. {
  2138. int update = command & FTRACE_UPDATE_TRACE_FUNC;
  2139. int err = 0;
  2140. /*
  2141. * If the ftrace_caller calls a ftrace_ops func directly,
  2142. * we need to make sure that it only traces functions it
  2143. * expects to trace. When doing the switch of functions,
  2144. * we need to update to the ftrace_ops_list_func first
  2145. * before the transition between old and new calls are set,
  2146. * as the ftrace_ops_list_func will check the ops hashes
  2147. * to make sure the ops are having the right functions
  2148. * traced.
  2149. */
  2150. if (update) {
  2151. err = ftrace_update_ftrace_func(ftrace_ops_list_func);
  2152. if (FTRACE_WARN_ON(err))
  2153. return;
  2154. }
  2155. if (command & FTRACE_UPDATE_CALLS)
  2156. ftrace_replace_code(1);
  2157. else if (command & FTRACE_DISABLE_CALLS)
  2158. ftrace_replace_code(0);
  2159. if (update && ftrace_trace_function != ftrace_ops_list_func) {
  2160. function_trace_op = set_function_trace_op;
  2161. smp_wmb();
  2162. /* If irqs are disabled, we are in stop machine */
  2163. if (!irqs_disabled())
  2164. smp_call_function(ftrace_sync_ipi, NULL, 1);
  2165. err = ftrace_update_ftrace_func(ftrace_trace_function);
  2166. if (FTRACE_WARN_ON(err))
  2167. return;
  2168. }
  2169. if (command & FTRACE_START_FUNC_RET)
  2170. err = ftrace_enable_ftrace_graph_caller();
  2171. else if (command & FTRACE_STOP_FUNC_RET)
  2172. err = ftrace_disable_ftrace_graph_caller();
  2173. FTRACE_WARN_ON(err);
  2174. }
  2175. static int __ftrace_modify_code(void *data)
  2176. {
  2177. int *command = data;
  2178. ftrace_modify_all_code(*command);
  2179. return 0;
  2180. }
  2181. /**
  2182. * ftrace_run_stop_machine, go back to the stop machine method
  2183. * @command: The command to tell ftrace what to do
  2184. *
  2185. * If an arch needs to fall back to the stop machine method, the
  2186. * it can call this function.
  2187. */
  2188. void ftrace_run_stop_machine(int command)
  2189. {
  2190. stop_machine(__ftrace_modify_code, &command, NULL);
  2191. }
  2192. /**
  2193. * arch_ftrace_update_code, modify the code to trace or not trace
  2194. * @command: The command that needs to be done
  2195. *
  2196. * Archs can override this function if it does not need to
  2197. * run stop_machine() to modify code.
  2198. */
  2199. void __weak arch_ftrace_update_code(int command)
  2200. {
  2201. ftrace_run_stop_machine(command);
  2202. }
  2203. static void ftrace_run_update_code(int command)
  2204. {
  2205. int ret;
  2206. ret = ftrace_arch_code_modify_prepare();
  2207. FTRACE_WARN_ON(ret);
  2208. if (ret)
  2209. return;
  2210. /*
  2211. * By default we use stop_machine() to modify the code.
  2212. * But archs can do what ever they want as long as it
  2213. * is safe. The stop_machine() is the safest, but also
  2214. * produces the most overhead.
  2215. */
  2216. arch_ftrace_update_code(command);
  2217. ret = ftrace_arch_code_modify_post_process();
  2218. FTRACE_WARN_ON(ret);
  2219. }
  2220. static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
  2221. struct ftrace_ops_hash *old_hash)
  2222. {
  2223. ops->flags |= FTRACE_OPS_FL_MODIFYING;
  2224. ops->old_hash.filter_hash = old_hash->filter_hash;
  2225. ops->old_hash.notrace_hash = old_hash->notrace_hash;
  2226. ftrace_run_update_code(command);
  2227. ops->old_hash.filter_hash = NULL;
  2228. ops->old_hash.notrace_hash = NULL;
  2229. ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
  2230. }
  2231. static ftrace_func_t saved_ftrace_func;
  2232. static int ftrace_start_up;
  2233. void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
  2234. {
  2235. }
  2236. static void ftrace_startup_enable(int command)
  2237. {
  2238. if (saved_ftrace_func != ftrace_trace_function) {
  2239. saved_ftrace_func = ftrace_trace_function;
  2240. command |= FTRACE_UPDATE_TRACE_FUNC;
  2241. }
  2242. if (!command || !ftrace_enabled)
  2243. return;
  2244. ftrace_run_update_code(command);
  2245. }
  2246. static void ftrace_startup_all(int command)
  2247. {
  2248. update_all_ops = true;
  2249. ftrace_startup_enable(command);
  2250. update_all_ops = false;
  2251. }
  2252. static int ftrace_startup(struct ftrace_ops *ops, int command)
  2253. {
  2254. int ret;
  2255. if (unlikely(ftrace_disabled))
  2256. return -ENODEV;
  2257. ret = __register_ftrace_function(ops);
  2258. if (ret)
  2259. return ret;
  2260. ftrace_start_up++;
  2261. /*
  2262. * Note that ftrace probes uses this to start up
  2263. * and modify functions it will probe. But we still
  2264. * set the ADDING flag for modification, as probes
  2265. * do not have trampolines. If they add them in the
  2266. * future, then the probes will need to distinguish
  2267. * between adding and updating probes.
  2268. */
  2269. ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
  2270. ret = ftrace_hash_ipmodify_enable(ops);
  2271. if (ret < 0) {
  2272. /* Rollback registration process */
  2273. __unregister_ftrace_function(ops);
  2274. ftrace_start_up--;
  2275. ops->flags &= ~FTRACE_OPS_FL_ENABLED;
  2276. return ret;
  2277. }
  2278. if (ftrace_hash_rec_enable(ops, 1))
  2279. command |= FTRACE_UPDATE_CALLS;
  2280. ftrace_startup_enable(command);
  2281. ops->flags &= ~FTRACE_OPS_FL_ADDING;
  2282. return 0;
  2283. }
  2284. static int ftrace_shutdown(struct ftrace_ops *ops, int command)
  2285. {
  2286. int ret;
  2287. if (unlikely(ftrace_disabled))
  2288. return -ENODEV;
  2289. ret = __unregister_ftrace_function(ops);
  2290. if (ret)
  2291. return ret;
  2292. ftrace_start_up--;
  2293. /*
  2294. * Just warn in case of unbalance, no need to kill ftrace, it's not
  2295. * critical but the ftrace_call callers may be never nopped again after
  2296. * further ftrace uses.
  2297. */
  2298. WARN_ON_ONCE(ftrace_start_up < 0);
  2299. /* Disabling ipmodify never fails */
  2300. ftrace_hash_ipmodify_disable(ops);
  2301. if (ftrace_hash_rec_disable(ops, 1))
  2302. command |= FTRACE_UPDATE_CALLS;
  2303. ops->flags &= ~FTRACE_OPS_FL_ENABLED;
  2304. if (saved_ftrace_func != ftrace_trace_function) {
  2305. saved_ftrace_func = ftrace_trace_function;
  2306. command |= FTRACE_UPDATE_TRACE_FUNC;
  2307. }
  2308. if (!command || !ftrace_enabled) {
  2309. /*
  2310. * If these are dynamic or per_cpu ops, they still
  2311. * need their data freed. Since, function tracing is
  2312. * not currently active, we can just free them
  2313. * without synchronizing all CPUs.
  2314. */
  2315. if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
  2316. goto free_ops;
  2317. return 0;
  2318. }
  2319. /*
  2320. * If the ops uses a trampoline, then it needs to be
  2321. * tested first on update.
  2322. */
  2323. ops->flags |= FTRACE_OPS_FL_REMOVING;
  2324. removed_ops = ops;
  2325. /* The trampoline logic checks the old hashes */
  2326. ops->old_hash.filter_hash = ops->func_hash->filter_hash;
  2327. ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
  2328. ftrace_run_update_code(command);
  2329. /*
  2330. * If there's no more ops registered with ftrace, run a
  2331. * sanity check to make sure all rec flags are cleared.
  2332. */
  2333. if (rcu_dereference_protected(ftrace_ops_list,
  2334. lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
  2335. struct ftrace_page *pg;
  2336. struct dyn_ftrace *rec;
  2337. do_for_each_ftrace_rec(pg, rec) {
  2338. if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
  2339. pr_warn(" %pS flags:%lx\n",
  2340. (void *)rec->ip, rec->flags);
  2341. } while_for_each_ftrace_rec();
  2342. }
  2343. ops->old_hash.filter_hash = NULL;
  2344. ops->old_hash.notrace_hash = NULL;
  2345. removed_ops = NULL;
  2346. ops->flags &= ~FTRACE_OPS_FL_REMOVING;
  2347. /*
  2348. * Dynamic ops may be freed, we must make sure that all
  2349. * callers are done before leaving this function.
  2350. * The same goes for freeing the per_cpu data of the per_cpu
  2351. * ops.
  2352. */
  2353. if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
  2354. /*
  2355. * We need to do a hard force of sched synchronization.
  2356. * This is because we use preempt_disable() to do RCU, but
  2357. * the function tracers can be called where RCU is not watching
  2358. * (like before user_exit()). We can not rely on the RCU
  2359. * infrastructure to do the synchronization, thus we must do it
  2360. * ourselves.
  2361. */
  2362. schedule_on_each_cpu(ftrace_sync);
  2363. /*
  2364. * When the kernel is preeptive, tasks can be preempted
  2365. * while on a ftrace trampoline. Just scheduling a task on
  2366. * a CPU is not good enough to flush them. Calling
  2367. * synchornize_rcu_tasks() will wait for those tasks to
  2368. * execute and either schedule voluntarily or enter user space.
  2369. */
  2370. if (IS_ENABLED(CONFIG_PREEMPT))
  2371. synchronize_rcu_tasks();
  2372. free_ops:
  2373. arch_ftrace_trampoline_free(ops);
  2374. }
  2375. return 0;
  2376. }
  2377. static void ftrace_startup_sysctl(void)
  2378. {
  2379. int command;
  2380. if (unlikely(ftrace_disabled))
  2381. return;
  2382. /* Force update next time */
  2383. saved_ftrace_func = NULL;
  2384. /* ftrace_start_up is true if we want ftrace running */
  2385. if (ftrace_start_up) {
  2386. command = FTRACE_UPDATE_CALLS;
  2387. if (ftrace_graph_active)
  2388. command |= FTRACE_START_FUNC_RET;
  2389. ftrace_startup_enable(command);
  2390. }
  2391. }
  2392. static void ftrace_shutdown_sysctl(void)
  2393. {
  2394. int command;
  2395. if (unlikely(ftrace_disabled))
  2396. return;
  2397. /* ftrace_start_up is true if ftrace is running */
  2398. if (ftrace_start_up) {
  2399. command = FTRACE_DISABLE_CALLS;
  2400. if (ftrace_graph_active)
  2401. command |= FTRACE_STOP_FUNC_RET;
  2402. ftrace_run_update_code(command);
  2403. }
  2404. }
  2405. static u64 ftrace_update_time;
  2406. unsigned long ftrace_update_tot_cnt;
  2407. static inline int ops_traces_mod(struct ftrace_ops *ops)
  2408. {
  2409. /*
  2410. * Filter_hash being empty will default to trace module.
  2411. * But notrace hash requires a test of individual module functions.
  2412. */
  2413. return ftrace_hash_empty(ops->func_hash->filter_hash) &&
  2414. ftrace_hash_empty(ops->func_hash->notrace_hash);
  2415. }
  2416. /*
  2417. * Check if the current ops references the record.
  2418. *
  2419. * If the ops traces all functions, then it was already accounted for.
  2420. * If the ops does not trace the current record function, skip it.
  2421. * If the ops ignores the function via notrace filter, skip it.
  2422. */
  2423. static inline bool
  2424. ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
  2425. {
  2426. /* If ops isn't enabled, ignore it */
  2427. if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
  2428. return false;
  2429. /* If ops traces all then it includes this function */
  2430. if (ops_traces_mod(ops))
  2431. return true;
  2432. /* The function must be in the filter */
  2433. if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
  2434. !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
  2435. return false;
  2436. /* If in notrace hash, we ignore it too */
  2437. if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
  2438. return false;
  2439. return true;
  2440. }
  2441. static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
  2442. {
  2443. struct ftrace_page *pg;
  2444. struct dyn_ftrace *p;
  2445. u64 start, stop;
  2446. unsigned long update_cnt = 0;
  2447. unsigned long rec_flags = 0;
  2448. int i;
  2449. start = ftrace_now(raw_smp_processor_id());
  2450. /*
  2451. * When a module is loaded, this function is called to convert
  2452. * the calls to mcount in its text to nops, and also to create
  2453. * an entry in the ftrace data. Now, if ftrace is activated
  2454. * after this call, but before the module sets its text to
  2455. * read-only, the modification of enabling ftrace can fail if
  2456. * the read-only is done while ftrace is converting the calls.
  2457. * To prevent this, the module's records are set as disabled
  2458. * and will be enabled after the call to set the module's text
  2459. * to read-only.
  2460. */
  2461. if (mod)
  2462. rec_flags |= FTRACE_FL_DISABLED;
  2463. for (pg = new_pgs; pg; pg = pg->next) {
  2464. for (i = 0; i < pg->index; i++) {
  2465. /* If something went wrong, bail without enabling anything */
  2466. if (unlikely(ftrace_disabled))
  2467. return -1;
  2468. p = &pg->records[i];
  2469. p->flags = rec_flags;
  2470. /*
  2471. * Do the initial record conversion from mcount jump
  2472. * to the NOP instructions.
  2473. */
  2474. if (!__is_defined(CC_USING_NOP_MCOUNT) &&
  2475. !ftrace_code_disable(mod, p))
  2476. break;
  2477. update_cnt++;
  2478. }
  2479. }
  2480. stop = ftrace_now(raw_smp_processor_id());
  2481. ftrace_update_time = stop - start;
  2482. ftrace_update_tot_cnt += update_cnt;
  2483. return 0;
  2484. }
  2485. static int ftrace_allocate_records(struct ftrace_page *pg, int count)
  2486. {
  2487. int order;
  2488. int cnt;
  2489. if (WARN_ON(!count))
  2490. return -EINVAL;
  2491. order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
  2492. /*
  2493. * We want to fill as much as possible. No more than a page
  2494. * may be empty.
  2495. */
  2496. while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
  2497. order--;
  2498. again:
  2499. pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
  2500. if (!pg->records) {
  2501. /* if we can't allocate this size, try something smaller */
  2502. if (!order)
  2503. return -ENOMEM;
  2504. order >>= 1;
  2505. goto again;
  2506. }
  2507. cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
  2508. pg->size = cnt;
  2509. if (cnt > count)
  2510. cnt = count;
  2511. return cnt;
  2512. }
  2513. static struct ftrace_page *
  2514. ftrace_allocate_pages(unsigned long num_to_init)
  2515. {
  2516. struct ftrace_page *start_pg;
  2517. struct ftrace_page *pg;
  2518. int order;
  2519. int cnt;
  2520. if (!num_to_init)
  2521. return 0;
  2522. start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
  2523. if (!pg)
  2524. return NULL;
  2525. /*
  2526. * Try to allocate as much as possible in one continues
  2527. * location that fills in all of the space. We want to
  2528. * waste as little space as possible.
  2529. */
  2530. for (;;) {
  2531. cnt = ftrace_allocate_records(pg, num_to_init);
  2532. if (cnt < 0)
  2533. goto free_pages;
  2534. num_to_init -= cnt;
  2535. if (!num_to_init)
  2536. break;
  2537. pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
  2538. if (!pg->next)
  2539. goto free_pages;
  2540. pg = pg->next;
  2541. }
  2542. return start_pg;
  2543. free_pages:
  2544. pg = start_pg;
  2545. while (pg) {
  2546. order = get_count_order(pg->size / ENTRIES_PER_PAGE);
  2547. free_pages((unsigned long)pg->records, order);
  2548. start_pg = pg->next;
  2549. kfree(pg);
  2550. pg = start_pg;
  2551. }
  2552. pr_info("ftrace: FAILED to allocate memory for functions\n");
  2553. return NULL;
  2554. }
  2555. #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
  2556. struct ftrace_iterator {
  2557. loff_t pos;
  2558. loff_t func_pos;
  2559. loff_t mod_pos;
  2560. struct ftrace_page *pg;
  2561. struct dyn_ftrace *func;
  2562. struct ftrace_func_probe *probe;
  2563. struct ftrace_func_entry *probe_entry;
  2564. struct trace_parser parser;
  2565. struct ftrace_hash *hash;
  2566. struct ftrace_ops *ops;
  2567. struct trace_array *tr;
  2568. struct list_head *mod_list;
  2569. int pidx;
  2570. int idx;
  2571. unsigned flags;
  2572. };
  2573. static void *
  2574. t_probe_next(struct seq_file *m, loff_t *pos)
  2575. {
  2576. struct ftrace_iterator *iter = m->private;
  2577. struct trace_array *tr = iter->ops->private;
  2578. struct list_head *func_probes;
  2579. struct ftrace_hash *hash;
  2580. struct list_head *next;
  2581. struct hlist_node *hnd = NULL;
  2582. struct hlist_head *hhd;
  2583. int size;
  2584. (*pos)++;
  2585. iter->pos = *pos;
  2586. if (!tr)
  2587. return NULL;
  2588. func_probes = &tr->func_probes;
  2589. if (list_empty(func_probes))
  2590. return NULL;
  2591. if (!iter->probe) {
  2592. next = func_probes->next;
  2593. iter->probe = list_entry(next, struct ftrace_func_probe, list);
  2594. }
  2595. if (iter->probe_entry)
  2596. hnd = &iter->probe_entry->hlist;
  2597. hash = iter->probe->ops.func_hash->filter_hash;
  2598. /*
  2599. * A probe being registered may temporarily have an empty hash
  2600. * and it's at the end of the func_probes list.
  2601. */
  2602. if (!hash || hash == EMPTY_HASH)
  2603. return NULL;
  2604. size = 1 << hash->size_bits;
  2605. retry:
  2606. if (iter->pidx >= size) {
  2607. if (iter->probe->list.next == func_probes)
  2608. return NULL;
  2609. next = iter->probe->list.next;
  2610. iter->probe = list_entry(next, struct ftrace_func_probe, list);
  2611. hash = iter->probe->ops.func_hash->filter_hash;
  2612. size = 1 << hash->size_bits;
  2613. iter->pidx = 0;
  2614. }
  2615. hhd = &hash->buckets[iter->pidx];
  2616. if (hlist_empty(hhd)) {
  2617. iter->pidx++;
  2618. hnd = NULL;
  2619. goto retry;
  2620. }
  2621. if (!hnd)
  2622. hnd = hhd->first;
  2623. else {
  2624. hnd = hnd->next;
  2625. if (!hnd) {
  2626. iter->pidx++;
  2627. goto retry;
  2628. }
  2629. }
  2630. if (WARN_ON_ONCE(!hnd))
  2631. return NULL;
  2632. iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
  2633. return iter;
  2634. }
  2635. static void *t_probe_start(struct seq_file *m, loff_t *pos)
  2636. {
  2637. struct ftrace_iterator *iter = m->private;
  2638. void *p = NULL;
  2639. loff_t l;
  2640. if (!(iter->flags & FTRACE_ITER_DO_PROBES))
  2641. return NULL;
  2642. if (iter->mod_pos > *pos)
  2643. return NULL;
  2644. iter->probe = NULL;
  2645. iter->probe_entry = NULL;
  2646. iter->pidx = 0;
  2647. for (l = 0; l <= (*pos - iter->mod_pos); ) {
  2648. p = t_probe_next(m, &l);
  2649. if (!p)
  2650. break;
  2651. }
  2652. if (!p)
  2653. return NULL;
  2654. /* Only set this if we have an item */
  2655. iter->flags |= FTRACE_ITER_PROBE;
  2656. return iter;
  2657. }
  2658. static int
  2659. t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
  2660. {
  2661. struct ftrace_func_entry *probe_entry;
  2662. struct ftrace_probe_ops *probe_ops;
  2663. struct ftrace_func_probe *probe;
  2664. probe = iter->probe;
  2665. probe_entry = iter->probe_entry;
  2666. if (WARN_ON_ONCE(!probe || !probe_entry))
  2667. return -EIO;
  2668. probe_ops = probe->probe_ops;
  2669. if (probe_ops->print)
  2670. return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
  2671. seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
  2672. (void *)probe_ops->func);
  2673. return 0;
  2674. }
  2675. static void *
  2676. t_mod_next(struct seq_file *m, loff_t *pos)
  2677. {
  2678. struct ftrace_iterator *iter = m->private;
  2679. struct trace_array *tr = iter->tr;
  2680. (*pos)++;
  2681. iter->pos = *pos;
  2682. iter->mod_list = iter->mod_list->next;
  2683. if (iter->mod_list == &tr->mod_trace ||
  2684. iter->mod_list == &tr->mod_notrace) {
  2685. iter->flags &= ~FTRACE_ITER_MOD;
  2686. return NULL;
  2687. }
  2688. iter->mod_pos = *pos;
  2689. return iter;
  2690. }
  2691. static void *t_mod_start(struct seq_file *m, loff_t *pos)
  2692. {
  2693. struct ftrace_iterator *iter = m->private;
  2694. void *p = NULL;
  2695. loff_t l;
  2696. if (iter->func_pos > *pos)
  2697. return NULL;
  2698. iter->mod_pos = iter->func_pos;
  2699. /* probes are only available if tr is set */
  2700. if (!iter->tr)
  2701. return NULL;
  2702. for (l = 0; l <= (*pos - iter->func_pos); ) {
  2703. p = t_mod_next(m, &l);
  2704. if (!p)
  2705. break;
  2706. }
  2707. if (!p) {
  2708. iter->flags &= ~FTRACE_ITER_MOD;
  2709. return t_probe_start(m, pos);
  2710. }
  2711. /* Only set this if we have an item */
  2712. iter->flags |= FTRACE_ITER_MOD;
  2713. return iter;
  2714. }
  2715. static int
  2716. t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
  2717. {
  2718. struct ftrace_mod_load *ftrace_mod;
  2719. struct trace_array *tr = iter->tr;
  2720. if (WARN_ON_ONCE(!iter->mod_list) ||
  2721. iter->mod_list == &tr->mod_trace ||
  2722. iter->mod_list == &tr->mod_notrace)
  2723. return -EIO;
  2724. ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);
  2725. if (ftrace_mod->func)
  2726. seq_printf(m, "%s", ftrace_mod->func);
  2727. else
  2728. seq_putc(m, '*');
  2729. seq_printf(m, ":mod:%s\n", ftrace_mod->module);
  2730. return 0;
  2731. }
  2732. static void *
  2733. t_func_next(struct seq_file *m, loff_t *pos)
  2734. {
  2735. struct ftrace_iterator *iter = m->private;
  2736. struct dyn_ftrace *rec = NULL;
  2737. (*pos)++;
  2738. retry:
  2739. if (iter->idx >= iter->pg->index) {
  2740. if (iter->pg->next) {
  2741. iter->pg = iter->pg->next;
  2742. iter->idx = 0;
  2743. goto retry;
  2744. }
  2745. } else {
  2746. rec = &iter->pg->records[iter->idx++];
  2747. if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
  2748. !ftrace_lookup_ip(iter->hash, rec->ip)) ||
  2749. ((iter->flags & FTRACE_ITER_ENABLED) &&
  2750. !(rec->flags & FTRACE_FL_ENABLED))) {
  2751. rec = NULL;
  2752. goto retry;
  2753. }
  2754. }
  2755. if (!rec)
  2756. return NULL;
  2757. iter->pos = iter->func_pos = *pos;
  2758. iter->func = rec;
  2759. return iter;
  2760. }
  2761. static void *
  2762. t_next(struct seq_file *m, void *v, loff_t *pos)
  2763. {
  2764. struct ftrace_iterator *iter = m->private;
  2765. loff_t l = *pos; /* t_probe_start() must use original pos */
  2766. void *ret;
  2767. if (unlikely(ftrace_disabled))
  2768. return NULL;
  2769. if (iter->flags & FTRACE_ITER_PROBE)
  2770. return t_probe_next(m, pos);
  2771. if (iter->flags & FTRACE_ITER_MOD)
  2772. return t_mod_next(m, pos);
  2773. if (iter->flags & FTRACE_ITER_PRINTALL) {
  2774. /* next must increment pos, and t_probe_start does not */
  2775. (*pos)++;
  2776. return t_mod_start(m, &l);
  2777. }
  2778. ret = t_func_next(m, pos);
  2779. if (!ret)
  2780. return t_mod_start(m, &l);
  2781. return ret;
  2782. }
  2783. static void reset_iter_read(struct ftrace_iterator *iter)
  2784. {
  2785. iter->pos = 0;
  2786. iter->func_pos = 0;
  2787. iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
  2788. }
  2789. static void *t_start(struct seq_file *m, loff_t *pos)
  2790. {
  2791. struct ftrace_iterator *iter = m->private;
  2792. void *p = NULL;
  2793. loff_t l;
  2794. mutex_lock(&ftrace_lock);
  2795. if (unlikely(ftrace_disabled))
  2796. return NULL;
  2797. /*
  2798. * If an lseek was done, then reset and start from beginning.
  2799. */
  2800. if (*pos < iter->pos)
  2801. reset_iter_read(iter);
  2802. /*
  2803. * For set_ftrace_filter reading, if we have the filter
  2804. * off, we can short cut and just print out that all
  2805. * functions are enabled.
  2806. */
  2807. if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
  2808. ftrace_hash_empty(iter->hash)) {
  2809. iter->func_pos = 1; /* Account for the message */
  2810. if (*pos > 0)
  2811. return t_mod_start(m, pos);
  2812. iter->flags |= FTRACE_ITER_PRINTALL;
  2813. /* reset in case of seek/pread */
  2814. iter->flags &= ~FTRACE_ITER_PROBE;
  2815. return iter;
  2816. }
  2817. if (iter->flags & FTRACE_ITER_MOD)
  2818. return t_mod_start(m, pos);
  2819. /*
  2820. * Unfortunately, we need to restart at ftrace_pages_start
  2821. * every time we let go of the ftrace_mutex. This is because
  2822. * those pointers can change without the lock.
  2823. */
  2824. iter->pg = ftrace_pages_start;
  2825. iter->idx = 0;
  2826. for (l = 0; l <= *pos; ) {
  2827. p = t_func_next(m, &l);
  2828. if (!p)
  2829. break;
  2830. }
  2831. if (!p)
  2832. return t_mod_start(m, pos);
  2833. return iter;
  2834. }
  2835. static void t_stop(struct seq_file *m, void *p)
  2836. {
  2837. mutex_unlock(&ftrace_lock);
  2838. }
  2839. void * __weak
  2840. arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
  2841. {
  2842. return NULL;
  2843. }
  2844. static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
  2845. struct dyn_ftrace *rec)
  2846. {
  2847. void *ptr;
  2848. ptr = arch_ftrace_trampoline_func(ops, rec);
  2849. if (ptr)
  2850. seq_printf(m, " ->%pS", ptr);
  2851. }
  2852. static int t_show(struct seq_file *m, void *v)
  2853. {
  2854. struct ftrace_iterator *iter = m->private;
  2855. struct dyn_ftrace *rec;
  2856. if (iter->flags & FTRACE_ITER_PROBE)
  2857. return t_probe_show(m, iter);
  2858. if (iter->flags & FTRACE_ITER_MOD)
  2859. return t_mod_show(m, iter);
  2860. if (iter->flags & FTRACE_ITER_PRINTALL) {
  2861. if (iter->flags & FTRACE_ITER_NOTRACE)
  2862. seq_puts(m, "#### no functions disabled ####\n");
  2863. else
  2864. seq_puts(m, "#### all functions enabled ####\n");
  2865. return 0;
  2866. }
  2867. rec = iter->func;
  2868. if (!rec)
  2869. return 0;
  2870. seq_printf(m, "%ps", (void *)rec->ip);
  2871. if (iter->flags & FTRACE_ITER_ENABLED) {
  2872. struct ftrace_ops *ops;
  2873. seq_printf(m, " (%ld)%s%s",
  2874. ftrace_rec_count(rec),
  2875. rec->flags & FTRACE_FL_REGS ? " R" : " ",
  2876. rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ");
  2877. if (rec->flags & FTRACE_FL_TRAMP_EN) {
  2878. ops = ftrace_find_tramp_ops_any(rec);
  2879. if (ops) {
  2880. do {
  2881. seq_printf(m, "\ttramp: %pS (%pS)",
  2882. (void *)ops->trampoline,
  2883. (void *)ops->func);
  2884. add_trampoline_func(m, ops, rec);
  2885. ops = ftrace_find_tramp_ops_next(rec, ops);
  2886. } while (ops);
  2887. } else
  2888. seq_puts(m, "\ttramp: ERROR!");
  2889. } else {
  2890. add_trampoline_func(m, NULL, rec);
  2891. }
  2892. }
  2893. seq_putc(m, '\n');
  2894. return 0;
  2895. }
  2896. static const struct seq_operations show_ftrace_seq_ops = {
  2897. .start = t_start,
  2898. .next = t_next,
  2899. .stop = t_stop,
  2900. .show = t_show,
  2901. };
  2902. static int
  2903. ftrace_avail_open(struct inode *inode, struct file *file)
  2904. {
  2905. struct ftrace_iterator *iter;
  2906. if (unlikely(ftrace_disabled))
  2907. return -ENODEV;
  2908. iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
  2909. if (!iter)
  2910. return -ENOMEM;
  2911. iter->pg = ftrace_pages_start;
  2912. iter->ops = &global_ops;
  2913. return 0;
  2914. }
  2915. static int
  2916. ftrace_enabled_open(struct inode *inode, struct file *file)
  2917. {
  2918. struct ftrace_iterator *iter;
  2919. iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
  2920. if (!iter)
  2921. return -ENOMEM;
  2922. iter->pg = ftrace_pages_start;
  2923. iter->flags = FTRACE_ITER_ENABLED;
  2924. iter->ops = &global_ops;
  2925. return 0;
  2926. }
  2927. /**
  2928. * ftrace_regex_open - initialize function tracer filter files
  2929. * @ops: The ftrace_ops that hold the hash filters
  2930. * @flag: The type of filter to process
  2931. * @inode: The inode, usually passed in to your open routine
  2932. * @file: The file, usually passed in to your open routine
  2933. *
  2934. * ftrace_regex_open() initializes the filter files for the
  2935. * @ops. Depending on @flag it may process the filter hash or
  2936. * the notrace hash of @ops. With this called from the open
  2937. * routine, you can use ftrace_filter_write() for the write
  2938. * routine if @flag has FTRACE_ITER_FILTER set, or
  2939. * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
  2940. * tracing_lseek() should be used as the lseek routine, and
  2941. * release must call ftrace_regex_release().
  2942. */
  2943. int
  2944. ftrace_regex_open(struct ftrace_ops *ops, int flag,
  2945. struct inode *inode, struct file *file)
  2946. {
  2947. struct ftrace_iterator *iter;
  2948. struct ftrace_hash *hash;
  2949. struct list_head *mod_head;
  2950. struct trace_array *tr = ops->private;
  2951. int ret = -ENOMEM;
  2952. ftrace_ops_init(ops);
  2953. if (unlikely(ftrace_disabled))
  2954. return -ENODEV;
  2955. if (tr && trace_array_get(tr) < 0)
  2956. return -ENODEV;
  2957. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  2958. if (!iter)
  2959. goto out;
  2960. if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX))
  2961. goto out;
  2962. iter->ops = ops;
  2963. iter->flags = flag;
  2964. iter->tr = tr;
  2965. mutex_lock(&ops->func_hash->regex_lock);
  2966. if (flag & FTRACE_ITER_NOTRACE) {
  2967. hash = ops->func_hash->notrace_hash;
  2968. mod_head = tr ? &tr->mod_notrace : NULL;
  2969. } else {
  2970. hash = ops->func_hash->filter_hash;
  2971. mod_head = tr ? &tr->mod_trace : NULL;
  2972. }
  2973. iter->mod_list = mod_head;
  2974. if (file->f_mode & FMODE_WRITE) {
  2975. const int size_bits = FTRACE_HASH_DEFAULT_BITS;
  2976. if (file->f_flags & O_TRUNC) {
  2977. iter->hash = alloc_ftrace_hash(size_bits);
  2978. clear_ftrace_mod_list(mod_head);
  2979. } else {
  2980. iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
  2981. }
  2982. if (!iter->hash) {
  2983. trace_parser_put(&iter->parser);
  2984. goto out_unlock;
  2985. }
  2986. } else
  2987. iter->hash = hash;
  2988. ret = 0;
  2989. if (file->f_mode & FMODE_READ) {
  2990. iter->pg = ftrace_pages_start;
  2991. ret = seq_open(file, &show_ftrace_seq_ops);
  2992. if (!ret) {
  2993. struct seq_file *m = file->private_data;
  2994. m->private = iter;
  2995. } else {
  2996. /* Failed */
  2997. free_ftrace_hash(iter->hash);
  2998. trace_parser_put(&iter->parser);
  2999. }
  3000. } else
  3001. file->private_data = iter;
  3002. out_unlock:
  3003. mutex_unlock(&ops->func_hash->regex_lock);
  3004. out:
  3005. if (ret) {
  3006. kfree(iter);
  3007. if (tr)
  3008. trace_array_put(tr);
  3009. }
  3010. return ret;
  3011. }
  3012. static int
  3013. ftrace_filter_open(struct inode *inode, struct file *file)
  3014. {
  3015. struct ftrace_ops *ops = inode->i_private;
  3016. return ftrace_regex_open(ops,
  3017. FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
  3018. inode, file);
  3019. }
  3020. static int
  3021. ftrace_notrace_open(struct inode *inode, struct file *file)
  3022. {
  3023. struct ftrace_ops *ops = inode->i_private;
  3024. return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
  3025. inode, file);
  3026. }
  3027. /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
  3028. struct ftrace_glob {
  3029. char *search;
  3030. unsigned len;
  3031. int type;
  3032. };
  3033. /*
  3034. * If symbols in an architecture don't correspond exactly to the user-visible
  3035. * name of what they represent, it is possible to define this function to
  3036. * perform the necessary adjustments.
  3037. */
  3038. char * __weak arch_ftrace_match_adjust(char *str, const char *search)
  3039. {
  3040. return str;
  3041. }
  3042. static int ftrace_match(char *str, struct ftrace_glob *g)
  3043. {
  3044. int matched = 0;
  3045. int slen;
  3046. str = arch_ftrace_match_adjust(str, g->search);
  3047. switch (g->type) {
  3048. case MATCH_FULL:
  3049. if (strcmp(str, g->search) == 0)
  3050. matched = 1;
  3051. break;
  3052. case MATCH_FRONT_ONLY:
  3053. if (strncmp(str, g->search, g->len) == 0)
  3054. matched = 1;
  3055. break;
  3056. case MATCH_MIDDLE_ONLY:
  3057. if (strstr(str, g->search))
  3058. matched = 1;
  3059. break;
  3060. case MATCH_END_ONLY:
  3061. slen = strlen(str);
  3062. if (slen >= g->len &&
  3063. memcmp(str + slen - g->len, g->search, g->len) == 0)
  3064. matched = 1;
  3065. break;
  3066. case MATCH_GLOB:
  3067. if (glob_match(g->search, str))
  3068. matched = 1;
  3069. break;
  3070. }
  3071. return matched;
  3072. }
  3073. static int
  3074. enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
  3075. {
  3076. struct ftrace_func_entry *entry;
  3077. int ret = 0;
  3078. entry = ftrace_lookup_ip(hash, rec->ip);
  3079. if (clear_filter) {
  3080. /* Do nothing if it doesn't exist */
  3081. if (!entry)
  3082. return 0;
  3083. free_hash_entry(hash, entry);
  3084. } else {
  3085. /* Do nothing if it exists */
  3086. if (entry)
  3087. return 0;
  3088. ret = add_hash_entry(hash, rec->ip);
  3089. }
  3090. return ret;
  3091. }
  3092. static int
  3093. ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
  3094. struct ftrace_glob *mod_g, int exclude_mod)
  3095. {
  3096. char str[KSYM_SYMBOL_LEN];
  3097. char *modname;
  3098. kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
  3099. if (mod_g) {
  3100. int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
  3101. /* blank module name to match all modules */
  3102. if (!mod_g->len) {
  3103. /* blank module globbing: modname xor exclude_mod */
  3104. if (!exclude_mod != !modname)
  3105. goto func_match;
  3106. return 0;
  3107. }
  3108. /*
  3109. * exclude_mod is set to trace everything but the given
  3110. * module. If it is set and the module matches, then
  3111. * return 0. If it is not set, and the module doesn't match
  3112. * also return 0. Otherwise, check the function to see if
  3113. * that matches.
  3114. */
  3115. if (!mod_matches == !exclude_mod)
  3116. return 0;
  3117. func_match:
  3118. /* blank search means to match all funcs in the mod */
  3119. if (!func_g->len)
  3120. return 1;
  3121. }
  3122. return ftrace_match(str, func_g);
  3123. }
  3124. static int
  3125. match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
  3126. {
  3127. struct ftrace_page *pg;
  3128. struct dyn_ftrace *rec;
  3129. struct ftrace_glob func_g = { .type = MATCH_FULL };
  3130. struct ftrace_glob mod_g = { .type = MATCH_FULL };
  3131. struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
  3132. int exclude_mod = 0;
  3133. int found = 0;
  3134. int ret;
  3135. int clear_filter = 0;
  3136. if (func) {
  3137. func_g.type = filter_parse_regex(func, len, &func_g.search,
  3138. &clear_filter);
  3139. func_g.len = strlen(func_g.search);
  3140. }
  3141. if (mod) {
  3142. mod_g.type = filter_parse_regex(mod, strlen(mod),
  3143. &mod_g.search, &exclude_mod);
  3144. mod_g.len = strlen(mod_g.search);
  3145. }
  3146. mutex_lock(&ftrace_lock);
  3147. if (unlikely(ftrace_disabled))
  3148. goto out_unlock;
  3149. do_for_each_ftrace_rec(pg, rec) {
  3150. if (rec->flags & FTRACE_FL_DISABLED)
  3151. continue;
  3152. if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
  3153. ret = enter_record(hash, rec, clear_filter);
  3154. if (ret < 0) {
  3155. found = ret;
  3156. goto out_unlock;
  3157. }
  3158. found = 1;
  3159. }
  3160. } while_for_each_ftrace_rec();
  3161. out_unlock:
  3162. mutex_unlock(&ftrace_lock);
  3163. return found;
  3164. }
  3165. static int
  3166. ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
  3167. {
  3168. return match_records(hash, buff, len, NULL);
  3169. }
  3170. static void ftrace_ops_update_code(struct ftrace_ops *ops,
  3171. struct ftrace_ops_hash *old_hash)
  3172. {
  3173. struct ftrace_ops *op;
  3174. if (!ftrace_enabled)
  3175. return;
  3176. if (ops->flags & FTRACE_OPS_FL_ENABLED) {
  3177. ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
  3178. return;
  3179. }
  3180. /*
  3181. * If this is the shared global_ops filter, then we need to
  3182. * check if there is another ops that shares it, is enabled.
  3183. * If so, we still need to run the modify code.
  3184. */
  3185. if (ops->func_hash != &global_ops.local_hash)
  3186. return;
  3187. do_for_each_ftrace_op(op, ftrace_ops_list) {
  3188. if (op->func_hash == &global_ops.local_hash &&
  3189. op->flags & FTRACE_OPS_FL_ENABLED) {
  3190. ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
  3191. /* Only need to do this once */
  3192. return;
  3193. }
  3194. } while_for_each_ftrace_op(op);
  3195. }
  3196. static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
  3197. struct ftrace_hash **orig_hash,
  3198. struct ftrace_hash *hash,
  3199. int enable)
  3200. {
  3201. struct ftrace_ops_hash old_hash_ops;
  3202. struct ftrace_hash *old_hash;
  3203. int ret;
  3204. old_hash = *orig_hash;
  3205. old_hash_ops.filter_hash = ops->func_hash->filter_hash;
  3206. old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
  3207. ret = ftrace_hash_move(ops, enable, orig_hash, hash);
  3208. if (!ret) {
  3209. ftrace_ops_update_code(ops, &old_hash_ops);
  3210. free_ftrace_hash_rcu(old_hash);
  3211. }
  3212. return ret;
  3213. }
  3214. static bool module_exists(const char *module)
  3215. {
  3216. /* All modules have the symbol __this_module */
  3217. const char this_mod[] = "__this_module";
  3218. char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2];
  3219. unsigned long val;
  3220. int n;
  3221. n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
  3222. if (n > sizeof(modname) - 1)
  3223. return false;
  3224. val = module_kallsyms_lookup_name(modname);
  3225. return val != 0;
  3226. }
  3227. static int cache_mod(struct trace_array *tr,
  3228. const char *func, char *module, int enable)
  3229. {
  3230. struct ftrace_mod_load *ftrace_mod, *n;
  3231. struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
  3232. int ret;
  3233. mutex_lock(&ftrace_lock);
  3234. /* We do not cache inverse filters */
  3235. if (func[0] == '!') {
  3236. func++;
  3237. ret = -EINVAL;
  3238. /* Look to remove this hash */
  3239. list_for_each_entry_safe(ftrace_mod, n, head, list) {
  3240. if (strcmp(ftrace_mod->module, module) != 0)
  3241. continue;
  3242. /* no func matches all */
  3243. if (strcmp(func, "*") == 0 ||
  3244. (ftrace_mod->func &&
  3245. strcmp(ftrace_mod->func, func) == 0)) {
  3246. ret = 0;
  3247. free_ftrace_mod(ftrace_mod);
  3248. continue;
  3249. }
  3250. }
  3251. goto out;
  3252. }
  3253. ret = -EINVAL;
  3254. /* We only care about modules that have not been loaded yet */
  3255. if (module_exists(module))
  3256. goto out;
  3257. /* Save this string off, and execute it when the module is loaded */
  3258. ret = ftrace_add_mod(tr, func, module, enable);
  3259. out:
  3260. mutex_unlock(&ftrace_lock);
  3261. return ret;
  3262. }
  3263. static int
  3264. ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
  3265. int reset, int enable);
  3266. #ifdef CONFIG_MODULES
  3267. static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
  3268. char *mod, bool enable)
  3269. {
  3270. struct ftrace_mod_load *ftrace_mod, *n;
  3271. struct ftrace_hash **orig_hash, *new_hash;
  3272. LIST_HEAD(process_mods);
  3273. char *func;
  3274. int ret;
  3275. mutex_lock(&ops->func_hash->regex_lock);
  3276. if (enable)
  3277. orig_hash = &ops->func_hash->filter_hash;
  3278. else
  3279. orig_hash = &ops->func_hash->notrace_hash;
  3280. new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
  3281. *orig_hash);
  3282. if (!new_hash)
  3283. goto out; /* warn? */
  3284. mutex_lock(&ftrace_lock);
  3285. list_for_each_entry_safe(ftrace_mod, n, head, list) {
  3286. if (strcmp(ftrace_mod->module, mod) != 0)
  3287. continue;
  3288. if (ftrace_mod->func)
  3289. func = kstrdup(ftrace_mod->func, GFP_KERNEL);
  3290. else
  3291. func = kstrdup("*", GFP_KERNEL);
  3292. if (!func) /* warn? */
  3293. continue;
  3294. list_del(&ftrace_mod->list);
  3295. list_add(&ftrace_mod->list, &process_mods);
  3296. /* Use the newly allocated func, as it may be "*" */
  3297. kfree(ftrace_mod->func);
  3298. ftrace_mod->func = func;
  3299. }
  3300. mutex_unlock(&ftrace_lock);
  3301. list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {
  3302. func = ftrace_mod->func;
  3303. /* Grabs ftrace_lock, which is why we have this extra step */
  3304. match_records(new_hash, func, strlen(func), mod);
  3305. free_ftrace_mod(ftrace_mod);
  3306. }
  3307. if (enable && list_empty(head))
  3308. new_hash->flags &= ~FTRACE_HASH_FL_MOD;
  3309. mutex_lock(&ftrace_lock);
  3310. ret = ftrace_hash_move_and_update_ops(ops, orig_hash,
  3311. new_hash, enable);
  3312. mutex_unlock(&ftrace_lock);
  3313. out:
  3314. mutex_unlock(&ops->func_hash->regex_lock);
  3315. free_ftrace_hash(new_hash);
  3316. }
  3317. static void process_cached_mods(const char *mod_name)
  3318. {
  3319. struct trace_array *tr;
  3320. char *mod;
  3321. mod = kstrdup(mod_name, GFP_KERNEL);
  3322. if (!mod)
  3323. return;
  3324. mutex_lock(&trace_types_lock);
  3325. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  3326. if (!list_empty(&tr->mod_trace))
  3327. process_mod_list(&tr->mod_trace, tr->ops, mod, true);
  3328. if (!list_empty(&tr->mod_notrace))
  3329. process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
  3330. }
  3331. mutex_unlock(&trace_types_lock);
  3332. kfree(mod);
  3333. }
  3334. #endif
  3335. /*
  3336. * We register the module command as a template to show others how
  3337. * to register the a command as well.
  3338. */
  3339. static int
  3340. ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
  3341. char *func_orig, char *cmd, char *module, int enable)
  3342. {
  3343. char *func;
  3344. int ret;
  3345. /* match_records() modifies func, and we need the original */
  3346. func = kstrdup(func_orig, GFP_KERNEL);
  3347. if (!func)
  3348. return -ENOMEM;
  3349. /*
  3350. * cmd == 'mod' because we only registered this func
  3351. * for the 'mod' ftrace_func_command.
  3352. * But if you register one func with multiple commands,
  3353. * you can tell which command was used by the cmd
  3354. * parameter.
  3355. */
  3356. ret = match_records(hash, func, strlen(func), module);
  3357. kfree(func);
  3358. if (!ret)
  3359. return cache_mod(tr, func_orig, module, enable);
  3360. if (ret < 0)
  3361. return ret;
  3362. return 0;
  3363. }
  3364. static struct ftrace_func_command ftrace_mod_cmd = {
  3365. .name = "mod",
  3366. .func = ftrace_mod_callback,
  3367. };
  3368. static int __init ftrace_mod_cmd_init(void)
  3369. {
  3370. return register_ftrace_command(&ftrace_mod_cmd);
  3371. }
  3372. core_initcall(ftrace_mod_cmd_init);
  3373. static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
  3374. struct ftrace_ops *op, struct pt_regs *pt_regs)
  3375. {
  3376. struct ftrace_probe_ops *probe_ops;
  3377. struct ftrace_func_probe *probe;
  3378. probe = container_of(op, struct ftrace_func_probe, ops);
  3379. probe_ops = probe->probe_ops;
  3380. /*
  3381. * Disable preemption for these calls to prevent a RCU grace
  3382. * period. This syncs the hash iteration and freeing of items
  3383. * on the hash. rcu_read_lock is too dangerous here.
  3384. */
  3385. preempt_disable_notrace();
  3386. probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
  3387. preempt_enable_notrace();
  3388. }
  3389. struct ftrace_func_map {
  3390. struct ftrace_func_entry entry;
  3391. void *data;
  3392. };
  3393. struct ftrace_func_mapper {
  3394. struct ftrace_hash hash;
  3395. };
  3396. /**
  3397. * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
  3398. *
  3399. * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
  3400. */
  3401. struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
  3402. {
  3403. struct ftrace_hash *hash;
  3404. /*
  3405. * The mapper is simply a ftrace_hash, but since the entries
  3406. * in the hash are not ftrace_func_entry type, we define it
  3407. * as a separate structure.
  3408. */
  3409. hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
  3410. return (struct ftrace_func_mapper *)hash;
  3411. }
  3412. /**
  3413. * ftrace_func_mapper_find_ip - Find some data mapped to an ip
  3414. * @mapper: The mapper that has the ip maps
  3415. * @ip: the instruction pointer to find the data for
  3416. *
  3417. * Returns the data mapped to @ip if found otherwise NULL. The return
  3418. * is actually the address of the mapper data pointer. The address is
  3419. * returned for use cases where the data is no bigger than a long, and
  3420. * the user can use the data pointer as its data instead of having to
  3421. * allocate more memory for the reference.
  3422. */
  3423. void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
  3424. unsigned long ip)
  3425. {
  3426. struct ftrace_func_entry *entry;
  3427. struct ftrace_func_map *map;
  3428. entry = ftrace_lookup_ip(&mapper->hash, ip);
  3429. if (!entry)
  3430. return NULL;
  3431. map = (struct ftrace_func_map *)entry;
  3432. return &map->data;
  3433. }
  3434. /**
  3435. * ftrace_func_mapper_add_ip - Map some data to an ip
  3436. * @mapper: The mapper that has the ip maps
  3437. * @ip: The instruction pointer address to map @data to
  3438. * @data: The data to map to @ip
  3439. *
  3440. * Returns 0 on succes otherwise an error.
  3441. */
  3442. int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
  3443. unsigned long ip, void *data)
  3444. {
  3445. struct ftrace_func_entry *entry;
  3446. struct ftrace_func_map *map;
  3447. entry = ftrace_lookup_ip(&mapper->hash, ip);
  3448. if (entry)
  3449. return -EBUSY;
  3450. map = kmalloc(sizeof(*map), GFP_KERNEL);
  3451. if (!map)
  3452. return -ENOMEM;
  3453. map->entry.ip = ip;
  3454. map->data = data;
  3455. __add_hash_entry(&mapper->hash, &map->entry);
  3456. return 0;
  3457. }
  3458. /**
  3459. * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
  3460. * @mapper: The mapper that has the ip maps
  3461. * @ip: The instruction pointer address to remove the data from
  3462. *
  3463. * Returns the data if it is found, otherwise NULL.
  3464. * Note, if the data pointer is used as the data itself, (see
  3465. * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
  3466. * if the data pointer was set to zero.
  3467. */
  3468. void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
  3469. unsigned long ip)
  3470. {
  3471. struct ftrace_func_entry *entry;
  3472. struct ftrace_func_map *map;
  3473. void *data;
  3474. entry = ftrace_lookup_ip(&mapper->hash, ip);
  3475. if (!entry)
  3476. return NULL;
  3477. map = (struct ftrace_func_map *)entry;
  3478. data = map->data;
  3479. remove_hash_entry(&mapper->hash, entry);
  3480. kfree(entry);
  3481. return data;
  3482. }
  3483. /**
  3484. * free_ftrace_func_mapper - free a mapping of ips and data
  3485. * @mapper: The mapper that has the ip maps
  3486. * @free_func: A function to be called on each data item.
  3487. *
  3488. * This is used to free the function mapper. The @free_func is optional
  3489. * and can be used if the data needs to be freed as well.
  3490. */
  3491. void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
  3492. ftrace_mapper_func free_func)
  3493. {
  3494. struct ftrace_func_entry *entry;
  3495. struct ftrace_func_map *map;
  3496. struct hlist_head *hhd;
  3497. int size, i;
  3498. if (!mapper)
  3499. return;
  3500. if (free_func && mapper->hash.count) {
  3501. size = 1 << mapper->hash.size_bits;
  3502. for (i = 0; i < size; i++) {
  3503. hhd = &mapper->hash.buckets[i];
  3504. hlist_for_each_entry(entry, hhd, hlist) {
  3505. map = (struct ftrace_func_map *)entry;
  3506. free_func(map);
  3507. }
  3508. }
  3509. }
  3510. free_ftrace_hash(&mapper->hash);
  3511. }
  3512. static void release_probe(struct ftrace_func_probe *probe)
  3513. {
  3514. struct ftrace_probe_ops *probe_ops;
  3515. mutex_lock(&ftrace_lock);
  3516. WARN_ON(probe->ref <= 0);
  3517. /* Subtract the ref that was used to protect this instance */
  3518. probe->ref--;
  3519. if (!probe->ref) {
  3520. probe_ops = probe->probe_ops;
  3521. /*
  3522. * Sending zero as ip tells probe_ops to free
  3523. * the probe->data itself
  3524. */
  3525. if (probe_ops->free)
  3526. probe_ops->free(probe_ops, probe->tr, 0, probe->data);
  3527. list_del(&probe->list);
  3528. kfree(probe);
  3529. }
  3530. mutex_unlock(&ftrace_lock);
  3531. }
  3532. static void acquire_probe_locked(struct ftrace_func_probe *probe)
  3533. {
  3534. /*
  3535. * Add one ref to keep it from being freed when releasing the
  3536. * ftrace_lock mutex.
  3537. */
  3538. probe->ref++;
  3539. }
  3540. int
  3541. register_ftrace_function_probe(char *glob, struct trace_array *tr,
  3542. struct ftrace_probe_ops *probe_ops,
  3543. void *data)
  3544. {
  3545. struct ftrace_func_entry *entry;
  3546. struct ftrace_func_probe *probe;
  3547. struct ftrace_hash **orig_hash;
  3548. struct ftrace_hash *old_hash;
  3549. struct ftrace_hash *hash;
  3550. int count = 0;
  3551. int size;
  3552. int ret;
  3553. int i;
  3554. if (WARN_ON(!tr))
  3555. return -EINVAL;
  3556. /* We do not support '!' for function probes */
  3557. if (WARN_ON(glob[0] == '!'))
  3558. return -EINVAL;
  3559. mutex_lock(&ftrace_lock);
  3560. /* Check if the probe_ops is already registered */
  3561. list_for_each_entry(probe, &tr->func_probes, list) {
  3562. if (probe->probe_ops == probe_ops)
  3563. break;
  3564. }
  3565. if (&probe->list == &tr->func_probes) {
  3566. probe = kzalloc(sizeof(*probe), GFP_KERNEL);
  3567. if (!probe) {
  3568. mutex_unlock(&ftrace_lock);
  3569. return -ENOMEM;
  3570. }
  3571. probe->probe_ops = probe_ops;
  3572. probe->ops.func = function_trace_probe_call;
  3573. probe->tr = tr;
  3574. ftrace_ops_init(&probe->ops);
  3575. list_add(&probe->list, &tr->func_probes);
  3576. }
  3577. acquire_probe_locked(probe);
  3578. mutex_unlock(&ftrace_lock);
  3579. /*
  3580. * Note, there's a small window here that the func_hash->filter_hash
  3581. * may be NULL or empty. Need to be carefule when reading the loop.
  3582. */
  3583. mutex_lock(&probe->ops.func_hash->regex_lock);
  3584. orig_hash = &probe->ops.func_hash->filter_hash;
  3585. old_hash = *orig_hash;
  3586. hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
  3587. if (!hash) {
  3588. ret = -ENOMEM;
  3589. goto out;
  3590. }
  3591. ret = ftrace_match_records(hash, glob, strlen(glob));
  3592. /* Nothing found? */
  3593. if (!ret)
  3594. ret = -EINVAL;
  3595. if (ret < 0)
  3596. goto out;
  3597. size = 1 << hash->size_bits;
  3598. for (i = 0; i < size; i++) {
  3599. hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
  3600. if (ftrace_lookup_ip(old_hash, entry->ip))
  3601. continue;
  3602. /*
  3603. * The caller might want to do something special
  3604. * for each function we find. We call the callback
  3605. * to give the caller an opportunity to do so.
  3606. */
  3607. if (probe_ops->init) {
  3608. ret = probe_ops->init(probe_ops, tr,
  3609. entry->ip, data,
  3610. &probe->data);
  3611. if (ret < 0) {
  3612. if (probe_ops->free && count)
  3613. probe_ops->free(probe_ops, tr,
  3614. 0, probe->data);
  3615. probe->data = NULL;
  3616. goto out;
  3617. }
  3618. }
  3619. count++;
  3620. }
  3621. }
  3622. mutex_lock(&ftrace_lock);
  3623. if (!count) {
  3624. /* Nothing was added? */
  3625. ret = -EINVAL;
  3626. goto out_unlock;
  3627. }
  3628. ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
  3629. hash, 1);
  3630. if (ret < 0)
  3631. goto err_unlock;
  3632. /* One ref for each new function traced */
  3633. probe->ref += count;
  3634. if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
  3635. ret = ftrace_startup(&probe->ops, 0);
  3636. out_unlock:
  3637. mutex_unlock(&ftrace_lock);
  3638. if (!ret)
  3639. ret = count;
  3640. out:
  3641. mutex_unlock(&probe->ops.func_hash->regex_lock);
  3642. free_ftrace_hash(hash);
  3643. release_probe(probe);
  3644. return ret;
  3645. err_unlock:
  3646. if (!probe_ops->free || !count)
  3647. goto out_unlock;
  3648. /* Failed to do the move, need to call the free functions */
  3649. for (i = 0; i < size; i++) {
  3650. hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
  3651. if (ftrace_lookup_ip(old_hash, entry->ip))
  3652. continue;
  3653. probe_ops->free(probe_ops, tr, entry->ip, probe->data);
  3654. }
  3655. }
  3656. goto out_unlock;
  3657. }
  3658. int
  3659. unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
  3660. struct ftrace_probe_ops *probe_ops)
  3661. {
  3662. struct ftrace_ops_hash old_hash_ops;
  3663. struct ftrace_func_entry *entry;
  3664. struct ftrace_func_probe *probe;
  3665. struct ftrace_glob func_g;
  3666. struct ftrace_hash **orig_hash;
  3667. struct ftrace_hash *old_hash;
  3668. struct ftrace_hash *hash = NULL;
  3669. struct hlist_node *tmp;
  3670. struct hlist_head hhd;
  3671. char str[KSYM_SYMBOL_LEN];
  3672. int count = 0;
  3673. int i, ret = -ENODEV;
  3674. int size;
  3675. if (!glob || !strlen(glob) || !strcmp(glob, "*"))
  3676. func_g.search = NULL;
  3677. else {
  3678. int not;
  3679. func_g.type = filter_parse_regex(glob, strlen(glob),
  3680. &func_g.search, &not);
  3681. func_g.len = strlen(func_g.search);
  3682. /* we do not support '!' for function probes */
  3683. if (WARN_ON(not))
  3684. return -EINVAL;
  3685. }
  3686. mutex_lock(&ftrace_lock);
  3687. /* Check if the probe_ops is already registered */
  3688. list_for_each_entry(probe, &tr->func_probes, list) {
  3689. if (probe->probe_ops == probe_ops)
  3690. break;
  3691. }
  3692. if (&probe->list == &tr->func_probes)
  3693. goto err_unlock_ftrace;
  3694. ret = -EINVAL;
  3695. if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
  3696. goto err_unlock_ftrace;
  3697. acquire_probe_locked(probe);
  3698. mutex_unlock(&ftrace_lock);
  3699. mutex_lock(&probe->ops.func_hash->regex_lock);
  3700. orig_hash = &probe->ops.func_hash->filter_hash;
  3701. old_hash = *orig_hash;
  3702. if (ftrace_hash_empty(old_hash))
  3703. goto out_unlock;
  3704. old_hash_ops.filter_hash = old_hash;
  3705. /* Probes only have filters */
  3706. old_hash_ops.notrace_hash = NULL;
  3707. ret = -ENOMEM;
  3708. hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
  3709. if (!hash)
  3710. goto out_unlock;
  3711. INIT_HLIST_HEAD(&hhd);
  3712. size = 1 << hash->size_bits;
  3713. for (i = 0; i < size; i++) {
  3714. hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
  3715. if (func_g.search) {
  3716. kallsyms_lookup(entry->ip, NULL, NULL,
  3717. NULL, str);
  3718. if (!ftrace_match(str, &func_g))
  3719. continue;
  3720. }
  3721. count++;
  3722. remove_hash_entry(hash, entry);
  3723. hlist_add_head(&entry->hlist, &hhd);
  3724. }
  3725. }
  3726. /* Nothing found? */
  3727. if (!count) {
  3728. ret = -EINVAL;
  3729. goto out_unlock;
  3730. }
  3731. mutex_lock(&ftrace_lock);
  3732. WARN_ON(probe->ref < count);
  3733. probe->ref -= count;
  3734. if (ftrace_hash_empty(hash))
  3735. ftrace_shutdown(&probe->ops, 0);
  3736. ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
  3737. hash, 1);
  3738. /* still need to update the function call sites */
  3739. if (ftrace_enabled && !ftrace_hash_empty(hash))
  3740. ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
  3741. &old_hash_ops);
  3742. synchronize_sched();
  3743. hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
  3744. hlist_del(&entry->hlist);
  3745. if (probe_ops->free)
  3746. probe_ops->free(probe_ops, tr, entry->ip, probe->data);
  3747. kfree(entry);
  3748. }
  3749. mutex_unlock(&ftrace_lock);
  3750. out_unlock:
  3751. mutex_unlock(&probe->ops.func_hash->regex_lock);
  3752. free_ftrace_hash(hash);
  3753. release_probe(probe);
  3754. return ret;
  3755. err_unlock_ftrace:
  3756. mutex_unlock(&ftrace_lock);
  3757. return ret;
  3758. }
  3759. void clear_ftrace_function_probes(struct trace_array *tr)
  3760. {
  3761. struct ftrace_func_probe *probe, *n;
  3762. list_for_each_entry_safe(probe, n, &tr->func_probes, list)
  3763. unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
  3764. }
  3765. static LIST_HEAD(ftrace_commands);
  3766. static DEFINE_MUTEX(ftrace_cmd_mutex);
  3767. /*
  3768. * Currently we only register ftrace commands from __init, so mark this
  3769. * __init too.
  3770. */
  3771. __init int register_ftrace_command(struct ftrace_func_command *cmd)
  3772. {
  3773. struct ftrace_func_command *p;
  3774. int ret = 0;
  3775. mutex_lock(&ftrace_cmd_mutex);
  3776. list_for_each_entry(p, &ftrace_commands, list) {
  3777. if (strcmp(cmd->name, p->name) == 0) {
  3778. ret = -EBUSY;
  3779. goto out_unlock;
  3780. }
  3781. }
  3782. list_add(&cmd->list, &ftrace_commands);
  3783. out_unlock:
  3784. mutex_unlock(&ftrace_cmd_mutex);
  3785. return ret;
  3786. }
  3787. /*
  3788. * Currently we only unregister ftrace commands from __init, so mark
  3789. * this __init too.
  3790. */
  3791. __init int unregister_ftrace_command(struct ftrace_func_command *cmd)
  3792. {
  3793. struct ftrace_func_command *p, *n;
  3794. int ret = -ENODEV;
  3795. mutex_lock(&ftrace_cmd_mutex);
  3796. list_for_each_entry_safe(p, n, &ftrace_commands, list) {
  3797. if (strcmp(cmd->name, p->name) == 0) {
  3798. ret = 0;
  3799. list_del_init(&p->list);
  3800. goto out_unlock;
  3801. }
  3802. }
  3803. out_unlock:
  3804. mutex_unlock(&ftrace_cmd_mutex);
  3805. return ret;
  3806. }
  3807. static int ftrace_process_regex(struct ftrace_iterator *iter,
  3808. char *buff, int len, int enable)
  3809. {
  3810. struct ftrace_hash *hash = iter->hash;
  3811. struct trace_array *tr = iter->ops->private;
  3812. char *func, *command, *next = buff;
  3813. struct ftrace_func_command *p;
  3814. int ret = -EINVAL;
  3815. func = strsep(&next, ":");
  3816. if (!next) {
  3817. ret = ftrace_match_records(hash, func, len);
  3818. if (!ret)
  3819. ret = -EINVAL;
  3820. if (ret < 0)
  3821. return ret;
  3822. return 0;
  3823. }
  3824. /* command found */
  3825. command = strsep(&next, ":");
  3826. mutex_lock(&ftrace_cmd_mutex);
  3827. list_for_each_entry(p, &ftrace_commands, list) {
  3828. if (strcmp(p->name, command) == 0) {
  3829. ret = p->func(tr, hash, func, command, next, enable);
  3830. goto out_unlock;
  3831. }
  3832. }
  3833. out_unlock:
  3834. mutex_unlock(&ftrace_cmd_mutex);
  3835. return ret;
  3836. }
  3837. static ssize_t
  3838. ftrace_regex_write(struct file *file, const char __user *ubuf,
  3839. size_t cnt, loff_t *ppos, int enable)
  3840. {
  3841. struct ftrace_iterator *iter;
  3842. struct trace_parser *parser;
  3843. ssize_t ret, read;
  3844. if (!cnt)
  3845. return 0;
  3846. if (file->f_mode & FMODE_READ) {
  3847. struct seq_file *m = file->private_data;
  3848. iter = m->private;
  3849. } else
  3850. iter = file->private_data;
  3851. if (unlikely(ftrace_disabled))
  3852. return -ENODEV;
  3853. /* iter->hash is a local copy, so we don't need regex_lock */
  3854. parser = &iter->parser;
  3855. read = trace_get_user(parser, ubuf, cnt, ppos);
  3856. if (read >= 0 && trace_parser_loaded(parser) &&
  3857. !trace_parser_cont(parser)) {
  3858. ret = ftrace_process_regex(iter, parser->buffer,
  3859. parser->idx, enable);
  3860. trace_parser_clear(parser);
  3861. if (ret < 0)
  3862. goto out;
  3863. }
  3864. ret = read;
  3865. out:
  3866. return ret;
  3867. }
  3868. ssize_t
  3869. ftrace_filter_write(struct file *file, const char __user *ubuf,
  3870. size_t cnt, loff_t *ppos)
  3871. {
  3872. return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
  3873. }
  3874. ssize_t
  3875. ftrace_notrace_write(struct file *file, const char __user *ubuf,
  3876. size_t cnt, loff_t *ppos)
  3877. {
  3878. return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
  3879. }
  3880. static int
  3881. ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
  3882. {
  3883. struct ftrace_func_entry *entry;
  3884. if (!ftrace_location(ip))
  3885. return -EINVAL;
  3886. if (remove) {
  3887. entry = ftrace_lookup_ip(hash, ip);
  3888. if (!entry)
  3889. return -ENOENT;
  3890. free_hash_entry(hash, entry);
  3891. return 0;
  3892. }
  3893. return add_hash_entry(hash, ip);
  3894. }
  3895. static int
  3896. ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
  3897. unsigned long ip, int remove, int reset, int enable)
  3898. {
  3899. struct ftrace_hash **orig_hash;
  3900. struct ftrace_hash *hash;
  3901. int ret;
  3902. if (unlikely(ftrace_disabled))
  3903. return -ENODEV;
  3904. mutex_lock(&ops->func_hash->regex_lock);
  3905. if (enable)
  3906. orig_hash = &ops->func_hash->filter_hash;
  3907. else
  3908. orig_hash = &ops->func_hash->notrace_hash;
  3909. if (reset)
  3910. hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
  3911. else
  3912. hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
  3913. if (!hash) {
  3914. ret = -ENOMEM;
  3915. goto out_regex_unlock;
  3916. }
  3917. if (buf && !ftrace_match_records(hash, buf, len)) {
  3918. ret = -EINVAL;
  3919. goto out_regex_unlock;
  3920. }
  3921. if (ip) {
  3922. ret = ftrace_match_addr(hash, ip, remove);
  3923. if (ret < 0)
  3924. goto out_regex_unlock;
  3925. }
  3926. mutex_lock(&ftrace_lock);
  3927. ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
  3928. mutex_unlock(&ftrace_lock);
  3929. out_regex_unlock:
  3930. mutex_unlock(&ops->func_hash->regex_lock);
  3931. free_ftrace_hash(hash);
  3932. return ret;
  3933. }
  3934. static int
  3935. ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
  3936. int reset, int enable)
  3937. {
  3938. return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
  3939. }
  3940. /**
  3941. * ftrace_set_filter_ip - set a function to filter on in ftrace by address
  3942. * @ops - the ops to set the filter with
  3943. * @ip - the address to add to or remove from the filter.
  3944. * @remove - non zero to remove the ip from the filter
  3945. * @reset - non zero to reset all filters before applying this filter.
  3946. *
  3947. * Filters denote which functions should be enabled when tracing is enabled
  3948. * If @ip is NULL, it failes to update filter.
  3949. */
  3950. int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
  3951. int remove, int reset)
  3952. {
  3953. ftrace_ops_init(ops);
  3954. return ftrace_set_addr(ops, ip, remove, reset, 1);
  3955. }
  3956. EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
  3957. /**
  3958. * ftrace_ops_set_global_filter - setup ops to use global filters
  3959. * @ops - the ops which will use the global filters
  3960. *
  3961. * ftrace users who need global function trace filtering should call this.
  3962. * It can set the global filter only if ops were not initialized before.
  3963. */
  3964. void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
  3965. {
  3966. if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
  3967. return;
  3968. ftrace_ops_init(ops);
  3969. ops->func_hash = &global_ops.local_hash;
  3970. }
  3971. EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
  3972. static int
  3973. ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
  3974. int reset, int enable)
  3975. {
  3976. return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
  3977. }
  3978. /**
  3979. * ftrace_set_filter - set a function to filter on in ftrace
  3980. * @ops - the ops to set the filter with
  3981. * @buf - the string that holds the function filter text.
  3982. * @len - the length of the string.
  3983. * @reset - non zero to reset all filters before applying this filter.
  3984. *
  3985. * Filters denote which functions should be enabled when tracing is enabled.
  3986. * If @buf is NULL and reset is set, all functions will be enabled for tracing.
  3987. */
  3988. int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
  3989. int len, int reset)
  3990. {
  3991. ftrace_ops_init(ops);
  3992. return ftrace_set_regex(ops, buf, len, reset, 1);
  3993. }
  3994. EXPORT_SYMBOL_GPL(ftrace_set_filter);
  3995. /**
  3996. * ftrace_set_notrace - set a function to not trace in ftrace
  3997. * @ops - the ops to set the notrace filter with
  3998. * @buf - the string that holds the function notrace text.
  3999. * @len - the length of the string.
  4000. * @reset - non zero to reset all filters before applying this filter.
  4001. *
  4002. * Notrace Filters denote which functions should not be enabled when tracing
  4003. * is enabled. If @buf is NULL and reset is set, all functions will be enabled
  4004. * for tracing.
  4005. */
  4006. int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
  4007. int len, int reset)
  4008. {
  4009. ftrace_ops_init(ops);
  4010. return ftrace_set_regex(ops, buf, len, reset, 0);
  4011. }
  4012. EXPORT_SYMBOL_GPL(ftrace_set_notrace);
  4013. /**
  4014. * ftrace_set_global_filter - set a function to filter on with global tracers
  4015. * @buf - the string that holds the function filter text.
  4016. * @len - the length of the string.
  4017. * @reset - non zero to reset all filters before applying this filter.
  4018. *
  4019. * Filters denote which functions should be enabled when tracing is enabled.
  4020. * If @buf is NULL and reset is set, all functions will be enabled for tracing.
  4021. */
  4022. void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
  4023. {
  4024. ftrace_set_regex(&global_ops, buf, len, reset, 1);
  4025. }
  4026. EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
  4027. /**
  4028. * ftrace_set_global_notrace - set a function to not trace with global tracers
  4029. * @buf - the string that holds the function notrace text.
  4030. * @len - the length of the string.
  4031. * @reset - non zero to reset all filters before applying this filter.
  4032. *
  4033. * Notrace Filters denote which functions should not be enabled when tracing
  4034. * is enabled. If @buf is NULL and reset is set, all functions will be enabled
  4035. * for tracing.
  4036. */
  4037. void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
  4038. {
  4039. ftrace_set_regex(&global_ops, buf, len, reset, 0);
  4040. }
  4041. EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
  4042. /*
  4043. * command line interface to allow users to set filters on boot up.
  4044. */
  4045. #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
  4046. static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
  4047. static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
  4048. /* Used by function selftest to not test if filter is set */
  4049. bool ftrace_filter_param __initdata;
  4050. static int __init set_ftrace_notrace(char *str)
  4051. {
  4052. ftrace_filter_param = true;
  4053. strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
  4054. return 1;
  4055. }
  4056. __setup("ftrace_notrace=", set_ftrace_notrace);
  4057. static int __init set_ftrace_filter(char *str)
  4058. {
  4059. ftrace_filter_param = true;
  4060. strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
  4061. return 1;
  4062. }
  4063. __setup("ftrace_filter=", set_ftrace_filter);
  4064. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  4065. static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
  4066. static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
  4067. static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
  4068. static int __init set_graph_function(char *str)
  4069. {
  4070. strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
  4071. return 1;
  4072. }
  4073. __setup("ftrace_graph_filter=", set_graph_function);
  4074. static int __init set_graph_notrace_function(char *str)
  4075. {
  4076. strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
  4077. return 1;
  4078. }
  4079. __setup("ftrace_graph_notrace=", set_graph_notrace_function);
  4080. static int __init set_graph_max_depth_function(char *str)
  4081. {
  4082. if (!str)
  4083. return 0;
  4084. fgraph_max_depth = simple_strtoul(str, NULL, 0);
  4085. return 1;
  4086. }
  4087. __setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
  4088. static void __init set_ftrace_early_graph(char *buf, int enable)
  4089. {
  4090. int ret;
  4091. char *func;
  4092. struct ftrace_hash *hash;
  4093. hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
  4094. if (WARN_ON(!hash))
  4095. return;
  4096. while (buf) {
  4097. func = strsep(&buf, ",");
  4098. /* we allow only one expression at a time */
  4099. ret = ftrace_graph_set_hash(hash, func);
  4100. if (ret)
  4101. printk(KERN_DEBUG "ftrace: function %s not "
  4102. "traceable\n", func);
  4103. }
  4104. if (enable)
  4105. ftrace_graph_hash = hash;
  4106. else
  4107. ftrace_graph_notrace_hash = hash;
  4108. }
  4109. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  4110. void __init
  4111. ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
  4112. {
  4113. char *func;
  4114. ftrace_ops_init(ops);
  4115. while (buf) {
  4116. func = strsep(&buf, ",");
  4117. ftrace_set_regex(ops, func, strlen(func), 0, enable);
  4118. }
  4119. }
  4120. static void __init set_ftrace_early_filters(void)
  4121. {
  4122. if (ftrace_filter_buf[0])
  4123. ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
  4124. if (ftrace_notrace_buf[0])
  4125. ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
  4126. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  4127. if (ftrace_graph_buf[0])
  4128. set_ftrace_early_graph(ftrace_graph_buf, 1);
  4129. if (ftrace_graph_notrace_buf[0])
  4130. set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
  4131. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  4132. }
  4133. int ftrace_regex_release(struct inode *inode, struct file *file)
  4134. {
  4135. struct seq_file *m = (struct seq_file *)file->private_data;
  4136. struct ftrace_iterator *iter;
  4137. struct ftrace_hash **orig_hash;
  4138. struct trace_parser *parser;
  4139. int filter_hash;
  4140. int ret;
  4141. if (file->f_mode & FMODE_READ) {
  4142. iter = m->private;
  4143. seq_release(inode, file);
  4144. } else
  4145. iter = file->private_data;
  4146. parser = &iter->parser;
  4147. if (trace_parser_loaded(parser)) {
  4148. int enable = !(iter->flags & FTRACE_ITER_NOTRACE);
  4149. ftrace_process_regex(iter, parser->buffer,
  4150. parser->idx, enable);
  4151. }
  4152. trace_parser_put(parser);
  4153. mutex_lock(&iter->ops->func_hash->regex_lock);
  4154. if (file->f_mode & FMODE_WRITE) {
  4155. filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
  4156. if (filter_hash) {
  4157. orig_hash = &iter->ops->func_hash->filter_hash;
  4158. if (iter->tr && !list_empty(&iter->tr->mod_trace))
  4159. iter->hash->flags |= FTRACE_HASH_FL_MOD;
  4160. } else
  4161. orig_hash = &iter->ops->func_hash->notrace_hash;
  4162. mutex_lock(&ftrace_lock);
  4163. ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
  4164. iter->hash, filter_hash);
  4165. mutex_unlock(&ftrace_lock);
  4166. } else {
  4167. /* For read only, the hash is the ops hash */
  4168. iter->hash = NULL;
  4169. }
  4170. mutex_unlock(&iter->ops->func_hash->regex_lock);
  4171. free_ftrace_hash(iter->hash);
  4172. if (iter->tr)
  4173. trace_array_put(iter->tr);
  4174. kfree(iter);
  4175. return 0;
  4176. }
  4177. static const struct file_operations ftrace_avail_fops = {
  4178. .open = ftrace_avail_open,
  4179. .read = seq_read,
  4180. .llseek = seq_lseek,
  4181. .release = seq_release_private,
  4182. };
  4183. static const struct file_operations ftrace_enabled_fops = {
  4184. .open = ftrace_enabled_open,
  4185. .read = seq_read,
  4186. .llseek = seq_lseek,
  4187. .release = seq_release_private,
  4188. };
  4189. static const struct file_operations ftrace_filter_fops = {
  4190. .open = ftrace_filter_open,
  4191. .read = seq_read,
  4192. .write = ftrace_filter_write,
  4193. .llseek = tracing_lseek,
  4194. .release = ftrace_regex_release,
  4195. };
  4196. static const struct file_operations ftrace_notrace_fops = {
  4197. .open = ftrace_notrace_open,
  4198. .read = seq_read,
  4199. .write = ftrace_notrace_write,
  4200. .llseek = tracing_lseek,
  4201. .release = ftrace_regex_release,
  4202. };
  4203. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  4204. static DEFINE_MUTEX(graph_lock);
  4205. struct ftrace_hash __rcu *ftrace_graph_hash = EMPTY_HASH;
  4206. struct ftrace_hash __rcu *ftrace_graph_notrace_hash = EMPTY_HASH;
  4207. enum graph_filter_type {
  4208. GRAPH_FILTER_NOTRACE = 0,
  4209. GRAPH_FILTER_FUNCTION,
  4210. };
  4211. #define FTRACE_GRAPH_EMPTY ((void *)1)
  4212. struct ftrace_graph_data {
  4213. struct ftrace_hash *hash;
  4214. struct ftrace_func_entry *entry;
  4215. int idx; /* for hash table iteration */
  4216. enum graph_filter_type type;
  4217. struct ftrace_hash *new_hash;
  4218. const struct seq_operations *seq_ops;
  4219. struct trace_parser parser;
  4220. };
  4221. static void *
  4222. __g_next(struct seq_file *m, loff_t *pos)
  4223. {
  4224. struct ftrace_graph_data *fgd = m->private;
  4225. struct ftrace_func_entry *entry = fgd->entry;
  4226. struct hlist_head *head;
  4227. int i, idx = fgd->idx;
  4228. if (*pos >= fgd->hash->count)
  4229. return NULL;
  4230. if (entry) {
  4231. hlist_for_each_entry_continue(entry, hlist) {
  4232. fgd->entry = entry;
  4233. return entry;
  4234. }
  4235. idx++;
  4236. }
  4237. for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
  4238. head = &fgd->hash->buckets[i];
  4239. hlist_for_each_entry(entry, head, hlist) {
  4240. fgd->entry = entry;
  4241. fgd->idx = i;
  4242. return entry;
  4243. }
  4244. }
  4245. return NULL;
  4246. }
  4247. static void *
  4248. g_next(struct seq_file *m, void *v, loff_t *pos)
  4249. {
  4250. (*pos)++;
  4251. return __g_next(m, pos);
  4252. }
  4253. static void *g_start(struct seq_file *m, loff_t *pos)
  4254. {
  4255. struct ftrace_graph_data *fgd = m->private;
  4256. mutex_lock(&graph_lock);
  4257. if (fgd->type == GRAPH_FILTER_FUNCTION)
  4258. fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
  4259. lockdep_is_held(&graph_lock));
  4260. else
  4261. fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
  4262. lockdep_is_held(&graph_lock));
  4263. /* Nothing, tell g_show to print all functions are enabled */
  4264. if (ftrace_hash_empty(fgd->hash) && !*pos)
  4265. return FTRACE_GRAPH_EMPTY;
  4266. fgd->idx = 0;
  4267. fgd->entry = NULL;
  4268. return __g_next(m, pos);
  4269. }
  4270. static void g_stop(struct seq_file *m, void *p)
  4271. {
  4272. mutex_unlock(&graph_lock);
  4273. }
  4274. static int g_show(struct seq_file *m, void *v)
  4275. {
  4276. struct ftrace_func_entry *entry = v;
  4277. if (!entry)
  4278. return 0;
  4279. if (entry == FTRACE_GRAPH_EMPTY) {
  4280. struct ftrace_graph_data *fgd = m->private;
  4281. if (fgd->type == GRAPH_FILTER_FUNCTION)
  4282. seq_puts(m, "#### all functions enabled ####\n");
  4283. else
  4284. seq_puts(m, "#### no functions disabled ####\n");
  4285. return 0;
  4286. }
  4287. seq_printf(m, "%ps\n", (void *)entry->ip);
  4288. return 0;
  4289. }
  4290. static const struct seq_operations ftrace_graph_seq_ops = {
  4291. .start = g_start,
  4292. .next = g_next,
  4293. .stop = g_stop,
  4294. .show = g_show,
  4295. };
  4296. static int
  4297. __ftrace_graph_open(struct inode *inode, struct file *file,
  4298. struct ftrace_graph_data *fgd)
  4299. {
  4300. int ret = 0;
  4301. struct ftrace_hash *new_hash = NULL;
  4302. if (file->f_mode & FMODE_WRITE) {
  4303. const int size_bits = FTRACE_HASH_DEFAULT_BITS;
  4304. if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
  4305. return -ENOMEM;
  4306. if (file->f_flags & O_TRUNC)
  4307. new_hash = alloc_ftrace_hash(size_bits);
  4308. else
  4309. new_hash = alloc_and_copy_ftrace_hash(size_bits,
  4310. fgd->hash);
  4311. if (!new_hash) {
  4312. ret = -ENOMEM;
  4313. goto out;
  4314. }
  4315. }
  4316. if (file->f_mode & FMODE_READ) {
  4317. ret = seq_open(file, &ftrace_graph_seq_ops);
  4318. if (!ret) {
  4319. struct seq_file *m = file->private_data;
  4320. m->private = fgd;
  4321. } else {
  4322. /* Failed */
  4323. free_ftrace_hash(new_hash);
  4324. new_hash = NULL;
  4325. }
  4326. } else
  4327. file->private_data = fgd;
  4328. out:
  4329. if (ret < 0 && file->f_mode & FMODE_WRITE)
  4330. trace_parser_put(&fgd->parser);
  4331. fgd->new_hash = new_hash;
  4332. /*
  4333. * All uses of fgd->hash must be taken with the graph_lock
  4334. * held. The graph_lock is going to be released, so force
  4335. * fgd->hash to be reinitialized when it is taken again.
  4336. */
  4337. fgd->hash = NULL;
  4338. return ret;
  4339. }
  4340. static int
  4341. ftrace_graph_open(struct inode *inode, struct file *file)
  4342. {
  4343. struct ftrace_graph_data *fgd;
  4344. int ret;
  4345. if (unlikely(ftrace_disabled))
  4346. return -ENODEV;
  4347. fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
  4348. if (fgd == NULL)
  4349. return -ENOMEM;
  4350. mutex_lock(&graph_lock);
  4351. fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
  4352. lockdep_is_held(&graph_lock));
  4353. fgd->type = GRAPH_FILTER_FUNCTION;
  4354. fgd->seq_ops = &ftrace_graph_seq_ops;
  4355. ret = __ftrace_graph_open(inode, file, fgd);
  4356. if (ret < 0)
  4357. kfree(fgd);
  4358. mutex_unlock(&graph_lock);
  4359. return ret;
  4360. }
  4361. static int
  4362. ftrace_graph_notrace_open(struct inode *inode, struct file *file)
  4363. {
  4364. struct ftrace_graph_data *fgd;
  4365. int ret;
  4366. if (unlikely(ftrace_disabled))
  4367. return -ENODEV;
  4368. fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
  4369. if (fgd == NULL)
  4370. return -ENOMEM;
  4371. mutex_lock(&graph_lock);
  4372. fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
  4373. lockdep_is_held(&graph_lock));
  4374. fgd->type = GRAPH_FILTER_NOTRACE;
  4375. fgd->seq_ops = &ftrace_graph_seq_ops;
  4376. ret = __ftrace_graph_open(inode, file, fgd);
  4377. if (ret < 0)
  4378. kfree(fgd);
  4379. mutex_unlock(&graph_lock);
  4380. return ret;
  4381. }
  4382. static int
  4383. ftrace_graph_release(struct inode *inode, struct file *file)
  4384. {
  4385. struct ftrace_graph_data *fgd;
  4386. struct ftrace_hash *old_hash, *new_hash;
  4387. struct trace_parser *parser;
  4388. int ret = 0;
  4389. if (file->f_mode & FMODE_READ) {
  4390. struct seq_file *m = file->private_data;
  4391. fgd = m->private;
  4392. seq_release(inode, file);
  4393. } else {
  4394. fgd = file->private_data;
  4395. }
  4396. if (file->f_mode & FMODE_WRITE) {
  4397. parser = &fgd->parser;
  4398. if (trace_parser_loaded((parser))) {
  4399. ret = ftrace_graph_set_hash(fgd->new_hash,
  4400. parser->buffer);
  4401. }
  4402. trace_parser_put(parser);
  4403. new_hash = __ftrace_hash_move(fgd->new_hash);
  4404. if (!new_hash) {
  4405. ret = -ENOMEM;
  4406. goto out;
  4407. }
  4408. mutex_lock(&graph_lock);
  4409. if (fgd->type == GRAPH_FILTER_FUNCTION) {
  4410. old_hash = rcu_dereference_protected(ftrace_graph_hash,
  4411. lockdep_is_held(&graph_lock));
  4412. rcu_assign_pointer(ftrace_graph_hash, new_hash);
  4413. } else {
  4414. old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
  4415. lockdep_is_held(&graph_lock));
  4416. rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
  4417. }
  4418. mutex_unlock(&graph_lock);
  4419. /*
  4420. * We need to do a hard force of sched synchronization.
  4421. * This is because we use preempt_disable() to do RCU, but
  4422. * the function tracers can be called where RCU is not watching
  4423. * (like before user_exit()). We can not rely on the RCU
  4424. * infrastructure to do the synchronization, thus we must do it
  4425. * ourselves.
  4426. */
  4427. schedule_on_each_cpu(ftrace_sync);
  4428. free_ftrace_hash(old_hash);
  4429. }
  4430. out:
  4431. free_ftrace_hash(fgd->new_hash);
  4432. kfree(fgd);
  4433. return ret;
  4434. }
  4435. static int
  4436. ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
  4437. {
  4438. struct ftrace_glob func_g;
  4439. struct dyn_ftrace *rec;
  4440. struct ftrace_page *pg;
  4441. struct ftrace_func_entry *entry;
  4442. int fail = 1;
  4443. int not;
  4444. /* decode regex */
  4445. func_g.type = filter_parse_regex(buffer, strlen(buffer),
  4446. &func_g.search, &not);
  4447. func_g.len = strlen(func_g.search);
  4448. mutex_lock(&ftrace_lock);
  4449. if (unlikely(ftrace_disabled)) {
  4450. mutex_unlock(&ftrace_lock);
  4451. return -ENODEV;
  4452. }
  4453. do_for_each_ftrace_rec(pg, rec) {
  4454. if (rec->flags & FTRACE_FL_DISABLED)
  4455. continue;
  4456. if (ftrace_match_record(rec, &func_g, NULL, 0)) {
  4457. entry = ftrace_lookup_ip(hash, rec->ip);
  4458. if (!not) {
  4459. fail = 0;
  4460. if (entry)
  4461. continue;
  4462. if (add_hash_entry(hash, rec->ip) < 0)
  4463. goto out;
  4464. } else {
  4465. if (entry) {
  4466. free_hash_entry(hash, entry);
  4467. fail = 0;
  4468. }
  4469. }
  4470. }
  4471. } while_for_each_ftrace_rec();
  4472. out:
  4473. mutex_unlock(&ftrace_lock);
  4474. if (fail)
  4475. return -EINVAL;
  4476. return 0;
  4477. }
  4478. static ssize_t
  4479. ftrace_graph_write(struct file *file, const char __user *ubuf,
  4480. size_t cnt, loff_t *ppos)
  4481. {
  4482. ssize_t read, ret = 0;
  4483. struct ftrace_graph_data *fgd = file->private_data;
  4484. struct trace_parser *parser;
  4485. if (!cnt)
  4486. return 0;
  4487. /* Read mode uses seq functions */
  4488. if (file->f_mode & FMODE_READ) {
  4489. struct seq_file *m = file->private_data;
  4490. fgd = m->private;
  4491. }
  4492. parser = &fgd->parser;
  4493. read = trace_get_user(parser, ubuf, cnt, ppos);
  4494. if (read >= 0 && trace_parser_loaded(parser) &&
  4495. !trace_parser_cont(parser)) {
  4496. ret = ftrace_graph_set_hash(fgd->new_hash,
  4497. parser->buffer);
  4498. trace_parser_clear(parser);
  4499. }
  4500. if (!ret)
  4501. ret = read;
  4502. return ret;
  4503. }
  4504. static const struct file_operations ftrace_graph_fops = {
  4505. .open = ftrace_graph_open,
  4506. .read = seq_read,
  4507. .write = ftrace_graph_write,
  4508. .llseek = tracing_lseek,
  4509. .release = ftrace_graph_release,
  4510. };
  4511. static const struct file_operations ftrace_graph_notrace_fops = {
  4512. .open = ftrace_graph_notrace_open,
  4513. .read = seq_read,
  4514. .write = ftrace_graph_write,
  4515. .llseek = tracing_lseek,
  4516. .release = ftrace_graph_release,
  4517. };
  4518. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  4519. void ftrace_create_filter_files(struct ftrace_ops *ops,
  4520. struct dentry *parent)
  4521. {
  4522. trace_create_file("set_ftrace_filter", 0644, parent,
  4523. ops, &ftrace_filter_fops);
  4524. trace_create_file("set_ftrace_notrace", 0644, parent,
  4525. ops, &ftrace_notrace_fops);
  4526. }
  4527. /*
  4528. * The name "destroy_filter_files" is really a misnomer. Although
  4529. * in the future, it may actualy delete the files, but this is
  4530. * really intended to make sure the ops passed in are disabled
  4531. * and that when this function returns, the caller is free to
  4532. * free the ops.
  4533. *
  4534. * The "destroy" name is only to match the "create" name that this
  4535. * should be paired with.
  4536. */
  4537. void ftrace_destroy_filter_files(struct ftrace_ops *ops)
  4538. {
  4539. mutex_lock(&ftrace_lock);
  4540. if (ops->flags & FTRACE_OPS_FL_ENABLED)
  4541. ftrace_shutdown(ops, 0);
  4542. ops->flags |= FTRACE_OPS_FL_DELETED;
  4543. ftrace_free_filter(ops);
  4544. mutex_unlock(&ftrace_lock);
  4545. }
  4546. static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
  4547. {
  4548. trace_create_file("available_filter_functions", 0444,
  4549. d_tracer, NULL, &ftrace_avail_fops);
  4550. trace_create_file("enabled_functions", 0444,
  4551. d_tracer, NULL, &ftrace_enabled_fops);
  4552. ftrace_create_filter_files(&global_ops, d_tracer);
  4553. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  4554. trace_create_file("set_graph_function", 0644, d_tracer,
  4555. NULL,
  4556. &ftrace_graph_fops);
  4557. trace_create_file("set_graph_notrace", 0644, d_tracer,
  4558. NULL,
  4559. &ftrace_graph_notrace_fops);
  4560. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  4561. return 0;
  4562. }
  4563. static int ftrace_cmp_ips(const void *a, const void *b)
  4564. {
  4565. const unsigned long *ipa = a;
  4566. const unsigned long *ipb = b;
  4567. if (*ipa > *ipb)
  4568. return 1;
  4569. if (*ipa < *ipb)
  4570. return -1;
  4571. return 0;
  4572. }
  4573. static int ftrace_process_locs(struct module *mod,
  4574. unsigned long *start,
  4575. unsigned long *end)
  4576. {
  4577. struct ftrace_page *start_pg;
  4578. struct ftrace_page *pg;
  4579. struct dyn_ftrace *rec;
  4580. unsigned long count;
  4581. unsigned long *p;
  4582. unsigned long addr;
  4583. unsigned long flags = 0; /* Shut up gcc */
  4584. int ret = -ENOMEM;
  4585. count = end - start;
  4586. if (!count)
  4587. return 0;
  4588. sort(start, count, sizeof(*start),
  4589. ftrace_cmp_ips, NULL);
  4590. start_pg = ftrace_allocate_pages(count);
  4591. if (!start_pg)
  4592. return -ENOMEM;
  4593. mutex_lock(&ftrace_lock);
  4594. /*
  4595. * Core and each module needs their own pages, as
  4596. * modules will free them when they are removed.
  4597. * Force a new page to be allocated for modules.
  4598. */
  4599. if (!mod) {
  4600. WARN_ON(ftrace_pages || ftrace_pages_start);
  4601. /* First initialization */
  4602. ftrace_pages = ftrace_pages_start = start_pg;
  4603. } else {
  4604. if (!ftrace_pages)
  4605. goto out;
  4606. if (WARN_ON(ftrace_pages->next)) {
  4607. /* Hmm, we have free pages? */
  4608. while (ftrace_pages->next)
  4609. ftrace_pages = ftrace_pages->next;
  4610. }
  4611. ftrace_pages->next = start_pg;
  4612. }
  4613. p = start;
  4614. pg = start_pg;
  4615. while (p < end) {
  4616. addr = ftrace_call_adjust(*p++);
  4617. /*
  4618. * Some architecture linkers will pad between
  4619. * the different mcount_loc sections of different
  4620. * object files to satisfy alignments.
  4621. * Skip any NULL pointers.
  4622. */
  4623. if (!addr)
  4624. continue;
  4625. if (pg->index == pg->size) {
  4626. /* We should have allocated enough */
  4627. if (WARN_ON(!pg->next))
  4628. break;
  4629. pg = pg->next;
  4630. }
  4631. rec = &pg->records[pg->index++];
  4632. rec->ip = addr;
  4633. }
  4634. /* We should have used all pages */
  4635. WARN_ON(pg->next);
  4636. /* Assign the last page to ftrace_pages */
  4637. ftrace_pages = pg;
  4638. /*
  4639. * We only need to disable interrupts on start up
  4640. * because we are modifying code that an interrupt
  4641. * may execute, and the modification is not atomic.
  4642. * But for modules, nothing runs the code we modify
  4643. * until we are finished with it, and there's no
  4644. * reason to cause large interrupt latencies while we do it.
  4645. */
  4646. if (!mod)
  4647. local_irq_save(flags);
  4648. ftrace_update_code(mod, start_pg);
  4649. if (!mod)
  4650. local_irq_restore(flags);
  4651. ret = 0;
  4652. out:
  4653. mutex_unlock(&ftrace_lock);
  4654. return ret;
  4655. }
  4656. struct ftrace_mod_func {
  4657. struct list_head list;
  4658. char *name;
  4659. unsigned long ip;
  4660. unsigned int size;
  4661. };
  4662. struct ftrace_mod_map {
  4663. struct rcu_head rcu;
  4664. struct list_head list;
  4665. struct module *mod;
  4666. unsigned long start_addr;
  4667. unsigned long end_addr;
  4668. struct list_head funcs;
  4669. unsigned int num_funcs;
  4670. };
  4671. #ifdef CONFIG_MODULES
  4672. #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
  4673. static LIST_HEAD(ftrace_mod_maps);
  4674. static int referenced_filters(struct dyn_ftrace *rec)
  4675. {
  4676. struct ftrace_ops *ops;
  4677. int cnt = 0;
  4678. for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
  4679. if (ops_references_rec(ops, rec)) {
  4680. cnt++;
  4681. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
  4682. rec->flags |= FTRACE_FL_REGS;
  4683. }
  4684. }
  4685. return cnt;
  4686. }
  4687. static void
  4688. clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
  4689. {
  4690. struct ftrace_func_entry *entry;
  4691. struct dyn_ftrace *rec;
  4692. int i;
  4693. if (ftrace_hash_empty(hash))
  4694. return;
  4695. for (i = 0; i < pg->index; i++) {
  4696. rec = &pg->records[i];
  4697. entry = __ftrace_lookup_ip(hash, rec->ip);
  4698. /*
  4699. * Do not allow this rec to match again.
  4700. * Yeah, it may waste some memory, but will be removed
  4701. * if/when the hash is modified again.
  4702. */
  4703. if (entry)
  4704. entry->ip = 0;
  4705. }
  4706. }
  4707. /* Clear any records from hashs */
  4708. static void clear_mod_from_hashes(struct ftrace_page *pg)
  4709. {
  4710. struct trace_array *tr;
  4711. mutex_lock(&trace_types_lock);
  4712. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  4713. if (!tr->ops || !tr->ops->func_hash)
  4714. continue;
  4715. mutex_lock(&tr->ops->func_hash->regex_lock);
  4716. clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
  4717. clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
  4718. mutex_unlock(&tr->ops->func_hash->regex_lock);
  4719. }
  4720. mutex_unlock(&trace_types_lock);
  4721. }
  4722. static void ftrace_free_mod_map(struct rcu_head *rcu)
  4723. {
  4724. struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
  4725. struct ftrace_mod_func *mod_func;
  4726. struct ftrace_mod_func *n;
  4727. /* All the contents of mod_map are now not visible to readers */
  4728. list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
  4729. kfree(mod_func->name);
  4730. list_del(&mod_func->list);
  4731. kfree(mod_func);
  4732. }
  4733. kfree(mod_map);
  4734. }
  4735. void ftrace_release_mod(struct module *mod)
  4736. {
  4737. struct ftrace_mod_map *mod_map;
  4738. struct ftrace_mod_map *n;
  4739. struct dyn_ftrace *rec;
  4740. struct ftrace_page **last_pg;
  4741. struct ftrace_page *tmp_page = NULL;
  4742. struct ftrace_page *pg;
  4743. int order;
  4744. mutex_lock(&ftrace_lock);
  4745. if (ftrace_disabled)
  4746. goto out_unlock;
  4747. list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
  4748. if (mod_map->mod == mod) {
  4749. list_del_rcu(&mod_map->list);
  4750. call_rcu_sched(&mod_map->rcu, ftrace_free_mod_map);
  4751. break;
  4752. }
  4753. }
  4754. /*
  4755. * Each module has its own ftrace_pages, remove
  4756. * them from the list.
  4757. */
  4758. last_pg = &ftrace_pages_start;
  4759. for (pg = ftrace_pages_start; pg; pg = *last_pg) {
  4760. rec = &pg->records[0];
  4761. if (within_module_core(rec->ip, mod) ||
  4762. within_module_init(rec->ip, mod)) {
  4763. /*
  4764. * As core pages are first, the first
  4765. * page should never be a module page.
  4766. */
  4767. if (WARN_ON(pg == ftrace_pages_start))
  4768. goto out_unlock;
  4769. /* Check if we are deleting the last page */
  4770. if (pg == ftrace_pages)
  4771. ftrace_pages = next_to_ftrace_page(last_pg);
  4772. ftrace_update_tot_cnt -= pg->index;
  4773. *last_pg = pg->next;
  4774. pg->next = tmp_page;
  4775. tmp_page = pg;
  4776. } else
  4777. last_pg = &pg->next;
  4778. }
  4779. out_unlock:
  4780. mutex_unlock(&ftrace_lock);
  4781. for (pg = tmp_page; pg; pg = tmp_page) {
  4782. /* Needs to be called outside of ftrace_lock */
  4783. clear_mod_from_hashes(pg);
  4784. order = get_count_order(pg->size / ENTRIES_PER_PAGE);
  4785. free_pages((unsigned long)pg->records, order);
  4786. tmp_page = pg->next;
  4787. kfree(pg);
  4788. }
  4789. }
  4790. void ftrace_module_enable(struct module *mod)
  4791. {
  4792. struct dyn_ftrace *rec;
  4793. struct ftrace_page *pg;
  4794. mutex_lock(&ftrace_lock);
  4795. if (ftrace_disabled)
  4796. goto out_unlock;
  4797. /*
  4798. * If the tracing is enabled, go ahead and enable the record.
  4799. *
  4800. * The reason not to enable the record immediatelly is the
  4801. * inherent check of ftrace_make_nop/ftrace_make_call for
  4802. * correct previous instructions. Making first the NOP
  4803. * conversion puts the module to the correct state, thus
  4804. * passing the ftrace_make_call check.
  4805. *
  4806. * We also delay this to after the module code already set the
  4807. * text to read-only, as we now need to set it back to read-write
  4808. * so that we can modify the text.
  4809. */
  4810. if (ftrace_start_up)
  4811. ftrace_arch_code_modify_prepare();
  4812. do_for_each_ftrace_rec(pg, rec) {
  4813. int cnt;
  4814. /*
  4815. * do_for_each_ftrace_rec() is a double loop.
  4816. * module text shares the pg. If a record is
  4817. * not part of this module, then skip this pg,
  4818. * which the "break" will do.
  4819. */
  4820. if (!within_module_core(rec->ip, mod) &&
  4821. !within_module_init(rec->ip, mod))
  4822. break;
  4823. cnt = 0;
  4824. /*
  4825. * When adding a module, we need to check if tracers are
  4826. * currently enabled and if they are, and can trace this record,
  4827. * we need to enable the module functions as well as update the
  4828. * reference counts for those function records.
  4829. */
  4830. if (ftrace_start_up)
  4831. cnt += referenced_filters(rec);
  4832. rec->flags &= ~FTRACE_FL_DISABLED;
  4833. rec->flags += cnt;
  4834. if (ftrace_start_up && cnt) {
  4835. int failed = __ftrace_replace_code(rec, 1);
  4836. if (failed) {
  4837. ftrace_bug(failed, rec);
  4838. goto out_loop;
  4839. }
  4840. }
  4841. } while_for_each_ftrace_rec();
  4842. out_loop:
  4843. if (ftrace_start_up)
  4844. ftrace_arch_code_modify_post_process();
  4845. out_unlock:
  4846. mutex_unlock(&ftrace_lock);
  4847. process_cached_mods(mod->name);
  4848. }
  4849. void ftrace_module_init(struct module *mod)
  4850. {
  4851. if (ftrace_disabled || !mod->num_ftrace_callsites)
  4852. return;
  4853. ftrace_process_locs(mod, mod->ftrace_callsites,
  4854. mod->ftrace_callsites + mod->num_ftrace_callsites);
  4855. }
  4856. static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
  4857. struct dyn_ftrace *rec)
  4858. {
  4859. struct ftrace_mod_func *mod_func;
  4860. unsigned long symsize;
  4861. unsigned long offset;
  4862. char str[KSYM_SYMBOL_LEN];
  4863. char *modname;
  4864. const char *ret;
  4865. ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
  4866. if (!ret)
  4867. return;
  4868. mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL);
  4869. if (!mod_func)
  4870. return;
  4871. mod_func->name = kstrdup(str, GFP_KERNEL);
  4872. if (!mod_func->name) {
  4873. kfree(mod_func);
  4874. return;
  4875. }
  4876. mod_func->ip = rec->ip - offset;
  4877. mod_func->size = symsize;
  4878. mod_map->num_funcs++;
  4879. list_add_rcu(&mod_func->list, &mod_map->funcs);
  4880. }
  4881. static struct ftrace_mod_map *
  4882. allocate_ftrace_mod_map(struct module *mod,
  4883. unsigned long start, unsigned long end)
  4884. {
  4885. struct ftrace_mod_map *mod_map;
  4886. mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL);
  4887. if (!mod_map)
  4888. return NULL;
  4889. mod_map->mod = mod;
  4890. mod_map->start_addr = start;
  4891. mod_map->end_addr = end;
  4892. mod_map->num_funcs = 0;
  4893. INIT_LIST_HEAD_RCU(&mod_map->funcs);
  4894. list_add_rcu(&mod_map->list, &ftrace_mod_maps);
  4895. return mod_map;
  4896. }
  4897. static const char *
  4898. ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
  4899. unsigned long addr, unsigned long *size,
  4900. unsigned long *off, char *sym)
  4901. {
  4902. struct ftrace_mod_func *found_func = NULL;
  4903. struct ftrace_mod_func *mod_func;
  4904. list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
  4905. if (addr >= mod_func->ip &&
  4906. addr < mod_func->ip + mod_func->size) {
  4907. found_func = mod_func;
  4908. break;
  4909. }
  4910. }
  4911. if (found_func) {
  4912. if (size)
  4913. *size = found_func->size;
  4914. if (off)
  4915. *off = addr - found_func->ip;
  4916. if (sym)
  4917. strlcpy(sym, found_func->name, KSYM_NAME_LEN);
  4918. return found_func->name;
  4919. }
  4920. return NULL;
  4921. }
  4922. const char *
  4923. ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
  4924. unsigned long *off, char **modname, char *sym)
  4925. {
  4926. struct ftrace_mod_map *mod_map;
  4927. const char *ret = NULL;
  4928. /* mod_map is freed via call_rcu_sched() */
  4929. preempt_disable();
  4930. list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
  4931. ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
  4932. if (ret) {
  4933. if (modname)
  4934. *modname = mod_map->mod->name;
  4935. break;
  4936. }
  4937. }
  4938. preempt_enable();
  4939. return ret;
  4940. }
  4941. int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
  4942. char *type, char *name,
  4943. char *module_name, int *exported)
  4944. {
  4945. struct ftrace_mod_map *mod_map;
  4946. struct ftrace_mod_func *mod_func;
  4947. preempt_disable();
  4948. list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
  4949. if (symnum >= mod_map->num_funcs) {
  4950. symnum -= mod_map->num_funcs;
  4951. continue;
  4952. }
  4953. list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
  4954. if (symnum > 1) {
  4955. symnum--;
  4956. continue;
  4957. }
  4958. *value = mod_func->ip;
  4959. *type = 'T';
  4960. strlcpy(name, mod_func->name, KSYM_NAME_LEN);
  4961. strlcpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
  4962. *exported = 1;
  4963. preempt_enable();
  4964. return 0;
  4965. }
  4966. WARN_ON(1);
  4967. break;
  4968. }
  4969. preempt_enable();
  4970. return -ERANGE;
  4971. }
  4972. #else
  4973. static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
  4974. struct dyn_ftrace *rec) { }
  4975. static inline struct ftrace_mod_map *
  4976. allocate_ftrace_mod_map(struct module *mod,
  4977. unsigned long start, unsigned long end)
  4978. {
  4979. return NULL;
  4980. }
  4981. #endif /* CONFIG_MODULES */
  4982. struct ftrace_init_func {
  4983. struct list_head list;
  4984. unsigned long ip;
  4985. };
  4986. /* Clear any init ips from hashes */
  4987. static void
  4988. clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
  4989. {
  4990. struct ftrace_func_entry *entry;
  4991. if (ftrace_hash_empty(hash))
  4992. return;
  4993. entry = __ftrace_lookup_ip(hash, func->ip);
  4994. /*
  4995. * Do not allow this rec to match again.
  4996. * Yeah, it may waste some memory, but will be removed
  4997. * if/when the hash is modified again.
  4998. */
  4999. if (entry)
  5000. entry->ip = 0;
  5001. }
  5002. static void
  5003. clear_func_from_hashes(struct ftrace_init_func *func)
  5004. {
  5005. struct trace_array *tr;
  5006. mutex_lock(&trace_types_lock);
  5007. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  5008. if (!tr->ops || !tr->ops->func_hash)
  5009. continue;
  5010. mutex_lock(&tr->ops->func_hash->regex_lock);
  5011. clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
  5012. clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
  5013. mutex_unlock(&tr->ops->func_hash->regex_lock);
  5014. }
  5015. mutex_unlock(&trace_types_lock);
  5016. }
  5017. static void add_to_clear_hash_list(struct list_head *clear_list,
  5018. struct dyn_ftrace *rec)
  5019. {
  5020. struct ftrace_init_func *func;
  5021. func = kmalloc(sizeof(*func), GFP_KERNEL);
  5022. if (!func) {
  5023. WARN_ONCE(1, "alloc failure, ftrace filter could be stale\n");
  5024. return;
  5025. }
  5026. func->ip = rec->ip;
  5027. list_add(&func->list, clear_list);
  5028. }
  5029. void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
  5030. {
  5031. unsigned long start = (unsigned long)(start_ptr);
  5032. unsigned long end = (unsigned long)(end_ptr);
  5033. struct ftrace_page **last_pg = &ftrace_pages_start;
  5034. struct ftrace_page *pg;
  5035. struct dyn_ftrace *rec;
  5036. struct dyn_ftrace key;
  5037. struct ftrace_mod_map *mod_map = NULL;
  5038. struct ftrace_init_func *func, *func_next;
  5039. struct list_head clear_hash;
  5040. int order;
  5041. INIT_LIST_HEAD(&clear_hash);
  5042. key.ip = start;
  5043. key.flags = end; /* overload flags, as it is unsigned long */
  5044. mutex_lock(&ftrace_lock);
  5045. /*
  5046. * If we are freeing module init memory, then check if
  5047. * any tracer is active. If so, we need to save a mapping of
  5048. * the module functions being freed with the address.
  5049. */
  5050. if (mod && ftrace_ops_list != &ftrace_list_end)
  5051. mod_map = allocate_ftrace_mod_map(mod, start, end);
  5052. for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
  5053. if (end < pg->records[0].ip ||
  5054. start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
  5055. continue;
  5056. again:
  5057. rec = bsearch(&key, pg->records, pg->index,
  5058. sizeof(struct dyn_ftrace),
  5059. ftrace_cmp_recs);
  5060. if (!rec)
  5061. continue;
  5062. /* rec will be cleared from hashes after ftrace_lock unlock */
  5063. add_to_clear_hash_list(&clear_hash, rec);
  5064. if (mod_map)
  5065. save_ftrace_mod_rec(mod_map, rec);
  5066. pg->index--;
  5067. ftrace_update_tot_cnt--;
  5068. if (!pg->index) {
  5069. *last_pg = pg->next;
  5070. order = get_count_order(pg->size / ENTRIES_PER_PAGE);
  5071. free_pages((unsigned long)pg->records, order);
  5072. kfree(pg);
  5073. pg = container_of(last_pg, struct ftrace_page, next);
  5074. if (!(*last_pg))
  5075. ftrace_pages = pg;
  5076. continue;
  5077. }
  5078. memmove(rec, rec + 1,
  5079. (pg->index - (rec - pg->records)) * sizeof(*rec));
  5080. /* More than one function may be in this block */
  5081. goto again;
  5082. }
  5083. mutex_unlock(&ftrace_lock);
  5084. list_for_each_entry_safe(func, func_next, &clear_hash, list) {
  5085. clear_func_from_hashes(func);
  5086. kfree(func);
  5087. }
  5088. }
  5089. void __init ftrace_free_init_mem(void)
  5090. {
  5091. void *start = (void *)(&__init_begin);
  5092. void *end = (void *)(&__init_end);
  5093. ftrace_free_mem(NULL, start, end);
  5094. }
  5095. void __init ftrace_init(void)
  5096. {
  5097. extern unsigned long __start_mcount_loc[];
  5098. extern unsigned long __stop_mcount_loc[];
  5099. unsigned long count, flags;
  5100. int ret;
  5101. local_irq_save(flags);
  5102. ret = ftrace_dyn_arch_init();
  5103. local_irq_restore(flags);
  5104. if (ret)
  5105. goto failed;
  5106. count = __stop_mcount_loc - __start_mcount_loc;
  5107. if (!count) {
  5108. pr_info("ftrace: No functions to be traced?\n");
  5109. goto failed;
  5110. }
  5111. pr_info("ftrace: allocating %ld entries in %ld pages\n",
  5112. count, count / ENTRIES_PER_PAGE + 1);
  5113. last_ftrace_enabled = ftrace_enabled = 1;
  5114. ret = ftrace_process_locs(NULL,
  5115. __start_mcount_loc,
  5116. __stop_mcount_loc);
  5117. set_ftrace_early_filters();
  5118. return;
  5119. failed:
  5120. ftrace_disabled = 1;
  5121. }
  5122. /* Do nothing if arch does not support this */
  5123. void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
  5124. {
  5125. }
  5126. static void ftrace_update_trampoline(struct ftrace_ops *ops)
  5127. {
  5128. arch_ftrace_update_trampoline(ops);
  5129. }
  5130. void ftrace_init_trace_array(struct trace_array *tr)
  5131. {
  5132. INIT_LIST_HEAD(&tr->func_probes);
  5133. INIT_LIST_HEAD(&tr->mod_trace);
  5134. INIT_LIST_HEAD(&tr->mod_notrace);
  5135. }
  5136. #else
  5137. static struct ftrace_ops global_ops = {
  5138. .func = ftrace_stub,
  5139. .flags = FTRACE_OPS_FL_RECURSION_SAFE |
  5140. FTRACE_OPS_FL_INITIALIZED |
  5141. FTRACE_OPS_FL_PID,
  5142. };
  5143. static int __init ftrace_nodyn_init(void)
  5144. {
  5145. ftrace_enabled = 1;
  5146. return 0;
  5147. }
  5148. core_initcall(ftrace_nodyn_init);
  5149. static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
  5150. static inline void ftrace_startup_enable(int command) { }
  5151. static inline void ftrace_startup_all(int command) { }
  5152. /* Keep as macros so we do not need to define the commands */
  5153. # define ftrace_startup(ops, command) \
  5154. ({ \
  5155. int ___ret = __register_ftrace_function(ops); \
  5156. if (!___ret) \
  5157. (ops)->flags |= FTRACE_OPS_FL_ENABLED; \
  5158. ___ret; \
  5159. })
  5160. # define ftrace_shutdown(ops, command) \
  5161. ({ \
  5162. int ___ret = __unregister_ftrace_function(ops); \
  5163. if (!___ret) \
  5164. (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \
  5165. ___ret; \
  5166. })
  5167. # define ftrace_startup_sysctl() do { } while (0)
  5168. # define ftrace_shutdown_sysctl() do { } while (0)
  5169. static inline int
  5170. ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
  5171. {
  5172. return 1;
  5173. }
  5174. static void ftrace_update_trampoline(struct ftrace_ops *ops)
  5175. {
  5176. }
  5177. #endif /* CONFIG_DYNAMIC_FTRACE */
  5178. __init void ftrace_init_global_array_ops(struct trace_array *tr)
  5179. {
  5180. tr->ops = &global_ops;
  5181. tr->ops->private = tr;
  5182. ftrace_init_trace_array(tr);
  5183. }
  5184. void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
  5185. {
  5186. /* If we filter on pids, update to use the pid function */
  5187. if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
  5188. if (WARN_ON(tr->ops->func != ftrace_stub))
  5189. printk("ftrace ops had %pS for function\n",
  5190. tr->ops->func);
  5191. }
  5192. tr->ops->func = func;
  5193. tr->ops->private = tr;
  5194. }
  5195. void ftrace_reset_array_ops(struct trace_array *tr)
  5196. {
  5197. tr->ops->func = ftrace_stub;
  5198. }
  5199. static nokprobe_inline void
  5200. __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
  5201. struct ftrace_ops *ignored, struct pt_regs *regs)
  5202. {
  5203. struct ftrace_ops *op;
  5204. int bit;
  5205. bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
  5206. if (bit < 0)
  5207. return;
  5208. /*
  5209. * Some of the ops may be dynamically allocated,
  5210. * they must be freed after a synchronize_sched().
  5211. */
  5212. preempt_disable_notrace();
  5213. do_for_each_ftrace_op(op, ftrace_ops_list) {
  5214. /*
  5215. * Check the following for each ops before calling their func:
  5216. * if RCU flag is set, then rcu_is_watching() must be true
  5217. * if PER_CPU is set, then ftrace_function_local_disable()
  5218. * must be false
  5219. * Otherwise test if the ip matches the ops filter
  5220. *
  5221. * If any of the above fails then the op->func() is not executed.
  5222. */
  5223. if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
  5224. ftrace_ops_test(op, ip, regs)) {
  5225. if (FTRACE_WARN_ON(!op->func)) {
  5226. pr_warn("op=%p %pS\n", op, op);
  5227. goto out;
  5228. }
  5229. op->func(ip, parent_ip, op, regs);
  5230. }
  5231. } while_for_each_ftrace_op(op);
  5232. out:
  5233. preempt_enable_notrace();
  5234. trace_clear_recursion(bit);
  5235. }
  5236. /*
  5237. * Some archs only support passing ip and parent_ip. Even though
  5238. * the list function ignores the op parameter, we do not want any
  5239. * C side effects, where a function is called without the caller
  5240. * sending a third parameter.
  5241. * Archs are to support both the regs and ftrace_ops at the same time.
  5242. * If they support ftrace_ops, it is assumed they support regs.
  5243. * If call backs want to use regs, they must either check for regs
  5244. * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
  5245. * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
  5246. * An architecture can pass partial regs with ftrace_ops and still
  5247. * set the ARCH_SUPPORTS_FTRACE_OPS.
  5248. */
  5249. #if ARCH_SUPPORTS_FTRACE_OPS
  5250. static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
  5251. struct ftrace_ops *op, struct pt_regs *regs)
  5252. {
  5253. __ftrace_ops_list_func(ip, parent_ip, NULL, regs);
  5254. }
  5255. NOKPROBE_SYMBOL(ftrace_ops_list_func);
  5256. #else
  5257. static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
  5258. {
  5259. __ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
  5260. }
  5261. NOKPROBE_SYMBOL(ftrace_ops_no_ops);
  5262. #endif
  5263. /*
  5264. * If there's only one function registered but it does not support
  5265. * recursion, needs RCU protection and/or requires per cpu handling, then
  5266. * this function will be called by the mcount trampoline.
  5267. */
  5268. static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
  5269. struct ftrace_ops *op, struct pt_regs *regs)
  5270. {
  5271. int bit;
  5272. bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
  5273. if (bit < 0)
  5274. return;
  5275. preempt_disable_notrace();
  5276. if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching())
  5277. op->func(ip, parent_ip, op, regs);
  5278. preempt_enable_notrace();
  5279. trace_clear_recursion(bit);
  5280. }
  5281. NOKPROBE_SYMBOL(ftrace_ops_assist_func);
  5282. /**
  5283. * ftrace_ops_get_func - get the function a trampoline should call
  5284. * @ops: the ops to get the function for
  5285. *
  5286. * Normally the mcount trampoline will call the ops->func, but there
  5287. * are times that it should not. For example, if the ops does not
  5288. * have its own recursion protection, then it should call the
  5289. * ftrace_ops_assist_func() instead.
  5290. *
  5291. * Returns the function that the trampoline should call for @ops.
  5292. */
  5293. ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
  5294. {
  5295. /*
  5296. * If the function does not handle recursion, needs to be RCU safe,
  5297. * or does per cpu logic, then we need to call the assist handler.
  5298. */
  5299. if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
  5300. ops->flags & FTRACE_OPS_FL_RCU)
  5301. return ftrace_ops_assist_func;
  5302. return ops->func;
  5303. }
  5304. static void
  5305. ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
  5306. struct task_struct *prev, struct task_struct *next)
  5307. {
  5308. struct trace_array *tr = data;
  5309. struct trace_pid_list *pid_list;
  5310. pid_list = rcu_dereference_sched(tr->function_pids);
  5311. this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
  5312. trace_ignore_this_task(pid_list, next));
  5313. }
  5314. static void
  5315. ftrace_pid_follow_sched_process_fork(void *data,
  5316. struct task_struct *self,
  5317. struct task_struct *task)
  5318. {
  5319. struct trace_pid_list *pid_list;
  5320. struct trace_array *tr = data;
  5321. pid_list = rcu_dereference_sched(tr->function_pids);
  5322. trace_filter_add_remove_task(pid_list, self, task);
  5323. }
  5324. static void
  5325. ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
  5326. {
  5327. struct trace_pid_list *pid_list;
  5328. struct trace_array *tr = data;
  5329. pid_list = rcu_dereference_sched(tr->function_pids);
  5330. trace_filter_add_remove_task(pid_list, NULL, task);
  5331. }
  5332. void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
  5333. {
  5334. if (enable) {
  5335. register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
  5336. tr);
  5337. register_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
  5338. tr);
  5339. } else {
  5340. unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
  5341. tr);
  5342. unregister_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
  5343. tr);
  5344. }
  5345. }
  5346. static void clear_ftrace_pids(struct trace_array *tr)
  5347. {
  5348. struct trace_pid_list *pid_list;
  5349. int cpu;
  5350. pid_list = rcu_dereference_protected(tr->function_pids,
  5351. lockdep_is_held(&ftrace_lock));
  5352. if (!pid_list)
  5353. return;
  5354. unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
  5355. for_each_possible_cpu(cpu)
  5356. per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
  5357. rcu_assign_pointer(tr->function_pids, NULL);
  5358. /* Wait till all users are no longer using pid filtering */
  5359. synchronize_sched();
  5360. trace_free_pid_list(pid_list);
  5361. }
  5362. void ftrace_clear_pids(struct trace_array *tr)
  5363. {
  5364. mutex_lock(&ftrace_lock);
  5365. clear_ftrace_pids(tr);
  5366. mutex_unlock(&ftrace_lock);
  5367. }
  5368. static void ftrace_pid_reset(struct trace_array *tr)
  5369. {
  5370. mutex_lock(&ftrace_lock);
  5371. clear_ftrace_pids(tr);
  5372. ftrace_update_pid_func();
  5373. ftrace_startup_all(0);
  5374. mutex_unlock(&ftrace_lock);
  5375. }
  5376. /* Greater than any max PID */
  5377. #define FTRACE_NO_PIDS (void *)(PID_MAX_LIMIT + 1)
  5378. static void *fpid_start(struct seq_file *m, loff_t *pos)
  5379. __acquires(RCU)
  5380. {
  5381. struct trace_pid_list *pid_list;
  5382. struct trace_array *tr = m->private;
  5383. mutex_lock(&ftrace_lock);
  5384. rcu_read_lock_sched();
  5385. pid_list = rcu_dereference_sched(tr->function_pids);
  5386. if (!pid_list)
  5387. return !(*pos) ? FTRACE_NO_PIDS : NULL;
  5388. return trace_pid_start(pid_list, pos);
  5389. }
  5390. static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
  5391. {
  5392. struct trace_array *tr = m->private;
  5393. struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
  5394. if (v == FTRACE_NO_PIDS) {
  5395. (*pos)++;
  5396. return NULL;
  5397. }
  5398. return trace_pid_next(pid_list, v, pos);
  5399. }
  5400. static void fpid_stop(struct seq_file *m, void *p)
  5401. __releases(RCU)
  5402. {
  5403. rcu_read_unlock_sched();
  5404. mutex_unlock(&ftrace_lock);
  5405. }
  5406. static int fpid_show(struct seq_file *m, void *v)
  5407. {
  5408. if (v == FTRACE_NO_PIDS) {
  5409. seq_puts(m, "no pid\n");
  5410. return 0;
  5411. }
  5412. return trace_pid_show(m, v);
  5413. }
  5414. static const struct seq_operations ftrace_pid_sops = {
  5415. .start = fpid_start,
  5416. .next = fpid_next,
  5417. .stop = fpid_stop,
  5418. .show = fpid_show,
  5419. };
  5420. static int
  5421. ftrace_pid_open(struct inode *inode, struct file *file)
  5422. {
  5423. struct trace_array *tr = inode->i_private;
  5424. struct seq_file *m;
  5425. int ret = 0;
  5426. if (trace_array_get(tr) < 0)
  5427. return -ENODEV;
  5428. if ((file->f_mode & FMODE_WRITE) &&
  5429. (file->f_flags & O_TRUNC))
  5430. ftrace_pid_reset(tr);
  5431. ret = seq_open(file, &ftrace_pid_sops);
  5432. if (ret < 0) {
  5433. trace_array_put(tr);
  5434. } else {
  5435. m = file->private_data;
  5436. /* copy tr over to seq ops */
  5437. m->private = tr;
  5438. }
  5439. return ret;
  5440. }
  5441. static void ignore_task_cpu(void *data)
  5442. {
  5443. struct trace_array *tr = data;
  5444. struct trace_pid_list *pid_list;
  5445. /*
  5446. * This function is called by on_each_cpu() while the
  5447. * event_mutex is held.
  5448. */
  5449. pid_list = rcu_dereference_protected(tr->function_pids,
  5450. mutex_is_locked(&ftrace_lock));
  5451. this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
  5452. trace_ignore_this_task(pid_list, current));
  5453. }
  5454. static ssize_t
  5455. ftrace_pid_write(struct file *filp, const char __user *ubuf,
  5456. size_t cnt, loff_t *ppos)
  5457. {
  5458. struct seq_file *m = filp->private_data;
  5459. struct trace_array *tr = m->private;
  5460. struct trace_pid_list *filtered_pids = NULL;
  5461. struct trace_pid_list *pid_list;
  5462. ssize_t ret;
  5463. if (!cnt)
  5464. return 0;
  5465. mutex_lock(&ftrace_lock);
  5466. filtered_pids = rcu_dereference_protected(tr->function_pids,
  5467. lockdep_is_held(&ftrace_lock));
  5468. ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
  5469. if (ret < 0)
  5470. goto out;
  5471. rcu_assign_pointer(tr->function_pids, pid_list);
  5472. if (filtered_pids) {
  5473. synchronize_sched();
  5474. trace_free_pid_list(filtered_pids);
  5475. } else if (pid_list) {
  5476. /* Register a probe to set whether to ignore the tracing of a task */
  5477. register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
  5478. }
  5479. /*
  5480. * Ignoring of pids is done at task switch. But we have to
  5481. * check for those tasks that are currently running.
  5482. * Always do this in case a pid was appended or removed.
  5483. */
  5484. on_each_cpu(ignore_task_cpu, tr, 1);
  5485. ftrace_update_pid_func();
  5486. ftrace_startup_all(0);
  5487. out:
  5488. mutex_unlock(&ftrace_lock);
  5489. if (ret > 0)
  5490. *ppos += ret;
  5491. return ret;
  5492. }
  5493. static int
  5494. ftrace_pid_release(struct inode *inode, struct file *file)
  5495. {
  5496. struct trace_array *tr = inode->i_private;
  5497. trace_array_put(tr);
  5498. return seq_release(inode, file);
  5499. }
  5500. static const struct file_operations ftrace_pid_fops = {
  5501. .open = ftrace_pid_open,
  5502. .write = ftrace_pid_write,
  5503. .read = seq_read,
  5504. .llseek = tracing_lseek,
  5505. .release = ftrace_pid_release,
  5506. };
  5507. void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
  5508. {
  5509. trace_create_file("set_ftrace_pid", 0644, d_tracer,
  5510. tr, &ftrace_pid_fops);
  5511. }
  5512. void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
  5513. struct dentry *d_tracer)
  5514. {
  5515. /* Only the top level directory has the dyn_tracefs and profile */
  5516. WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
  5517. ftrace_init_dyn_tracefs(d_tracer);
  5518. ftrace_profile_tracefs(d_tracer);
  5519. }
  5520. /**
  5521. * ftrace_kill - kill ftrace
  5522. *
  5523. * This function should be used by panic code. It stops ftrace
  5524. * but in a not so nice way. If you need to simply kill ftrace
  5525. * from a non-atomic section, use ftrace_kill.
  5526. */
  5527. void ftrace_kill(void)
  5528. {
  5529. ftrace_disabled = 1;
  5530. ftrace_enabled = 0;
  5531. ftrace_trace_function = ftrace_stub;
  5532. }
  5533. /**
  5534. * Test if ftrace is dead or not.
  5535. */
  5536. int ftrace_is_dead(void)
  5537. {
  5538. return ftrace_disabled;
  5539. }
  5540. /**
  5541. * register_ftrace_function - register a function for profiling
  5542. * @ops - ops structure that holds the function for profiling.
  5543. *
  5544. * Register a function to be called by all functions in the
  5545. * kernel.
  5546. *
  5547. * Note: @ops->func and all the functions it calls must be labeled
  5548. * with "notrace", otherwise it will go into a
  5549. * recursive loop.
  5550. */
  5551. int register_ftrace_function(struct ftrace_ops *ops)
  5552. {
  5553. int ret = -1;
  5554. ftrace_ops_init(ops);
  5555. mutex_lock(&ftrace_lock);
  5556. ret = ftrace_startup(ops, 0);
  5557. mutex_unlock(&ftrace_lock);
  5558. return ret;
  5559. }
  5560. EXPORT_SYMBOL_GPL(register_ftrace_function);
  5561. /**
  5562. * unregister_ftrace_function - unregister a function for profiling.
  5563. * @ops - ops structure that holds the function to unregister
  5564. *
  5565. * Unregister a function that was added to be called by ftrace profiling.
  5566. */
  5567. int unregister_ftrace_function(struct ftrace_ops *ops)
  5568. {
  5569. int ret;
  5570. mutex_lock(&ftrace_lock);
  5571. ret = ftrace_shutdown(ops, 0);
  5572. mutex_unlock(&ftrace_lock);
  5573. return ret;
  5574. }
  5575. EXPORT_SYMBOL_GPL(unregister_ftrace_function);
  5576. int
  5577. ftrace_enable_sysctl(struct ctl_table *table, int write,
  5578. void __user *buffer, size_t *lenp,
  5579. loff_t *ppos)
  5580. {
  5581. int ret = -ENODEV;
  5582. mutex_lock(&ftrace_lock);
  5583. if (unlikely(ftrace_disabled))
  5584. goto out;
  5585. ret = proc_dointvec(table, write, buffer, lenp, ppos);
  5586. if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
  5587. goto out;
  5588. last_ftrace_enabled = !!ftrace_enabled;
  5589. if (ftrace_enabled) {
  5590. /* we are starting ftrace again */
  5591. if (rcu_dereference_protected(ftrace_ops_list,
  5592. lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
  5593. update_ftrace_function();
  5594. ftrace_startup_sysctl();
  5595. } else {
  5596. /* stopping ftrace calls (just send to ftrace_stub) */
  5597. ftrace_trace_function = ftrace_stub;
  5598. ftrace_shutdown_sysctl();
  5599. }
  5600. out:
  5601. mutex_unlock(&ftrace_lock);
  5602. return ret;
  5603. }
  5604. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  5605. static struct ftrace_ops graph_ops = {
  5606. .func = ftrace_stub,
  5607. .flags = FTRACE_OPS_FL_RECURSION_SAFE |
  5608. FTRACE_OPS_FL_INITIALIZED |
  5609. FTRACE_OPS_FL_PID |
  5610. FTRACE_OPS_FL_STUB,
  5611. #ifdef FTRACE_GRAPH_TRAMP_ADDR
  5612. .trampoline = FTRACE_GRAPH_TRAMP_ADDR,
  5613. /* trampoline_size is only needed for dynamically allocated tramps */
  5614. #endif
  5615. ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
  5616. };
  5617. void ftrace_graph_sleep_time_control(bool enable)
  5618. {
  5619. fgraph_sleep_time = enable;
  5620. }
  5621. void ftrace_graph_graph_time_control(bool enable)
  5622. {
  5623. fgraph_graph_time = enable;
  5624. }
  5625. int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
  5626. {
  5627. return 0;
  5628. }
  5629. /* The callbacks that hook a function */
  5630. trace_func_graph_ret_t ftrace_graph_return =
  5631. (trace_func_graph_ret_t)ftrace_stub;
  5632. trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
  5633. static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
  5634. /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
  5635. static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
  5636. {
  5637. int i;
  5638. int ret = 0;
  5639. int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
  5640. struct task_struct *g, *t;
  5641. for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
  5642. ret_stack_list[i] =
  5643. kmalloc_array(FTRACE_RETFUNC_DEPTH,
  5644. sizeof(struct ftrace_ret_stack),
  5645. GFP_KERNEL);
  5646. if (!ret_stack_list[i]) {
  5647. start = 0;
  5648. end = i;
  5649. ret = -ENOMEM;
  5650. goto free;
  5651. }
  5652. }
  5653. read_lock(&tasklist_lock);
  5654. do_each_thread(g, t) {
  5655. if (start == end) {
  5656. ret = -EAGAIN;
  5657. goto unlock;
  5658. }
  5659. if (t->ret_stack == NULL) {
  5660. atomic_set(&t->trace_overrun, 0);
  5661. t->curr_ret_stack = -1;
  5662. t->curr_ret_depth = -1;
  5663. /* Make sure the tasks see the -1 first: */
  5664. smp_wmb();
  5665. t->ret_stack = ret_stack_list[start++];
  5666. }
  5667. } while_each_thread(g, t);
  5668. unlock:
  5669. read_unlock(&tasklist_lock);
  5670. free:
  5671. for (i = start; i < end; i++)
  5672. kfree(ret_stack_list[i]);
  5673. return ret;
  5674. }
  5675. static void
  5676. ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
  5677. struct task_struct *prev, struct task_struct *next)
  5678. {
  5679. unsigned long long timestamp;
  5680. int index;
  5681. /*
  5682. * Does the user want to count the time a function was asleep.
  5683. * If so, do not update the time stamps.
  5684. */
  5685. if (fgraph_sleep_time)
  5686. return;
  5687. timestamp = trace_clock_local();
  5688. prev->ftrace_timestamp = timestamp;
  5689. /* only process tasks that we timestamped */
  5690. if (!next->ftrace_timestamp)
  5691. return;
  5692. /*
  5693. * Update all the counters in next to make up for the
  5694. * time next was sleeping.
  5695. */
  5696. timestamp -= next->ftrace_timestamp;
  5697. for (index = next->curr_ret_stack; index >= 0; index--)
  5698. next->ret_stack[index].calltime += timestamp;
  5699. }
  5700. /* Allocate a return stack for each task */
  5701. static int start_graph_tracing(void)
  5702. {
  5703. struct ftrace_ret_stack **ret_stack_list;
  5704. int ret, cpu;
  5705. ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE,
  5706. sizeof(struct ftrace_ret_stack *),
  5707. GFP_KERNEL);
  5708. if (!ret_stack_list)
  5709. return -ENOMEM;
  5710. /* The cpu_boot init_task->ret_stack will never be freed */
  5711. for_each_online_cpu(cpu) {
  5712. if (!idle_task(cpu)->ret_stack)
  5713. ftrace_graph_init_idle_task(idle_task(cpu), cpu);
  5714. }
  5715. do {
  5716. ret = alloc_retstack_tasklist(ret_stack_list);
  5717. } while (ret == -EAGAIN);
  5718. if (!ret) {
  5719. ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
  5720. if (ret)
  5721. pr_info("ftrace_graph: Couldn't activate tracepoint"
  5722. " probe to kernel_sched_switch\n");
  5723. }
  5724. kfree(ret_stack_list);
  5725. return ret;
  5726. }
  5727. /*
  5728. * Hibernation protection.
  5729. * The state of the current task is too much unstable during
  5730. * suspend/restore to disk. We want to protect against that.
  5731. */
  5732. static int
  5733. ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
  5734. void *unused)
  5735. {
  5736. switch (state) {
  5737. case PM_HIBERNATION_PREPARE:
  5738. pause_graph_tracing();
  5739. break;
  5740. case PM_POST_HIBERNATION:
  5741. unpause_graph_tracing();
  5742. break;
  5743. }
  5744. return NOTIFY_DONE;
  5745. }
  5746. static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
  5747. {
  5748. if (!ftrace_ops_test(&global_ops, trace->func, NULL))
  5749. return 0;
  5750. return __ftrace_graph_entry(trace);
  5751. }
  5752. /*
  5753. * The function graph tracer should only trace the functions defined
  5754. * by set_ftrace_filter and set_ftrace_notrace. If another function
  5755. * tracer ops is registered, the graph tracer requires testing the
  5756. * function against the global ops, and not just trace any function
  5757. * that any ftrace_ops registered.
  5758. */
  5759. static void update_function_graph_func(void)
  5760. {
  5761. struct ftrace_ops *op;
  5762. bool do_test = false;
  5763. /*
  5764. * The graph and global ops share the same set of functions
  5765. * to test. If any other ops is on the list, then
  5766. * the graph tracing needs to test if its the function
  5767. * it should call.
  5768. */
  5769. do_for_each_ftrace_op(op, ftrace_ops_list) {
  5770. if (op != &global_ops && op != &graph_ops &&
  5771. op != &ftrace_list_end) {
  5772. do_test = true;
  5773. /* in double loop, break out with goto */
  5774. goto out;
  5775. }
  5776. } while_for_each_ftrace_op(op);
  5777. out:
  5778. if (do_test)
  5779. ftrace_graph_entry = ftrace_graph_entry_test;
  5780. else
  5781. ftrace_graph_entry = __ftrace_graph_entry;
  5782. }
  5783. static struct notifier_block ftrace_suspend_notifier = {
  5784. .notifier_call = ftrace_suspend_notifier_call,
  5785. };
  5786. int register_ftrace_graph(trace_func_graph_ret_t retfunc,
  5787. trace_func_graph_ent_t entryfunc)
  5788. {
  5789. int ret = 0;
  5790. mutex_lock(&ftrace_lock);
  5791. /* we currently allow only one tracer registered at a time */
  5792. if (ftrace_graph_active) {
  5793. ret = -EBUSY;
  5794. goto out;
  5795. }
  5796. register_pm_notifier(&ftrace_suspend_notifier);
  5797. ftrace_graph_active++;
  5798. ret = start_graph_tracing();
  5799. if (ret) {
  5800. ftrace_graph_active--;
  5801. goto out;
  5802. }
  5803. ftrace_graph_return = retfunc;
  5804. /*
  5805. * Update the indirect function to the entryfunc, and the
  5806. * function that gets called to the entry_test first. Then
  5807. * call the update fgraph entry function to determine if
  5808. * the entryfunc should be called directly or not.
  5809. */
  5810. __ftrace_graph_entry = entryfunc;
  5811. ftrace_graph_entry = ftrace_graph_entry_test;
  5812. update_function_graph_func();
  5813. ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
  5814. out:
  5815. mutex_unlock(&ftrace_lock);
  5816. return ret;
  5817. }
  5818. void unregister_ftrace_graph(void)
  5819. {
  5820. mutex_lock(&ftrace_lock);
  5821. if (unlikely(!ftrace_graph_active))
  5822. goto out;
  5823. ftrace_graph_active--;
  5824. ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
  5825. ftrace_graph_entry = ftrace_graph_entry_stub;
  5826. __ftrace_graph_entry = ftrace_graph_entry_stub;
  5827. ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
  5828. unregister_pm_notifier(&ftrace_suspend_notifier);
  5829. unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
  5830. out:
  5831. mutex_unlock(&ftrace_lock);
  5832. }
  5833. static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
  5834. static void
  5835. graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
  5836. {
  5837. atomic_set(&t->trace_overrun, 0);
  5838. t->ftrace_timestamp = 0;
  5839. /* make curr_ret_stack visible before we add the ret_stack */
  5840. smp_wmb();
  5841. t->ret_stack = ret_stack;
  5842. }
  5843. /*
  5844. * Allocate a return stack for the idle task. May be the first
  5845. * time through, or it may be done by CPU hotplug online.
  5846. */
  5847. void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
  5848. {
  5849. t->curr_ret_stack = -1;
  5850. t->curr_ret_depth = -1;
  5851. /*
  5852. * The idle task has no parent, it either has its own
  5853. * stack or no stack at all.
  5854. */
  5855. if (t->ret_stack)
  5856. WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
  5857. if (ftrace_graph_active) {
  5858. struct ftrace_ret_stack *ret_stack;
  5859. ret_stack = per_cpu(idle_ret_stack, cpu);
  5860. if (!ret_stack) {
  5861. ret_stack =
  5862. kmalloc_array(FTRACE_RETFUNC_DEPTH,
  5863. sizeof(struct ftrace_ret_stack),
  5864. GFP_KERNEL);
  5865. if (!ret_stack)
  5866. return;
  5867. per_cpu(idle_ret_stack, cpu) = ret_stack;
  5868. }
  5869. graph_init_task(t, ret_stack);
  5870. }
  5871. }
  5872. /* Allocate a return stack for newly created task */
  5873. void ftrace_graph_init_task(struct task_struct *t)
  5874. {
  5875. /* Make sure we do not use the parent ret_stack */
  5876. t->ret_stack = NULL;
  5877. t->curr_ret_stack = -1;
  5878. t->curr_ret_depth = -1;
  5879. if (ftrace_graph_active) {
  5880. struct ftrace_ret_stack *ret_stack;
  5881. ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH,
  5882. sizeof(struct ftrace_ret_stack),
  5883. GFP_KERNEL);
  5884. if (!ret_stack)
  5885. return;
  5886. graph_init_task(t, ret_stack);
  5887. }
  5888. }
  5889. void ftrace_graph_exit_task(struct task_struct *t)
  5890. {
  5891. struct ftrace_ret_stack *ret_stack = t->ret_stack;
  5892. t->ret_stack = NULL;
  5893. /* NULL must become visible to IRQs before we free it: */
  5894. barrier();
  5895. kfree(ret_stack);
  5896. }
  5897. #endif