inode.c 183 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/fs/ext4/inode.c
  4. *
  5. * Copyright (C) 1992, 1993, 1994, 1995
  6. * Remy Card (card@masi.ibp.fr)
  7. * Laboratoire MASI - Institut Blaise Pascal
  8. * Universite Pierre et Marie Curie (Paris VI)
  9. *
  10. * from
  11. *
  12. * linux/fs/minix/inode.c
  13. *
  14. * Copyright (C) 1991, 1992 Linus Torvalds
  15. *
  16. * 64-bit file support on 64-bit platforms by Jakub Jelinek
  17. * (jj@sunsite.ms.mff.cuni.cz)
  18. *
  19. * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
  20. */
  21. #include <linux/fs.h>
  22. #include <linux/time.h>
  23. #include <linux/highuid.h>
  24. #include <linux/pagemap.h>
  25. #include <linux/dax.h>
  26. #include <linux/quotaops.h>
  27. #include <linux/string.h>
  28. #include <linux/buffer_head.h>
  29. #include <linux/writeback.h>
  30. #include <linux/pagevec.h>
  31. #include <linux/mpage.h>
  32. #include <linux/namei.h>
  33. #include <linux/uio.h>
  34. #include <linux/bio.h>
  35. #include <linux/workqueue.h>
  36. #include <linux/kernel.h>
  37. #include <linux/printk.h>
  38. #include <linux/slab.h>
  39. #include <linux/bitops.h>
  40. #include <linux/iomap.h>
  41. #include <linux/iversion.h>
  42. #include "ext4_jbd2.h"
  43. #include "xattr.h"
  44. #include "acl.h"
  45. #include "truncate.h"
  46. #include <trace/events/ext4.h>
  47. #define MPAGE_DA_EXTENT_TAIL 0x01
  48. static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
  49. struct ext4_inode_info *ei)
  50. {
  51. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  52. __u32 csum;
  53. __u16 dummy_csum = 0;
  54. int offset = offsetof(struct ext4_inode, i_checksum_lo);
  55. unsigned int csum_size = sizeof(dummy_csum);
  56. csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);
  57. csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);
  58. offset += csum_size;
  59. csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
  60. EXT4_GOOD_OLD_INODE_SIZE - offset);
  61. if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
  62. offset = offsetof(struct ext4_inode, i_checksum_hi);
  63. csum = ext4_chksum(sbi, csum, (__u8 *)raw +
  64. EXT4_GOOD_OLD_INODE_SIZE,
  65. offset - EXT4_GOOD_OLD_INODE_SIZE);
  66. if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
  67. csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
  68. csum_size);
  69. offset += csum_size;
  70. }
  71. csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
  72. EXT4_INODE_SIZE(inode->i_sb) - offset);
  73. }
  74. return csum;
  75. }
  76. static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
  77. struct ext4_inode_info *ei)
  78. {
  79. __u32 provided, calculated;
  80. if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
  81. cpu_to_le32(EXT4_OS_LINUX) ||
  82. !ext4_has_metadata_csum(inode->i_sb))
  83. return 1;
  84. provided = le16_to_cpu(raw->i_checksum_lo);
  85. calculated = ext4_inode_csum(inode, raw, ei);
  86. if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
  87. EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
  88. provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
  89. else
  90. calculated &= 0xFFFF;
  91. return provided == calculated;
  92. }
  93. static void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
  94. struct ext4_inode_info *ei)
  95. {
  96. __u32 csum;
  97. if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
  98. cpu_to_le32(EXT4_OS_LINUX) ||
  99. !ext4_has_metadata_csum(inode->i_sb))
  100. return;
  101. csum = ext4_inode_csum(inode, raw, ei);
  102. raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
  103. if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
  104. EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
  105. raw->i_checksum_hi = cpu_to_le16(csum >> 16);
  106. }
  107. static inline int ext4_begin_ordered_truncate(struct inode *inode,
  108. loff_t new_size)
  109. {
  110. trace_ext4_begin_ordered_truncate(inode, new_size);
  111. /*
  112. * If jinode is zero, then we never opened the file for
  113. * writing, so there's no need to call
  114. * jbd2_journal_begin_ordered_truncate() since there's no
  115. * outstanding writes we need to flush.
  116. */
  117. if (!EXT4_I(inode)->jinode)
  118. return 0;
  119. return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
  120. EXT4_I(inode)->jinode,
  121. new_size);
  122. }
  123. static void ext4_invalidatepage(struct page *page, unsigned int offset,
  124. unsigned int length);
  125. static int __ext4_journalled_writepage(struct page *page, unsigned int len);
  126. static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh);
  127. static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
  128. int pextents);
  129. /*
  130. * Test whether an inode is a fast symlink.
  131. * A fast symlink has its symlink data stored in ext4_inode_info->i_data.
  132. */
  133. int ext4_inode_is_fast_symlink(struct inode *inode)
  134. {
  135. if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
  136. int ea_blocks = EXT4_I(inode)->i_file_acl ?
  137. EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
  138. if (ext4_has_inline_data(inode))
  139. return 0;
  140. return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
  141. }
  142. return S_ISLNK(inode->i_mode) && inode->i_size &&
  143. (inode->i_size < EXT4_N_BLOCKS * 4);
  144. }
  145. /*
  146. * Restart the transaction associated with *handle. This does a commit,
  147. * so before we call here everything must be consistently dirtied against
  148. * this transaction.
  149. */
  150. int ext4_truncate_restart_trans(handle_t *handle, struct inode *inode,
  151. int nblocks)
  152. {
  153. int ret;
  154. /*
  155. * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this
  156. * moment, get_block can be called only for blocks inside i_size since
  157. * page cache has been already dropped and writes are blocked by
  158. * i_mutex. So we can safely drop the i_data_sem here.
  159. */
  160. BUG_ON(EXT4_JOURNAL(inode) == NULL);
  161. jbd_debug(2, "restarting handle %p\n", handle);
  162. up_write(&EXT4_I(inode)->i_data_sem);
  163. ret = ext4_journal_restart(handle, nblocks);
  164. down_write(&EXT4_I(inode)->i_data_sem);
  165. ext4_discard_preallocations(inode);
  166. return ret;
  167. }
  168. /*
  169. * Called at the last iput() if i_nlink is zero.
  170. */
  171. void ext4_evict_inode(struct inode *inode)
  172. {
  173. handle_t *handle;
  174. int err;
  175. /*
  176. * Credits for final inode cleanup and freeing:
  177. * sb + inode (ext4_orphan_del()), block bitmap, group descriptor
  178. * (xattr block freeing), bitmap, group descriptor (inode freeing)
  179. */
  180. int extra_credits = 6;
  181. struct ext4_xattr_inode_array *ea_inode_array = NULL;
  182. bool freeze_protected = false;
  183. trace_ext4_evict_inode(inode);
  184. if (inode->i_nlink) {
  185. /*
  186. * When journalling data dirty buffers are tracked only in the
  187. * journal. So although mm thinks everything is clean and
  188. * ready for reaping the inode might still have some pages to
  189. * write in the running transaction or waiting to be
  190. * checkpointed. Thus calling jbd2_journal_invalidatepage()
  191. * (via truncate_inode_pages()) to discard these buffers can
  192. * cause data loss. Also even if we did not discard these
  193. * buffers, we would have no way to find them after the inode
  194. * is reaped and thus user could see stale data if he tries to
  195. * read them before the transaction is checkpointed. So be
  196. * careful and force everything to disk here... We use
  197. * ei->i_datasync_tid to store the newest transaction
  198. * containing inode's data.
  199. *
  200. * Note that directories do not have this problem because they
  201. * don't use page cache.
  202. */
  203. if (inode->i_ino != EXT4_JOURNAL_INO &&
  204. ext4_should_journal_data(inode) &&
  205. (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode)) &&
  206. inode->i_data.nrpages) {
  207. journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
  208. tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;
  209. jbd2_complete_transaction(journal, commit_tid);
  210. filemap_write_and_wait(&inode->i_data);
  211. }
  212. truncate_inode_pages_final(&inode->i_data);
  213. goto no_delete;
  214. }
  215. if (is_bad_inode(inode))
  216. goto no_delete;
  217. dquot_initialize(inode);
  218. if (ext4_should_order_data(inode))
  219. ext4_begin_ordered_truncate(inode, 0);
  220. truncate_inode_pages_final(&inode->i_data);
  221. /*
  222. * Protect us against freezing - iput() caller didn't have to have any
  223. * protection against it. When we are in a running transaction though,
  224. * we are already protected against freezing and we cannot grab further
  225. * protection due to lock ordering constraints.
  226. */
  227. if (!ext4_journal_current_handle()) {
  228. sb_start_intwrite(inode->i_sb);
  229. freeze_protected = true;
  230. }
  231. if (!IS_NOQUOTA(inode))
  232. extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
  233. /*
  234. * Block bitmap, group descriptor, and inode are accounted in both
  235. * ext4_blocks_for_truncate() and extra_credits. So subtract 3.
  236. */
  237. handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
  238. ext4_blocks_for_truncate(inode) + extra_credits - 3);
  239. if (IS_ERR(handle)) {
  240. ext4_std_error(inode->i_sb, PTR_ERR(handle));
  241. /*
  242. * If we're going to skip the normal cleanup, we still need to
  243. * make sure that the in-core orphan linked list is properly
  244. * cleaned up.
  245. */
  246. ext4_orphan_del(NULL, inode);
  247. if (freeze_protected)
  248. sb_end_intwrite(inode->i_sb);
  249. goto no_delete;
  250. }
  251. if (IS_SYNC(inode))
  252. ext4_handle_sync(handle);
  253. /*
  254. * Set inode->i_size to 0 before calling ext4_truncate(). We need
  255. * special handling of symlinks here because i_size is used to
  256. * determine whether ext4_inode_info->i_data contains symlink data or
  257. * block mappings. Setting i_size to 0 will remove its fast symlink
  258. * status. Erase i_data so that it becomes a valid empty block map.
  259. */
  260. if (ext4_inode_is_fast_symlink(inode))
  261. memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
  262. inode->i_size = 0;
  263. err = ext4_mark_inode_dirty(handle, inode);
  264. if (err) {
  265. ext4_warning(inode->i_sb,
  266. "couldn't mark inode dirty (err %d)", err);
  267. goto stop_handle;
  268. }
  269. if (inode->i_blocks) {
  270. err = ext4_truncate(inode);
  271. if (err) {
  272. ext4_error(inode->i_sb,
  273. "couldn't truncate inode %lu (err %d)",
  274. inode->i_ino, err);
  275. goto stop_handle;
  276. }
  277. }
  278. /* Remove xattr references. */
  279. err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
  280. extra_credits);
  281. if (err) {
  282. ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
  283. stop_handle:
  284. ext4_journal_stop(handle);
  285. ext4_orphan_del(NULL, inode);
  286. if (freeze_protected)
  287. sb_end_intwrite(inode->i_sb);
  288. ext4_xattr_inode_array_free(ea_inode_array);
  289. goto no_delete;
  290. }
  291. /*
  292. * Kill off the orphan record which ext4_truncate created.
  293. * AKPM: I think this can be inside the above `if'.
  294. * Note that ext4_orphan_del() has to be able to cope with the
  295. * deletion of a non-existent orphan - this is because we don't
  296. * know if ext4_truncate() actually created an orphan record.
  297. * (Well, we could do this if we need to, but heck - it works)
  298. */
  299. ext4_orphan_del(handle, inode);
  300. EXT4_I(inode)->i_dtime = (__u32)ktime_get_real_seconds();
  301. /*
  302. * One subtle ordering requirement: if anything has gone wrong
  303. * (transaction abort, IO errors, whatever), then we can still
  304. * do these next steps (the fs will already have been marked as
  305. * having errors), but we can't free the inode if the mark_dirty
  306. * fails.
  307. */
  308. if (ext4_mark_inode_dirty(handle, inode))
  309. /* If that failed, just do the required in-core inode clear. */
  310. ext4_clear_inode(inode);
  311. else
  312. ext4_free_inode(handle, inode);
  313. ext4_journal_stop(handle);
  314. if (freeze_protected)
  315. sb_end_intwrite(inode->i_sb);
  316. ext4_xattr_inode_array_free(ea_inode_array);
  317. return;
  318. no_delete:
  319. ext4_clear_inode(inode); /* We must guarantee clearing of inode... */
  320. }
  321. #ifdef CONFIG_QUOTA
  322. qsize_t *ext4_get_reserved_space(struct inode *inode)
  323. {
  324. return &EXT4_I(inode)->i_reserved_quota;
  325. }
  326. #endif
  327. /*
  328. * Called with i_data_sem down, which is important since we can call
  329. * ext4_discard_preallocations() from here.
  330. */
  331. void ext4_da_update_reserve_space(struct inode *inode,
  332. int used, int quota_claim)
  333. {
  334. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  335. struct ext4_inode_info *ei = EXT4_I(inode);
  336. spin_lock(&ei->i_block_reservation_lock);
  337. trace_ext4_da_update_reserve_space(inode, used, quota_claim);
  338. if (unlikely(used > ei->i_reserved_data_blocks)) {
  339. ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
  340. "with only %d reserved data blocks",
  341. __func__, inode->i_ino, used,
  342. ei->i_reserved_data_blocks);
  343. WARN_ON(1);
  344. used = ei->i_reserved_data_blocks;
  345. }
  346. /* Update per-inode reservations */
  347. ei->i_reserved_data_blocks -= used;
  348. percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
  349. spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
  350. /* Update quota subsystem for data blocks */
  351. if (quota_claim)
  352. dquot_claim_block(inode, EXT4_C2B(sbi, used));
  353. else {
  354. /*
  355. * We did fallocate with an offset that is already delayed
  356. * allocated. So on delayed allocated writeback we should
  357. * not re-claim the quota for fallocated blocks.
  358. */
  359. dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
  360. }
  361. /*
  362. * If we have done all the pending block allocations and if
  363. * there aren't any writers on the inode, we can discard the
  364. * inode's preallocations.
  365. */
  366. if ((ei->i_reserved_data_blocks == 0) &&
  367. (atomic_read(&inode->i_writecount) == 0))
  368. ext4_discard_preallocations(inode);
  369. }
  370. static int __check_block_validity(struct inode *inode, const char *func,
  371. unsigned int line,
  372. struct ext4_map_blocks *map)
  373. {
  374. if (ext4_has_feature_journal(inode->i_sb) &&
  375. (inode->i_ino ==
  376. le32_to_cpu(EXT4_SB(inode->i_sb)->s_es->s_journal_inum)))
  377. return 0;
  378. if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {
  379. ext4_error_inode(inode, func, line, map->m_pblk,
  380. "lblock %lu mapped to illegal pblock %llu "
  381. "(length %d)", (unsigned long) map->m_lblk,
  382. map->m_pblk, map->m_len);
  383. return -EFSCORRUPTED;
  384. }
  385. return 0;
  386. }
  387. int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
  388. ext4_lblk_t len)
  389. {
  390. int ret;
  391. if (ext4_encrypted_inode(inode))
  392. return fscrypt_zeroout_range(inode, lblk, pblk, len);
  393. ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
  394. if (ret > 0)
  395. ret = 0;
  396. return ret;
  397. }
  398. #define check_block_validity(inode, map) \
  399. __check_block_validity((inode), __func__, __LINE__, (map))
  400. #ifdef ES_AGGRESSIVE_TEST
  401. static void ext4_map_blocks_es_recheck(handle_t *handle,
  402. struct inode *inode,
  403. struct ext4_map_blocks *es_map,
  404. struct ext4_map_blocks *map,
  405. int flags)
  406. {
  407. int retval;
  408. map->m_flags = 0;
  409. /*
  410. * There is a race window that the result is not the same.
  411. * e.g. xfstests #223 when dioread_nolock enables. The reason
  412. * is that we lookup a block mapping in extent status tree with
  413. * out taking i_data_sem. So at the time the unwritten extent
  414. * could be converted.
  415. */
  416. down_read(&EXT4_I(inode)->i_data_sem);
  417. if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
  418. retval = ext4_ext_map_blocks(handle, inode, map, flags &
  419. EXT4_GET_BLOCKS_KEEP_SIZE);
  420. } else {
  421. retval = ext4_ind_map_blocks(handle, inode, map, flags &
  422. EXT4_GET_BLOCKS_KEEP_SIZE);
  423. }
  424. up_read((&EXT4_I(inode)->i_data_sem));
  425. /*
  426. * We don't check m_len because extent will be collpased in status
  427. * tree. So the m_len might not equal.
  428. */
  429. if (es_map->m_lblk != map->m_lblk ||
  430. es_map->m_flags != map->m_flags ||
  431. es_map->m_pblk != map->m_pblk) {
  432. printk("ES cache assertion failed for inode: %lu "
  433. "es_cached ex [%d/%d/%llu/%x] != "
  434. "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
  435. inode->i_ino, es_map->m_lblk, es_map->m_len,
  436. es_map->m_pblk, es_map->m_flags, map->m_lblk,
  437. map->m_len, map->m_pblk, map->m_flags,
  438. retval, flags);
  439. }
  440. }
  441. #endif /* ES_AGGRESSIVE_TEST */
  442. /*
  443. * The ext4_map_blocks() function tries to look up the requested blocks,
  444. * and returns if the blocks are already mapped.
  445. *
  446. * Otherwise it takes the write lock of the i_data_sem and allocate blocks
  447. * and store the allocated blocks in the result buffer head and mark it
  448. * mapped.
  449. *
  450. * If file type is extents based, it will call ext4_ext_map_blocks(),
  451. * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
  452. * based files
  453. *
  454. * On success, it returns the number of blocks being mapped or allocated. if
  455. * create==0 and the blocks are pre-allocated and unwritten, the resulting @map
  456. * is marked as unwritten. If the create == 1, it will mark @map as mapped.
  457. *
  458. * It returns 0 if plain look up failed (blocks have not been allocated), in
  459. * that case, @map is returned as unmapped but we still do fill map->m_len to
  460. * indicate the length of a hole starting at map->m_lblk.
  461. *
  462. * It returns the error in case of allocation failure.
  463. */
  464. int ext4_map_blocks(handle_t *handle, struct inode *inode,
  465. struct ext4_map_blocks *map, int flags)
  466. {
  467. struct extent_status es;
  468. int retval;
  469. int ret = 0;
  470. #ifdef ES_AGGRESSIVE_TEST
  471. struct ext4_map_blocks orig_map;
  472. memcpy(&orig_map, map, sizeof(*map));
  473. #endif
  474. map->m_flags = 0;
  475. ext_debug("ext4_map_blocks(): inode %lu, flag %d, max_blocks %u,"
  476. "logical block %lu\n", inode->i_ino, flags, map->m_len,
  477. (unsigned long) map->m_lblk);
  478. /*
  479. * ext4_map_blocks returns an int, and m_len is an unsigned int
  480. */
  481. if (unlikely(map->m_len > INT_MAX))
  482. map->m_len = INT_MAX;
  483. /* We can handle the block number less than EXT_MAX_BLOCKS */
  484. if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
  485. return -EFSCORRUPTED;
  486. /* Lookup extent status tree firstly */
  487. if (ext4_es_lookup_extent(inode, map->m_lblk, &es)) {
  488. if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
  489. map->m_pblk = ext4_es_pblock(&es) +
  490. map->m_lblk - es.es_lblk;
  491. map->m_flags |= ext4_es_is_written(&es) ?
  492. EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
  493. retval = es.es_len - (map->m_lblk - es.es_lblk);
  494. if (retval > map->m_len)
  495. retval = map->m_len;
  496. map->m_len = retval;
  497. } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
  498. map->m_pblk = 0;
  499. retval = es.es_len - (map->m_lblk - es.es_lblk);
  500. if (retval > map->m_len)
  501. retval = map->m_len;
  502. map->m_len = retval;
  503. retval = 0;
  504. } else {
  505. BUG_ON(1);
  506. }
  507. #ifdef ES_AGGRESSIVE_TEST
  508. ext4_map_blocks_es_recheck(handle, inode, map,
  509. &orig_map, flags);
  510. #endif
  511. goto found;
  512. }
  513. /*
  514. * Try to see if we can get the block without requesting a new
  515. * file system block.
  516. */
  517. down_read(&EXT4_I(inode)->i_data_sem);
  518. if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
  519. retval = ext4_ext_map_blocks(handle, inode, map, flags &
  520. EXT4_GET_BLOCKS_KEEP_SIZE);
  521. } else {
  522. retval = ext4_ind_map_blocks(handle, inode, map, flags &
  523. EXT4_GET_BLOCKS_KEEP_SIZE);
  524. }
  525. if (retval > 0) {
  526. unsigned int status;
  527. if (unlikely(retval != map->m_len)) {
  528. ext4_warning(inode->i_sb,
  529. "ES len assertion failed for inode "
  530. "%lu: retval %d != map->m_len %d",
  531. inode->i_ino, retval, map->m_len);
  532. WARN_ON(1);
  533. }
  534. status = map->m_flags & EXT4_MAP_UNWRITTEN ?
  535. EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
  536. if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
  537. !(status & EXTENT_STATUS_WRITTEN) &&
  538. ext4_find_delalloc_range(inode, map->m_lblk,
  539. map->m_lblk + map->m_len - 1))
  540. status |= EXTENT_STATUS_DELAYED;
  541. ret = ext4_es_insert_extent(inode, map->m_lblk,
  542. map->m_len, map->m_pblk, status);
  543. if (ret < 0)
  544. retval = ret;
  545. }
  546. up_read((&EXT4_I(inode)->i_data_sem));
  547. found:
  548. if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
  549. ret = check_block_validity(inode, map);
  550. if (ret != 0)
  551. return ret;
  552. }
  553. /* If it is only a block(s) look up */
  554. if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
  555. return retval;
  556. /*
  557. * Returns if the blocks have already allocated
  558. *
  559. * Note that if blocks have been preallocated
  560. * ext4_ext_get_block() returns the create = 0
  561. * with buffer head unmapped.
  562. */
  563. if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
  564. /*
  565. * If we need to convert extent to unwritten
  566. * we continue and do the actual work in
  567. * ext4_ext_map_blocks()
  568. */
  569. if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
  570. return retval;
  571. /*
  572. * Here we clear m_flags because after allocating an new extent,
  573. * it will be set again.
  574. */
  575. map->m_flags &= ~EXT4_MAP_FLAGS;
  576. /*
  577. * New blocks allocate and/or writing to unwritten extent
  578. * will possibly result in updating i_data, so we take
  579. * the write lock of i_data_sem, and call get_block()
  580. * with create == 1 flag.
  581. */
  582. down_write(&EXT4_I(inode)->i_data_sem);
  583. /*
  584. * We need to check for EXT4 here because migrate
  585. * could have changed the inode type in between
  586. */
  587. if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
  588. retval = ext4_ext_map_blocks(handle, inode, map, flags);
  589. } else {
  590. retval = ext4_ind_map_blocks(handle, inode, map, flags);
  591. if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
  592. /*
  593. * We allocated new blocks which will result in
  594. * i_data's format changing. Force the migrate
  595. * to fail by clearing migrate flags
  596. */
  597. ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
  598. }
  599. /*
  600. * Update reserved blocks/metadata blocks after successful
  601. * block allocation which had been deferred till now. We don't
  602. * support fallocate for non extent files. So we can update
  603. * reserve space here.
  604. */
  605. if ((retval > 0) &&
  606. (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE))
  607. ext4_da_update_reserve_space(inode, retval, 1);
  608. }
  609. if (retval > 0) {
  610. unsigned int status;
  611. if (unlikely(retval != map->m_len)) {
  612. ext4_warning(inode->i_sb,
  613. "ES len assertion failed for inode "
  614. "%lu: retval %d != map->m_len %d",
  615. inode->i_ino, retval, map->m_len);
  616. WARN_ON(1);
  617. }
  618. /*
  619. * We have to zeroout blocks before inserting them into extent
  620. * status tree. Otherwise someone could look them up there and
  621. * use them before they are really zeroed. We also have to
  622. * unmap metadata before zeroing as otherwise writeback can
  623. * overwrite zeros with stale data from block device.
  624. */
  625. if (flags & EXT4_GET_BLOCKS_ZERO &&
  626. map->m_flags & EXT4_MAP_MAPPED &&
  627. map->m_flags & EXT4_MAP_NEW) {
  628. clean_bdev_aliases(inode->i_sb->s_bdev, map->m_pblk,
  629. map->m_len);
  630. ret = ext4_issue_zeroout(inode, map->m_lblk,
  631. map->m_pblk, map->m_len);
  632. if (ret) {
  633. retval = ret;
  634. goto out_sem;
  635. }
  636. }
  637. /*
  638. * If the extent has been zeroed out, we don't need to update
  639. * extent status tree.
  640. */
  641. if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
  642. ext4_es_lookup_extent(inode, map->m_lblk, &es)) {
  643. if (ext4_es_is_written(&es))
  644. goto out_sem;
  645. }
  646. status = map->m_flags & EXT4_MAP_UNWRITTEN ?
  647. EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
  648. if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
  649. !(status & EXTENT_STATUS_WRITTEN) &&
  650. ext4_find_delalloc_range(inode, map->m_lblk,
  651. map->m_lblk + map->m_len - 1))
  652. status |= EXTENT_STATUS_DELAYED;
  653. ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
  654. map->m_pblk, status);
  655. if (ret < 0) {
  656. retval = ret;
  657. goto out_sem;
  658. }
  659. }
  660. out_sem:
  661. up_write((&EXT4_I(inode)->i_data_sem));
  662. if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
  663. ret = check_block_validity(inode, map);
  664. if (ret != 0)
  665. return ret;
  666. /*
  667. * Inodes with freshly allocated blocks where contents will be
  668. * visible after transaction commit must be on transaction's
  669. * ordered data list.
  670. */
  671. if (map->m_flags & EXT4_MAP_NEW &&
  672. !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
  673. !(flags & EXT4_GET_BLOCKS_ZERO) &&
  674. !ext4_is_quota_file(inode) &&
  675. ext4_should_order_data(inode)) {
  676. loff_t start_byte =
  677. (loff_t)map->m_lblk << inode->i_blkbits;
  678. loff_t length = (loff_t)map->m_len << inode->i_blkbits;
  679. if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
  680. ret = ext4_jbd2_inode_add_wait(handle, inode,
  681. start_byte, length);
  682. else
  683. ret = ext4_jbd2_inode_add_write(handle, inode,
  684. start_byte, length);
  685. if (ret)
  686. return ret;
  687. }
  688. }
  689. return retval;
  690. }
  691. /*
  692. * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
  693. * we have to be careful as someone else may be manipulating b_state as well.
  694. */
  695. static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
  696. {
  697. unsigned long old_state;
  698. unsigned long new_state;
  699. flags &= EXT4_MAP_FLAGS;
  700. /* Dummy buffer_head? Set non-atomically. */
  701. if (!bh->b_page) {
  702. bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
  703. return;
  704. }
  705. /*
  706. * Someone else may be modifying b_state. Be careful! This is ugly but
  707. * once we get rid of using bh as a container for mapping information
  708. * to pass to / from get_block functions, this can go away.
  709. */
  710. do {
  711. old_state = READ_ONCE(bh->b_state);
  712. new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
  713. } while (unlikely(
  714. cmpxchg(&bh->b_state, old_state, new_state) != old_state));
  715. }
  716. static int _ext4_get_block(struct inode *inode, sector_t iblock,
  717. struct buffer_head *bh, int flags)
  718. {
  719. struct ext4_map_blocks map;
  720. int ret = 0;
  721. if (ext4_has_inline_data(inode))
  722. return -ERANGE;
  723. map.m_lblk = iblock;
  724. map.m_len = bh->b_size >> inode->i_blkbits;
  725. ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
  726. flags);
  727. if (ret > 0) {
  728. map_bh(bh, inode->i_sb, map.m_pblk);
  729. ext4_update_bh_state(bh, map.m_flags);
  730. bh->b_size = inode->i_sb->s_blocksize * map.m_len;
  731. ret = 0;
  732. } else if (ret == 0) {
  733. /* hole case, need to fill in bh->b_size */
  734. bh->b_size = inode->i_sb->s_blocksize * map.m_len;
  735. }
  736. return ret;
  737. }
  738. int ext4_get_block(struct inode *inode, sector_t iblock,
  739. struct buffer_head *bh, int create)
  740. {
  741. return _ext4_get_block(inode, iblock, bh,
  742. create ? EXT4_GET_BLOCKS_CREATE : 0);
  743. }
  744. /*
  745. * Get block function used when preparing for buffered write if we require
  746. * creating an unwritten extent if blocks haven't been allocated. The extent
  747. * will be converted to written after the IO is complete.
  748. */
  749. int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
  750. struct buffer_head *bh_result, int create)
  751. {
  752. ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
  753. inode->i_ino, create);
  754. return _ext4_get_block(inode, iblock, bh_result,
  755. EXT4_GET_BLOCKS_IO_CREATE_EXT);
  756. }
  757. /* Maximum number of blocks we map for direct IO at once. */
  758. #define DIO_MAX_BLOCKS 4096
  759. /*
  760. * Get blocks function for the cases that need to start a transaction -
  761. * generally difference cases of direct IO and DAX IO. It also handles retries
  762. * in case of ENOSPC.
  763. */
  764. static int ext4_get_block_trans(struct inode *inode, sector_t iblock,
  765. struct buffer_head *bh_result, int flags)
  766. {
  767. int dio_credits;
  768. handle_t *handle;
  769. int retries = 0;
  770. int ret;
  771. /* Trim mapping request to maximum we can map at once for DIO */
  772. if (bh_result->b_size >> inode->i_blkbits > DIO_MAX_BLOCKS)
  773. bh_result->b_size = DIO_MAX_BLOCKS << inode->i_blkbits;
  774. dio_credits = ext4_chunk_trans_blocks(inode,
  775. bh_result->b_size >> inode->i_blkbits);
  776. retry:
  777. handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
  778. if (IS_ERR(handle))
  779. return PTR_ERR(handle);
  780. ret = _ext4_get_block(inode, iblock, bh_result, flags);
  781. ext4_journal_stop(handle);
  782. if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
  783. goto retry;
  784. return ret;
  785. }
  786. /* Get block function for DIO reads and writes to inodes without extents */
  787. int ext4_dio_get_block(struct inode *inode, sector_t iblock,
  788. struct buffer_head *bh, int create)
  789. {
  790. /* We don't expect handle for direct IO */
  791. WARN_ON_ONCE(ext4_journal_current_handle());
  792. if (!create)
  793. return _ext4_get_block(inode, iblock, bh, 0);
  794. return ext4_get_block_trans(inode, iblock, bh, EXT4_GET_BLOCKS_CREATE);
  795. }
  796. /*
  797. * Get block function for AIO DIO writes when we create unwritten extent if
  798. * blocks are not allocated yet. The extent will be converted to written
  799. * after IO is complete.
  800. */
  801. static int ext4_dio_get_block_unwritten_async(struct inode *inode,
  802. sector_t iblock, struct buffer_head *bh_result, int create)
  803. {
  804. int ret;
  805. /* We don't expect handle for direct IO */
  806. WARN_ON_ONCE(ext4_journal_current_handle());
  807. ret = ext4_get_block_trans(inode, iblock, bh_result,
  808. EXT4_GET_BLOCKS_IO_CREATE_EXT);
  809. /*
  810. * When doing DIO using unwritten extents, we need io_end to convert
  811. * unwritten extents to written on IO completion. We allocate io_end
  812. * once we spot unwritten extent and store it in b_private. Generic
  813. * DIO code keeps b_private set and furthermore passes the value to
  814. * our completion callback in 'private' argument.
  815. */
  816. if (!ret && buffer_unwritten(bh_result)) {
  817. if (!bh_result->b_private) {
  818. ext4_io_end_t *io_end;
  819. io_end = ext4_init_io_end(inode, GFP_KERNEL);
  820. if (!io_end)
  821. return -ENOMEM;
  822. bh_result->b_private = io_end;
  823. ext4_set_io_unwritten_flag(inode, io_end);
  824. }
  825. set_buffer_defer_completion(bh_result);
  826. }
  827. return ret;
  828. }
  829. /*
  830. * Get block function for non-AIO DIO writes when we create unwritten extent if
  831. * blocks are not allocated yet. The extent will be converted to written
  832. * after IO is complete by ext4_direct_IO_write().
  833. */
  834. static int ext4_dio_get_block_unwritten_sync(struct inode *inode,
  835. sector_t iblock, struct buffer_head *bh_result, int create)
  836. {
  837. int ret;
  838. /* We don't expect handle for direct IO */
  839. WARN_ON_ONCE(ext4_journal_current_handle());
  840. ret = ext4_get_block_trans(inode, iblock, bh_result,
  841. EXT4_GET_BLOCKS_IO_CREATE_EXT);
  842. /*
  843. * Mark inode as having pending DIO writes to unwritten extents.
  844. * ext4_direct_IO_write() checks this flag and converts extents to
  845. * written.
  846. */
  847. if (!ret && buffer_unwritten(bh_result))
  848. ext4_set_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
  849. return ret;
  850. }
  851. static int ext4_dio_get_block_overwrite(struct inode *inode, sector_t iblock,
  852. struct buffer_head *bh_result, int create)
  853. {
  854. int ret;
  855. ext4_debug("ext4_dio_get_block_overwrite: inode %lu, create flag %d\n",
  856. inode->i_ino, create);
  857. /* We don't expect handle for direct IO */
  858. WARN_ON_ONCE(ext4_journal_current_handle());
  859. ret = _ext4_get_block(inode, iblock, bh_result, 0);
  860. /*
  861. * Blocks should have been preallocated! ext4_file_write_iter() checks
  862. * that.
  863. */
  864. WARN_ON_ONCE(!buffer_mapped(bh_result) || buffer_unwritten(bh_result));
  865. return ret;
  866. }
  867. /*
  868. * `handle' can be NULL if create is zero
  869. */
  870. struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
  871. ext4_lblk_t block, int map_flags)
  872. {
  873. struct ext4_map_blocks map;
  874. struct buffer_head *bh;
  875. int create = map_flags & EXT4_GET_BLOCKS_CREATE;
  876. int err;
  877. J_ASSERT(handle != NULL || create == 0);
  878. map.m_lblk = block;
  879. map.m_len = 1;
  880. err = ext4_map_blocks(handle, inode, &map, map_flags);
  881. if (err == 0)
  882. return create ? ERR_PTR(-ENOSPC) : NULL;
  883. if (err < 0)
  884. return ERR_PTR(err);
  885. bh = sb_getblk(inode->i_sb, map.m_pblk);
  886. if (unlikely(!bh))
  887. return ERR_PTR(-ENOMEM);
  888. if (map.m_flags & EXT4_MAP_NEW) {
  889. J_ASSERT(create != 0);
  890. J_ASSERT(handle != NULL);
  891. /*
  892. * Now that we do not always journal data, we should
  893. * keep in mind whether this should always journal the
  894. * new buffer as metadata. For now, regular file
  895. * writes use ext4_get_block instead, so it's not a
  896. * problem.
  897. */
  898. lock_buffer(bh);
  899. BUFFER_TRACE(bh, "call get_create_access");
  900. err = ext4_journal_get_create_access(handle, bh);
  901. if (unlikely(err)) {
  902. unlock_buffer(bh);
  903. goto errout;
  904. }
  905. if (!buffer_uptodate(bh)) {
  906. memset(bh->b_data, 0, inode->i_sb->s_blocksize);
  907. set_buffer_uptodate(bh);
  908. }
  909. unlock_buffer(bh);
  910. BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
  911. err = ext4_handle_dirty_metadata(handle, inode, bh);
  912. if (unlikely(err))
  913. goto errout;
  914. } else
  915. BUFFER_TRACE(bh, "not a new buffer");
  916. return bh;
  917. errout:
  918. brelse(bh);
  919. return ERR_PTR(err);
  920. }
  921. struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
  922. ext4_lblk_t block, int map_flags)
  923. {
  924. struct buffer_head *bh;
  925. bh = ext4_getblk(handle, inode, block, map_flags);
  926. if (IS_ERR(bh))
  927. return bh;
  928. if (!bh || buffer_uptodate(bh))
  929. return bh;
  930. ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &bh);
  931. wait_on_buffer(bh);
  932. if (buffer_uptodate(bh))
  933. return bh;
  934. put_bh(bh);
  935. return ERR_PTR(-EIO);
  936. }
  937. /* Read a contiguous batch of blocks. */
  938. int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
  939. bool wait, struct buffer_head **bhs)
  940. {
  941. int i, err;
  942. for (i = 0; i < bh_count; i++) {
  943. bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */);
  944. if (IS_ERR(bhs[i])) {
  945. err = PTR_ERR(bhs[i]);
  946. bh_count = i;
  947. goto out_brelse;
  948. }
  949. }
  950. for (i = 0; i < bh_count; i++)
  951. /* Note that NULL bhs[i] is valid because of holes. */
  952. if (bhs[i] && !buffer_uptodate(bhs[i]))
  953. ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1,
  954. &bhs[i]);
  955. if (!wait)
  956. return 0;
  957. for (i = 0; i < bh_count; i++)
  958. if (bhs[i])
  959. wait_on_buffer(bhs[i]);
  960. for (i = 0; i < bh_count; i++) {
  961. if (bhs[i] && !buffer_uptodate(bhs[i])) {
  962. err = -EIO;
  963. goto out_brelse;
  964. }
  965. }
  966. return 0;
  967. out_brelse:
  968. for (i = 0; i < bh_count; i++) {
  969. brelse(bhs[i]);
  970. bhs[i] = NULL;
  971. }
  972. return err;
  973. }
  974. int ext4_walk_page_buffers(handle_t *handle,
  975. struct buffer_head *head,
  976. unsigned from,
  977. unsigned to,
  978. int *partial,
  979. int (*fn)(handle_t *handle,
  980. struct buffer_head *bh))
  981. {
  982. struct buffer_head *bh;
  983. unsigned block_start, block_end;
  984. unsigned blocksize = head->b_size;
  985. int err, ret = 0;
  986. struct buffer_head *next;
  987. for (bh = head, block_start = 0;
  988. ret == 0 && (bh != head || !block_start);
  989. block_start = block_end, bh = next) {
  990. next = bh->b_this_page;
  991. block_end = block_start + blocksize;
  992. if (block_end <= from || block_start >= to) {
  993. if (partial && !buffer_uptodate(bh))
  994. *partial = 1;
  995. continue;
  996. }
  997. err = (*fn)(handle, bh);
  998. if (!ret)
  999. ret = err;
  1000. }
  1001. return ret;
  1002. }
  1003. /*
  1004. * To preserve ordering, it is essential that the hole instantiation and
  1005. * the data write be encapsulated in a single transaction. We cannot
  1006. * close off a transaction and start a new one between the ext4_get_block()
  1007. * and the commit_write(). So doing the jbd2_journal_start at the start of
  1008. * prepare_write() is the right place.
  1009. *
  1010. * Also, this function can nest inside ext4_writepage(). In that case, we
  1011. * *know* that ext4_writepage() has generated enough buffer credits to do the
  1012. * whole page. So we won't block on the journal in that case, which is good,
  1013. * because the caller may be PF_MEMALLOC.
  1014. *
  1015. * By accident, ext4 can be reentered when a transaction is open via
  1016. * quota file writes. If we were to commit the transaction while thus
  1017. * reentered, there can be a deadlock - we would be holding a quota
  1018. * lock, and the commit would never complete if another thread had a
  1019. * transaction open and was blocking on the quota lock - a ranking
  1020. * violation.
  1021. *
  1022. * So what we do is to rely on the fact that jbd2_journal_stop/journal_start
  1023. * will _not_ run commit under these circumstances because handle->h_ref
  1024. * is elevated. We'll still have enough credits for the tiny quotafile
  1025. * write.
  1026. */
  1027. int do_journal_get_write_access(handle_t *handle,
  1028. struct buffer_head *bh)
  1029. {
  1030. int dirty = buffer_dirty(bh);
  1031. int ret;
  1032. if (!buffer_mapped(bh) || buffer_freed(bh))
  1033. return 0;
  1034. /*
  1035. * __block_write_begin() could have dirtied some buffers. Clean
  1036. * the dirty bit as jbd2_journal_get_write_access() could complain
  1037. * otherwise about fs integrity issues. Setting of the dirty bit
  1038. * by __block_write_begin() isn't a real problem here as we clear
  1039. * the bit before releasing a page lock and thus writeback cannot
  1040. * ever write the buffer.
  1041. */
  1042. if (dirty)
  1043. clear_buffer_dirty(bh);
  1044. BUFFER_TRACE(bh, "get write access");
  1045. ret = ext4_journal_get_write_access(handle, bh);
  1046. if (!ret && dirty)
  1047. ret = ext4_handle_dirty_metadata(handle, NULL, bh);
  1048. return ret;
  1049. }
  1050. #ifdef CONFIG_EXT4_FS_ENCRYPTION
  1051. static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
  1052. get_block_t *get_block)
  1053. {
  1054. unsigned from = pos & (PAGE_SIZE - 1);
  1055. unsigned to = from + len;
  1056. struct inode *inode = page->mapping->host;
  1057. unsigned block_start, block_end;
  1058. sector_t block;
  1059. int err = 0;
  1060. unsigned blocksize = inode->i_sb->s_blocksize;
  1061. unsigned bbits;
  1062. struct buffer_head *bh, *head, *wait[2], **wait_bh = wait;
  1063. bool decrypt = false;
  1064. BUG_ON(!PageLocked(page));
  1065. BUG_ON(from > PAGE_SIZE);
  1066. BUG_ON(to > PAGE_SIZE);
  1067. BUG_ON(from > to);
  1068. if (!page_has_buffers(page))
  1069. create_empty_buffers(page, blocksize, 0);
  1070. head = page_buffers(page);
  1071. bbits = ilog2(blocksize);
  1072. block = (sector_t)page->index << (PAGE_SHIFT - bbits);
  1073. for (bh = head, block_start = 0; bh != head || !block_start;
  1074. block++, block_start = block_end, bh = bh->b_this_page) {
  1075. block_end = block_start + blocksize;
  1076. if (block_end <= from || block_start >= to) {
  1077. if (PageUptodate(page)) {
  1078. if (!buffer_uptodate(bh))
  1079. set_buffer_uptodate(bh);
  1080. }
  1081. continue;
  1082. }
  1083. if (buffer_new(bh))
  1084. clear_buffer_new(bh);
  1085. if (!buffer_mapped(bh)) {
  1086. WARN_ON(bh->b_size != blocksize);
  1087. err = get_block(inode, block, bh, 1);
  1088. if (err)
  1089. break;
  1090. if (buffer_new(bh)) {
  1091. clean_bdev_bh_alias(bh);
  1092. if (PageUptodate(page)) {
  1093. clear_buffer_new(bh);
  1094. set_buffer_uptodate(bh);
  1095. mark_buffer_dirty(bh);
  1096. continue;
  1097. }
  1098. if (block_end > to || block_start < from)
  1099. zero_user_segments(page, to, block_end,
  1100. block_start, from);
  1101. continue;
  1102. }
  1103. }
  1104. if (PageUptodate(page)) {
  1105. if (!buffer_uptodate(bh))
  1106. set_buffer_uptodate(bh);
  1107. continue;
  1108. }
  1109. if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
  1110. !buffer_unwritten(bh) &&
  1111. (block_start < from || block_end > to)) {
  1112. ll_rw_block(REQ_OP_READ, 0, 1, &bh);
  1113. *wait_bh++ = bh;
  1114. decrypt = ext4_encrypted_inode(inode) &&
  1115. S_ISREG(inode->i_mode);
  1116. }
  1117. }
  1118. /*
  1119. * If we issued read requests, let them complete.
  1120. */
  1121. while (wait_bh > wait) {
  1122. wait_on_buffer(*--wait_bh);
  1123. if (!buffer_uptodate(*wait_bh))
  1124. err = -EIO;
  1125. }
  1126. if (unlikely(err))
  1127. page_zero_new_buffers(page, from, to);
  1128. else if (decrypt)
  1129. err = fscrypt_decrypt_page(page->mapping->host, page,
  1130. PAGE_SIZE, 0, page->index);
  1131. return err;
  1132. }
  1133. #endif
  1134. static int ext4_write_begin(struct file *file, struct address_space *mapping,
  1135. loff_t pos, unsigned len, unsigned flags,
  1136. struct page **pagep, void **fsdata)
  1137. {
  1138. struct inode *inode = mapping->host;
  1139. int ret, needed_blocks;
  1140. handle_t *handle;
  1141. int retries = 0;
  1142. struct page *page;
  1143. pgoff_t index;
  1144. unsigned from, to;
  1145. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
  1146. return -EIO;
  1147. trace_ext4_write_begin(inode, pos, len, flags);
  1148. /*
  1149. * Reserve one block more for addition to orphan list in case
  1150. * we allocate blocks but write fails for some reason
  1151. */
  1152. needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
  1153. index = pos >> PAGE_SHIFT;
  1154. from = pos & (PAGE_SIZE - 1);
  1155. to = from + len;
  1156. if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
  1157. ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
  1158. flags, pagep);
  1159. if (ret < 0)
  1160. return ret;
  1161. if (ret == 1)
  1162. return 0;
  1163. }
  1164. /*
  1165. * grab_cache_page_write_begin() can take a long time if the
  1166. * system is thrashing due to memory pressure, or if the page
  1167. * is being written back. So grab it first before we start
  1168. * the transaction handle. This also allows us to allocate
  1169. * the page (if needed) without using GFP_NOFS.
  1170. */
  1171. retry_grab:
  1172. page = grab_cache_page_write_begin(mapping, index, flags);
  1173. if (!page)
  1174. return -ENOMEM;
  1175. unlock_page(page);
  1176. retry_journal:
  1177. handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
  1178. if (IS_ERR(handle)) {
  1179. put_page(page);
  1180. return PTR_ERR(handle);
  1181. }
  1182. lock_page(page);
  1183. if (page->mapping != mapping) {
  1184. /* The page got truncated from under us */
  1185. unlock_page(page);
  1186. put_page(page);
  1187. ext4_journal_stop(handle);
  1188. goto retry_grab;
  1189. }
  1190. /* In case writeback began while the page was unlocked */
  1191. wait_for_stable_page(page);
  1192. #ifdef CONFIG_EXT4_FS_ENCRYPTION
  1193. if (ext4_should_dioread_nolock(inode))
  1194. ret = ext4_block_write_begin(page, pos, len,
  1195. ext4_get_block_unwritten);
  1196. else
  1197. ret = ext4_block_write_begin(page, pos, len,
  1198. ext4_get_block);
  1199. #else
  1200. if (ext4_should_dioread_nolock(inode))
  1201. ret = __block_write_begin(page, pos, len,
  1202. ext4_get_block_unwritten);
  1203. else
  1204. ret = __block_write_begin(page, pos, len, ext4_get_block);
  1205. #endif
  1206. if (!ret && ext4_should_journal_data(inode)) {
  1207. ret = ext4_walk_page_buffers(handle, page_buffers(page),
  1208. from, to, NULL,
  1209. do_journal_get_write_access);
  1210. }
  1211. if (ret) {
  1212. unlock_page(page);
  1213. /*
  1214. * __block_write_begin may have instantiated a few blocks
  1215. * outside i_size. Trim these off again. Don't need
  1216. * i_size_read because we hold i_mutex.
  1217. *
  1218. * Add inode to orphan list in case we crash before
  1219. * truncate finishes
  1220. */
  1221. if (pos + len > inode->i_size && ext4_can_truncate(inode))
  1222. ext4_orphan_add(handle, inode);
  1223. ext4_journal_stop(handle);
  1224. if (pos + len > inode->i_size) {
  1225. ext4_truncate_failed_write(inode);
  1226. /*
  1227. * If truncate failed early the inode might
  1228. * still be on the orphan list; we need to
  1229. * make sure the inode is removed from the
  1230. * orphan list in that case.
  1231. */
  1232. if (inode->i_nlink)
  1233. ext4_orphan_del(NULL, inode);
  1234. }
  1235. if (ret == -ENOSPC &&
  1236. ext4_should_retry_alloc(inode->i_sb, &retries))
  1237. goto retry_journal;
  1238. put_page(page);
  1239. return ret;
  1240. }
  1241. *pagep = page;
  1242. return ret;
  1243. }
  1244. /* For write_end() in data=journal mode */
  1245. static int write_end_fn(handle_t *handle, struct buffer_head *bh)
  1246. {
  1247. int ret;
  1248. if (!buffer_mapped(bh) || buffer_freed(bh))
  1249. return 0;
  1250. set_buffer_uptodate(bh);
  1251. ret = ext4_handle_dirty_metadata(handle, NULL, bh);
  1252. clear_buffer_meta(bh);
  1253. clear_buffer_prio(bh);
  1254. return ret;
  1255. }
  1256. /*
  1257. * We need to pick up the new inode size which generic_commit_write gave us
  1258. * `file' can be NULL - eg, when called from page_symlink().
  1259. *
  1260. * ext4 never places buffers on inode->i_mapping->private_list. metadata
  1261. * buffers are managed internally.
  1262. */
  1263. static int ext4_write_end(struct file *file,
  1264. struct address_space *mapping,
  1265. loff_t pos, unsigned len, unsigned copied,
  1266. struct page *page, void *fsdata)
  1267. {
  1268. handle_t *handle = ext4_journal_current_handle();
  1269. struct inode *inode = mapping->host;
  1270. loff_t old_size = inode->i_size;
  1271. int ret = 0, ret2;
  1272. int i_size_changed = 0;
  1273. int inline_data = ext4_has_inline_data(inode);
  1274. trace_ext4_write_end(inode, pos, len, copied);
  1275. if (inline_data) {
  1276. ret = ext4_write_inline_data_end(inode, pos, len,
  1277. copied, page);
  1278. if (ret < 0) {
  1279. unlock_page(page);
  1280. put_page(page);
  1281. goto errout;
  1282. }
  1283. copied = ret;
  1284. } else
  1285. copied = block_write_end(file, mapping, pos,
  1286. len, copied, page, fsdata);
  1287. /*
  1288. * it's important to update i_size while still holding page lock:
  1289. * page writeout could otherwise come in and zero beyond i_size.
  1290. */
  1291. i_size_changed = ext4_update_inode_size(inode, pos + copied);
  1292. unlock_page(page);
  1293. put_page(page);
  1294. if (old_size < pos)
  1295. pagecache_isize_extended(inode, old_size, pos);
  1296. /*
  1297. * Don't mark the inode dirty under page lock. First, it unnecessarily
  1298. * makes the holding time of page lock longer. Second, it forces lock
  1299. * ordering of page lock and transaction start for journaling
  1300. * filesystems.
  1301. */
  1302. if (i_size_changed || inline_data)
  1303. ext4_mark_inode_dirty(handle, inode);
  1304. if (pos + len > inode->i_size && ext4_can_truncate(inode))
  1305. /* if we have allocated more blocks and copied
  1306. * less. We will have blocks allocated outside
  1307. * inode->i_size. So truncate them
  1308. */
  1309. ext4_orphan_add(handle, inode);
  1310. errout:
  1311. ret2 = ext4_journal_stop(handle);
  1312. if (!ret)
  1313. ret = ret2;
  1314. if (pos + len > inode->i_size) {
  1315. ext4_truncate_failed_write(inode);
  1316. /*
  1317. * If truncate failed early the inode might still be
  1318. * on the orphan list; we need to make sure the inode
  1319. * is removed from the orphan list in that case.
  1320. */
  1321. if (inode->i_nlink)
  1322. ext4_orphan_del(NULL, inode);
  1323. }
  1324. return ret ? ret : copied;
  1325. }
  1326. /*
  1327. * This is a private version of page_zero_new_buffers() which doesn't
  1328. * set the buffer to be dirty, since in data=journalled mode we need
  1329. * to call ext4_handle_dirty_metadata() instead.
  1330. */
  1331. static void ext4_journalled_zero_new_buffers(handle_t *handle,
  1332. struct page *page,
  1333. unsigned from, unsigned to)
  1334. {
  1335. unsigned int block_start = 0, block_end;
  1336. struct buffer_head *head, *bh;
  1337. bh = head = page_buffers(page);
  1338. do {
  1339. block_end = block_start + bh->b_size;
  1340. if (buffer_new(bh)) {
  1341. if (block_end > from && block_start < to) {
  1342. if (!PageUptodate(page)) {
  1343. unsigned start, size;
  1344. start = max(from, block_start);
  1345. size = min(to, block_end) - start;
  1346. zero_user(page, start, size);
  1347. write_end_fn(handle, bh);
  1348. }
  1349. clear_buffer_new(bh);
  1350. }
  1351. }
  1352. block_start = block_end;
  1353. bh = bh->b_this_page;
  1354. } while (bh != head);
  1355. }
  1356. static int ext4_journalled_write_end(struct file *file,
  1357. struct address_space *mapping,
  1358. loff_t pos, unsigned len, unsigned copied,
  1359. struct page *page, void *fsdata)
  1360. {
  1361. handle_t *handle = ext4_journal_current_handle();
  1362. struct inode *inode = mapping->host;
  1363. loff_t old_size = inode->i_size;
  1364. int ret = 0, ret2;
  1365. int partial = 0;
  1366. unsigned from, to;
  1367. int size_changed = 0;
  1368. int inline_data = ext4_has_inline_data(inode);
  1369. trace_ext4_journalled_write_end(inode, pos, len, copied);
  1370. from = pos & (PAGE_SIZE - 1);
  1371. to = from + len;
  1372. BUG_ON(!ext4_handle_valid(handle));
  1373. if (inline_data) {
  1374. ret = ext4_write_inline_data_end(inode, pos, len,
  1375. copied, page);
  1376. if (ret < 0) {
  1377. unlock_page(page);
  1378. put_page(page);
  1379. goto errout;
  1380. }
  1381. copied = ret;
  1382. } else if (unlikely(copied < len) && !PageUptodate(page)) {
  1383. copied = 0;
  1384. ext4_journalled_zero_new_buffers(handle, page, from, to);
  1385. } else {
  1386. if (unlikely(copied < len))
  1387. ext4_journalled_zero_new_buffers(handle, page,
  1388. from + copied, to);
  1389. ret = ext4_walk_page_buffers(handle, page_buffers(page), from,
  1390. from + copied, &partial,
  1391. write_end_fn);
  1392. if (!partial)
  1393. SetPageUptodate(page);
  1394. }
  1395. size_changed = ext4_update_inode_size(inode, pos + copied);
  1396. ext4_set_inode_state(inode, EXT4_STATE_JDATA);
  1397. EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
  1398. unlock_page(page);
  1399. put_page(page);
  1400. if (old_size < pos)
  1401. pagecache_isize_extended(inode, old_size, pos);
  1402. if (size_changed || inline_data) {
  1403. ret2 = ext4_mark_inode_dirty(handle, inode);
  1404. if (!ret)
  1405. ret = ret2;
  1406. }
  1407. if (pos + len > inode->i_size && ext4_can_truncate(inode))
  1408. /* if we have allocated more blocks and copied
  1409. * less. We will have blocks allocated outside
  1410. * inode->i_size. So truncate them
  1411. */
  1412. ext4_orphan_add(handle, inode);
  1413. errout:
  1414. ret2 = ext4_journal_stop(handle);
  1415. if (!ret)
  1416. ret = ret2;
  1417. if (pos + len > inode->i_size) {
  1418. ext4_truncate_failed_write(inode);
  1419. /*
  1420. * If truncate failed early the inode might still be
  1421. * on the orphan list; we need to make sure the inode
  1422. * is removed from the orphan list in that case.
  1423. */
  1424. if (inode->i_nlink)
  1425. ext4_orphan_del(NULL, inode);
  1426. }
  1427. return ret ? ret : copied;
  1428. }
  1429. /*
  1430. * Reserve space for a single cluster
  1431. */
  1432. static int ext4_da_reserve_space(struct inode *inode)
  1433. {
  1434. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  1435. struct ext4_inode_info *ei = EXT4_I(inode);
  1436. int ret;
  1437. /*
  1438. * We will charge metadata quota at writeout time; this saves
  1439. * us from metadata over-estimation, though we may go over by
  1440. * a small amount in the end. Here we just reserve for data.
  1441. */
  1442. ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
  1443. if (ret)
  1444. return ret;
  1445. spin_lock(&ei->i_block_reservation_lock);
  1446. if (ext4_claim_free_clusters(sbi, 1, 0)) {
  1447. spin_unlock(&ei->i_block_reservation_lock);
  1448. dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
  1449. return -ENOSPC;
  1450. }
  1451. ei->i_reserved_data_blocks++;
  1452. trace_ext4_da_reserve_space(inode);
  1453. spin_unlock(&ei->i_block_reservation_lock);
  1454. return 0; /* success */
  1455. }
  1456. static void ext4_da_release_space(struct inode *inode, int to_free)
  1457. {
  1458. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  1459. struct ext4_inode_info *ei = EXT4_I(inode);
  1460. if (!to_free)
  1461. return; /* Nothing to release, exit */
  1462. spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
  1463. trace_ext4_da_release_space(inode, to_free);
  1464. if (unlikely(to_free > ei->i_reserved_data_blocks)) {
  1465. /*
  1466. * if there aren't enough reserved blocks, then the
  1467. * counter is messed up somewhere. Since this
  1468. * function is called from invalidate page, it's
  1469. * harmless to return without any action.
  1470. */
  1471. ext4_warning(inode->i_sb, "ext4_da_release_space: "
  1472. "ino %lu, to_free %d with only %d reserved "
  1473. "data blocks", inode->i_ino, to_free,
  1474. ei->i_reserved_data_blocks);
  1475. WARN_ON(1);
  1476. to_free = ei->i_reserved_data_blocks;
  1477. }
  1478. ei->i_reserved_data_blocks -= to_free;
  1479. /* update fs dirty data blocks counter */
  1480. percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
  1481. spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
  1482. dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
  1483. }
  1484. static void ext4_da_page_release_reservation(struct page *page,
  1485. unsigned int offset,
  1486. unsigned int length)
  1487. {
  1488. int to_release = 0, contiguous_blks = 0;
  1489. struct buffer_head *head, *bh;
  1490. unsigned int curr_off = 0;
  1491. struct inode *inode = page->mapping->host;
  1492. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  1493. unsigned int stop = offset + length;
  1494. int num_clusters;
  1495. ext4_fsblk_t lblk;
  1496. BUG_ON(stop > PAGE_SIZE || stop < length);
  1497. head = page_buffers(page);
  1498. bh = head;
  1499. do {
  1500. unsigned int next_off = curr_off + bh->b_size;
  1501. if (next_off > stop)
  1502. break;
  1503. if ((offset <= curr_off) && (buffer_delay(bh))) {
  1504. to_release++;
  1505. contiguous_blks++;
  1506. clear_buffer_delay(bh);
  1507. } else if (contiguous_blks) {
  1508. lblk = page->index <<
  1509. (PAGE_SHIFT - inode->i_blkbits);
  1510. lblk += (curr_off >> inode->i_blkbits) -
  1511. contiguous_blks;
  1512. ext4_es_remove_extent(inode, lblk, contiguous_blks);
  1513. contiguous_blks = 0;
  1514. }
  1515. curr_off = next_off;
  1516. } while ((bh = bh->b_this_page) != head);
  1517. if (contiguous_blks) {
  1518. lblk = page->index << (PAGE_SHIFT - inode->i_blkbits);
  1519. lblk += (curr_off >> inode->i_blkbits) - contiguous_blks;
  1520. ext4_es_remove_extent(inode, lblk, contiguous_blks);
  1521. }
  1522. /* If we have released all the blocks belonging to a cluster, then we
  1523. * need to release the reserved space for that cluster. */
  1524. num_clusters = EXT4_NUM_B2C(sbi, to_release);
  1525. while (num_clusters > 0) {
  1526. lblk = (page->index << (PAGE_SHIFT - inode->i_blkbits)) +
  1527. ((num_clusters - 1) << sbi->s_cluster_bits);
  1528. if (sbi->s_cluster_ratio == 1 ||
  1529. !ext4_find_delalloc_cluster(inode, lblk))
  1530. ext4_da_release_space(inode, 1);
  1531. num_clusters--;
  1532. }
  1533. }
  1534. /*
  1535. * Delayed allocation stuff
  1536. */
  1537. struct mpage_da_data {
  1538. struct inode *inode;
  1539. struct writeback_control *wbc;
  1540. pgoff_t first_page; /* The first page to write */
  1541. pgoff_t next_page; /* Current page to examine */
  1542. pgoff_t last_page; /* Last page to examine */
  1543. /*
  1544. * Extent to map - this can be after first_page because that can be
  1545. * fully mapped. We somewhat abuse m_flags to store whether the extent
  1546. * is delalloc or unwritten.
  1547. */
  1548. struct ext4_map_blocks map;
  1549. struct ext4_io_submit io_submit; /* IO submission data */
  1550. unsigned int do_map:1;
  1551. };
  1552. static void mpage_release_unused_pages(struct mpage_da_data *mpd,
  1553. bool invalidate)
  1554. {
  1555. int nr_pages, i;
  1556. pgoff_t index, end;
  1557. struct pagevec pvec;
  1558. struct inode *inode = mpd->inode;
  1559. struct address_space *mapping = inode->i_mapping;
  1560. /* This is necessary when next_page == 0. */
  1561. if (mpd->first_page >= mpd->next_page)
  1562. return;
  1563. index = mpd->first_page;
  1564. end = mpd->next_page - 1;
  1565. if (invalidate) {
  1566. ext4_lblk_t start, last;
  1567. start = index << (PAGE_SHIFT - inode->i_blkbits);
  1568. last = end << (PAGE_SHIFT - inode->i_blkbits);
  1569. ext4_es_remove_extent(inode, start, last - start + 1);
  1570. }
  1571. pagevec_init(&pvec);
  1572. while (index <= end) {
  1573. nr_pages = pagevec_lookup_range(&pvec, mapping, &index, end);
  1574. if (nr_pages == 0)
  1575. break;
  1576. for (i = 0; i < nr_pages; i++) {
  1577. struct page *page = pvec.pages[i];
  1578. BUG_ON(!PageLocked(page));
  1579. BUG_ON(PageWriteback(page));
  1580. if (invalidate) {
  1581. if (page_mapped(page))
  1582. clear_page_dirty_for_io(page);
  1583. block_invalidatepage(page, 0, PAGE_SIZE);
  1584. ClearPageUptodate(page);
  1585. }
  1586. unlock_page(page);
  1587. }
  1588. pagevec_release(&pvec);
  1589. }
  1590. }
  1591. static void ext4_print_free_blocks(struct inode *inode)
  1592. {
  1593. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  1594. struct super_block *sb = inode->i_sb;
  1595. struct ext4_inode_info *ei = EXT4_I(inode);
  1596. ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
  1597. EXT4_C2B(EXT4_SB(inode->i_sb),
  1598. ext4_count_free_clusters(sb)));
  1599. ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
  1600. ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
  1601. (long long) EXT4_C2B(EXT4_SB(sb),
  1602. percpu_counter_sum(&sbi->s_freeclusters_counter)));
  1603. ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
  1604. (long long) EXT4_C2B(EXT4_SB(sb),
  1605. percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
  1606. ext4_msg(sb, KERN_CRIT, "Block reservation details");
  1607. ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
  1608. ei->i_reserved_data_blocks);
  1609. return;
  1610. }
  1611. static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh)
  1612. {
  1613. return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
  1614. }
  1615. /*
  1616. * This function is grabs code from the very beginning of
  1617. * ext4_map_blocks, but assumes that the caller is from delayed write
  1618. * time. This function looks up the requested blocks and sets the
  1619. * buffer delay bit under the protection of i_data_sem.
  1620. */
  1621. static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
  1622. struct ext4_map_blocks *map,
  1623. struct buffer_head *bh)
  1624. {
  1625. struct extent_status es;
  1626. int retval;
  1627. sector_t invalid_block = ~((sector_t) 0xffff);
  1628. #ifdef ES_AGGRESSIVE_TEST
  1629. struct ext4_map_blocks orig_map;
  1630. memcpy(&orig_map, map, sizeof(*map));
  1631. #endif
  1632. if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
  1633. invalid_block = ~0;
  1634. map->m_flags = 0;
  1635. ext_debug("ext4_da_map_blocks(): inode %lu, max_blocks %u,"
  1636. "logical block %lu\n", inode->i_ino, map->m_len,
  1637. (unsigned long) map->m_lblk);
  1638. /* Lookup extent status tree firstly */
  1639. if (ext4_es_lookup_extent(inode, iblock, &es)) {
  1640. if (ext4_es_is_hole(&es)) {
  1641. retval = 0;
  1642. down_read(&EXT4_I(inode)->i_data_sem);
  1643. goto add_delayed;
  1644. }
  1645. /*
  1646. * Delayed extent could be allocated by fallocate.
  1647. * So we need to check it.
  1648. */
  1649. if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) {
  1650. map_bh(bh, inode->i_sb, invalid_block);
  1651. set_buffer_new(bh);
  1652. set_buffer_delay(bh);
  1653. return 0;
  1654. }
  1655. map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk;
  1656. retval = es.es_len - (iblock - es.es_lblk);
  1657. if (retval > map->m_len)
  1658. retval = map->m_len;
  1659. map->m_len = retval;
  1660. if (ext4_es_is_written(&es))
  1661. map->m_flags |= EXT4_MAP_MAPPED;
  1662. else if (ext4_es_is_unwritten(&es))
  1663. map->m_flags |= EXT4_MAP_UNWRITTEN;
  1664. else
  1665. BUG_ON(1);
  1666. #ifdef ES_AGGRESSIVE_TEST
  1667. ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
  1668. #endif
  1669. return retval;
  1670. }
  1671. /*
  1672. * Try to see if we can get the block without requesting a new
  1673. * file system block.
  1674. */
  1675. down_read(&EXT4_I(inode)->i_data_sem);
  1676. if (ext4_has_inline_data(inode))
  1677. retval = 0;
  1678. else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
  1679. retval = ext4_ext_map_blocks(NULL, inode, map, 0);
  1680. else
  1681. retval = ext4_ind_map_blocks(NULL, inode, map, 0);
  1682. add_delayed:
  1683. if (retval == 0) {
  1684. int ret;
  1685. /*
  1686. * XXX: __block_prepare_write() unmaps passed block,
  1687. * is it OK?
  1688. */
  1689. /*
  1690. * If the block was allocated from previously allocated cluster,
  1691. * then we don't need to reserve it again. However we still need
  1692. * to reserve metadata for every block we're going to write.
  1693. */
  1694. if (EXT4_SB(inode->i_sb)->s_cluster_ratio == 1 ||
  1695. !ext4_find_delalloc_cluster(inode, map->m_lblk)) {
  1696. ret = ext4_da_reserve_space(inode);
  1697. if (ret) {
  1698. /* not enough space to reserve */
  1699. retval = ret;
  1700. goto out_unlock;
  1701. }
  1702. }
  1703. ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
  1704. ~0, EXTENT_STATUS_DELAYED);
  1705. if (ret) {
  1706. retval = ret;
  1707. goto out_unlock;
  1708. }
  1709. map_bh(bh, inode->i_sb, invalid_block);
  1710. set_buffer_new(bh);
  1711. set_buffer_delay(bh);
  1712. } else if (retval > 0) {
  1713. int ret;
  1714. unsigned int status;
  1715. if (unlikely(retval != map->m_len)) {
  1716. ext4_warning(inode->i_sb,
  1717. "ES len assertion failed for inode "
  1718. "%lu: retval %d != map->m_len %d",
  1719. inode->i_ino, retval, map->m_len);
  1720. WARN_ON(1);
  1721. }
  1722. status = map->m_flags & EXT4_MAP_UNWRITTEN ?
  1723. EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
  1724. ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
  1725. map->m_pblk, status);
  1726. if (ret != 0)
  1727. retval = ret;
  1728. }
  1729. out_unlock:
  1730. up_read((&EXT4_I(inode)->i_data_sem));
  1731. return retval;
  1732. }
  1733. /*
  1734. * This is a special get_block_t callback which is used by
  1735. * ext4_da_write_begin(). It will either return mapped block or
  1736. * reserve space for a single block.
  1737. *
  1738. * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
  1739. * We also have b_blocknr = -1 and b_bdev initialized properly
  1740. *
  1741. * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
  1742. * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
  1743. * initialized properly.
  1744. */
  1745. int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
  1746. struct buffer_head *bh, int create)
  1747. {
  1748. struct ext4_map_blocks map;
  1749. int ret = 0;
  1750. BUG_ON(create == 0);
  1751. BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
  1752. map.m_lblk = iblock;
  1753. map.m_len = 1;
  1754. /*
  1755. * first, we need to know whether the block is allocated already
  1756. * preallocated blocks are unmapped but should treated
  1757. * the same as allocated blocks.
  1758. */
  1759. ret = ext4_da_map_blocks(inode, iblock, &map, bh);
  1760. if (ret <= 0)
  1761. return ret;
  1762. map_bh(bh, inode->i_sb, map.m_pblk);
  1763. ext4_update_bh_state(bh, map.m_flags);
  1764. if (buffer_unwritten(bh)) {
  1765. /* A delayed write to unwritten bh should be marked
  1766. * new and mapped. Mapped ensures that we don't do
  1767. * get_block multiple times when we write to the same
  1768. * offset and new ensures that we do proper zero out
  1769. * for partial write.
  1770. */
  1771. set_buffer_new(bh);
  1772. set_buffer_mapped(bh);
  1773. }
  1774. return 0;
  1775. }
  1776. static int bget_one(handle_t *handle, struct buffer_head *bh)
  1777. {
  1778. get_bh(bh);
  1779. return 0;
  1780. }
  1781. static int bput_one(handle_t *handle, struct buffer_head *bh)
  1782. {
  1783. put_bh(bh);
  1784. return 0;
  1785. }
  1786. static int __ext4_journalled_writepage(struct page *page,
  1787. unsigned int len)
  1788. {
  1789. struct address_space *mapping = page->mapping;
  1790. struct inode *inode = mapping->host;
  1791. struct buffer_head *page_bufs = NULL;
  1792. handle_t *handle = NULL;
  1793. int ret = 0, err = 0;
  1794. int inline_data = ext4_has_inline_data(inode);
  1795. struct buffer_head *inode_bh = NULL;
  1796. ClearPageChecked(page);
  1797. if (inline_data) {
  1798. BUG_ON(page->index != 0);
  1799. BUG_ON(len > ext4_get_max_inline_size(inode));
  1800. inode_bh = ext4_journalled_write_inline_data(inode, len, page);
  1801. if (inode_bh == NULL)
  1802. goto out;
  1803. } else {
  1804. page_bufs = page_buffers(page);
  1805. if (!page_bufs) {
  1806. BUG();
  1807. goto out;
  1808. }
  1809. ext4_walk_page_buffers(handle, page_bufs, 0, len,
  1810. NULL, bget_one);
  1811. }
  1812. /*
  1813. * We need to release the page lock before we start the
  1814. * journal, so grab a reference so the page won't disappear
  1815. * out from under us.
  1816. */
  1817. get_page(page);
  1818. unlock_page(page);
  1819. handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
  1820. ext4_writepage_trans_blocks(inode));
  1821. if (IS_ERR(handle)) {
  1822. ret = PTR_ERR(handle);
  1823. put_page(page);
  1824. goto out_no_pagelock;
  1825. }
  1826. BUG_ON(!ext4_handle_valid(handle));
  1827. lock_page(page);
  1828. put_page(page);
  1829. if (page->mapping != mapping) {
  1830. /* The page got truncated from under us */
  1831. ext4_journal_stop(handle);
  1832. ret = 0;
  1833. goto out;
  1834. }
  1835. if (inline_data) {
  1836. ret = ext4_mark_inode_dirty(handle, inode);
  1837. } else {
  1838. ret = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
  1839. do_journal_get_write_access);
  1840. err = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
  1841. write_end_fn);
  1842. }
  1843. if (ret == 0)
  1844. ret = err;
  1845. EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
  1846. err = ext4_journal_stop(handle);
  1847. if (!ret)
  1848. ret = err;
  1849. ext4_set_inode_state(inode, EXT4_STATE_JDATA);
  1850. out:
  1851. unlock_page(page);
  1852. out_no_pagelock:
  1853. if (!inline_data && page_bufs)
  1854. ext4_walk_page_buffers(NULL, page_bufs, 0, len,
  1855. NULL, bput_one);
  1856. brelse(inode_bh);
  1857. return ret;
  1858. }
  1859. /*
  1860. * Note that we don't need to start a transaction unless we're journaling data
  1861. * because we should have holes filled from ext4_page_mkwrite(). We even don't
  1862. * need to file the inode to the transaction's list in ordered mode because if
  1863. * we are writing back data added by write(), the inode is already there and if
  1864. * we are writing back data modified via mmap(), no one guarantees in which
  1865. * transaction the data will hit the disk. In case we are journaling data, we
  1866. * cannot start transaction directly because transaction start ranks above page
  1867. * lock so we have to do some magic.
  1868. *
  1869. * This function can get called via...
  1870. * - ext4_writepages after taking page lock (have journal handle)
  1871. * - journal_submit_inode_data_buffers (no journal handle)
  1872. * - shrink_page_list via the kswapd/direct reclaim (no journal handle)
  1873. * - grab_page_cache when doing write_begin (have journal handle)
  1874. *
  1875. * We don't do any block allocation in this function. If we have page with
  1876. * multiple blocks we need to write those buffer_heads that are mapped. This
  1877. * is important for mmaped based write. So if we do with blocksize 1K
  1878. * truncate(f, 1024);
  1879. * a = mmap(f, 0, 4096);
  1880. * a[0] = 'a';
  1881. * truncate(f, 4096);
  1882. * we have in the page first buffer_head mapped via page_mkwrite call back
  1883. * but other buffer_heads would be unmapped but dirty (dirty done via the
  1884. * do_wp_page). So writepage should write the first block. If we modify
  1885. * the mmap area beyond 1024 we will again get a page_fault and the
  1886. * page_mkwrite callback will do the block allocation and mark the
  1887. * buffer_heads mapped.
  1888. *
  1889. * We redirty the page if we have any buffer_heads that is either delay or
  1890. * unwritten in the page.
  1891. *
  1892. * We can get recursively called as show below.
  1893. *
  1894. * ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() ->
  1895. * ext4_writepage()
  1896. *
  1897. * But since we don't do any block allocation we should not deadlock.
  1898. * Page also have the dirty flag cleared so we don't get recurive page_lock.
  1899. */
  1900. static int ext4_writepage(struct page *page,
  1901. struct writeback_control *wbc)
  1902. {
  1903. int ret = 0;
  1904. loff_t size;
  1905. unsigned int len;
  1906. struct buffer_head *page_bufs = NULL;
  1907. struct inode *inode = page->mapping->host;
  1908. struct ext4_io_submit io_submit;
  1909. bool keep_towrite = false;
  1910. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
  1911. inode->i_mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
  1912. unlock_page(page);
  1913. return -EIO;
  1914. }
  1915. trace_ext4_writepage(page);
  1916. size = i_size_read(inode);
  1917. if (page->index == size >> PAGE_SHIFT)
  1918. len = size & ~PAGE_MASK;
  1919. else
  1920. len = PAGE_SIZE;
  1921. page_bufs = page_buffers(page);
  1922. /*
  1923. * We cannot do block allocation or other extent handling in this
  1924. * function. If there are buffers needing that, we have to redirty
  1925. * the page. But we may reach here when we do a journal commit via
  1926. * journal_submit_inode_data_buffers() and in that case we must write
  1927. * allocated buffers to achieve data=ordered mode guarantees.
  1928. *
  1929. * Also, if there is only one buffer per page (the fs block
  1930. * size == the page size), if one buffer needs block
  1931. * allocation or needs to modify the extent tree to clear the
  1932. * unwritten flag, we know that the page can't be written at
  1933. * all, so we might as well refuse the write immediately.
  1934. * Unfortunately if the block size != page size, we can't as
  1935. * easily detect this case using ext4_walk_page_buffers(), but
  1936. * for the extremely common case, this is an optimization that
  1937. * skips a useless round trip through ext4_bio_write_page().
  1938. */
  1939. if (ext4_walk_page_buffers(NULL, page_bufs, 0, len, NULL,
  1940. ext4_bh_delay_or_unwritten)) {
  1941. redirty_page_for_writepage(wbc, page);
  1942. if ((current->flags & PF_MEMALLOC) ||
  1943. (inode->i_sb->s_blocksize == PAGE_SIZE)) {
  1944. /*
  1945. * For memory cleaning there's no point in writing only
  1946. * some buffers. So just bail out. Warn if we came here
  1947. * from direct reclaim.
  1948. */
  1949. WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD))
  1950. == PF_MEMALLOC);
  1951. unlock_page(page);
  1952. return 0;
  1953. }
  1954. keep_towrite = true;
  1955. }
  1956. if (PageChecked(page) && ext4_should_journal_data(inode))
  1957. /*
  1958. * It's mmapped pagecache. Add buffers and journal it. There
  1959. * doesn't seem much point in redirtying the page here.
  1960. */
  1961. return __ext4_journalled_writepage(page, len);
  1962. ext4_io_submit_init(&io_submit, wbc);
  1963. io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
  1964. if (!io_submit.io_end) {
  1965. redirty_page_for_writepage(wbc, page);
  1966. unlock_page(page);
  1967. return -ENOMEM;
  1968. }
  1969. ret = ext4_bio_write_page(&io_submit, page, len, wbc, keep_towrite);
  1970. ext4_io_submit(&io_submit);
  1971. /* Drop io_end reference we got from init */
  1972. ext4_put_io_end_defer(io_submit.io_end);
  1973. return ret;
  1974. }
  1975. static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page)
  1976. {
  1977. int len;
  1978. loff_t size;
  1979. int err;
  1980. BUG_ON(page->index != mpd->first_page);
  1981. clear_page_dirty_for_io(page);
  1982. /*
  1983. * We have to be very careful here! Nothing protects writeback path
  1984. * against i_size changes and the page can be writeably mapped into
  1985. * page tables. So an application can be growing i_size and writing
  1986. * data through mmap while writeback runs. clear_page_dirty_for_io()
  1987. * write-protects our page in page tables and the page cannot get
  1988. * written to again until we release page lock. So only after
  1989. * clear_page_dirty_for_io() we are safe to sample i_size for
  1990. * ext4_bio_write_page() to zero-out tail of the written page. We rely
  1991. * on the barrier provided by TestClearPageDirty in
  1992. * clear_page_dirty_for_io() to make sure i_size is really sampled only
  1993. * after page tables are updated.
  1994. */
  1995. size = i_size_read(mpd->inode);
  1996. if (page->index == size >> PAGE_SHIFT)
  1997. len = size & ~PAGE_MASK;
  1998. else
  1999. len = PAGE_SIZE;
  2000. err = ext4_bio_write_page(&mpd->io_submit, page, len, mpd->wbc, false);
  2001. if (!err)
  2002. mpd->wbc->nr_to_write--;
  2003. mpd->first_page++;
  2004. return err;
  2005. }
  2006. #define BH_FLAGS ((1 << BH_Unwritten) | (1 << BH_Delay))
  2007. /*
  2008. * mballoc gives us at most this number of blocks...
  2009. * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
  2010. * The rest of mballoc seems to handle chunks up to full group size.
  2011. */
  2012. #define MAX_WRITEPAGES_EXTENT_LEN 2048
  2013. /*
  2014. * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
  2015. *
  2016. * @mpd - extent of blocks
  2017. * @lblk - logical number of the block in the file
  2018. * @bh - buffer head we want to add to the extent
  2019. *
  2020. * The function is used to collect contig. blocks in the same state. If the
  2021. * buffer doesn't require mapping for writeback and we haven't started the
  2022. * extent of buffers to map yet, the function returns 'true' immediately - the
  2023. * caller can write the buffer right away. Otherwise the function returns true
  2024. * if the block has been added to the extent, false if the block couldn't be
  2025. * added.
  2026. */
  2027. static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
  2028. struct buffer_head *bh)
  2029. {
  2030. struct ext4_map_blocks *map = &mpd->map;
  2031. /* Buffer that doesn't need mapping for writeback? */
  2032. if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
  2033. (!buffer_delay(bh) && !buffer_unwritten(bh))) {
  2034. /* So far no extent to map => we write the buffer right away */
  2035. if (map->m_len == 0)
  2036. return true;
  2037. return false;
  2038. }
  2039. /* First block in the extent? */
  2040. if (map->m_len == 0) {
  2041. /* We cannot map unless handle is started... */
  2042. if (!mpd->do_map)
  2043. return false;
  2044. map->m_lblk = lblk;
  2045. map->m_len = 1;
  2046. map->m_flags = bh->b_state & BH_FLAGS;
  2047. return true;
  2048. }
  2049. /* Don't go larger than mballoc is willing to allocate */
  2050. if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
  2051. return false;
  2052. /* Can we merge the block to our big extent? */
  2053. if (lblk == map->m_lblk + map->m_len &&
  2054. (bh->b_state & BH_FLAGS) == map->m_flags) {
  2055. map->m_len++;
  2056. return true;
  2057. }
  2058. return false;
  2059. }
  2060. /*
  2061. * mpage_process_page_bufs - submit page buffers for IO or add them to extent
  2062. *
  2063. * @mpd - extent of blocks for mapping
  2064. * @head - the first buffer in the page
  2065. * @bh - buffer we should start processing from
  2066. * @lblk - logical number of the block in the file corresponding to @bh
  2067. *
  2068. * Walk through page buffers from @bh upto @head (exclusive) and either submit
  2069. * the page for IO if all buffers in this page were mapped and there's no
  2070. * accumulated extent of buffers to map or add buffers in the page to the
  2071. * extent of buffers to map. The function returns 1 if the caller can continue
  2072. * by processing the next page, 0 if it should stop adding buffers to the
  2073. * extent to map because we cannot extend it anymore. It can also return value
  2074. * < 0 in case of error during IO submission.
  2075. */
  2076. static int mpage_process_page_bufs(struct mpage_da_data *mpd,
  2077. struct buffer_head *head,
  2078. struct buffer_head *bh,
  2079. ext4_lblk_t lblk)
  2080. {
  2081. struct inode *inode = mpd->inode;
  2082. int err;
  2083. ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
  2084. >> inode->i_blkbits;
  2085. do {
  2086. BUG_ON(buffer_locked(bh));
  2087. if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
  2088. /* Found extent to map? */
  2089. if (mpd->map.m_len)
  2090. return 0;
  2091. /* Buffer needs mapping and handle is not started? */
  2092. if (!mpd->do_map)
  2093. return 0;
  2094. /* Everything mapped so far and we hit EOF */
  2095. break;
  2096. }
  2097. } while (lblk++, (bh = bh->b_this_page) != head);
  2098. /* So far everything mapped? Submit the page for IO. */
  2099. if (mpd->map.m_len == 0) {
  2100. err = mpage_submit_page(mpd, head->b_page);
  2101. if (err < 0)
  2102. return err;
  2103. }
  2104. return lblk < blocks;
  2105. }
  2106. /*
  2107. * mpage_map_buffers - update buffers corresponding to changed extent and
  2108. * submit fully mapped pages for IO
  2109. *
  2110. * @mpd - description of extent to map, on return next extent to map
  2111. *
  2112. * Scan buffers corresponding to changed extent (we expect corresponding pages
  2113. * to be already locked) and update buffer state according to new extent state.
  2114. * We map delalloc buffers to their physical location, clear unwritten bits,
  2115. * and mark buffers as uninit when we perform writes to unwritten extents
  2116. * and do extent conversion after IO is finished. If the last page is not fully
  2117. * mapped, we update @map to the next extent in the last page that needs
  2118. * mapping. Otherwise we submit the page for IO.
  2119. */
  2120. static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
  2121. {
  2122. struct pagevec pvec;
  2123. int nr_pages, i;
  2124. struct inode *inode = mpd->inode;
  2125. struct buffer_head *head, *bh;
  2126. int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
  2127. pgoff_t start, end;
  2128. ext4_lblk_t lblk;
  2129. sector_t pblock;
  2130. int err;
  2131. start = mpd->map.m_lblk >> bpp_bits;
  2132. end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
  2133. lblk = start << bpp_bits;
  2134. pblock = mpd->map.m_pblk;
  2135. pagevec_init(&pvec);
  2136. while (start <= end) {
  2137. nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping,
  2138. &start, end);
  2139. if (nr_pages == 0)
  2140. break;
  2141. for (i = 0; i < nr_pages; i++) {
  2142. struct page *page = pvec.pages[i];
  2143. bh = head = page_buffers(page);
  2144. do {
  2145. if (lblk < mpd->map.m_lblk)
  2146. continue;
  2147. if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
  2148. /*
  2149. * Buffer after end of mapped extent.
  2150. * Find next buffer in the page to map.
  2151. */
  2152. mpd->map.m_len = 0;
  2153. mpd->map.m_flags = 0;
  2154. /*
  2155. * FIXME: If dioread_nolock supports
  2156. * blocksize < pagesize, we need to make
  2157. * sure we add size mapped so far to
  2158. * io_end->size as the following call
  2159. * can submit the page for IO.
  2160. */
  2161. err = mpage_process_page_bufs(mpd, head,
  2162. bh, lblk);
  2163. pagevec_release(&pvec);
  2164. if (err > 0)
  2165. err = 0;
  2166. return err;
  2167. }
  2168. if (buffer_delay(bh)) {
  2169. clear_buffer_delay(bh);
  2170. bh->b_blocknr = pblock++;
  2171. }
  2172. clear_buffer_unwritten(bh);
  2173. } while (lblk++, (bh = bh->b_this_page) != head);
  2174. /*
  2175. * FIXME: This is going to break if dioread_nolock
  2176. * supports blocksize < pagesize as we will try to
  2177. * convert potentially unmapped parts of inode.
  2178. */
  2179. mpd->io_submit.io_end->size += PAGE_SIZE;
  2180. /* Page fully mapped - let IO run! */
  2181. err = mpage_submit_page(mpd, page);
  2182. if (err < 0) {
  2183. pagevec_release(&pvec);
  2184. return err;
  2185. }
  2186. }
  2187. pagevec_release(&pvec);
  2188. }
  2189. /* Extent fully mapped and matches with page boundary. We are done. */
  2190. mpd->map.m_len = 0;
  2191. mpd->map.m_flags = 0;
  2192. return 0;
  2193. }
  2194. static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
  2195. {
  2196. struct inode *inode = mpd->inode;
  2197. struct ext4_map_blocks *map = &mpd->map;
  2198. int get_blocks_flags;
  2199. int err, dioread_nolock;
  2200. trace_ext4_da_write_pages_extent(inode, map);
  2201. /*
  2202. * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
  2203. * to convert an unwritten extent to be initialized (in the case
  2204. * where we have written into one or more preallocated blocks). It is
  2205. * possible that we're going to need more metadata blocks than
  2206. * previously reserved. However we must not fail because we're in
  2207. * writeback and there is nothing we can do about it so it might result
  2208. * in data loss. So use reserved blocks to allocate metadata if
  2209. * possible.
  2210. *
  2211. * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if
  2212. * the blocks in question are delalloc blocks. This indicates
  2213. * that the blocks and quotas has already been checked when
  2214. * the data was copied into the page cache.
  2215. */
  2216. get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
  2217. EXT4_GET_BLOCKS_METADATA_NOFAIL |
  2218. EXT4_GET_BLOCKS_IO_SUBMIT;
  2219. dioread_nolock = ext4_should_dioread_nolock(inode);
  2220. if (dioread_nolock)
  2221. get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
  2222. if (map->m_flags & (1 << BH_Delay))
  2223. get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
  2224. err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
  2225. if (err < 0)
  2226. return err;
  2227. if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
  2228. if (!mpd->io_submit.io_end->handle &&
  2229. ext4_handle_valid(handle)) {
  2230. mpd->io_submit.io_end->handle = handle->h_rsv_handle;
  2231. handle->h_rsv_handle = NULL;
  2232. }
  2233. ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
  2234. }
  2235. BUG_ON(map->m_len == 0);
  2236. if (map->m_flags & EXT4_MAP_NEW) {
  2237. clean_bdev_aliases(inode->i_sb->s_bdev, map->m_pblk,
  2238. map->m_len);
  2239. }
  2240. return 0;
  2241. }
  2242. /*
  2243. * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
  2244. * mpd->len and submit pages underlying it for IO
  2245. *
  2246. * @handle - handle for journal operations
  2247. * @mpd - extent to map
  2248. * @give_up_on_write - we set this to true iff there is a fatal error and there
  2249. * is no hope of writing the data. The caller should discard
  2250. * dirty pages to avoid infinite loops.
  2251. *
  2252. * The function maps extent starting at mpd->lblk of length mpd->len. If it is
  2253. * delayed, blocks are allocated, if it is unwritten, we may need to convert
  2254. * them to initialized or split the described range from larger unwritten
  2255. * extent. Note that we need not map all the described range since allocation
  2256. * can return less blocks or the range is covered by more unwritten extents. We
  2257. * cannot map more because we are limited by reserved transaction credits. On
  2258. * the other hand we always make sure that the last touched page is fully
  2259. * mapped so that it can be written out (and thus forward progress is
  2260. * guaranteed). After mapping we submit all mapped pages for IO.
  2261. */
  2262. static int mpage_map_and_submit_extent(handle_t *handle,
  2263. struct mpage_da_data *mpd,
  2264. bool *give_up_on_write)
  2265. {
  2266. struct inode *inode = mpd->inode;
  2267. struct ext4_map_blocks *map = &mpd->map;
  2268. int err;
  2269. loff_t disksize;
  2270. int progress = 0;
  2271. mpd->io_submit.io_end->offset =
  2272. ((loff_t)map->m_lblk) << inode->i_blkbits;
  2273. do {
  2274. err = mpage_map_one_extent(handle, mpd);
  2275. if (err < 0) {
  2276. struct super_block *sb = inode->i_sb;
  2277. if (ext4_forced_shutdown(EXT4_SB(sb)) ||
  2278. EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)
  2279. goto invalidate_dirty_pages;
  2280. /*
  2281. * Let the uper layers retry transient errors.
  2282. * In the case of ENOSPC, if ext4_count_free_blocks()
  2283. * is non-zero, a commit should free up blocks.
  2284. */
  2285. if ((err == -ENOMEM) ||
  2286. (err == -ENOSPC && ext4_count_free_clusters(sb))) {
  2287. if (progress)
  2288. goto update_disksize;
  2289. return err;
  2290. }
  2291. ext4_msg(sb, KERN_CRIT,
  2292. "Delayed block allocation failed for "
  2293. "inode %lu at logical offset %llu with"
  2294. " max blocks %u with error %d",
  2295. inode->i_ino,
  2296. (unsigned long long)map->m_lblk,
  2297. (unsigned)map->m_len, -err);
  2298. ext4_msg(sb, KERN_CRIT,
  2299. "This should not happen!! Data will "
  2300. "be lost\n");
  2301. if (err == -ENOSPC)
  2302. ext4_print_free_blocks(inode);
  2303. invalidate_dirty_pages:
  2304. *give_up_on_write = true;
  2305. return err;
  2306. }
  2307. progress = 1;
  2308. /*
  2309. * Update buffer state, submit mapped pages, and get us new
  2310. * extent to map
  2311. */
  2312. err = mpage_map_and_submit_buffers(mpd);
  2313. if (err < 0)
  2314. goto update_disksize;
  2315. } while (map->m_len);
  2316. update_disksize:
  2317. /*
  2318. * Update on-disk size after IO is submitted. Races with
  2319. * truncate are avoided by checking i_size under i_data_sem.
  2320. */
  2321. disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
  2322. if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
  2323. int err2;
  2324. loff_t i_size;
  2325. down_write(&EXT4_I(inode)->i_data_sem);
  2326. i_size = i_size_read(inode);
  2327. if (disksize > i_size)
  2328. disksize = i_size;
  2329. if (disksize > EXT4_I(inode)->i_disksize)
  2330. EXT4_I(inode)->i_disksize = disksize;
  2331. up_write(&EXT4_I(inode)->i_data_sem);
  2332. err2 = ext4_mark_inode_dirty(handle, inode);
  2333. if (err2)
  2334. ext4_error(inode->i_sb,
  2335. "Failed to mark inode %lu dirty",
  2336. inode->i_ino);
  2337. if (!err)
  2338. err = err2;
  2339. }
  2340. return err;
  2341. }
  2342. /*
  2343. * Calculate the total number of credits to reserve for one writepages
  2344. * iteration. This is called from ext4_writepages(). We map an extent of
  2345. * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping
  2346. * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +
  2347. * bpp - 1 blocks in bpp different extents.
  2348. */
  2349. static int ext4_da_writepages_trans_blocks(struct inode *inode)
  2350. {
  2351. int bpp = ext4_journal_blocks_per_page(inode);
  2352. return ext4_meta_trans_blocks(inode,
  2353. MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);
  2354. }
  2355. /*
  2356. * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
  2357. * and underlying extent to map
  2358. *
  2359. * @mpd - where to look for pages
  2360. *
  2361. * Walk dirty pages in the mapping. If they are fully mapped, submit them for
  2362. * IO immediately. When we find a page which isn't mapped we start accumulating
  2363. * extent of buffers underlying these pages that needs mapping (formed by
  2364. * either delayed or unwritten buffers). We also lock the pages containing
  2365. * these buffers. The extent found is returned in @mpd structure (starting at
  2366. * mpd->lblk with length mpd->len blocks).
  2367. *
  2368. * Note that this function can attach bios to one io_end structure which are
  2369. * neither logically nor physically contiguous. Although it may seem as an
  2370. * unnecessary complication, it is actually inevitable in blocksize < pagesize
  2371. * case as we need to track IO to all buffers underlying a page in one io_end.
  2372. */
  2373. static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
  2374. {
  2375. struct address_space *mapping = mpd->inode->i_mapping;
  2376. struct pagevec pvec;
  2377. unsigned int nr_pages;
  2378. long left = mpd->wbc->nr_to_write;
  2379. pgoff_t index = mpd->first_page;
  2380. pgoff_t end = mpd->last_page;
  2381. int tag;
  2382. int i, err = 0;
  2383. int blkbits = mpd->inode->i_blkbits;
  2384. ext4_lblk_t lblk;
  2385. struct buffer_head *head;
  2386. if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
  2387. tag = PAGECACHE_TAG_TOWRITE;
  2388. else
  2389. tag = PAGECACHE_TAG_DIRTY;
  2390. pagevec_init(&pvec);
  2391. mpd->map.m_len = 0;
  2392. mpd->next_page = index;
  2393. while (index <= end) {
  2394. nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
  2395. tag);
  2396. if (nr_pages == 0)
  2397. goto out;
  2398. for (i = 0; i < nr_pages; i++) {
  2399. struct page *page = pvec.pages[i];
  2400. /*
  2401. * Accumulated enough dirty pages? This doesn't apply
  2402. * to WB_SYNC_ALL mode. For integrity sync we have to
  2403. * keep going because someone may be concurrently
  2404. * dirtying pages, and we might have synced a lot of
  2405. * newly appeared dirty pages, but have not synced all
  2406. * of the old dirty pages.
  2407. */
  2408. if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0)
  2409. goto out;
  2410. /* If we can't merge this page, we are done. */
  2411. if (mpd->map.m_len > 0 && mpd->next_page != page->index)
  2412. goto out;
  2413. lock_page(page);
  2414. /*
  2415. * If the page is no longer dirty, or its mapping no
  2416. * longer corresponds to inode we are writing (which
  2417. * means it has been truncated or invalidated), or the
  2418. * page is already under writeback and we are not doing
  2419. * a data integrity writeback, skip the page
  2420. */
  2421. if (!PageDirty(page) ||
  2422. (PageWriteback(page) &&
  2423. (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
  2424. unlikely(page->mapping != mapping)) {
  2425. unlock_page(page);
  2426. continue;
  2427. }
  2428. wait_on_page_writeback(page);
  2429. BUG_ON(PageWriteback(page));
  2430. if (mpd->map.m_len == 0)
  2431. mpd->first_page = page->index;
  2432. mpd->next_page = page->index + 1;
  2433. /* Add all dirty buffers to mpd */
  2434. lblk = ((ext4_lblk_t)page->index) <<
  2435. (PAGE_SHIFT - blkbits);
  2436. head = page_buffers(page);
  2437. err = mpage_process_page_bufs(mpd, head, head, lblk);
  2438. if (err <= 0)
  2439. goto out;
  2440. err = 0;
  2441. left--;
  2442. }
  2443. pagevec_release(&pvec);
  2444. cond_resched();
  2445. }
  2446. return 0;
  2447. out:
  2448. pagevec_release(&pvec);
  2449. return err;
  2450. }
  2451. static int ext4_writepages(struct address_space *mapping,
  2452. struct writeback_control *wbc)
  2453. {
  2454. pgoff_t writeback_index = 0;
  2455. long nr_to_write = wbc->nr_to_write;
  2456. int range_whole = 0;
  2457. int cycled = 1;
  2458. handle_t *handle = NULL;
  2459. struct mpage_da_data mpd;
  2460. struct inode *inode = mapping->host;
  2461. int needed_blocks, rsv_blocks = 0, ret = 0;
  2462. struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
  2463. bool done;
  2464. struct blk_plug plug;
  2465. bool give_up_on_write = false;
  2466. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
  2467. return -EIO;
  2468. percpu_down_read(&sbi->s_writepages_rwsem);
  2469. trace_ext4_writepages(inode, wbc);
  2470. /*
  2471. * No pages to write? This is mainly a kludge to avoid starting
  2472. * a transaction for special inodes like journal inode on last iput()
  2473. * because that could violate lock ordering on umount
  2474. */
  2475. if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
  2476. goto out_writepages;
  2477. if (ext4_should_journal_data(inode)) {
  2478. ret = generic_writepages(mapping, wbc);
  2479. goto out_writepages;
  2480. }
  2481. /*
  2482. * If the filesystem has aborted, it is read-only, so return
  2483. * right away instead of dumping stack traces later on that
  2484. * will obscure the real source of the problem. We test
  2485. * EXT4_MF_FS_ABORTED instead of sb->s_flag's SB_RDONLY because
  2486. * the latter could be true if the filesystem is mounted
  2487. * read-only, and in that case, ext4_writepages should
  2488. * *never* be called, so if that ever happens, we would want
  2489. * the stack trace.
  2490. */
  2491. if (unlikely(ext4_forced_shutdown(EXT4_SB(mapping->host->i_sb)) ||
  2492. sbi->s_mount_flags & EXT4_MF_FS_ABORTED)) {
  2493. ret = -EROFS;
  2494. goto out_writepages;
  2495. }
  2496. if (ext4_should_dioread_nolock(inode)) {
  2497. /*
  2498. * We may need to convert up to one extent per block in
  2499. * the page and we may dirty the inode.
  2500. */
  2501. rsv_blocks = 1 + ext4_chunk_trans_blocks(inode,
  2502. PAGE_SIZE >> inode->i_blkbits);
  2503. }
  2504. /*
  2505. * If we have inline data and arrive here, it means that
  2506. * we will soon create the block for the 1st page, so
  2507. * we'd better clear the inline data here.
  2508. */
  2509. if (ext4_has_inline_data(inode)) {
  2510. /* Just inode will be modified... */
  2511. handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
  2512. if (IS_ERR(handle)) {
  2513. ret = PTR_ERR(handle);
  2514. goto out_writepages;
  2515. }
  2516. BUG_ON(ext4_test_inode_state(inode,
  2517. EXT4_STATE_MAY_INLINE_DATA));
  2518. ext4_destroy_inline_data(handle, inode);
  2519. ext4_journal_stop(handle);
  2520. }
  2521. if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
  2522. range_whole = 1;
  2523. if (wbc->range_cyclic) {
  2524. writeback_index = mapping->writeback_index;
  2525. if (writeback_index)
  2526. cycled = 0;
  2527. mpd.first_page = writeback_index;
  2528. mpd.last_page = -1;
  2529. } else {
  2530. mpd.first_page = wbc->range_start >> PAGE_SHIFT;
  2531. mpd.last_page = wbc->range_end >> PAGE_SHIFT;
  2532. }
  2533. mpd.inode = inode;
  2534. mpd.wbc = wbc;
  2535. ext4_io_submit_init(&mpd.io_submit, wbc);
  2536. retry:
  2537. if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
  2538. tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page);
  2539. done = false;
  2540. blk_start_plug(&plug);
  2541. /*
  2542. * First writeback pages that don't need mapping - we can avoid
  2543. * starting a transaction unnecessarily and also avoid being blocked
  2544. * in the block layer on device congestion while having transaction
  2545. * started.
  2546. */
  2547. mpd.do_map = 0;
  2548. mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
  2549. if (!mpd.io_submit.io_end) {
  2550. ret = -ENOMEM;
  2551. goto unplug;
  2552. }
  2553. ret = mpage_prepare_extent_to_map(&mpd);
  2554. /* Submit prepared bio */
  2555. ext4_io_submit(&mpd.io_submit);
  2556. ext4_put_io_end_defer(mpd.io_submit.io_end);
  2557. mpd.io_submit.io_end = NULL;
  2558. /* Unlock pages we didn't use */
  2559. mpage_release_unused_pages(&mpd, false);
  2560. if (ret < 0)
  2561. goto unplug;
  2562. while (!done && mpd.first_page <= mpd.last_page) {
  2563. /* For each extent of pages we use new io_end */
  2564. mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
  2565. if (!mpd.io_submit.io_end) {
  2566. ret = -ENOMEM;
  2567. break;
  2568. }
  2569. /*
  2570. * We have two constraints: We find one extent to map and we
  2571. * must always write out whole page (makes a difference when
  2572. * blocksize < pagesize) so that we don't block on IO when we
  2573. * try to write out the rest of the page. Journalled mode is
  2574. * not supported by delalloc.
  2575. */
  2576. BUG_ON(ext4_should_journal_data(inode));
  2577. needed_blocks = ext4_da_writepages_trans_blocks(inode);
  2578. /* start a new transaction */
  2579. handle = ext4_journal_start_with_reserve(inode,
  2580. EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
  2581. if (IS_ERR(handle)) {
  2582. ret = PTR_ERR(handle);
  2583. ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
  2584. "%ld pages, ino %lu; err %d", __func__,
  2585. wbc->nr_to_write, inode->i_ino, ret);
  2586. /* Release allocated io_end */
  2587. ext4_put_io_end(mpd.io_submit.io_end);
  2588. mpd.io_submit.io_end = NULL;
  2589. break;
  2590. }
  2591. mpd.do_map = 1;
  2592. trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc);
  2593. ret = mpage_prepare_extent_to_map(&mpd);
  2594. if (!ret) {
  2595. if (mpd.map.m_len)
  2596. ret = mpage_map_and_submit_extent(handle, &mpd,
  2597. &give_up_on_write);
  2598. else {
  2599. /*
  2600. * We scanned the whole range (or exhausted
  2601. * nr_to_write), submitted what was mapped and
  2602. * didn't find anything needing mapping. We are
  2603. * done.
  2604. */
  2605. done = true;
  2606. }
  2607. }
  2608. /*
  2609. * Caution: If the handle is synchronous,
  2610. * ext4_journal_stop() can wait for transaction commit
  2611. * to finish which may depend on writeback of pages to
  2612. * complete or on page lock to be released. In that
  2613. * case, we have to wait until after after we have
  2614. * submitted all the IO, released page locks we hold,
  2615. * and dropped io_end reference (for extent conversion
  2616. * to be able to complete) before stopping the handle.
  2617. */
  2618. if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
  2619. ext4_journal_stop(handle);
  2620. handle = NULL;
  2621. mpd.do_map = 0;
  2622. }
  2623. /* Submit prepared bio */
  2624. ext4_io_submit(&mpd.io_submit);
  2625. /* Unlock pages we didn't use */
  2626. mpage_release_unused_pages(&mpd, give_up_on_write);
  2627. /*
  2628. * Drop our io_end reference we got from init. We have
  2629. * to be careful and use deferred io_end finishing if
  2630. * we are still holding the transaction as we can
  2631. * release the last reference to io_end which may end
  2632. * up doing unwritten extent conversion.
  2633. */
  2634. if (handle) {
  2635. ext4_put_io_end_defer(mpd.io_submit.io_end);
  2636. ext4_journal_stop(handle);
  2637. } else
  2638. ext4_put_io_end(mpd.io_submit.io_end);
  2639. mpd.io_submit.io_end = NULL;
  2640. if (ret == -ENOSPC && sbi->s_journal) {
  2641. /*
  2642. * Commit the transaction which would
  2643. * free blocks released in the transaction
  2644. * and try again
  2645. */
  2646. jbd2_journal_force_commit_nested(sbi->s_journal);
  2647. ret = 0;
  2648. continue;
  2649. }
  2650. /* Fatal error - ENOMEM, EIO... */
  2651. if (ret)
  2652. break;
  2653. }
  2654. unplug:
  2655. blk_finish_plug(&plug);
  2656. if (!ret && !cycled && wbc->nr_to_write > 0) {
  2657. cycled = 1;
  2658. mpd.last_page = writeback_index - 1;
  2659. mpd.first_page = 0;
  2660. goto retry;
  2661. }
  2662. /* Update index */
  2663. if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
  2664. /*
  2665. * Set the writeback_index so that range_cyclic
  2666. * mode will write it back later
  2667. */
  2668. mapping->writeback_index = mpd.first_page;
  2669. out_writepages:
  2670. trace_ext4_writepages_result(inode, wbc, ret,
  2671. nr_to_write - wbc->nr_to_write);
  2672. percpu_up_read(&sbi->s_writepages_rwsem);
  2673. return ret;
  2674. }
  2675. static int ext4_dax_writepages(struct address_space *mapping,
  2676. struct writeback_control *wbc)
  2677. {
  2678. int ret;
  2679. long nr_to_write = wbc->nr_to_write;
  2680. struct inode *inode = mapping->host;
  2681. struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
  2682. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
  2683. return -EIO;
  2684. percpu_down_read(&sbi->s_writepages_rwsem);
  2685. trace_ext4_writepages(inode, wbc);
  2686. ret = dax_writeback_mapping_range(mapping, inode->i_sb->s_bdev, wbc);
  2687. trace_ext4_writepages_result(inode, wbc, ret,
  2688. nr_to_write - wbc->nr_to_write);
  2689. percpu_up_read(&sbi->s_writepages_rwsem);
  2690. return ret;
  2691. }
  2692. static int ext4_nonda_switch(struct super_block *sb)
  2693. {
  2694. s64 free_clusters, dirty_clusters;
  2695. struct ext4_sb_info *sbi = EXT4_SB(sb);
  2696. /*
  2697. * switch to non delalloc mode if we are running low
  2698. * on free block. The free block accounting via percpu
  2699. * counters can get slightly wrong with percpu_counter_batch getting
  2700. * accumulated on each CPU without updating global counters
  2701. * Delalloc need an accurate free block accounting. So switch
  2702. * to non delalloc when we are near to error range.
  2703. */
  2704. free_clusters =
  2705. percpu_counter_read_positive(&sbi->s_freeclusters_counter);
  2706. dirty_clusters =
  2707. percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
  2708. /*
  2709. * Start pushing delalloc when 1/2 of free blocks are dirty.
  2710. */
  2711. if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
  2712. try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
  2713. if (2 * free_clusters < 3 * dirty_clusters ||
  2714. free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
  2715. /*
  2716. * free block count is less than 150% of dirty blocks
  2717. * or free blocks is less than watermark
  2718. */
  2719. return 1;
  2720. }
  2721. return 0;
  2722. }
  2723. /* We always reserve for an inode update; the superblock could be there too */
  2724. static int ext4_da_write_credits(struct inode *inode, loff_t pos, unsigned len)
  2725. {
  2726. if (likely(ext4_has_feature_large_file(inode->i_sb)))
  2727. return 1;
  2728. if (pos + len <= 0x7fffffffULL)
  2729. return 1;
  2730. /* We might need to update the superblock to set LARGE_FILE */
  2731. return 2;
  2732. }
  2733. static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
  2734. loff_t pos, unsigned len, unsigned flags,
  2735. struct page **pagep, void **fsdata)
  2736. {
  2737. int ret, retries = 0;
  2738. struct page *page;
  2739. pgoff_t index;
  2740. struct inode *inode = mapping->host;
  2741. handle_t *handle;
  2742. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
  2743. return -EIO;
  2744. index = pos >> PAGE_SHIFT;
  2745. if (ext4_nonda_switch(inode->i_sb) ||
  2746. S_ISLNK(inode->i_mode)) {
  2747. *fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
  2748. return ext4_write_begin(file, mapping, pos,
  2749. len, flags, pagep, fsdata);
  2750. }
  2751. *fsdata = (void *)0;
  2752. trace_ext4_da_write_begin(inode, pos, len, flags);
  2753. if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
  2754. ret = ext4_da_write_inline_data_begin(mapping, inode,
  2755. pos, len, flags,
  2756. pagep, fsdata);
  2757. if (ret < 0)
  2758. return ret;
  2759. if (ret == 1)
  2760. return 0;
  2761. }
  2762. /*
  2763. * grab_cache_page_write_begin() can take a long time if the
  2764. * system is thrashing due to memory pressure, or if the page
  2765. * is being written back. So grab it first before we start
  2766. * the transaction handle. This also allows us to allocate
  2767. * the page (if needed) without using GFP_NOFS.
  2768. */
  2769. retry_grab:
  2770. page = grab_cache_page_write_begin(mapping, index, flags);
  2771. if (!page)
  2772. return -ENOMEM;
  2773. unlock_page(page);
  2774. /*
  2775. * With delayed allocation, we don't log the i_disksize update
  2776. * if there is delayed block allocation. But we still need
  2777. * to journalling the i_disksize update if writes to the end
  2778. * of file which has an already mapped buffer.
  2779. */
  2780. retry_journal:
  2781. handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
  2782. ext4_da_write_credits(inode, pos, len));
  2783. if (IS_ERR(handle)) {
  2784. put_page(page);
  2785. return PTR_ERR(handle);
  2786. }
  2787. lock_page(page);
  2788. if (page->mapping != mapping) {
  2789. /* The page got truncated from under us */
  2790. unlock_page(page);
  2791. put_page(page);
  2792. ext4_journal_stop(handle);
  2793. goto retry_grab;
  2794. }
  2795. /* In case writeback began while the page was unlocked */
  2796. wait_for_stable_page(page);
  2797. #ifdef CONFIG_EXT4_FS_ENCRYPTION
  2798. ret = ext4_block_write_begin(page, pos, len,
  2799. ext4_da_get_block_prep);
  2800. #else
  2801. ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
  2802. #endif
  2803. if (ret < 0) {
  2804. unlock_page(page);
  2805. ext4_journal_stop(handle);
  2806. /*
  2807. * block_write_begin may have instantiated a few blocks
  2808. * outside i_size. Trim these off again. Don't need
  2809. * i_size_read because we hold i_mutex.
  2810. */
  2811. if (pos + len > inode->i_size)
  2812. ext4_truncate_failed_write(inode);
  2813. if (ret == -ENOSPC &&
  2814. ext4_should_retry_alloc(inode->i_sb, &retries))
  2815. goto retry_journal;
  2816. put_page(page);
  2817. return ret;
  2818. }
  2819. *pagep = page;
  2820. return ret;
  2821. }
  2822. /*
  2823. * Check if we should update i_disksize
  2824. * when write to the end of file but not require block allocation
  2825. */
  2826. static int ext4_da_should_update_i_disksize(struct page *page,
  2827. unsigned long offset)
  2828. {
  2829. struct buffer_head *bh;
  2830. struct inode *inode = page->mapping->host;
  2831. unsigned int idx;
  2832. int i;
  2833. bh = page_buffers(page);
  2834. idx = offset >> inode->i_blkbits;
  2835. for (i = 0; i < idx; i++)
  2836. bh = bh->b_this_page;
  2837. if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
  2838. return 0;
  2839. return 1;
  2840. }
  2841. static int ext4_da_write_end(struct file *file,
  2842. struct address_space *mapping,
  2843. loff_t pos, unsigned len, unsigned copied,
  2844. struct page *page, void *fsdata)
  2845. {
  2846. struct inode *inode = mapping->host;
  2847. int ret = 0, ret2;
  2848. handle_t *handle = ext4_journal_current_handle();
  2849. loff_t new_i_size;
  2850. unsigned long start, end;
  2851. int write_mode = (int)(unsigned long)fsdata;
  2852. if (write_mode == FALL_BACK_TO_NONDELALLOC)
  2853. return ext4_write_end(file, mapping, pos,
  2854. len, copied, page, fsdata);
  2855. trace_ext4_da_write_end(inode, pos, len, copied);
  2856. start = pos & (PAGE_SIZE - 1);
  2857. end = start + copied - 1;
  2858. /*
  2859. * generic_write_end() will run mark_inode_dirty() if i_size
  2860. * changes. So let's piggyback the i_disksize mark_inode_dirty
  2861. * into that.
  2862. */
  2863. new_i_size = pos + copied;
  2864. if (copied && new_i_size > EXT4_I(inode)->i_disksize) {
  2865. if (ext4_has_inline_data(inode) ||
  2866. ext4_da_should_update_i_disksize(page, end)) {
  2867. ext4_update_i_disksize(inode, new_i_size);
  2868. /* We need to mark inode dirty even if
  2869. * new_i_size is less that inode->i_size
  2870. * bu greater than i_disksize.(hint delalloc)
  2871. */
  2872. ext4_mark_inode_dirty(handle, inode);
  2873. }
  2874. }
  2875. if (write_mode != CONVERT_INLINE_DATA &&
  2876. ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
  2877. ext4_has_inline_data(inode))
  2878. ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied,
  2879. page);
  2880. else
  2881. ret2 = generic_write_end(file, mapping, pos, len, copied,
  2882. page, fsdata);
  2883. copied = ret2;
  2884. if (ret2 < 0)
  2885. ret = ret2;
  2886. ret2 = ext4_journal_stop(handle);
  2887. if (!ret)
  2888. ret = ret2;
  2889. return ret ? ret : copied;
  2890. }
  2891. static void ext4_da_invalidatepage(struct page *page, unsigned int offset,
  2892. unsigned int length)
  2893. {
  2894. /*
  2895. * Drop reserved blocks
  2896. */
  2897. BUG_ON(!PageLocked(page));
  2898. if (!page_has_buffers(page))
  2899. goto out;
  2900. ext4_da_page_release_reservation(page, offset, length);
  2901. out:
  2902. ext4_invalidatepage(page, offset, length);
  2903. return;
  2904. }
  2905. /*
  2906. * Force all delayed allocation blocks to be allocated for a given inode.
  2907. */
  2908. int ext4_alloc_da_blocks(struct inode *inode)
  2909. {
  2910. trace_ext4_alloc_da_blocks(inode);
  2911. if (!EXT4_I(inode)->i_reserved_data_blocks)
  2912. return 0;
  2913. /*
  2914. * We do something simple for now. The filemap_flush() will
  2915. * also start triggering a write of the data blocks, which is
  2916. * not strictly speaking necessary (and for users of
  2917. * laptop_mode, not even desirable). However, to do otherwise
  2918. * would require replicating code paths in:
  2919. *
  2920. * ext4_writepages() ->
  2921. * write_cache_pages() ---> (via passed in callback function)
  2922. * __mpage_da_writepage() -->
  2923. * mpage_add_bh_to_extent()
  2924. * mpage_da_map_blocks()
  2925. *
  2926. * The problem is that write_cache_pages(), located in
  2927. * mm/page-writeback.c, marks pages clean in preparation for
  2928. * doing I/O, which is not desirable if we're not planning on
  2929. * doing I/O at all.
  2930. *
  2931. * We could call write_cache_pages(), and then redirty all of
  2932. * the pages by calling redirty_page_for_writepage() but that
  2933. * would be ugly in the extreme. So instead we would need to
  2934. * replicate parts of the code in the above functions,
  2935. * simplifying them because we wouldn't actually intend to
  2936. * write out the pages, but rather only collect contiguous
  2937. * logical block extents, call the multi-block allocator, and
  2938. * then update the buffer heads with the block allocations.
  2939. *
  2940. * For now, though, we'll cheat by calling filemap_flush(),
  2941. * which will map the blocks, and start the I/O, but not
  2942. * actually wait for the I/O to complete.
  2943. */
  2944. return filemap_flush(inode->i_mapping);
  2945. }
  2946. /*
  2947. * bmap() is special. It gets used by applications such as lilo and by
  2948. * the swapper to find the on-disk block of a specific piece of data.
  2949. *
  2950. * Naturally, this is dangerous if the block concerned is still in the
  2951. * journal. If somebody makes a swapfile on an ext4 data-journaling
  2952. * filesystem and enables swap, then they may get a nasty shock when the
  2953. * data getting swapped to that swapfile suddenly gets overwritten by
  2954. * the original zero's written out previously to the journal and
  2955. * awaiting writeback in the kernel's buffer cache.
  2956. *
  2957. * So, if we see any bmap calls here on a modified, data-journaled file,
  2958. * take extra steps to flush any blocks which might be in the cache.
  2959. */
  2960. static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
  2961. {
  2962. struct inode *inode = mapping->host;
  2963. journal_t *journal;
  2964. int err;
  2965. /*
  2966. * We can get here for an inline file via the FIBMAP ioctl
  2967. */
  2968. if (ext4_has_inline_data(inode))
  2969. return 0;
  2970. if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
  2971. test_opt(inode->i_sb, DELALLOC)) {
  2972. /*
  2973. * With delalloc we want to sync the file
  2974. * so that we can make sure we allocate
  2975. * blocks for file
  2976. */
  2977. filemap_write_and_wait(mapping);
  2978. }
  2979. if (EXT4_JOURNAL(inode) &&
  2980. ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
  2981. /*
  2982. * This is a REALLY heavyweight approach, but the use of
  2983. * bmap on dirty files is expected to be extremely rare:
  2984. * only if we run lilo or swapon on a freshly made file
  2985. * do we expect this to happen.
  2986. *
  2987. * (bmap requires CAP_SYS_RAWIO so this does not
  2988. * represent an unprivileged user DOS attack --- we'd be
  2989. * in trouble if mortal users could trigger this path at
  2990. * will.)
  2991. *
  2992. * NB. EXT4_STATE_JDATA is not set on files other than
  2993. * regular files. If somebody wants to bmap a directory
  2994. * or symlink and gets confused because the buffer
  2995. * hasn't yet been flushed to disk, they deserve
  2996. * everything they get.
  2997. */
  2998. ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
  2999. journal = EXT4_JOURNAL(inode);
  3000. jbd2_journal_lock_updates(journal);
  3001. err = jbd2_journal_flush(journal);
  3002. jbd2_journal_unlock_updates(journal);
  3003. if (err)
  3004. return 0;
  3005. }
  3006. return generic_block_bmap(mapping, block, ext4_get_block);
  3007. }
  3008. static int ext4_readpage(struct file *file, struct page *page)
  3009. {
  3010. int ret = -EAGAIN;
  3011. struct inode *inode = page->mapping->host;
  3012. trace_ext4_readpage(page);
  3013. if (ext4_has_inline_data(inode))
  3014. ret = ext4_readpage_inline(inode, page);
  3015. if (ret == -EAGAIN)
  3016. return ext4_mpage_readpages(page->mapping, NULL, page, 1,
  3017. false);
  3018. return ret;
  3019. }
  3020. static int
  3021. ext4_readpages(struct file *file, struct address_space *mapping,
  3022. struct list_head *pages, unsigned nr_pages)
  3023. {
  3024. struct inode *inode = mapping->host;
  3025. /* If the file has inline data, no need to do readpages. */
  3026. if (ext4_has_inline_data(inode))
  3027. return 0;
  3028. return ext4_mpage_readpages(mapping, pages, NULL, nr_pages, true);
  3029. }
  3030. static void ext4_invalidatepage(struct page *page, unsigned int offset,
  3031. unsigned int length)
  3032. {
  3033. trace_ext4_invalidatepage(page, offset, length);
  3034. /* No journalling happens on data buffers when this function is used */
  3035. WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page)));
  3036. block_invalidatepage(page, offset, length);
  3037. }
  3038. static int __ext4_journalled_invalidatepage(struct page *page,
  3039. unsigned int offset,
  3040. unsigned int length)
  3041. {
  3042. journal_t *journal = EXT4_JOURNAL(page->mapping->host);
  3043. trace_ext4_journalled_invalidatepage(page, offset, length);
  3044. /*
  3045. * If it's a full truncate we just forget about the pending dirtying
  3046. */
  3047. if (offset == 0 && length == PAGE_SIZE)
  3048. ClearPageChecked(page);
  3049. return jbd2_journal_invalidatepage(journal, page, offset, length);
  3050. }
  3051. /* Wrapper for aops... */
  3052. static void ext4_journalled_invalidatepage(struct page *page,
  3053. unsigned int offset,
  3054. unsigned int length)
  3055. {
  3056. WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0);
  3057. }
  3058. static int ext4_releasepage(struct page *page, gfp_t wait)
  3059. {
  3060. journal_t *journal = EXT4_JOURNAL(page->mapping->host);
  3061. trace_ext4_releasepage(page);
  3062. /* Page has dirty journalled data -> cannot release */
  3063. if (PageChecked(page))
  3064. return 0;
  3065. if (journal)
  3066. return jbd2_journal_try_to_free_buffers(journal, page, wait);
  3067. else
  3068. return try_to_free_buffers(page);
  3069. }
  3070. static bool ext4_inode_datasync_dirty(struct inode *inode)
  3071. {
  3072. journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
  3073. if (journal)
  3074. return !jbd2_transaction_committed(journal,
  3075. EXT4_I(inode)->i_datasync_tid);
  3076. /* Any metadata buffers to write? */
  3077. if (!list_empty(&inode->i_mapping->private_list))
  3078. return true;
  3079. return inode->i_state & I_DIRTY_DATASYNC;
  3080. }
  3081. static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
  3082. unsigned flags, struct iomap *iomap)
  3083. {
  3084. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  3085. unsigned int blkbits = inode->i_blkbits;
  3086. unsigned long first_block, last_block;
  3087. struct ext4_map_blocks map;
  3088. bool delalloc = false;
  3089. int ret;
  3090. if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
  3091. return -EINVAL;
  3092. first_block = offset >> blkbits;
  3093. last_block = min_t(loff_t, (offset + length - 1) >> blkbits,
  3094. EXT4_MAX_LOGICAL_BLOCK);
  3095. if (flags & IOMAP_REPORT) {
  3096. if (ext4_has_inline_data(inode)) {
  3097. ret = ext4_inline_data_iomap(inode, iomap);
  3098. if (ret != -EAGAIN) {
  3099. if (ret == 0 && offset >= iomap->length)
  3100. ret = -ENOENT;
  3101. return ret;
  3102. }
  3103. }
  3104. } else {
  3105. if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
  3106. return -ERANGE;
  3107. }
  3108. map.m_lblk = first_block;
  3109. map.m_len = last_block - first_block + 1;
  3110. if (flags & IOMAP_REPORT) {
  3111. ret = ext4_map_blocks(NULL, inode, &map, 0);
  3112. if (ret < 0)
  3113. return ret;
  3114. if (ret == 0) {
  3115. ext4_lblk_t end = map.m_lblk + map.m_len - 1;
  3116. struct extent_status es;
  3117. ext4_es_find_delayed_extent_range(inode, map.m_lblk, end, &es);
  3118. if (!es.es_len || es.es_lblk > end) {
  3119. /* entire range is a hole */
  3120. } else if (es.es_lblk > map.m_lblk) {
  3121. /* range starts with a hole */
  3122. map.m_len = es.es_lblk - map.m_lblk;
  3123. } else {
  3124. ext4_lblk_t offs = 0;
  3125. if (es.es_lblk < map.m_lblk)
  3126. offs = map.m_lblk - es.es_lblk;
  3127. map.m_lblk = es.es_lblk + offs;
  3128. map.m_len = es.es_len - offs;
  3129. delalloc = true;
  3130. }
  3131. }
  3132. } else if (flags & IOMAP_WRITE) {
  3133. int dio_credits;
  3134. handle_t *handle;
  3135. int retries = 0;
  3136. /* Trim mapping request to maximum we can map at once for DIO */
  3137. if (map.m_len > DIO_MAX_BLOCKS)
  3138. map.m_len = DIO_MAX_BLOCKS;
  3139. dio_credits = ext4_chunk_trans_blocks(inode, map.m_len);
  3140. retry:
  3141. /*
  3142. * Either we allocate blocks and then we don't get unwritten
  3143. * extent so we have reserved enough credits, or the blocks
  3144. * are already allocated and unwritten and in that case
  3145. * extent conversion fits in the credits as well.
  3146. */
  3147. handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
  3148. dio_credits);
  3149. if (IS_ERR(handle))
  3150. return PTR_ERR(handle);
  3151. ret = ext4_map_blocks(handle, inode, &map,
  3152. EXT4_GET_BLOCKS_CREATE_ZERO);
  3153. if (ret < 0) {
  3154. ext4_journal_stop(handle);
  3155. if (ret == -ENOSPC &&
  3156. ext4_should_retry_alloc(inode->i_sb, &retries))
  3157. goto retry;
  3158. return ret;
  3159. }
  3160. /*
  3161. * If we added blocks beyond i_size, we need to make sure they
  3162. * will get truncated if we crash before updating i_size in
  3163. * ext4_iomap_end(). For faults we don't need to do that (and
  3164. * even cannot because for orphan list operations inode_lock is
  3165. * required) - if we happen to instantiate block beyond i_size,
  3166. * it is because we race with truncate which has already added
  3167. * the inode to the orphan list.
  3168. */
  3169. if (!(flags & IOMAP_FAULT) && first_block + map.m_len >
  3170. (i_size_read(inode) + (1 << blkbits) - 1) >> blkbits) {
  3171. int err;
  3172. err = ext4_orphan_add(handle, inode);
  3173. if (err < 0) {
  3174. ext4_journal_stop(handle);
  3175. return err;
  3176. }
  3177. }
  3178. ext4_journal_stop(handle);
  3179. } else {
  3180. ret = ext4_map_blocks(NULL, inode, &map, 0);
  3181. if (ret < 0)
  3182. return ret;
  3183. }
  3184. /*
  3185. * Writes that span EOF might trigger an I/O size update on completion,
  3186. * so consider them to be dirty for the purposes of O_DSYNC, even if
  3187. * there is no other metadata changes being made or are pending here.
  3188. */
  3189. iomap->flags = 0;
  3190. if (ext4_inode_datasync_dirty(inode) ||
  3191. offset + length > i_size_read(inode))
  3192. iomap->flags |= IOMAP_F_DIRTY;
  3193. iomap->bdev = inode->i_sb->s_bdev;
  3194. iomap->dax_dev = sbi->s_daxdev;
  3195. iomap->offset = (u64)first_block << blkbits;
  3196. iomap->length = (u64)map.m_len << blkbits;
  3197. if (ret == 0) {
  3198. iomap->type = delalloc ? IOMAP_DELALLOC : IOMAP_HOLE;
  3199. iomap->addr = IOMAP_NULL_ADDR;
  3200. } else {
  3201. if (map.m_flags & EXT4_MAP_MAPPED) {
  3202. iomap->type = IOMAP_MAPPED;
  3203. } else if (map.m_flags & EXT4_MAP_UNWRITTEN) {
  3204. iomap->type = IOMAP_UNWRITTEN;
  3205. } else {
  3206. WARN_ON_ONCE(1);
  3207. return -EIO;
  3208. }
  3209. iomap->addr = (u64)map.m_pblk << blkbits;
  3210. }
  3211. if (map.m_flags & EXT4_MAP_NEW)
  3212. iomap->flags |= IOMAP_F_NEW;
  3213. return 0;
  3214. }
  3215. static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
  3216. ssize_t written, unsigned flags, struct iomap *iomap)
  3217. {
  3218. int ret = 0;
  3219. handle_t *handle;
  3220. int blkbits = inode->i_blkbits;
  3221. bool truncate = false;
  3222. if (!(flags & IOMAP_WRITE) || (flags & IOMAP_FAULT))
  3223. return 0;
  3224. handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
  3225. if (IS_ERR(handle)) {
  3226. ret = PTR_ERR(handle);
  3227. goto orphan_del;
  3228. }
  3229. if (ext4_update_inode_size(inode, offset + written))
  3230. ext4_mark_inode_dirty(handle, inode);
  3231. /*
  3232. * We may need to truncate allocated but not written blocks beyond EOF.
  3233. */
  3234. if (iomap->offset + iomap->length >
  3235. ALIGN(inode->i_size, 1 << blkbits)) {
  3236. ext4_lblk_t written_blk, end_blk;
  3237. written_blk = (offset + written) >> blkbits;
  3238. end_blk = (offset + length) >> blkbits;
  3239. if (written_blk < end_blk && ext4_can_truncate(inode))
  3240. truncate = true;
  3241. }
  3242. /*
  3243. * Remove inode from orphan list if we were extending a inode and
  3244. * everything went fine.
  3245. */
  3246. if (!truncate && inode->i_nlink &&
  3247. !list_empty(&EXT4_I(inode)->i_orphan))
  3248. ext4_orphan_del(handle, inode);
  3249. ext4_journal_stop(handle);
  3250. if (truncate) {
  3251. ext4_truncate_failed_write(inode);
  3252. orphan_del:
  3253. /*
  3254. * If truncate failed early the inode might still be on the
  3255. * orphan list; we need to make sure the inode is removed from
  3256. * the orphan list in that case.
  3257. */
  3258. if (inode->i_nlink)
  3259. ext4_orphan_del(NULL, inode);
  3260. }
  3261. return ret;
  3262. }
  3263. const struct iomap_ops ext4_iomap_ops = {
  3264. .iomap_begin = ext4_iomap_begin,
  3265. .iomap_end = ext4_iomap_end,
  3266. };
  3267. static int ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
  3268. ssize_t size, void *private)
  3269. {
  3270. ext4_io_end_t *io_end = private;
  3271. /* if not async direct IO just return */
  3272. if (!io_end)
  3273. return 0;
  3274. ext_debug("ext4_end_io_dio(): io_end 0x%p "
  3275. "for inode %lu, iocb 0x%p, offset %llu, size %zd\n",
  3276. io_end, io_end->inode->i_ino, iocb, offset, size);
  3277. /*
  3278. * Error during AIO DIO. We cannot convert unwritten extents as the
  3279. * data was not written. Just clear the unwritten flag and drop io_end.
  3280. */
  3281. if (size <= 0) {
  3282. ext4_clear_io_unwritten_flag(io_end);
  3283. size = 0;
  3284. }
  3285. io_end->offset = offset;
  3286. io_end->size = size;
  3287. ext4_put_io_end(io_end);
  3288. return 0;
  3289. }
  3290. /*
  3291. * Handling of direct IO writes.
  3292. *
  3293. * For ext4 extent files, ext4 will do direct-io write even to holes,
  3294. * preallocated extents, and those write extend the file, no need to
  3295. * fall back to buffered IO.
  3296. *
  3297. * For holes, we fallocate those blocks, mark them as unwritten
  3298. * If those blocks were preallocated, we mark sure they are split, but
  3299. * still keep the range to write as unwritten.
  3300. *
  3301. * The unwritten extents will be converted to written when DIO is completed.
  3302. * For async direct IO, since the IO may still pending when return, we
  3303. * set up an end_io call back function, which will do the conversion
  3304. * when async direct IO completed.
  3305. *
  3306. * If the O_DIRECT write will extend the file then add this inode to the
  3307. * orphan list. So recovery will truncate it back to the original size
  3308. * if the machine crashes during the write.
  3309. *
  3310. */
  3311. static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter)
  3312. {
  3313. struct file *file = iocb->ki_filp;
  3314. struct inode *inode = file->f_mapping->host;
  3315. struct ext4_inode_info *ei = EXT4_I(inode);
  3316. ssize_t ret;
  3317. loff_t offset = iocb->ki_pos;
  3318. size_t count = iov_iter_count(iter);
  3319. int overwrite = 0;
  3320. get_block_t *get_block_func = NULL;
  3321. int dio_flags = 0;
  3322. loff_t final_size = offset + count;
  3323. int orphan = 0;
  3324. handle_t *handle;
  3325. if (final_size > inode->i_size || final_size > ei->i_disksize) {
  3326. /* Credits for sb + inode write */
  3327. handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
  3328. if (IS_ERR(handle)) {
  3329. ret = PTR_ERR(handle);
  3330. goto out;
  3331. }
  3332. ret = ext4_orphan_add(handle, inode);
  3333. if (ret) {
  3334. ext4_journal_stop(handle);
  3335. goto out;
  3336. }
  3337. orphan = 1;
  3338. ext4_update_i_disksize(inode, inode->i_size);
  3339. ext4_journal_stop(handle);
  3340. }
  3341. BUG_ON(iocb->private == NULL);
  3342. /*
  3343. * Make all waiters for direct IO properly wait also for extent
  3344. * conversion. This also disallows race between truncate() and
  3345. * overwrite DIO as i_dio_count needs to be incremented under i_mutex.
  3346. */
  3347. inode_dio_begin(inode);
  3348. /* If we do a overwrite dio, i_mutex locking can be released */
  3349. overwrite = *((int *)iocb->private);
  3350. if (overwrite)
  3351. inode_unlock(inode);
  3352. /*
  3353. * For extent mapped files we could direct write to holes and fallocate.
  3354. *
  3355. * Allocated blocks to fill the hole are marked as unwritten to prevent
  3356. * parallel buffered read to expose the stale data before DIO complete
  3357. * the data IO.
  3358. *
  3359. * As to previously fallocated extents, ext4 get_block will just simply
  3360. * mark the buffer mapped but still keep the extents unwritten.
  3361. *
  3362. * For non AIO case, we will convert those unwritten extents to written
  3363. * after return back from blockdev_direct_IO. That way we save us from
  3364. * allocating io_end structure and also the overhead of offloading
  3365. * the extent convertion to a workqueue.
  3366. *
  3367. * For async DIO, the conversion needs to be deferred when the
  3368. * IO is completed. The ext4 end_io callback function will be
  3369. * called to take care of the conversion work. Here for async
  3370. * case, we allocate an io_end structure to hook to the iocb.
  3371. */
  3372. iocb->private = NULL;
  3373. if (overwrite)
  3374. get_block_func = ext4_dio_get_block_overwrite;
  3375. else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) ||
  3376. round_down(offset, i_blocksize(inode)) >= inode->i_size) {
  3377. get_block_func = ext4_dio_get_block;
  3378. dio_flags = DIO_LOCKING | DIO_SKIP_HOLES;
  3379. } else if (is_sync_kiocb(iocb)) {
  3380. get_block_func = ext4_dio_get_block_unwritten_sync;
  3381. dio_flags = DIO_LOCKING;
  3382. } else {
  3383. get_block_func = ext4_dio_get_block_unwritten_async;
  3384. dio_flags = DIO_LOCKING;
  3385. }
  3386. ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter,
  3387. get_block_func, ext4_end_io_dio, NULL,
  3388. dio_flags);
  3389. if (ret > 0 && !overwrite && ext4_test_inode_state(inode,
  3390. EXT4_STATE_DIO_UNWRITTEN)) {
  3391. int err;
  3392. /*
  3393. * for non AIO case, since the IO is already
  3394. * completed, we could do the conversion right here
  3395. */
  3396. err = ext4_convert_unwritten_extents(NULL, inode,
  3397. offset, ret);
  3398. if (err < 0)
  3399. ret = err;
  3400. ext4_clear_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
  3401. }
  3402. inode_dio_end(inode);
  3403. /* take i_mutex locking again if we do a ovewrite dio */
  3404. if (overwrite)
  3405. inode_lock(inode);
  3406. if (ret < 0 && final_size > inode->i_size)
  3407. ext4_truncate_failed_write(inode);
  3408. /* Handle extending of i_size after direct IO write */
  3409. if (orphan) {
  3410. int err;
  3411. /* Credits for sb + inode write */
  3412. handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
  3413. if (IS_ERR(handle)) {
  3414. /*
  3415. * We wrote the data but cannot extend
  3416. * i_size. Bail out. In async io case, we do
  3417. * not return error here because we have
  3418. * already submmitted the corresponding
  3419. * bio. Returning error here makes the caller
  3420. * think that this IO is done and failed
  3421. * resulting in race with bio's completion
  3422. * handler.
  3423. */
  3424. if (!ret)
  3425. ret = PTR_ERR(handle);
  3426. if (inode->i_nlink)
  3427. ext4_orphan_del(NULL, inode);
  3428. goto out;
  3429. }
  3430. if (inode->i_nlink)
  3431. ext4_orphan_del(handle, inode);
  3432. if (ret > 0) {
  3433. loff_t end = offset + ret;
  3434. if (end > inode->i_size || end > ei->i_disksize) {
  3435. ext4_update_i_disksize(inode, end);
  3436. if (end > inode->i_size)
  3437. i_size_write(inode, end);
  3438. /*
  3439. * We're going to return a positive `ret'
  3440. * here due to non-zero-length I/O, so there's
  3441. * no way of reporting error returns from
  3442. * ext4_mark_inode_dirty() to userspace. So
  3443. * ignore it.
  3444. */
  3445. ext4_mark_inode_dirty(handle, inode);
  3446. }
  3447. }
  3448. err = ext4_journal_stop(handle);
  3449. if (ret == 0)
  3450. ret = err;
  3451. }
  3452. out:
  3453. return ret;
  3454. }
  3455. static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter)
  3456. {
  3457. struct address_space *mapping = iocb->ki_filp->f_mapping;
  3458. struct inode *inode = mapping->host;
  3459. size_t count = iov_iter_count(iter);
  3460. ssize_t ret;
  3461. loff_t offset = iocb->ki_pos;
  3462. loff_t size = i_size_read(inode);
  3463. if (offset >= size)
  3464. return 0;
  3465. /*
  3466. * Shared inode_lock is enough for us - it protects against concurrent
  3467. * writes & truncates and since we take care of writing back page cache,
  3468. * we are protected against page writeback as well.
  3469. */
  3470. if (iocb->ki_flags & IOCB_NOWAIT) {
  3471. if (!inode_trylock_shared(inode))
  3472. return -EAGAIN;
  3473. } else {
  3474. inode_lock_shared(inode);
  3475. }
  3476. ret = filemap_write_and_wait_range(mapping, iocb->ki_pos,
  3477. iocb->ki_pos + count - 1);
  3478. if (ret)
  3479. goto out_unlock;
  3480. ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
  3481. iter, ext4_dio_get_block, NULL, NULL, 0);
  3482. out_unlock:
  3483. inode_unlock_shared(inode);
  3484. return ret;
  3485. }
  3486. static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
  3487. {
  3488. struct file *file = iocb->ki_filp;
  3489. struct inode *inode = file->f_mapping->host;
  3490. size_t count = iov_iter_count(iter);
  3491. loff_t offset = iocb->ki_pos;
  3492. ssize_t ret;
  3493. #ifdef CONFIG_EXT4_FS_ENCRYPTION
  3494. if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode))
  3495. return 0;
  3496. #endif
  3497. /*
  3498. * If we are doing data journalling we don't support O_DIRECT
  3499. */
  3500. if (ext4_should_journal_data(inode))
  3501. return 0;
  3502. /* Let buffer I/O handle the inline data case. */
  3503. if (ext4_has_inline_data(inode))
  3504. return 0;
  3505. trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter));
  3506. if (iov_iter_rw(iter) == READ)
  3507. ret = ext4_direct_IO_read(iocb, iter);
  3508. else
  3509. ret = ext4_direct_IO_write(iocb, iter);
  3510. trace_ext4_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), ret);
  3511. return ret;
  3512. }
  3513. /*
  3514. * Pages can be marked dirty completely asynchronously from ext4's journalling
  3515. * activity. By filemap_sync_pte(), try_to_unmap_one(), etc. We cannot do
  3516. * much here because ->set_page_dirty is called under VFS locks. The page is
  3517. * not necessarily locked.
  3518. *
  3519. * We cannot just dirty the page and leave attached buffers clean, because the
  3520. * buffers' dirty state is "definitive". We cannot just set the buffers dirty
  3521. * or jbddirty because all the journalling code will explode.
  3522. *
  3523. * So what we do is to mark the page "pending dirty" and next time writepage
  3524. * is called, propagate that into the buffers appropriately.
  3525. */
  3526. static int ext4_journalled_set_page_dirty(struct page *page)
  3527. {
  3528. SetPageChecked(page);
  3529. return __set_page_dirty_nobuffers(page);
  3530. }
  3531. static int ext4_set_page_dirty(struct page *page)
  3532. {
  3533. WARN_ON_ONCE(!PageLocked(page) && !PageDirty(page));
  3534. WARN_ON_ONCE(!page_has_buffers(page));
  3535. return __set_page_dirty_buffers(page);
  3536. }
  3537. static const struct address_space_operations ext4_aops = {
  3538. .readpage = ext4_readpage,
  3539. .readpages = ext4_readpages,
  3540. .writepage = ext4_writepage,
  3541. .writepages = ext4_writepages,
  3542. .write_begin = ext4_write_begin,
  3543. .write_end = ext4_write_end,
  3544. .set_page_dirty = ext4_set_page_dirty,
  3545. .bmap = ext4_bmap,
  3546. .invalidatepage = ext4_invalidatepage,
  3547. .releasepage = ext4_releasepage,
  3548. .direct_IO = ext4_direct_IO,
  3549. .migratepage = buffer_migrate_page,
  3550. .is_partially_uptodate = block_is_partially_uptodate,
  3551. .error_remove_page = generic_error_remove_page,
  3552. };
  3553. static const struct address_space_operations ext4_journalled_aops = {
  3554. .readpage = ext4_readpage,
  3555. .readpages = ext4_readpages,
  3556. .writepage = ext4_writepage,
  3557. .writepages = ext4_writepages,
  3558. .write_begin = ext4_write_begin,
  3559. .write_end = ext4_journalled_write_end,
  3560. .set_page_dirty = ext4_journalled_set_page_dirty,
  3561. .bmap = ext4_bmap,
  3562. .invalidatepage = ext4_journalled_invalidatepage,
  3563. .releasepage = ext4_releasepage,
  3564. .direct_IO = ext4_direct_IO,
  3565. .is_partially_uptodate = block_is_partially_uptodate,
  3566. .error_remove_page = generic_error_remove_page,
  3567. };
  3568. static const struct address_space_operations ext4_da_aops = {
  3569. .readpage = ext4_readpage,
  3570. .readpages = ext4_readpages,
  3571. .writepage = ext4_writepage,
  3572. .writepages = ext4_writepages,
  3573. .write_begin = ext4_da_write_begin,
  3574. .write_end = ext4_da_write_end,
  3575. .set_page_dirty = ext4_set_page_dirty,
  3576. .bmap = ext4_bmap,
  3577. .invalidatepage = ext4_da_invalidatepage,
  3578. .releasepage = ext4_releasepage,
  3579. .direct_IO = ext4_direct_IO,
  3580. .migratepage = buffer_migrate_page,
  3581. .is_partially_uptodate = block_is_partially_uptodate,
  3582. .error_remove_page = generic_error_remove_page,
  3583. };
  3584. static const struct address_space_operations ext4_dax_aops = {
  3585. .writepages = ext4_dax_writepages,
  3586. .direct_IO = noop_direct_IO,
  3587. .set_page_dirty = noop_set_page_dirty,
  3588. .bmap = ext4_bmap,
  3589. .invalidatepage = noop_invalidatepage,
  3590. };
  3591. void ext4_set_aops(struct inode *inode)
  3592. {
  3593. switch (ext4_inode_journal_mode(inode)) {
  3594. case EXT4_INODE_ORDERED_DATA_MODE:
  3595. case EXT4_INODE_WRITEBACK_DATA_MODE:
  3596. break;
  3597. case EXT4_INODE_JOURNAL_DATA_MODE:
  3598. inode->i_mapping->a_ops = &ext4_journalled_aops;
  3599. return;
  3600. default:
  3601. BUG();
  3602. }
  3603. if (IS_DAX(inode))
  3604. inode->i_mapping->a_ops = &ext4_dax_aops;
  3605. else if (test_opt(inode->i_sb, DELALLOC))
  3606. inode->i_mapping->a_ops = &ext4_da_aops;
  3607. else
  3608. inode->i_mapping->a_ops = &ext4_aops;
  3609. }
  3610. static int __ext4_block_zero_page_range(handle_t *handle,
  3611. struct address_space *mapping, loff_t from, loff_t length)
  3612. {
  3613. ext4_fsblk_t index = from >> PAGE_SHIFT;
  3614. unsigned offset = from & (PAGE_SIZE-1);
  3615. unsigned blocksize, pos;
  3616. ext4_lblk_t iblock;
  3617. struct inode *inode = mapping->host;
  3618. struct buffer_head *bh;
  3619. struct page *page;
  3620. int err = 0;
  3621. page = find_or_create_page(mapping, from >> PAGE_SHIFT,
  3622. mapping_gfp_constraint(mapping, ~__GFP_FS));
  3623. if (!page)
  3624. return -ENOMEM;
  3625. blocksize = inode->i_sb->s_blocksize;
  3626. iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
  3627. if (!page_has_buffers(page))
  3628. create_empty_buffers(page, blocksize, 0);
  3629. /* Find the buffer that contains "offset" */
  3630. bh = page_buffers(page);
  3631. pos = blocksize;
  3632. while (offset >= pos) {
  3633. bh = bh->b_this_page;
  3634. iblock++;
  3635. pos += blocksize;
  3636. }
  3637. if (buffer_freed(bh)) {
  3638. BUFFER_TRACE(bh, "freed: skip");
  3639. goto unlock;
  3640. }
  3641. if (!buffer_mapped(bh)) {
  3642. BUFFER_TRACE(bh, "unmapped");
  3643. ext4_get_block(inode, iblock, bh, 0);
  3644. /* unmapped? It's a hole - nothing to do */
  3645. if (!buffer_mapped(bh)) {
  3646. BUFFER_TRACE(bh, "still unmapped");
  3647. goto unlock;
  3648. }
  3649. }
  3650. /* Ok, it's mapped. Make sure it's up-to-date */
  3651. if (PageUptodate(page))
  3652. set_buffer_uptodate(bh);
  3653. if (!buffer_uptodate(bh)) {
  3654. err = -EIO;
  3655. ll_rw_block(REQ_OP_READ, 0, 1, &bh);
  3656. wait_on_buffer(bh);
  3657. /* Uhhuh. Read error. Complain and punt. */
  3658. if (!buffer_uptodate(bh))
  3659. goto unlock;
  3660. if (S_ISREG(inode->i_mode) &&
  3661. ext4_encrypted_inode(inode)) {
  3662. /* We expect the key to be set. */
  3663. BUG_ON(!fscrypt_has_encryption_key(inode));
  3664. BUG_ON(blocksize != PAGE_SIZE);
  3665. WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host,
  3666. page, PAGE_SIZE, 0, page->index));
  3667. }
  3668. }
  3669. if (ext4_should_journal_data(inode)) {
  3670. BUFFER_TRACE(bh, "get write access");
  3671. err = ext4_journal_get_write_access(handle, bh);
  3672. if (err)
  3673. goto unlock;
  3674. }
  3675. zero_user(page, offset, length);
  3676. BUFFER_TRACE(bh, "zeroed end of block");
  3677. if (ext4_should_journal_data(inode)) {
  3678. err = ext4_handle_dirty_metadata(handle, inode, bh);
  3679. } else {
  3680. err = 0;
  3681. mark_buffer_dirty(bh);
  3682. if (ext4_should_order_data(inode))
  3683. err = ext4_jbd2_inode_add_write(handle, inode, from,
  3684. length);
  3685. }
  3686. unlock:
  3687. unlock_page(page);
  3688. put_page(page);
  3689. return err;
  3690. }
  3691. /*
  3692. * ext4_block_zero_page_range() zeros out a mapping of length 'length'
  3693. * starting from file offset 'from'. The range to be zero'd must
  3694. * be contained with in one block. If the specified range exceeds
  3695. * the end of the block it will be shortened to end of the block
  3696. * that cooresponds to 'from'
  3697. */
  3698. static int ext4_block_zero_page_range(handle_t *handle,
  3699. struct address_space *mapping, loff_t from, loff_t length)
  3700. {
  3701. struct inode *inode = mapping->host;
  3702. unsigned offset = from & (PAGE_SIZE-1);
  3703. unsigned blocksize = inode->i_sb->s_blocksize;
  3704. unsigned max = blocksize - (offset & (blocksize - 1));
  3705. /*
  3706. * correct length if it does not fall between
  3707. * 'from' and the end of the block
  3708. */
  3709. if (length > max || length < 0)
  3710. length = max;
  3711. if (IS_DAX(inode)) {
  3712. return iomap_zero_range(inode, from, length, NULL,
  3713. &ext4_iomap_ops);
  3714. }
  3715. return __ext4_block_zero_page_range(handle, mapping, from, length);
  3716. }
  3717. /*
  3718. * ext4_block_truncate_page() zeroes out a mapping from file offset `from'
  3719. * up to the end of the block which corresponds to `from'.
  3720. * This required during truncate. We need to physically zero the tail end
  3721. * of that block so it doesn't yield old data if the file is later grown.
  3722. */
  3723. static int ext4_block_truncate_page(handle_t *handle,
  3724. struct address_space *mapping, loff_t from)
  3725. {
  3726. unsigned offset = from & (PAGE_SIZE-1);
  3727. unsigned length;
  3728. unsigned blocksize;
  3729. struct inode *inode = mapping->host;
  3730. /* If we are processing an encrypted inode during orphan list handling */
  3731. if (ext4_encrypted_inode(inode) && !fscrypt_has_encryption_key(inode))
  3732. return 0;
  3733. blocksize = inode->i_sb->s_blocksize;
  3734. length = blocksize - (offset & (blocksize - 1));
  3735. return ext4_block_zero_page_range(handle, mapping, from, length);
  3736. }
  3737. int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
  3738. loff_t lstart, loff_t length)
  3739. {
  3740. struct super_block *sb = inode->i_sb;
  3741. struct address_space *mapping = inode->i_mapping;
  3742. unsigned partial_start, partial_end;
  3743. ext4_fsblk_t start, end;
  3744. loff_t byte_end = (lstart + length - 1);
  3745. int err = 0;
  3746. partial_start = lstart & (sb->s_blocksize - 1);
  3747. partial_end = byte_end & (sb->s_blocksize - 1);
  3748. start = lstart >> sb->s_blocksize_bits;
  3749. end = byte_end >> sb->s_blocksize_bits;
  3750. /* Handle partial zero within the single block */
  3751. if (start == end &&
  3752. (partial_start || (partial_end != sb->s_blocksize - 1))) {
  3753. err = ext4_block_zero_page_range(handle, mapping,
  3754. lstart, length);
  3755. return err;
  3756. }
  3757. /* Handle partial zero out on the start of the range */
  3758. if (partial_start) {
  3759. err = ext4_block_zero_page_range(handle, mapping,
  3760. lstart, sb->s_blocksize);
  3761. if (err)
  3762. return err;
  3763. }
  3764. /* Handle partial zero out on the end of the range */
  3765. if (partial_end != sb->s_blocksize - 1)
  3766. err = ext4_block_zero_page_range(handle, mapping,
  3767. byte_end - partial_end,
  3768. partial_end + 1);
  3769. return err;
  3770. }
  3771. int ext4_can_truncate(struct inode *inode)
  3772. {
  3773. if (S_ISREG(inode->i_mode))
  3774. return 1;
  3775. if (S_ISDIR(inode->i_mode))
  3776. return 1;
  3777. if (S_ISLNK(inode->i_mode))
  3778. return !ext4_inode_is_fast_symlink(inode);
  3779. return 0;
  3780. }
  3781. /*
  3782. * We have to make sure i_disksize gets properly updated before we truncate
  3783. * page cache due to hole punching or zero range. Otherwise i_disksize update
  3784. * can get lost as it may have been postponed to submission of writeback but
  3785. * that will never happen after we truncate page cache.
  3786. */
  3787. int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
  3788. loff_t len)
  3789. {
  3790. handle_t *handle;
  3791. loff_t size = i_size_read(inode);
  3792. WARN_ON(!inode_is_locked(inode));
  3793. if (offset > size || offset + len < size)
  3794. return 0;
  3795. if (EXT4_I(inode)->i_disksize >= size)
  3796. return 0;
  3797. handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
  3798. if (IS_ERR(handle))
  3799. return PTR_ERR(handle);
  3800. ext4_update_i_disksize(inode, size);
  3801. ext4_mark_inode_dirty(handle, inode);
  3802. ext4_journal_stop(handle);
  3803. return 0;
  3804. }
  3805. static void ext4_wait_dax_page(struct ext4_inode_info *ei)
  3806. {
  3807. up_write(&ei->i_mmap_sem);
  3808. schedule();
  3809. down_write(&ei->i_mmap_sem);
  3810. }
  3811. int ext4_break_layouts(struct inode *inode)
  3812. {
  3813. struct ext4_inode_info *ei = EXT4_I(inode);
  3814. struct page *page;
  3815. int error;
  3816. if (WARN_ON_ONCE(!rwsem_is_locked(&ei->i_mmap_sem)))
  3817. return -EINVAL;
  3818. do {
  3819. page = dax_layout_busy_page(inode->i_mapping);
  3820. if (!page)
  3821. return 0;
  3822. error = ___wait_var_event(&page->_refcount,
  3823. atomic_read(&page->_refcount) == 1,
  3824. TASK_INTERRUPTIBLE, 0, 0,
  3825. ext4_wait_dax_page(ei));
  3826. } while (error == 0);
  3827. return error;
  3828. }
  3829. /*
  3830. * ext4_punch_hole: punches a hole in a file by releasing the blocks
  3831. * associated with the given offset and length
  3832. *
  3833. * @inode: File inode
  3834. * @offset: The offset where the hole will begin
  3835. * @len: The length of the hole
  3836. *
  3837. * Returns: 0 on success or negative on failure
  3838. */
  3839. int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
  3840. {
  3841. struct super_block *sb = inode->i_sb;
  3842. ext4_lblk_t first_block, stop_block;
  3843. struct address_space *mapping = inode->i_mapping;
  3844. loff_t first_block_offset, last_block_offset;
  3845. handle_t *handle;
  3846. unsigned int credits;
  3847. int ret = 0;
  3848. if (!S_ISREG(inode->i_mode))
  3849. return -EOPNOTSUPP;
  3850. trace_ext4_punch_hole(inode, offset, length, 0);
  3851. ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
  3852. if (ext4_has_inline_data(inode)) {
  3853. down_write(&EXT4_I(inode)->i_mmap_sem);
  3854. ret = ext4_convert_inline_data(inode);
  3855. up_write(&EXT4_I(inode)->i_mmap_sem);
  3856. if (ret)
  3857. return ret;
  3858. }
  3859. /*
  3860. * Write out all dirty pages to avoid race conditions
  3861. * Then release them.
  3862. */
  3863. if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
  3864. ret = filemap_write_and_wait_range(mapping, offset,
  3865. offset + length - 1);
  3866. if (ret)
  3867. return ret;
  3868. }
  3869. inode_lock(inode);
  3870. /* No need to punch hole beyond i_size */
  3871. if (offset >= inode->i_size)
  3872. goto out_mutex;
  3873. /*
  3874. * If the hole extends beyond i_size, set the hole
  3875. * to end after the page that contains i_size
  3876. */
  3877. if (offset + length > inode->i_size) {
  3878. length = inode->i_size +
  3879. PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -
  3880. offset;
  3881. }
  3882. if (offset & (sb->s_blocksize - 1) ||
  3883. (offset + length) & (sb->s_blocksize - 1)) {
  3884. /*
  3885. * Attach jinode to inode for jbd2 if we do any zeroing of
  3886. * partial block
  3887. */
  3888. ret = ext4_inode_attach_jinode(inode);
  3889. if (ret < 0)
  3890. goto out_mutex;
  3891. }
  3892. /* Wait all existing dio workers, newcomers will block on i_mutex */
  3893. inode_dio_wait(inode);
  3894. /*
  3895. * Prevent page faults from reinstantiating pages we have released from
  3896. * page cache.
  3897. */
  3898. down_write(&EXT4_I(inode)->i_mmap_sem);
  3899. ret = ext4_break_layouts(inode);
  3900. if (ret)
  3901. goto out_dio;
  3902. first_block_offset = round_up(offset, sb->s_blocksize);
  3903. last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;
  3904. /* Now release the pages and zero block aligned part of pages*/
  3905. if (last_block_offset > first_block_offset) {
  3906. ret = ext4_update_disksize_before_punch(inode, offset, length);
  3907. if (ret)
  3908. goto out_dio;
  3909. truncate_pagecache_range(inode, first_block_offset,
  3910. last_block_offset);
  3911. }
  3912. if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
  3913. credits = ext4_writepage_trans_blocks(inode);
  3914. else
  3915. credits = ext4_blocks_for_truncate(inode);
  3916. handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
  3917. if (IS_ERR(handle)) {
  3918. ret = PTR_ERR(handle);
  3919. ext4_std_error(sb, ret);
  3920. goto out_dio;
  3921. }
  3922. ret = ext4_zero_partial_blocks(handle, inode, offset,
  3923. length);
  3924. if (ret)
  3925. goto out_stop;
  3926. first_block = (offset + sb->s_blocksize - 1) >>
  3927. EXT4_BLOCK_SIZE_BITS(sb);
  3928. stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);
  3929. /* If there are blocks to remove, do it */
  3930. if (stop_block > first_block) {
  3931. down_write(&EXT4_I(inode)->i_data_sem);
  3932. ext4_discard_preallocations(inode);
  3933. ret = ext4_es_remove_extent(inode, first_block,
  3934. stop_block - first_block);
  3935. if (ret) {
  3936. up_write(&EXT4_I(inode)->i_data_sem);
  3937. goto out_stop;
  3938. }
  3939. if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
  3940. ret = ext4_ext_remove_space(inode, first_block,
  3941. stop_block - 1);
  3942. else
  3943. ret = ext4_ind_remove_space(handle, inode, first_block,
  3944. stop_block);
  3945. up_write(&EXT4_I(inode)->i_data_sem);
  3946. }
  3947. if (IS_SYNC(inode))
  3948. ext4_handle_sync(handle);
  3949. inode->i_mtime = inode->i_ctime = current_time(inode);
  3950. ext4_mark_inode_dirty(handle, inode);
  3951. if (ret >= 0)
  3952. ext4_update_inode_fsync_trans(handle, inode, 1);
  3953. out_stop:
  3954. ext4_journal_stop(handle);
  3955. out_dio:
  3956. up_write(&EXT4_I(inode)->i_mmap_sem);
  3957. out_mutex:
  3958. inode_unlock(inode);
  3959. return ret;
  3960. }
  3961. int ext4_inode_attach_jinode(struct inode *inode)
  3962. {
  3963. struct ext4_inode_info *ei = EXT4_I(inode);
  3964. struct jbd2_inode *jinode;
  3965. if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
  3966. return 0;
  3967. jinode = jbd2_alloc_inode(GFP_KERNEL);
  3968. spin_lock(&inode->i_lock);
  3969. if (!ei->jinode) {
  3970. if (!jinode) {
  3971. spin_unlock(&inode->i_lock);
  3972. return -ENOMEM;
  3973. }
  3974. ei->jinode = jinode;
  3975. jbd2_journal_init_jbd_inode(ei->jinode, inode);
  3976. jinode = NULL;
  3977. }
  3978. spin_unlock(&inode->i_lock);
  3979. if (unlikely(jinode != NULL))
  3980. jbd2_free_inode(jinode);
  3981. return 0;
  3982. }
  3983. /*
  3984. * ext4_truncate()
  3985. *
  3986. * We block out ext4_get_block() block instantiations across the entire
  3987. * transaction, and VFS/VM ensures that ext4_truncate() cannot run
  3988. * simultaneously on behalf of the same inode.
  3989. *
  3990. * As we work through the truncate and commit bits of it to the journal there
  3991. * is one core, guiding principle: the file's tree must always be consistent on
  3992. * disk. We must be able to restart the truncate after a crash.
  3993. *
  3994. * The file's tree may be transiently inconsistent in memory (although it
  3995. * probably isn't), but whenever we close off and commit a journal transaction,
  3996. * the contents of (the filesystem + the journal) must be consistent and
  3997. * restartable. It's pretty simple, really: bottom up, right to left (although
  3998. * left-to-right works OK too).
  3999. *
  4000. * Note that at recovery time, journal replay occurs *before* the restart of
  4001. * truncate against the orphan inode list.
  4002. *
  4003. * The committed inode has the new, desired i_size (which is the same as
  4004. * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see
  4005. * that this inode's truncate did not complete and it will again call
  4006. * ext4_truncate() to have another go. So there will be instantiated blocks
  4007. * to the right of the truncation point in a crashed ext4 filesystem. But
  4008. * that's fine - as long as they are linked from the inode, the post-crash
  4009. * ext4_truncate() run will find them and release them.
  4010. */
  4011. int ext4_truncate(struct inode *inode)
  4012. {
  4013. struct ext4_inode_info *ei = EXT4_I(inode);
  4014. unsigned int credits;
  4015. int err = 0;
  4016. handle_t *handle;
  4017. struct address_space *mapping = inode->i_mapping;
  4018. /*
  4019. * There is a possibility that we're either freeing the inode
  4020. * or it's a completely new inode. In those cases we might not
  4021. * have i_mutex locked because it's not necessary.
  4022. */
  4023. if (!(inode->i_state & (I_NEW|I_FREEING)))
  4024. WARN_ON(!inode_is_locked(inode));
  4025. trace_ext4_truncate_enter(inode);
  4026. if (!ext4_can_truncate(inode))
  4027. return 0;
  4028. ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
  4029. if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
  4030. ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
  4031. if (ext4_has_inline_data(inode)) {
  4032. int has_inline = 1;
  4033. err = ext4_inline_data_truncate(inode, &has_inline);
  4034. if (err)
  4035. return err;
  4036. if (has_inline)
  4037. return 0;
  4038. }
  4039. /* If we zero-out tail of the page, we have to create jinode for jbd2 */
  4040. if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
  4041. if (ext4_inode_attach_jinode(inode) < 0)
  4042. return 0;
  4043. }
  4044. if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
  4045. credits = ext4_writepage_trans_blocks(inode);
  4046. else
  4047. credits = ext4_blocks_for_truncate(inode);
  4048. handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
  4049. if (IS_ERR(handle))
  4050. return PTR_ERR(handle);
  4051. if (inode->i_size & (inode->i_sb->s_blocksize - 1))
  4052. ext4_block_truncate_page(handle, mapping, inode->i_size);
  4053. /*
  4054. * We add the inode to the orphan list, so that if this
  4055. * truncate spans multiple transactions, and we crash, we will
  4056. * resume the truncate when the filesystem recovers. It also
  4057. * marks the inode dirty, to catch the new size.
  4058. *
  4059. * Implication: the file must always be in a sane, consistent
  4060. * truncatable state while each transaction commits.
  4061. */
  4062. err = ext4_orphan_add(handle, inode);
  4063. if (err)
  4064. goto out_stop;
  4065. down_write(&EXT4_I(inode)->i_data_sem);
  4066. ext4_discard_preallocations(inode);
  4067. if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
  4068. err = ext4_ext_truncate(handle, inode);
  4069. else
  4070. ext4_ind_truncate(handle, inode);
  4071. up_write(&ei->i_data_sem);
  4072. if (err)
  4073. goto out_stop;
  4074. if (IS_SYNC(inode))
  4075. ext4_handle_sync(handle);
  4076. out_stop:
  4077. /*
  4078. * If this was a simple ftruncate() and the file will remain alive,
  4079. * then we need to clear up the orphan record which we created above.
  4080. * However, if this was a real unlink then we were called by
  4081. * ext4_evict_inode(), and we allow that function to clean up the
  4082. * orphan info for us.
  4083. */
  4084. if (inode->i_nlink)
  4085. ext4_orphan_del(handle, inode);
  4086. inode->i_mtime = inode->i_ctime = current_time(inode);
  4087. ext4_mark_inode_dirty(handle, inode);
  4088. ext4_journal_stop(handle);
  4089. trace_ext4_truncate_exit(inode);
  4090. return err;
  4091. }
  4092. /*
  4093. * ext4_get_inode_loc returns with an extra refcount against the inode's
  4094. * underlying buffer_head on success. If 'in_mem' is true, we have all
  4095. * data in memory that is needed to recreate the on-disk version of this
  4096. * inode.
  4097. */
  4098. static int __ext4_get_inode_loc(struct inode *inode,
  4099. struct ext4_iloc *iloc, int in_mem)
  4100. {
  4101. struct ext4_group_desc *gdp;
  4102. struct buffer_head *bh;
  4103. struct super_block *sb = inode->i_sb;
  4104. ext4_fsblk_t block;
  4105. int inodes_per_block, inode_offset;
  4106. iloc->bh = NULL;
  4107. if (inode->i_ino < EXT4_ROOT_INO ||
  4108. inode->i_ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
  4109. return -EFSCORRUPTED;
  4110. iloc->block_group = (inode->i_ino - 1) / EXT4_INODES_PER_GROUP(sb);
  4111. gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
  4112. if (!gdp)
  4113. return -EIO;
  4114. /*
  4115. * Figure out the offset within the block group inode table
  4116. */
  4117. inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
  4118. inode_offset = ((inode->i_ino - 1) %
  4119. EXT4_INODES_PER_GROUP(sb));
  4120. block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block);
  4121. iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
  4122. bh = sb_getblk(sb, block);
  4123. if (unlikely(!bh))
  4124. return -ENOMEM;
  4125. if (!buffer_uptodate(bh)) {
  4126. lock_buffer(bh);
  4127. /*
  4128. * If the buffer has the write error flag, we have failed
  4129. * to write out another inode in the same block. In this
  4130. * case, we don't have to read the block because we may
  4131. * read the old inode data successfully.
  4132. */
  4133. if (buffer_write_io_error(bh) && !buffer_uptodate(bh))
  4134. set_buffer_uptodate(bh);
  4135. if (buffer_uptodate(bh)) {
  4136. /* someone brought it uptodate while we waited */
  4137. unlock_buffer(bh);
  4138. goto has_buffer;
  4139. }
  4140. /*
  4141. * If we have all information of the inode in memory and this
  4142. * is the only valid inode in the block, we need not read the
  4143. * block.
  4144. */
  4145. if (in_mem) {
  4146. struct buffer_head *bitmap_bh;
  4147. int i, start;
  4148. start = inode_offset & ~(inodes_per_block - 1);
  4149. /* Is the inode bitmap in cache? */
  4150. bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
  4151. if (unlikely(!bitmap_bh))
  4152. goto make_io;
  4153. /*
  4154. * If the inode bitmap isn't in cache then the
  4155. * optimisation may end up performing two reads instead
  4156. * of one, so skip it.
  4157. */
  4158. if (!buffer_uptodate(bitmap_bh)) {
  4159. brelse(bitmap_bh);
  4160. goto make_io;
  4161. }
  4162. for (i = start; i < start + inodes_per_block; i++) {
  4163. if (i == inode_offset)
  4164. continue;
  4165. if (ext4_test_bit(i, bitmap_bh->b_data))
  4166. break;
  4167. }
  4168. brelse(bitmap_bh);
  4169. if (i == start + inodes_per_block) {
  4170. /* all other inodes are free, so skip I/O */
  4171. memset(bh->b_data, 0, bh->b_size);
  4172. set_buffer_uptodate(bh);
  4173. unlock_buffer(bh);
  4174. goto has_buffer;
  4175. }
  4176. }
  4177. make_io:
  4178. /*
  4179. * If we need to do any I/O, try to pre-readahead extra
  4180. * blocks from the inode table.
  4181. */
  4182. if (EXT4_SB(sb)->s_inode_readahead_blks) {
  4183. ext4_fsblk_t b, end, table;
  4184. unsigned num;
  4185. __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
  4186. table = ext4_inode_table(sb, gdp);
  4187. /* s_inode_readahead_blks is always a power of 2 */
  4188. b = block & ~((ext4_fsblk_t) ra_blks - 1);
  4189. if (table > b)
  4190. b = table;
  4191. end = b + ra_blks;
  4192. num = EXT4_INODES_PER_GROUP(sb);
  4193. if (ext4_has_group_desc_csum(sb))
  4194. num -= ext4_itable_unused_count(sb, gdp);
  4195. table += num / inodes_per_block;
  4196. if (end > table)
  4197. end = table;
  4198. while (b <= end)
  4199. sb_breadahead_unmovable(sb, b++);
  4200. }
  4201. /*
  4202. * There are other valid inodes in the buffer, this inode
  4203. * has in-inode xattrs, or we don't have this inode in memory.
  4204. * Read the block from disk.
  4205. */
  4206. trace_ext4_load_inode(inode);
  4207. get_bh(bh);
  4208. bh->b_end_io = end_buffer_read_sync;
  4209. submit_bh(REQ_OP_READ, REQ_META | REQ_PRIO, bh);
  4210. wait_on_buffer(bh);
  4211. if (!buffer_uptodate(bh)) {
  4212. EXT4_ERROR_INODE_BLOCK(inode, block,
  4213. "unable to read itable block");
  4214. brelse(bh);
  4215. return -EIO;
  4216. }
  4217. }
  4218. has_buffer:
  4219. iloc->bh = bh;
  4220. return 0;
  4221. }
  4222. int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
  4223. {
  4224. /* We have all inode data except xattrs in memory here. */
  4225. return __ext4_get_inode_loc(inode, iloc,
  4226. !ext4_test_inode_state(inode, EXT4_STATE_XATTR));
  4227. }
  4228. static bool ext4_should_use_dax(struct inode *inode)
  4229. {
  4230. if (!test_opt(inode->i_sb, DAX))
  4231. return false;
  4232. if (!S_ISREG(inode->i_mode))
  4233. return false;
  4234. if (ext4_should_journal_data(inode))
  4235. return false;
  4236. if (ext4_has_inline_data(inode))
  4237. return false;
  4238. if (ext4_encrypted_inode(inode))
  4239. return false;
  4240. return true;
  4241. }
  4242. void ext4_set_inode_flags(struct inode *inode)
  4243. {
  4244. unsigned int flags = EXT4_I(inode)->i_flags;
  4245. unsigned int new_fl = 0;
  4246. if (flags & EXT4_SYNC_FL)
  4247. new_fl |= S_SYNC;
  4248. if (flags & EXT4_APPEND_FL)
  4249. new_fl |= S_APPEND;
  4250. if (flags & EXT4_IMMUTABLE_FL)
  4251. new_fl |= S_IMMUTABLE;
  4252. if (flags & EXT4_NOATIME_FL)
  4253. new_fl |= S_NOATIME;
  4254. if (flags & EXT4_DIRSYNC_FL)
  4255. new_fl |= S_DIRSYNC;
  4256. if (ext4_should_use_dax(inode))
  4257. new_fl |= S_DAX;
  4258. if (flags & EXT4_ENCRYPT_FL)
  4259. new_fl |= S_ENCRYPTED;
  4260. inode_set_flags(inode, new_fl,
  4261. S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
  4262. S_ENCRYPTED);
  4263. }
  4264. static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
  4265. struct ext4_inode_info *ei)
  4266. {
  4267. blkcnt_t i_blocks ;
  4268. struct inode *inode = &(ei->vfs_inode);
  4269. struct super_block *sb = inode->i_sb;
  4270. if (ext4_has_feature_huge_file(sb)) {
  4271. /* we are using combined 48 bit field */
  4272. i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
  4273. le32_to_cpu(raw_inode->i_blocks_lo);
  4274. if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
  4275. /* i_blocks represent file system block size */
  4276. return i_blocks << (inode->i_blkbits - 9);
  4277. } else {
  4278. return i_blocks;
  4279. }
  4280. } else {
  4281. return le32_to_cpu(raw_inode->i_blocks_lo);
  4282. }
  4283. }
  4284. static inline int ext4_iget_extra_inode(struct inode *inode,
  4285. struct ext4_inode *raw_inode,
  4286. struct ext4_inode_info *ei)
  4287. {
  4288. __le32 *magic = (void *)raw_inode +
  4289. EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
  4290. if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize + sizeof(__le32) <=
  4291. EXT4_INODE_SIZE(inode->i_sb) &&
  4292. *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
  4293. ext4_set_inode_state(inode, EXT4_STATE_XATTR);
  4294. return ext4_find_inline_data_nolock(inode);
  4295. } else
  4296. EXT4_I(inode)->i_inline_off = 0;
  4297. return 0;
  4298. }
  4299. int ext4_get_projid(struct inode *inode, kprojid_t *projid)
  4300. {
  4301. if (!ext4_has_feature_project(inode->i_sb))
  4302. return -EOPNOTSUPP;
  4303. *projid = EXT4_I(inode)->i_projid;
  4304. return 0;
  4305. }
  4306. /*
  4307. * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of
  4308. * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag
  4309. * set.
  4310. */
  4311. static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
  4312. {
  4313. if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
  4314. inode_set_iversion_raw(inode, val);
  4315. else
  4316. inode_set_iversion_queried(inode, val);
  4317. }
  4318. static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
  4319. {
  4320. if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
  4321. return inode_peek_iversion_raw(inode);
  4322. else
  4323. return inode_peek_iversion(inode);
  4324. }
  4325. struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
  4326. ext4_iget_flags flags, const char *function,
  4327. unsigned int line)
  4328. {
  4329. struct ext4_iloc iloc;
  4330. struct ext4_inode *raw_inode;
  4331. struct ext4_inode_info *ei;
  4332. struct inode *inode;
  4333. journal_t *journal = EXT4_SB(sb)->s_journal;
  4334. long ret;
  4335. loff_t size;
  4336. int block;
  4337. uid_t i_uid;
  4338. gid_t i_gid;
  4339. projid_t i_projid;
  4340. if ((!(flags & EXT4_IGET_SPECIAL) &&
  4341. (ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO)) ||
  4342. (ino < EXT4_ROOT_INO) ||
  4343. (ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))) {
  4344. if (flags & EXT4_IGET_HANDLE)
  4345. return ERR_PTR(-ESTALE);
  4346. __ext4_error(sb, function, line,
  4347. "inode #%lu: comm %s: iget: illegal inode #",
  4348. ino, current->comm);
  4349. return ERR_PTR(-EFSCORRUPTED);
  4350. }
  4351. inode = iget_locked(sb, ino);
  4352. if (!inode)
  4353. return ERR_PTR(-ENOMEM);
  4354. if (!(inode->i_state & I_NEW))
  4355. return inode;
  4356. ei = EXT4_I(inode);
  4357. iloc.bh = NULL;
  4358. ret = __ext4_get_inode_loc(inode, &iloc, 0);
  4359. if (ret < 0)
  4360. goto bad_inode;
  4361. raw_inode = ext4_raw_inode(&iloc);
  4362. if ((ino == EXT4_ROOT_INO) && (raw_inode->i_links_count == 0)) {
  4363. ext4_error_inode(inode, function, line, 0,
  4364. "iget: root inode unallocated");
  4365. ret = -EFSCORRUPTED;
  4366. goto bad_inode;
  4367. }
  4368. if ((flags & EXT4_IGET_HANDLE) &&
  4369. (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
  4370. ret = -ESTALE;
  4371. goto bad_inode;
  4372. }
  4373. if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
  4374. ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
  4375. if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
  4376. EXT4_INODE_SIZE(inode->i_sb) ||
  4377. (ei->i_extra_isize & 3)) {
  4378. ext4_error_inode(inode, function, line, 0,
  4379. "iget: bad extra_isize %u "
  4380. "(inode size %u)",
  4381. ei->i_extra_isize,
  4382. EXT4_INODE_SIZE(inode->i_sb));
  4383. ret = -EFSCORRUPTED;
  4384. goto bad_inode;
  4385. }
  4386. } else
  4387. ei->i_extra_isize = 0;
  4388. /* Precompute checksum seed for inode metadata */
  4389. if (ext4_has_metadata_csum(sb)) {
  4390. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  4391. __u32 csum;
  4392. __le32 inum = cpu_to_le32(inode->i_ino);
  4393. __le32 gen = raw_inode->i_generation;
  4394. csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum,
  4395. sizeof(inum));
  4396. ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen,
  4397. sizeof(gen));
  4398. }
  4399. if (!ext4_inode_csum_verify(inode, raw_inode, ei)) {
  4400. ext4_error_inode(inode, function, line, 0,
  4401. "iget: checksum invalid");
  4402. ret = -EFSBADCRC;
  4403. goto bad_inode;
  4404. }
  4405. inode->i_mode = le16_to_cpu(raw_inode->i_mode);
  4406. i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
  4407. i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
  4408. if (ext4_has_feature_project(sb) &&
  4409. EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
  4410. EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
  4411. i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
  4412. else
  4413. i_projid = EXT4_DEF_PROJID;
  4414. if (!(test_opt(inode->i_sb, NO_UID32))) {
  4415. i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
  4416. i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
  4417. }
  4418. i_uid_write(inode, i_uid);
  4419. i_gid_write(inode, i_gid);
  4420. ei->i_projid = make_kprojid(&init_user_ns, i_projid);
  4421. set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
  4422. ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */
  4423. ei->i_inline_off = 0;
  4424. ei->i_dir_start_lookup = 0;
  4425. ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
  4426. /* We now have enough fields to check if the inode was active or not.
  4427. * This is needed because nfsd might try to access dead inodes
  4428. * the test is that same one that e2fsck uses
  4429. * NeilBrown 1999oct15
  4430. */
  4431. if (inode->i_nlink == 0) {
  4432. if ((inode->i_mode == 0 ||
  4433. !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
  4434. ino != EXT4_BOOT_LOADER_INO) {
  4435. /* this inode is deleted */
  4436. ret = -ESTALE;
  4437. goto bad_inode;
  4438. }
  4439. /* The only unlinked inodes we let through here have
  4440. * valid i_mode and are being read by the orphan
  4441. * recovery code: that's fine, we're about to complete
  4442. * the process of deleting those.
  4443. * OR it is the EXT4_BOOT_LOADER_INO which is
  4444. * not initialized on a new filesystem. */
  4445. }
  4446. ei->i_flags = le32_to_cpu(raw_inode->i_flags);
  4447. ext4_set_inode_flags(inode);
  4448. inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
  4449. ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
  4450. if (ext4_has_feature_64bit(sb))
  4451. ei->i_file_acl |=
  4452. ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
  4453. inode->i_size = ext4_isize(sb, raw_inode);
  4454. if ((size = i_size_read(inode)) < 0) {
  4455. ext4_error_inode(inode, function, line, 0,
  4456. "iget: bad i_size value: %lld", size);
  4457. ret = -EFSCORRUPTED;
  4458. goto bad_inode;
  4459. }
  4460. /*
  4461. * If dir_index is not enabled but there's dir with INDEX flag set,
  4462. * we'd normally treat htree data as empty space. But with metadata
  4463. * checksumming that corrupts checksums so forbid that.
  4464. */
  4465. if (!ext4_has_feature_dir_index(sb) && ext4_has_metadata_csum(sb) &&
  4466. ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {
  4467. ext4_error_inode(inode, function, line, 0,
  4468. "iget: Dir with htree data on filesystem without dir_index feature.");
  4469. ret = -EFSCORRUPTED;
  4470. goto bad_inode;
  4471. }
  4472. ei->i_disksize = inode->i_size;
  4473. #ifdef CONFIG_QUOTA
  4474. ei->i_reserved_quota = 0;
  4475. #endif
  4476. inode->i_generation = le32_to_cpu(raw_inode->i_generation);
  4477. ei->i_block_group = iloc.block_group;
  4478. ei->i_last_alloc_group = ~0;
  4479. /*
  4480. * NOTE! The in-memory inode i_data array is in little-endian order
  4481. * even on big-endian machines: we do NOT byteswap the block numbers!
  4482. */
  4483. for (block = 0; block < EXT4_N_BLOCKS; block++)
  4484. ei->i_data[block] = raw_inode->i_block[block];
  4485. INIT_LIST_HEAD(&ei->i_orphan);
  4486. /*
  4487. * Set transaction id's of transactions that have to be committed
  4488. * to finish f[data]sync. We set them to currently running transaction
  4489. * as we cannot be sure that the inode or some of its metadata isn't
  4490. * part of the transaction - the inode could have been reclaimed and
  4491. * now it is reread from disk.
  4492. */
  4493. if (journal) {
  4494. transaction_t *transaction;
  4495. tid_t tid;
  4496. read_lock(&journal->j_state_lock);
  4497. if (journal->j_running_transaction)
  4498. transaction = journal->j_running_transaction;
  4499. else
  4500. transaction = journal->j_committing_transaction;
  4501. if (transaction)
  4502. tid = transaction->t_tid;
  4503. else
  4504. tid = journal->j_commit_sequence;
  4505. read_unlock(&journal->j_state_lock);
  4506. ei->i_sync_tid = tid;
  4507. ei->i_datasync_tid = tid;
  4508. }
  4509. if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
  4510. if (ei->i_extra_isize == 0) {
  4511. /* The extra space is currently unused. Use it. */
  4512. BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
  4513. ei->i_extra_isize = sizeof(struct ext4_inode) -
  4514. EXT4_GOOD_OLD_INODE_SIZE;
  4515. } else {
  4516. ret = ext4_iget_extra_inode(inode, raw_inode, ei);
  4517. if (ret)
  4518. goto bad_inode;
  4519. }
  4520. }
  4521. EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode);
  4522. EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode);
  4523. EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode);
  4524. EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
  4525. if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
  4526. u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
  4527. if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
  4528. if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
  4529. ivers |=
  4530. (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
  4531. }
  4532. ext4_inode_set_iversion_queried(inode, ivers);
  4533. }
  4534. ret = 0;
  4535. if (ei->i_file_acl &&
  4536. !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {
  4537. ext4_error_inode(inode, function, line, 0,
  4538. "iget: bad extended attribute block %llu",
  4539. ei->i_file_acl);
  4540. ret = -EFSCORRUPTED;
  4541. goto bad_inode;
  4542. } else if (!ext4_has_inline_data(inode)) {
  4543. /* validate the block references in the inode */
  4544. if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
  4545. (S_ISLNK(inode->i_mode) &&
  4546. !ext4_inode_is_fast_symlink(inode))) {
  4547. if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
  4548. ret = ext4_ext_check_inode(inode);
  4549. else
  4550. ret = ext4_ind_check_inode(inode);
  4551. }
  4552. }
  4553. if (ret)
  4554. goto bad_inode;
  4555. if (S_ISREG(inode->i_mode)) {
  4556. inode->i_op = &ext4_file_inode_operations;
  4557. inode->i_fop = &ext4_file_operations;
  4558. ext4_set_aops(inode);
  4559. } else if (S_ISDIR(inode->i_mode)) {
  4560. inode->i_op = &ext4_dir_inode_operations;
  4561. inode->i_fop = &ext4_dir_operations;
  4562. } else if (S_ISLNK(inode->i_mode)) {
  4563. /* VFS does not allow setting these so must be corruption */
  4564. if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
  4565. ext4_error_inode(inode, function, line, 0,
  4566. "iget: immutable or append flags "
  4567. "not allowed on symlinks");
  4568. ret = -EFSCORRUPTED;
  4569. goto bad_inode;
  4570. }
  4571. if (ext4_encrypted_inode(inode)) {
  4572. inode->i_op = &ext4_encrypted_symlink_inode_operations;
  4573. ext4_set_aops(inode);
  4574. } else if (ext4_inode_is_fast_symlink(inode)) {
  4575. inode->i_link = (char *)ei->i_data;
  4576. inode->i_op = &ext4_fast_symlink_inode_operations;
  4577. nd_terminate_link(ei->i_data, inode->i_size,
  4578. sizeof(ei->i_data) - 1);
  4579. } else {
  4580. inode->i_op = &ext4_symlink_inode_operations;
  4581. ext4_set_aops(inode);
  4582. }
  4583. inode_nohighmem(inode);
  4584. } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
  4585. S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
  4586. inode->i_op = &ext4_special_inode_operations;
  4587. if (raw_inode->i_block[0])
  4588. init_special_inode(inode, inode->i_mode,
  4589. old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
  4590. else
  4591. init_special_inode(inode, inode->i_mode,
  4592. new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
  4593. } else if (ino == EXT4_BOOT_LOADER_INO) {
  4594. make_bad_inode(inode);
  4595. } else {
  4596. ret = -EFSCORRUPTED;
  4597. ext4_error_inode(inode, function, line, 0,
  4598. "iget: bogus i_mode (%o)", inode->i_mode);
  4599. goto bad_inode;
  4600. }
  4601. brelse(iloc.bh);
  4602. unlock_new_inode(inode);
  4603. return inode;
  4604. bad_inode:
  4605. brelse(iloc.bh);
  4606. iget_failed(inode);
  4607. return ERR_PTR(ret);
  4608. }
  4609. static int ext4_inode_blocks_set(handle_t *handle,
  4610. struct ext4_inode *raw_inode,
  4611. struct ext4_inode_info *ei)
  4612. {
  4613. struct inode *inode = &(ei->vfs_inode);
  4614. u64 i_blocks = READ_ONCE(inode->i_blocks);
  4615. struct super_block *sb = inode->i_sb;
  4616. if (i_blocks <= ~0U) {
  4617. /*
  4618. * i_blocks can be represented in a 32 bit variable
  4619. * as multiple of 512 bytes
  4620. */
  4621. raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
  4622. raw_inode->i_blocks_high = 0;
  4623. ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
  4624. return 0;
  4625. }
  4626. if (!ext4_has_feature_huge_file(sb))
  4627. return -EFBIG;
  4628. if (i_blocks <= 0xffffffffffffULL) {
  4629. /*
  4630. * i_blocks can be represented in a 48 bit variable
  4631. * as multiple of 512 bytes
  4632. */
  4633. raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
  4634. raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
  4635. ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
  4636. } else {
  4637. ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
  4638. /* i_block is stored in file system block size */
  4639. i_blocks = i_blocks >> (inode->i_blkbits - 9);
  4640. raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
  4641. raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
  4642. }
  4643. return 0;
  4644. }
  4645. struct other_inode {
  4646. unsigned long orig_ino;
  4647. struct ext4_inode *raw_inode;
  4648. };
  4649. static int other_inode_match(struct inode * inode, unsigned long ino,
  4650. void *data)
  4651. {
  4652. struct other_inode *oi = (struct other_inode *) data;
  4653. if ((inode->i_ino != ino) ||
  4654. (inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW |
  4655. I_DIRTY_INODE)) ||
  4656. ((inode->i_state & I_DIRTY_TIME) == 0))
  4657. return 0;
  4658. spin_lock(&inode->i_lock);
  4659. if (((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW |
  4660. I_DIRTY_INODE)) == 0) &&
  4661. (inode->i_state & I_DIRTY_TIME)) {
  4662. struct ext4_inode_info *ei = EXT4_I(inode);
  4663. inode->i_state &= ~I_DIRTY_TIME;
  4664. spin_unlock(&inode->i_lock);
  4665. spin_lock(&ei->i_raw_lock);
  4666. EXT4_INODE_SET_XTIME(i_ctime, inode, oi->raw_inode);
  4667. EXT4_INODE_SET_XTIME(i_mtime, inode, oi->raw_inode);
  4668. EXT4_INODE_SET_XTIME(i_atime, inode, oi->raw_inode);
  4669. ext4_inode_csum_set(inode, oi->raw_inode, ei);
  4670. spin_unlock(&ei->i_raw_lock);
  4671. trace_ext4_other_inode_update_time(inode, oi->orig_ino);
  4672. return -1;
  4673. }
  4674. spin_unlock(&inode->i_lock);
  4675. return -1;
  4676. }
  4677. /*
  4678. * Opportunistically update the other time fields for other inodes in
  4679. * the same inode table block.
  4680. */
  4681. static void ext4_update_other_inodes_time(struct super_block *sb,
  4682. unsigned long orig_ino, char *buf)
  4683. {
  4684. struct other_inode oi;
  4685. unsigned long ino;
  4686. int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
  4687. int inode_size = EXT4_INODE_SIZE(sb);
  4688. oi.orig_ino = orig_ino;
  4689. /*
  4690. * Calculate the first inode in the inode table block. Inode
  4691. * numbers are one-based. That is, the first inode in a block
  4692. * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
  4693. */
  4694. ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
  4695. for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
  4696. if (ino == orig_ino)
  4697. continue;
  4698. oi.raw_inode = (struct ext4_inode *) buf;
  4699. (void) find_inode_nowait(sb, ino, other_inode_match, &oi);
  4700. }
  4701. }
  4702. /*
  4703. * Post the struct inode info into an on-disk inode location in the
  4704. * buffer-cache. This gobbles the caller's reference to the
  4705. * buffer_head in the inode location struct.
  4706. *
  4707. * The caller must have write access to iloc->bh.
  4708. */
  4709. static int ext4_do_update_inode(handle_t *handle,
  4710. struct inode *inode,
  4711. struct ext4_iloc *iloc)
  4712. {
  4713. struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
  4714. struct ext4_inode_info *ei = EXT4_I(inode);
  4715. struct buffer_head *bh = iloc->bh;
  4716. struct super_block *sb = inode->i_sb;
  4717. int err = 0, block;
  4718. int need_datasync = 0, set_large_file = 0;
  4719. uid_t i_uid;
  4720. gid_t i_gid;
  4721. projid_t i_projid;
  4722. spin_lock(&ei->i_raw_lock);
  4723. /* For fields not tracked in the in-memory inode,
  4724. * initialise them to zero for new inodes. */
  4725. if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
  4726. memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
  4727. err = ext4_inode_blocks_set(handle, raw_inode, ei);
  4728. if (err) {
  4729. spin_unlock(&ei->i_raw_lock);
  4730. goto out_brelse;
  4731. }
  4732. raw_inode->i_mode = cpu_to_le16(inode->i_mode);
  4733. i_uid = i_uid_read(inode);
  4734. i_gid = i_gid_read(inode);
  4735. i_projid = from_kprojid(&init_user_ns, ei->i_projid);
  4736. if (!(test_opt(inode->i_sb, NO_UID32))) {
  4737. raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
  4738. raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
  4739. /*
  4740. * Fix up interoperability with old kernels. Otherwise, old inodes get
  4741. * re-used with the upper 16 bits of the uid/gid intact
  4742. */
  4743. if (ei->i_dtime && list_empty(&ei->i_orphan)) {
  4744. raw_inode->i_uid_high = 0;
  4745. raw_inode->i_gid_high = 0;
  4746. } else {
  4747. raw_inode->i_uid_high =
  4748. cpu_to_le16(high_16_bits(i_uid));
  4749. raw_inode->i_gid_high =
  4750. cpu_to_le16(high_16_bits(i_gid));
  4751. }
  4752. } else {
  4753. raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
  4754. raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
  4755. raw_inode->i_uid_high = 0;
  4756. raw_inode->i_gid_high = 0;
  4757. }
  4758. raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
  4759. EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
  4760. EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
  4761. EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
  4762. EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
  4763. raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
  4764. raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
  4765. if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
  4766. raw_inode->i_file_acl_high =
  4767. cpu_to_le16(ei->i_file_acl >> 32);
  4768. raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
  4769. if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode)) {
  4770. ext4_isize_set(raw_inode, ei->i_disksize);
  4771. need_datasync = 1;
  4772. }
  4773. if (ei->i_disksize > 0x7fffffffULL) {
  4774. if (!ext4_has_feature_large_file(sb) ||
  4775. EXT4_SB(sb)->s_es->s_rev_level ==
  4776. cpu_to_le32(EXT4_GOOD_OLD_REV))
  4777. set_large_file = 1;
  4778. }
  4779. raw_inode->i_generation = cpu_to_le32(inode->i_generation);
  4780. if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
  4781. if (old_valid_dev(inode->i_rdev)) {
  4782. raw_inode->i_block[0] =
  4783. cpu_to_le32(old_encode_dev(inode->i_rdev));
  4784. raw_inode->i_block[1] = 0;
  4785. } else {
  4786. raw_inode->i_block[0] = 0;
  4787. raw_inode->i_block[1] =
  4788. cpu_to_le32(new_encode_dev(inode->i_rdev));
  4789. raw_inode->i_block[2] = 0;
  4790. }
  4791. } else if (!ext4_has_inline_data(inode)) {
  4792. for (block = 0; block < EXT4_N_BLOCKS; block++)
  4793. raw_inode->i_block[block] = ei->i_data[block];
  4794. }
  4795. if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
  4796. u64 ivers = ext4_inode_peek_iversion(inode);
  4797. raw_inode->i_disk_version = cpu_to_le32(ivers);
  4798. if (ei->i_extra_isize) {
  4799. if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
  4800. raw_inode->i_version_hi =
  4801. cpu_to_le32(ivers >> 32);
  4802. raw_inode->i_extra_isize =
  4803. cpu_to_le16(ei->i_extra_isize);
  4804. }
  4805. }
  4806. BUG_ON(!ext4_has_feature_project(inode->i_sb) &&
  4807. i_projid != EXT4_DEF_PROJID);
  4808. if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
  4809. EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
  4810. raw_inode->i_projid = cpu_to_le32(i_projid);
  4811. ext4_inode_csum_set(inode, raw_inode, ei);
  4812. spin_unlock(&ei->i_raw_lock);
  4813. if (inode->i_sb->s_flags & SB_LAZYTIME)
  4814. ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
  4815. bh->b_data);
  4816. BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
  4817. err = ext4_handle_dirty_metadata(handle, NULL, bh);
  4818. if (err)
  4819. goto out_brelse;
  4820. ext4_clear_inode_state(inode, EXT4_STATE_NEW);
  4821. if (set_large_file) {
  4822. BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
  4823. err = ext4_journal_get_write_access(handle, EXT4_SB(sb)->s_sbh);
  4824. if (err)
  4825. goto out_brelse;
  4826. ext4_set_feature_large_file(sb);
  4827. ext4_handle_sync(handle);
  4828. err = ext4_handle_dirty_super(handle, sb);
  4829. }
  4830. ext4_update_inode_fsync_trans(handle, inode, need_datasync);
  4831. out_brelse:
  4832. brelse(bh);
  4833. ext4_std_error(inode->i_sb, err);
  4834. return err;
  4835. }
  4836. /*
  4837. * ext4_write_inode()
  4838. *
  4839. * We are called from a few places:
  4840. *
  4841. * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
  4842. * Here, there will be no transaction running. We wait for any running
  4843. * transaction to commit.
  4844. *
  4845. * - Within flush work (sys_sync(), kupdate and such).
  4846. * We wait on commit, if told to.
  4847. *
  4848. * - Within iput_final() -> write_inode_now()
  4849. * We wait on commit, if told to.
  4850. *
  4851. * In all cases it is actually safe for us to return without doing anything,
  4852. * because the inode has been copied into a raw inode buffer in
  4853. * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL
  4854. * writeback.
  4855. *
  4856. * Note that we are absolutely dependent upon all inode dirtiers doing the
  4857. * right thing: they *must* call mark_inode_dirty() after dirtying info in
  4858. * which we are interested.
  4859. *
  4860. * It would be a bug for them to not do this. The code:
  4861. *
  4862. * mark_inode_dirty(inode)
  4863. * stuff();
  4864. * inode->i_size = expr;
  4865. *
  4866. * is in error because write_inode() could occur while `stuff()' is running,
  4867. * and the new i_size will be lost. Plus the inode will no longer be on the
  4868. * superblock's dirty inode list.
  4869. */
  4870. int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
  4871. {
  4872. int err;
  4873. if (WARN_ON_ONCE(current->flags & PF_MEMALLOC) ||
  4874. sb_rdonly(inode->i_sb))
  4875. return 0;
  4876. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
  4877. return -EIO;
  4878. if (EXT4_SB(inode->i_sb)->s_journal) {
  4879. if (ext4_journal_current_handle()) {
  4880. jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n");
  4881. dump_stack();
  4882. return -EIO;
  4883. }
  4884. /*
  4885. * No need to force transaction in WB_SYNC_NONE mode. Also
  4886. * ext4_sync_fs() will force the commit after everything is
  4887. * written.
  4888. */
  4889. if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
  4890. return 0;
  4891. err = jbd2_complete_transaction(EXT4_SB(inode->i_sb)->s_journal,
  4892. EXT4_I(inode)->i_sync_tid);
  4893. } else {
  4894. struct ext4_iloc iloc;
  4895. err = __ext4_get_inode_loc(inode, &iloc, 0);
  4896. if (err)
  4897. return err;
  4898. /*
  4899. * sync(2) will flush the whole buffer cache. No need to do
  4900. * it here separately for each inode.
  4901. */
  4902. if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
  4903. sync_dirty_buffer(iloc.bh);
  4904. if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
  4905. EXT4_ERROR_INODE_BLOCK(inode, iloc.bh->b_blocknr,
  4906. "IO error syncing inode");
  4907. err = -EIO;
  4908. }
  4909. brelse(iloc.bh);
  4910. }
  4911. return err;
  4912. }
  4913. /*
  4914. * In data=journal mode ext4_journalled_invalidatepage() may fail to invalidate
  4915. * buffers that are attached to a page stradding i_size and are undergoing
  4916. * commit. In that case we have to wait for commit to finish and try again.
  4917. */
  4918. static void ext4_wait_for_tail_page_commit(struct inode *inode)
  4919. {
  4920. struct page *page;
  4921. unsigned offset;
  4922. journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
  4923. tid_t commit_tid = 0;
  4924. int ret;
  4925. offset = inode->i_size & (PAGE_SIZE - 1);
  4926. /*
  4927. * If the page is fully truncated, we don't need to wait for any commit
  4928. * (and we even should not as __ext4_journalled_invalidatepage() may
  4929. * strip all buffers from the page but keep the page dirty which can then
  4930. * confuse e.g. concurrent ext4_writepage() seeing dirty page without
  4931. * buffers). Also we don't need to wait for any commit if all buffers in
  4932. * the page remain valid. This is most beneficial for the common case of
  4933. * blocksize == PAGESIZE.
  4934. */
  4935. if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))
  4936. return;
  4937. while (1) {
  4938. page = find_lock_page(inode->i_mapping,
  4939. inode->i_size >> PAGE_SHIFT);
  4940. if (!page)
  4941. return;
  4942. ret = __ext4_journalled_invalidatepage(page, offset,
  4943. PAGE_SIZE - offset);
  4944. unlock_page(page);
  4945. put_page(page);
  4946. if (ret != -EBUSY)
  4947. return;
  4948. commit_tid = 0;
  4949. read_lock(&journal->j_state_lock);
  4950. if (journal->j_committing_transaction)
  4951. commit_tid = journal->j_committing_transaction->t_tid;
  4952. read_unlock(&journal->j_state_lock);
  4953. if (commit_tid)
  4954. jbd2_log_wait_commit(journal, commit_tid);
  4955. }
  4956. }
  4957. /*
  4958. * ext4_setattr()
  4959. *
  4960. * Called from notify_change.
  4961. *
  4962. * We want to trap VFS attempts to truncate the file as soon as
  4963. * possible. In particular, we want to make sure that when the VFS
  4964. * shrinks i_size, we put the inode on the orphan list and modify
  4965. * i_disksize immediately, so that during the subsequent flushing of
  4966. * dirty pages and freeing of disk blocks, we can guarantee that any
  4967. * commit will leave the blocks being flushed in an unused state on
  4968. * disk. (On recovery, the inode will get truncated and the blocks will
  4969. * be freed, so we have a strong guarantee that no future commit will
  4970. * leave these blocks visible to the user.)
  4971. *
  4972. * Another thing we have to assure is that if we are in ordered mode
  4973. * and inode is still attached to the committing transaction, we must
  4974. * we start writeout of all the dirty pages which are being truncated.
  4975. * This way we are sure that all the data written in the previous
  4976. * transaction are already on disk (truncate waits for pages under
  4977. * writeback).
  4978. *
  4979. * Called with inode->i_mutex down.
  4980. */
  4981. int ext4_setattr(struct dentry *dentry, struct iattr *attr)
  4982. {
  4983. struct inode *inode = d_inode(dentry);
  4984. int error, rc = 0;
  4985. int orphan = 0;
  4986. const unsigned int ia_valid = attr->ia_valid;
  4987. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
  4988. return -EIO;
  4989. if (unlikely(IS_IMMUTABLE(inode)))
  4990. return -EPERM;
  4991. if (unlikely(IS_APPEND(inode) &&
  4992. (ia_valid & (ATTR_MODE | ATTR_UID |
  4993. ATTR_GID | ATTR_TIMES_SET))))
  4994. return -EPERM;
  4995. error = setattr_prepare(dentry, attr);
  4996. if (error)
  4997. return error;
  4998. error = fscrypt_prepare_setattr(dentry, attr);
  4999. if (error)
  5000. return error;
  5001. if (is_quota_modification(inode, attr)) {
  5002. error = dquot_initialize(inode);
  5003. if (error)
  5004. return error;
  5005. }
  5006. if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) ||
  5007. (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) {
  5008. handle_t *handle;
  5009. /* (user+group)*(old+new) structure, inode write (sb,
  5010. * inode block, ? - but truncate inode update has it) */
  5011. handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
  5012. (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
  5013. EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
  5014. if (IS_ERR(handle)) {
  5015. error = PTR_ERR(handle);
  5016. goto err_out;
  5017. }
  5018. /* dquot_transfer() calls back ext4_get_inode_usage() which
  5019. * counts xattr inode references.
  5020. */
  5021. down_read(&EXT4_I(inode)->xattr_sem);
  5022. error = dquot_transfer(inode, attr);
  5023. up_read(&EXT4_I(inode)->xattr_sem);
  5024. if (error) {
  5025. ext4_journal_stop(handle);
  5026. return error;
  5027. }
  5028. /* Update corresponding info in inode so that everything is in
  5029. * one transaction */
  5030. if (attr->ia_valid & ATTR_UID)
  5031. inode->i_uid = attr->ia_uid;
  5032. if (attr->ia_valid & ATTR_GID)
  5033. inode->i_gid = attr->ia_gid;
  5034. error = ext4_mark_inode_dirty(handle, inode);
  5035. ext4_journal_stop(handle);
  5036. }
  5037. if (attr->ia_valid & ATTR_SIZE) {
  5038. handle_t *handle;
  5039. loff_t oldsize = inode->i_size;
  5040. int shrink = (attr->ia_size <= inode->i_size);
  5041. if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
  5042. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  5043. if (attr->ia_size > sbi->s_bitmap_maxbytes)
  5044. return -EFBIG;
  5045. }
  5046. if (!S_ISREG(inode->i_mode))
  5047. return -EINVAL;
  5048. if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size)
  5049. inode_inc_iversion(inode);
  5050. if (ext4_should_order_data(inode) &&
  5051. (attr->ia_size < inode->i_size)) {
  5052. error = ext4_begin_ordered_truncate(inode,
  5053. attr->ia_size);
  5054. if (error)
  5055. goto err_out;
  5056. }
  5057. if (attr->ia_size != inode->i_size) {
  5058. handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
  5059. if (IS_ERR(handle)) {
  5060. error = PTR_ERR(handle);
  5061. goto err_out;
  5062. }
  5063. if (ext4_handle_valid(handle) && shrink) {
  5064. error = ext4_orphan_add(handle, inode);
  5065. orphan = 1;
  5066. }
  5067. /*
  5068. * Update c/mtime on truncate up, ext4_truncate() will
  5069. * update c/mtime in shrink case below
  5070. */
  5071. if (!shrink) {
  5072. inode->i_mtime = current_time(inode);
  5073. inode->i_ctime = inode->i_mtime;
  5074. }
  5075. down_write(&EXT4_I(inode)->i_data_sem);
  5076. EXT4_I(inode)->i_disksize = attr->ia_size;
  5077. rc = ext4_mark_inode_dirty(handle, inode);
  5078. if (!error)
  5079. error = rc;
  5080. /*
  5081. * We have to update i_size under i_data_sem together
  5082. * with i_disksize to avoid races with writeback code
  5083. * running ext4_wb_update_i_disksize().
  5084. */
  5085. if (!error)
  5086. i_size_write(inode, attr->ia_size);
  5087. up_write(&EXT4_I(inode)->i_data_sem);
  5088. ext4_journal_stop(handle);
  5089. if (error) {
  5090. if (orphan && inode->i_nlink)
  5091. ext4_orphan_del(NULL, inode);
  5092. goto err_out;
  5093. }
  5094. }
  5095. if (!shrink) {
  5096. pagecache_isize_extended(inode, oldsize, inode->i_size);
  5097. } else {
  5098. /*
  5099. * Blocks are going to be removed from the inode. Wait
  5100. * for dio in flight.
  5101. */
  5102. inode_dio_wait(inode);
  5103. }
  5104. if (orphan && ext4_should_journal_data(inode))
  5105. ext4_wait_for_tail_page_commit(inode);
  5106. down_write(&EXT4_I(inode)->i_mmap_sem);
  5107. rc = ext4_break_layouts(inode);
  5108. if (rc) {
  5109. up_write(&EXT4_I(inode)->i_mmap_sem);
  5110. error = rc;
  5111. goto err_out;
  5112. }
  5113. /*
  5114. * Truncate pagecache after we've waited for commit
  5115. * in data=journal mode to make pages freeable.
  5116. */
  5117. truncate_pagecache(inode, inode->i_size);
  5118. if (shrink) {
  5119. rc = ext4_truncate(inode);
  5120. if (rc)
  5121. error = rc;
  5122. }
  5123. up_write(&EXT4_I(inode)->i_mmap_sem);
  5124. }
  5125. if (!error) {
  5126. setattr_copy(inode, attr);
  5127. mark_inode_dirty(inode);
  5128. }
  5129. /*
  5130. * If the call to ext4_truncate failed to get a transaction handle at
  5131. * all, we need to clean up the in-core orphan list manually.
  5132. */
  5133. if (orphan && inode->i_nlink)
  5134. ext4_orphan_del(NULL, inode);
  5135. if (!error && (ia_valid & ATTR_MODE))
  5136. rc = posix_acl_chmod(inode, inode->i_mode);
  5137. err_out:
  5138. ext4_std_error(inode->i_sb, error);
  5139. if (!error)
  5140. error = rc;
  5141. return error;
  5142. }
  5143. int ext4_getattr(const struct path *path, struct kstat *stat,
  5144. u32 request_mask, unsigned int query_flags)
  5145. {
  5146. struct inode *inode = d_inode(path->dentry);
  5147. struct ext4_inode *raw_inode;
  5148. struct ext4_inode_info *ei = EXT4_I(inode);
  5149. unsigned int flags;
  5150. if (EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
  5151. stat->result_mask |= STATX_BTIME;
  5152. stat->btime.tv_sec = ei->i_crtime.tv_sec;
  5153. stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
  5154. }
  5155. flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
  5156. if (flags & EXT4_APPEND_FL)
  5157. stat->attributes |= STATX_ATTR_APPEND;
  5158. if (flags & EXT4_COMPR_FL)
  5159. stat->attributes |= STATX_ATTR_COMPRESSED;
  5160. if (flags & EXT4_ENCRYPT_FL)
  5161. stat->attributes |= STATX_ATTR_ENCRYPTED;
  5162. if (flags & EXT4_IMMUTABLE_FL)
  5163. stat->attributes |= STATX_ATTR_IMMUTABLE;
  5164. if (flags & EXT4_NODUMP_FL)
  5165. stat->attributes |= STATX_ATTR_NODUMP;
  5166. stat->attributes_mask |= (STATX_ATTR_APPEND |
  5167. STATX_ATTR_COMPRESSED |
  5168. STATX_ATTR_ENCRYPTED |
  5169. STATX_ATTR_IMMUTABLE |
  5170. STATX_ATTR_NODUMP);
  5171. generic_fillattr(inode, stat);
  5172. return 0;
  5173. }
  5174. int ext4_file_getattr(const struct path *path, struct kstat *stat,
  5175. u32 request_mask, unsigned int query_flags)
  5176. {
  5177. struct inode *inode = d_inode(path->dentry);
  5178. u64 delalloc_blocks;
  5179. ext4_getattr(path, stat, request_mask, query_flags);
  5180. /*
  5181. * If there is inline data in the inode, the inode will normally not
  5182. * have data blocks allocated (it may have an external xattr block).
  5183. * Report at least one sector for such files, so tools like tar, rsync,
  5184. * others don't incorrectly think the file is completely sparse.
  5185. */
  5186. if (unlikely(ext4_has_inline_data(inode)))
  5187. stat->blocks += (stat->size + 511) >> 9;
  5188. /*
  5189. * We can't update i_blocks if the block allocation is delayed
  5190. * otherwise in the case of system crash before the real block
  5191. * allocation is done, we will have i_blocks inconsistent with
  5192. * on-disk file blocks.
  5193. * We always keep i_blocks updated together with real
  5194. * allocation. But to not confuse with user, stat
  5195. * will return the blocks that include the delayed allocation
  5196. * blocks for this file.
  5197. */
  5198. delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
  5199. EXT4_I(inode)->i_reserved_data_blocks);
  5200. stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
  5201. return 0;
  5202. }
  5203. static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
  5204. int pextents)
  5205. {
  5206. if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
  5207. return ext4_ind_trans_blocks(inode, lblocks);
  5208. return ext4_ext_index_trans_blocks(inode, pextents);
  5209. }
  5210. /*
  5211. * Account for index blocks, block groups bitmaps and block group
  5212. * descriptor blocks if modify datablocks and index blocks
  5213. * worse case, the indexs blocks spread over different block groups
  5214. *
  5215. * If datablocks are discontiguous, they are possible to spread over
  5216. * different block groups too. If they are contiguous, with flexbg,
  5217. * they could still across block group boundary.
  5218. *
  5219. * Also account for superblock, inode, quota and xattr blocks
  5220. */
  5221. static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
  5222. int pextents)
  5223. {
  5224. ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
  5225. int gdpblocks;
  5226. int idxblocks;
  5227. int ret = 0;
  5228. /*
  5229. * How many index blocks need to touch to map @lblocks logical blocks
  5230. * to @pextents physical extents?
  5231. */
  5232. idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
  5233. ret = idxblocks;
  5234. /*
  5235. * Now let's see how many group bitmaps and group descriptors need
  5236. * to account
  5237. */
  5238. groups = idxblocks + pextents;
  5239. gdpblocks = groups;
  5240. if (groups > ngroups)
  5241. groups = ngroups;
  5242. if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
  5243. gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
  5244. /* bitmaps and block group descriptor blocks */
  5245. ret += groups + gdpblocks;
  5246. /* Blocks for super block, inode, quota and xattr blocks */
  5247. ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
  5248. return ret;
  5249. }
  5250. /*
  5251. * Calculate the total number of credits to reserve to fit
  5252. * the modification of a single pages into a single transaction,
  5253. * which may include multiple chunks of block allocations.
  5254. *
  5255. * This could be called via ext4_write_begin()
  5256. *
  5257. * We need to consider the worse case, when
  5258. * one new block per extent.
  5259. */
  5260. int ext4_writepage_trans_blocks(struct inode *inode)
  5261. {
  5262. int bpp = ext4_journal_blocks_per_page(inode);
  5263. int ret;
  5264. ret = ext4_meta_trans_blocks(inode, bpp, bpp);
  5265. /* Account for data blocks for journalled mode */
  5266. if (ext4_should_journal_data(inode))
  5267. ret += bpp;
  5268. return ret;
  5269. }
  5270. /*
  5271. * Calculate the journal credits for a chunk of data modification.
  5272. *
  5273. * This is called from DIO, fallocate or whoever calling
  5274. * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
  5275. *
  5276. * journal buffers for data blocks are not included here, as DIO
  5277. * and fallocate do no need to journal data buffers.
  5278. */
  5279. int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
  5280. {
  5281. return ext4_meta_trans_blocks(inode, nrblocks, 1);
  5282. }
  5283. /*
  5284. * The caller must have previously called ext4_reserve_inode_write().
  5285. * Give this, we know that the caller already has write access to iloc->bh.
  5286. */
  5287. int ext4_mark_iloc_dirty(handle_t *handle,
  5288. struct inode *inode, struct ext4_iloc *iloc)
  5289. {
  5290. int err = 0;
  5291. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
  5292. put_bh(iloc->bh);
  5293. return -EIO;
  5294. }
  5295. if (IS_I_VERSION(inode))
  5296. inode_inc_iversion(inode);
  5297. /* the do_update_inode consumes one bh->b_count */
  5298. get_bh(iloc->bh);
  5299. /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
  5300. err = ext4_do_update_inode(handle, inode, iloc);
  5301. put_bh(iloc->bh);
  5302. return err;
  5303. }
  5304. /*
  5305. * On success, We end up with an outstanding reference count against
  5306. * iloc->bh. This _must_ be cleaned up later.
  5307. */
  5308. int
  5309. ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
  5310. struct ext4_iloc *iloc)
  5311. {
  5312. int err;
  5313. if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
  5314. return -EIO;
  5315. err = ext4_get_inode_loc(inode, iloc);
  5316. if (!err) {
  5317. BUFFER_TRACE(iloc->bh, "get_write_access");
  5318. err = ext4_journal_get_write_access(handle, iloc->bh);
  5319. if (err) {
  5320. brelse(iloc->bh);
  5321. iloc->bh = NULL;
  5322. }
  5323. }
  5324. ext4_std_error(inode->i_sb, err);
  5325. return err;
  5326. }
  5327. static int __ext4_expand_extra_isize(struct inode *inode,
  5328. unsigned int new_extra_isize,
  5329. struct ext4_iloc *iloc,
  5330. handle_t *handle, int *no_expand)
  5331. {
  5332. struct ext4_inode *raw_inode;
  5333. struct ext4_xattr_ibody_header *header;
  5334. unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
  5335. struct ext4_inode_info *ei = EXT4_I(inode);
  5336. int error;
  5337. /* this was checked at iget time, but double check for good measure */
  5338. if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
  5339. (ei->i_extra_isize & 3)) {
  5340. EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
  5341. ei->i_extra_isize,
  5342. EXT4_INODE_SIZE(inode->i_sb));
  5343. return -EFSCORRUPTED;
  5344. }
  5345. if ((new_extra_isize < ei->i_extra_isize) ||
  5346. (new_extra_isize < 4) ||
  5347. (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
  5348. return -EINVAL; /* Should never happen */
  5349. raw_inode = ext4_raw_inode(iloc);
  5350. header = IHDR(inode, raw_inode);
  5351. /* No extended attributes present */
  5352. if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
  5353. header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
  5354. memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
  5355. EXT4_I(inode)->i_extra_isize, 0,
  5356. new_extra_isize - EXT4_I(inode)->i_extra_isize);
  5357. EXT4_I(inode)->i_extra_isize = new_extra_isize;
  5358. return 0;
  5359. }
  5360. /* try to expand with EAs present */
  5361. error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
  5362. raw_inode, handle);
  5363. if (error) {
  5364. /*
  5365. * Inode size expansion failed; don't try again
  5366. */
  5367. *no_expand = 1;
  5368. }
  5369. return error;
  5370. }
  5371. /*
  5372. * Expand an inode by new_extra_isize bytes.
  5373. * Returns 0 on success or negative error number on failure.
  5374. */
  5375. static int ext4_try_to_expand_extra_isize(struct inode *inode,
  5376. unsigned int new_extra_isize,
  5377. struct ext4_iloc iloc,
  5378. handle_t *handle)
  5379. {
  5380. int no_expand;
  5381. int error;
  5382. if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
  5383. return -EOVERFLOW;
  5384. /*
  5385. * In nojournal mode, we can immediately attempt to expand
  5386. * the inode. When journaled, we first need to obtain extra
  5387. * buffer credits since we may write into the EA block
  5388. * with this same handle. If journal_extend fails, then it will
  5389. * only result in a minor loss of functionality for that inode.
  5390. * If this is felt to be critical, then e2fsck should be run to
  5391. * force a large enough s_min_extra_isize.
  5392. */
  5393. if (ext4_handle_valid(handle) &&
  5394. jbd2_journal_extend(handle,
  5395. EXT4_DATA_TRANS_BLOCKS(inode->i_sb)) != 0)
  5396. return -ENOSPC;
  5397. if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
  5398. return -EBUSY;
  5399. error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
  5400. handle, &no_expand);
  5401. ext4_write_unlock_xattr(inode, &no_expand);
  5402. return error;
  5403. }
  5404. int ext4_expand_extra_isize(struct inode *inode,
  5405. unsigned int new_extra_isize,
  5406. struct ext4_iloc *iloc)
  5407. {
  5408. handle_t *handle;
  5409. int no_expand;
  5410. int error, rc;
  5411. if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
  5412. brelse(iloc->bh);
  5413. return -EOVERFLOW;
  5414. }
  5415. handle = ext4_journal_start(inode, EXT4_HT_INODE,
  5416. EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
  5417. if (IS_ERR(handle)) {
  5418. error = PTR_ERR(handle);
  5419. brelse(iloc->bh);
  5420. return error;
  5421. }
  5422. ext4_write_lock_xattr(inode, &no_expand);
  5423. BUFFER_TRACE(iloc->bh, "get_write_access");
  5424. error = ext4_journal_get_write_access(handle, iloc->bh);
  5425. if (error) {
  5426. brelse(iloc->bh);
  5427. goto out_unlock;
  5428. }
  5429. error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
  5430. handle, &no_expand);
  5431. rc = ext4_mark_iloc_dirty(handle, inode, iloc);
  5432. if (!error)
  5433. error = rc;
  5434. out_unlock:
  5435. ext4_write_unlock_xattr(inode, &no_expand);
  5436. ext4_journal_stop(handle);
  5437. return error;
  5438. }
  5439. /*
  5440. * What we do here is to mark the in-core inode as clean with respect to inode
  5441. * dirtiness (it may still be data-dirty).
  5442. * This means that the in-core inode may be reaped by prune_icache
  5443. * without having to perform any I/O. This is a very good thing,
  5444. * because *any* task may call prune_icache - even ones which
  5445. * have a transaction open against a different journal.
  5446. *
  5447. * Is this cheating? Not really. Sure, we haven't written the
  5448. * inode out, but prune_icache isn't a user-visible syncing function.
  5449. * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
  5450. * we start and wait on commits.
  5451. */
  5452. int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode)
  5453. {
  5454. struct ext4_iloc iloc;
  5455. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  5456. int err;
  5457. might_sleep();
  5458. trace_ext4_mark_inode_dirty(inode, _RET_IP_);
  5459. err = ext4_reserve_inode_write(handle, inode, &iloc);
  5460. if (err)
  5461. return err;
  5462. if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
  5463. ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
  5464. iloc, handle);
  5465. return ext4_mark_iloc_dirty(handle, inode, &iloc);
  5466. }
  5467. /*
  5468. * ext4_dirty_inode() is called from __mark_inode_dirty()
  5469. *
  5470. * We're really interested in the case where a file is being extended.
  5471. * i_size has been changed by generic_commit_write() and we thus need
  5472. * to include the updated inode in the current transaction.
  5473. *
  5474. * Also, dquot_alloc_block() will always dirty the inode when blocks
  5475. * are allocated to the file.
  5476. *
  5477. * If the inode is marked synchronous, we don't honour that here - doing
  5478. * so would cause a commit on atime updates, which we don't bother doing.
  5479. * We handle synchronous inodes at the highest possible level.
  5480. *
  5481. * If only the I_DIRTY_TIME flag is set, we can skip everything. If
  5482. * I_DIRTY_TIME and I_DIRTY_SYNC is set, the only inode fields we need
  5483. * to copy into the on-disk inode structure are the timestamp files.
  5484. */
  5485. void ext4_dirty_inode(struct inode *inode, int flags)
  5486. {
  5487. handle_t *handle;
  5488. if (flags == I_DIRTY_TIME)
  5489. return;
  5490. handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
  5491. if (IS_ERR(handle))
  5492. goto out;
  5493. ext4_mark_inode_dirty(handle, inode);
  5494. ext4_journal_stop(handle);
  5495. out:
  5496. return;
  5497. }
  5498. #if 0
  5499. /*
  5500. * Bind an inode's backing buffer_head into this transaction, to prevent
  5501. * it from being flushed to disk early. Unlike
  5502. * ext4_reserve_inode_write, this leaves behind no bh reference and
  5503. * returns no iloc structure, so the caller needs to repeat the iloc
  5504. * lookup to mark the inode dirty later.
  5505. */
  5506. static int ext4_pin_inode(handle_t *handle, struct inode *inode)
  5507. {
  5508. struct ext4_iloc iloc;
  5509. int err = 0;
  5510. if (handle) {
  5511. err = ext4_get_inode_loc(inode, &iloc);
  5512. if (!err) {
  5513. BUFFER_TRACE(iloc.bh, "get_write_access");
  5514. err = jbd2_journal_get_write_access(handle, iloc.bh);
  5515. if (!err)
  5516. err = ext4_handle_dirty_metadata(handle,
  5517. NULL,
  5518. iloc.bh);
  5519. brelse(iloc.bh);
  5520. }
  5521. }
  5522. ext4_std_error(inode->i_sb, err);
  5523. return err;
  5524. }
  5525. #endif
  5526. int ext4_change_inode_journal_flag(struct inode *inode, int val)
  5527. {
  5528. journal_t *journal;
  5529. handle_t *handle;
  5530. int err;
  5531. struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
  5532. /*
  5533. * We have to be very careful here: changing a data block's
  5534. * journaling status dynamically is dangerous. If we write a
  5535. * data block to the journal, change the status and then delete
  5536. * that block, we risk forgetting to revoke the old log record
  5537. * from the journal and so a subsequent replay can corrupt data.
  5538. * So, first we make sure that the journal is empty and that
  5539. * nobody is changing anything.
  5540. */
  5541. journal = EXT4_JOURNAL(inode);
  5542. if (!journal)
  5543. return 0;
  5544. if (is_journal_aborted(journal))
  5545. return -EROFS;
  5546. /* Wait for all existing dio workers */
  5547. inode_dio_wait(inode);
  5548. /*
  5549. * Before flushing the journal and switching inode's aops, we have
  5550. * to flush all dirty data the inode has. There can be outstanding
  5551. * delayed allocations, there can be unwritten extents created by
  5552. * fallocate or buffered writes in dioread_nolock mode covered by
  5553. * dirty data which can be converted only after flushing the dirty
  5554. * data (and journalled aops don't know how to handle these cases).
  5555. */
  5556. if (val) {
  5557. down_write(&EXT4_I(inode)->i_mmap_sem);
  5558. err = filemap_write_and_wait(inode->i_mapping);
  5559. if (err < 0) {
  5560. up_write(&EXT4_I(inode)->i_mmap_sem);
  5561. return err;
  5562. }
  5563. }
  5564. percpu_down_write(&sbi->s_writepages_rwsem);
  5565. jbd2_journal_lock_updates(journal);
  5566. /*
  5567. * OK, there are no updates running now, and all cached data is
  5568. * synced to disk. We are now in a completely consistent state
  5569. * which doesn't have anything in the journal, and we know that
  5570. * no filesystem updates are running, so it is safe to modify
  5571. * the inode's in-core data-journaling state flag now.
  5572. */
  5573. if (val)
  5574. ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
  5575. else {
  5576. err = jbd2_journal_flush(journal);
  5577. if (err < 0) {
  5578. jbd2_journal_unlock_updates(journal);
  5579. percpu_up_write(&sbi->s_writepages_rwsem);
  5580. return err;
  5581. }
  5582. ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
  5583. }
  5584. ext4_set_aops(inode);
  5585. jbd2_journal_unlock_updates(journal);
  5586. percpu_up_write(&sbi->s_writepages_rwsem);
  5587. if (val)
  5588. up_write(&EXT4_I(inode)->i_mmap_sem);
  5589. /* Finally we can mark the inode as dirty. */
  5590. handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
  5591. if (IS_ERR(handle))
  5592. return PTR_ERR(handle);
  5593. err = ext4_mark_inode_dirty(handle, inode);
  5594. ext4_handle_sync(handle);
  5595. ext4_journal_stop(handle);
  5596. ext4_std_error(inode->i_sb, err);
  5597. return err;
  5598. }
  5599. static int ext4_bh_unmapped(handle_t *handle, struct buffer_head *bh)
  5600. {
  5601. return !buffer_mapped(bh);
  5602. }
  5603. int ext4_page_mkwrite(struct vm_fault *vmf)
  5604. {
  5605. struct vm_area_struct *vma = vmf->vma;
  5606. struct page *page = vmf->page;
  5607. loff_t size;
  5608. unsigned long len;
  5609. int ret;
  5610. struct file *file = vma->vm_file;
  5611. struct inode *inode = file_inode(file);
  5612. struct address_space *mapping = inode->i_mapping;
  5613. handle_t *handle;
  5614. get_block_t *get_block;
  5615. int retries = 0;
  5616. if (unlikely(IS_IMMUTABLE(inode)))
  5617. return VM_FAULT_SIGBUS;
  5618. sb_start_pagefault(inode->i_sb);
  5619. file_update_time(vma->vm_file);
  5620. down_read(&EXT4_I(inode)->i_mmap_sem);
  5621. ret = ext4_convert_inline_data(inode);
  5622. if (ret)
  5623. goto out_ret;
  5624. /* Delalloc case is easy... */
  5625. if (test_opt(inode->i_sb, DELALLOC) &&
  5626. !ext4_should_journal_data(inode) &&
  5627. !ext4_nonda_switch(inode->i_sb)) {
  5628. do {
  5629. ret = block_page_mkwrite(vma, vmf,
  5630. ext4_da_get_block_prep);
  5631. } while (ret == -ENOSPC &&
  5632. ext4_should_retry_alloc(inode->i_sb, &retries));
  5633. goto out_ret;
  5634. }
  5635. lock_page(page);
  5636. size = i_size_read(inode);
  5637. /* Page got truncated from under us? */
  5638. if (page->mapping != mapping || page_offset(page) > size) {
  5639. unlock_page(page);
  5640. ret = VM_FAULT_NOPAGE;
  5641. goto out;
  5642. }
  5643. if (page->index == size >> PAGE_SHIFT)
  5644. len = size & ~PAGE_MASK;
  5645. else
  5646. len = PAGE_SIZE;
  5647. /*
  5648. * Return if we have all the buffers mapped. This avoids the need to do
  5649. * journal_start/journal_stop which can block and take a long time
  5650. */
  5651. if (page_has_buffers(page)) {
  5652. if (!ext4_walk_page_buffers(NULL, page_buffers(page),
  5653. 0, len, NULL,
  5654. ext4_bh_unmapped)) {
  5655. /* Wait so that we don't change page under IO */
  5656. wait_for_stable_page(page);
  5657. ret = VM_FAULT_LOCKED;
  5658. goto out;
  5659. }
  5660. }
  5661. unlock_page(page);
  5662. /* OK, we need to fill the hole... */
  5663. if (ext4_should_dioread_nolock(inode))
  5664. get_block = ext4_get_block_unwritten;
  5665. else
  5666. get_block = ext4_get_block;
  5667. retry_alloc:
  5668. handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
  5669. ext4_writepage_trans_blocks(inode));
  5670. if (IS_ERR(handle)) {
  5671. ret = VM_FAULT_SIGBUS;
  5672. goto out;
  5673. }
  5674. ret = block_page_mkwrite(vma, vmf, get_block);
  5675. if (!ret && ext4_should_journal_data(inode)) {
  5676. if (ext4_walk_page_buffers(handle, page_buffers(page), 0,
  5677. PAGE_SIZE, NULL, do_journal_get_write_access)) {
  5678. unlock_page(page);
  5679. ret = VM_FAULT_SIGBUS;
  5680. ext4_journal_stop(handle);
  5681. goto out;
  5682. }
  5683. ext4_set_inode_state(inode, EXT4_STATE_JDATA);
  5684. }
  5685. ext4_journal_stop(handle);
  5686. if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
  5687. goto retry_alloc;
  5688. out_ret:
  5689. ret = block_page_mkwrite_return(ret);
  5690. out:
  5691. up_read(&EXT4_I(inode)->i_mmap_sem);
  5692. sb_end_pagefault(inode->i_sb);
  5693. return ret;
  5694. }
  5695. int ext4_filemap_fault(struct vm_fault *vmf)
  5696. {
  5697. struct inode *inode = file_inode(vmf->vma->vm_file);
  5698. int err;
  5699. down_read(&EXT4_I(inode)->i_mmap_sem);
  5700. err = filemap_fault(vmf);
  5701. up_read(&EXT4_I(inode)->i_mmap_sem);
  5702. return err;
  5703. }