xfs_icache.c 46 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  4. * All Rights Reserved.
  5. */
  6. #include "xfs.h"
  7. #include "xfs_fs.h"
  8. #include "xfs_format.h"
  9. #include "xfs_log_format.h"
  10. #include "xfs_trans_resv.h"
  11. #include "xfs_sb.h"
  12. #include "xfs_mount.h"
  13. #include "xfs_inode.h"
  14. #include "xfs_error.h"
  15. #include "xfs_trans.h"
  16. #include "xfs_trans_priv.h"
  17. #include "xfs_inode_item.h"
  18. #include "xfs_quota.h"
  19. #include "xfs_trace.h"
  20. #include "xfs_icache.h"
  21. #include "xfs_bmap_util.h"
  22. #include "xfs_dquot_item.h"
  23. #include "xfs_dquot.h"
  24. #include "xfs_reflink.h"
  25. #include <linux/kthread.h>
  26. #include <linux/freezer.h>
  27. #include <linux/iversion.h>
  28. /*
  29. * Allocate and initialise an xfs_inode.
  30. */
  31. struct xfs_inode *
  32. xfs_inode_alloc(
  33. struct xfs_mount *mp,
  34. xfs_ino_t ino)
  35. {
  36. struct xfs_inode *ip;
  37. /*
  38. * if this didn't occur in transactions, we could use
  39. * KM_MAYFAIL and return NULL here on ENOMEM. Set the
  40. * code up to do this anyway.
  41. */
  42. ip = kmem_zone_alloc(xfs_inode_zone, KM_SLEEP);
  43. if (!ip)
  44. return NULL;
  45. if (inode_init_always(mp->m_super, VFS_I(ip))) {
  46. kmem_zone_free(xfs_inode_zone, ip);
  47. return NULL;
  48. }
  49. /* VFS doesn't initialise i_mode! */
  50. VFS_I(ip)->i_mode = 0;
  51. XFS_STATS_INC(mp, vn_active);
  52. ASSERT(atomic_read(&ip->i_pincount) == 0);
  53. ASSERT(!xfs_isiflocked(ip));
  54. ASSERT(ip->i_ino == 0);
  55. /* initialise the xfs inode */
  56. ip->i_ino = ino;
  57. ip->i_mount = mp;
  58. memset(&ip->i_imap, 0, sizeof(struct xfs_imap));
  59. ip->i_afp = NULL;
  60. ip->i_cowfp = NULL;
  61. ip->i_cnextents = 0;
  62. ip->i_cformat = XFS_DINODE_FMT_EXTENTS;
  63. memset(&ip->i_df, 0, sizeof(ip->i_df));
  64. ip->i_flags = 0;
  65. ip->i_delayed_blks = 0;
  66. memset(&ip->i_d, 0, sizeof(ip->i_d));
  67. return ip;
  68. }
  69. STATIC void
  70. xfs_inode_free_callback(
  71. struct rcu_head *head)
  72. {
  73. struct inode *inode = container_of(head, struct inode, i_rcu);
  74. struct xfs_inode *ip = XFS_I(inode);
  75. switch (VFS_I(ip)->i_mode & S_IFMT) {
  76. case S_IFREG:
  77. case S_IFDIR:
  78. case S_IFLNK:
  79. xfs_idestroy_fork(ip, XFS_DATA_FORK);
  80. break;
  81. }
  82. if (ip->i_afp)
  83. xfs_idestroy_fork(ip, XFS_ATTR_FORK);
  84. if (ip->i_cowfp)
  85. xfs_idestroy_fork(ip, XFS_COW_FORK);
  86. if (ip->i_itemp) {
  87. ASSERT(!test_bit(XFS_LI_IN_AIL,
  88. &ip->i_itemp->ili_item.li_flags));
  89. xfs_inode_item_destroy(ip);
  90. ip->i_itemp = NULL;
  91. }
  92. kmem_zone_free(xfs_inode_zone, ip);
  93. }
  94. static void
  95. __xfs_inode_free(
  96. struct xfs_inode *ip)
  97. {
  98. /* asserts to verify all state is correct here */
  99. ASSERT(atomic_read(&ip->i_pincount) == 0);
  100. XFS_STATS_DEC(ip->i_mount, vn_active);
  101. call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback);
  102. }
  103. void
  104. xfs_inode_free(
  105. struct xfs_inode *ip)
  106. {
  107. ASSERT(!xfs_isiflocked(ip));
  108. /*
  109. * Because we use RCU freeing we need to ensure the inode always
  110. * appears to be reclaimed with an invalid inode number when in the
  111. * free state. The ip->i_flags_lock provides the barrier against lookup
  112. * races.
  113. */
  114. spin_lock(&ip->i_flags_lock);
  115. ip->i_flags = XFS_IRECLAIM;
  116. ip->i_ino = 0;
  117. spin_unlock(&ip->i_flags_lock);
  118. __xfs_inode_free(ip);
  119. }
  120. /*
  121. * Queue a new inode reclaim pass if there are reclaimable inodes and there
  122. * isn't a reclaim pass already in progress. By default it runs every 5s based
  123. * on the xfs periodic sync default of 30s. Perhaps this should have it's own
  124. * tunable, but that can be done if this method proves to be ineffective or too
  125. * aggressive.
  126. */
  127. static void
  128. xfs_reclaim_work_queue(
  129. struct xfs_mount *mp)
  130. {
  131. rcu_read_lock();
  132. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
  133. queue_delayed_work(mp->m_reclaim_workqueue, &mp->m_reclaim_work,
  134. msecs_to_jiffies(xfs_syncd_centisecs / 6 * 10));
  135. }
  136. rcu_read_unlock();
  137. }
  138. /*
  139. * This is a fast pass over the inode cache to try to get reclaim moving on as
  140. * many inodes as possible in a short period of time. It kicks itself every few
  141. * seconds, as well as being kicked by the inode cache shrinker when memory
  142. * goes low. It scans as quickly as possible avoiding locked inodes or those
  143. * already being flushed, and once done schedules a future pass.
  144. */
  145. void
  146. xfs_reclaim_worker(
  147. struct work_struct *work)
  148. {
  149. struct xfs_mount *mp = container_of(to_delayed_work(work),
  150. struct xfs_mount, m_reclaim_work);
  151. xfs_reclaim_inodes(mp, SYNC_TRYLOCK);
  152. xfs_reclaim_work_queue(mp);
  153. }
  154. static void
  155. xfs_perag_set_reclaim_tag(
  156. struct xfs_perag *pag)
  157. {
  158. struct xfs_mount *mp = pag->pag_mount;
  159. lockdep_assert_held(&pag->pag_ici_lock);
  160. if (pag->pag_ici_reclaimable++)
  161. return;
  162. /* propagate the reclaim tag up into the perag radix tree */
  163. spin_lock(&mp->m_perag_lock);
  164. radix_tree_tag_set(&mp->m_perag_tree, pag->pag_agno,
  165. XFS_ICI_RECLAIM_TAG);
  166. spin_unlock(&mp->m_perag_lock);
  167. /* schedule periodic background inode reclaim */
  168. xfs_reclaim_work_queue(mp);
  169. trace_xfs_perag_set_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
  170. }
  171. static void
  172. xfs_perag_clear_reclaim_tag(
  173. struct xfs_perag *pag)
  174. {
  175. struct xfs_mount *mp = pag->pag_mount;
  176. lockdep_assert_held(&pag->pag_ici_lock);
  177. if (--pag->pag_ici_reclaimable)
  178. return;
  179. /* clear the reclaim tag from the perag radix tree */
  180. spin_lock(&mp->m_perag_lock);
  181. radix_tree_tag_clear(&mp->m_perag_tree, pag->pag_agno,
  182. XFS_ICI_RECLAIM_TAG);
  183. spin_unlock(&mp->m_perag_lock);
  184. trace_xfs_perag_clear_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
  185. }
  186. /*
  187. * We set the inode flag atomically with the radix tree tag.
  188. * Once we get tag lookups on the radix tree, this inode flag
  189. * can go away.
  190. */
  191. void
  192. xfs_inode_set_reclaim_tag(
  193. struct xfs_inode *ip)
  194. {
  195. struct xfs_mount *mp = ip->i_mount;
  196. struct xfs_perag *pag;
  197. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  198. spin_lock(&pag->pag_ici_lock);
  199. spin_lock(&ip->i_flags_lock);
  200. radix_tree_tag_set(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, ip->i_ino),
  201. XFS_ICI_RECLAIM_TAG);
  202. xfs_perag_set_reclaim_tag(pag);
  203. __xfs_iflags_set(ip, XFS_IRECLAIMABLE);
  204. spin_unlock(&ip->i_flags_lock);
  205. spin_unlock(&pag->pag_ici_lock);
  206. xfs_perag_put(pag);
  207. }
  208. STATIC void
  209. xfs_inode_clear_reclaim_tag(
  210. struct xfs_perag *pag,
  211. xfs_ino_t ino)
  212. {
  213. radix_tree_tag_clear(&pag->pag_ici_root,
  214. XFS_INO_TO_AGINO(pag->pag_mount, ino),
  215. XFS_ICI_RECLAIM_TAG);
  216. xfs_perag_clear_reclaim_tag(pag);
  217. }
  218. static void
  219. xfs_inew_wait(
  220. struct xfs_inode *ip)
  221. {
  222. wait_queue_head_t *wq = bit_waitqueue(&ip->i_flags, __XFS_INEW_BIT);
  223. DEFINE_WAIT_BIT(wait, &ip->i_flags, __XFS_INEW_BIT);
  224. do {
  225. prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
  226. if (!xfs_iflags_test(ip, XFS_INEW))
  227. break;
  228. schedule();
  229. } while (true);
  230. finish_wait(wq, &wait.wq_entry);
  231. }
  232. /*
  233. * When we recycle a reclaimable inode, we need to re-initialise the VFS inode
  234. * part of the structure. This is made more complex by the fact we store
  235. * information about the on-disk values in the VFS inode and so we can't just
  236. * overwrite the values unconditionally. Hence we save the parameters we
  237. * need to retain across reinitialisation, and rewrite them into the VFS inode
  238. * after reinitialisation even if it fails.
  239. */
  240. static int
  241. xfs_reinit_inode(
  242. struct xfs_mount *mp,
  243. struct inode *inode)
  244. {
  245. int error;
  246. uint32_t nlink = inode->i_nlink;
  247. uint32_t generation = inode->i_generation;
  248. uint64_t version = inode_peek_iversion(inode);
  249. umode_t mode = inode->i_mode;
  250. dev_t dev = inode->i_rdev;
  251. error = inode_init_always(mp->m_super, inode);
  252. set_nlink(inode, nlink);
  253. inode->i_generation = generation;
  254. inode_set_iversion_queried(inode, version);
  255. inode->i_mode = mode;
  256. inode->i_rdev = dev;
  257. return error;
  258. }
  259. /*
  260. * If we are allocating a new inode, then check what was returned is
  261. * actually a free, empty inode. If we are not allocating an inode,
  262. * then check we didn't find a free inode.
  263. *
  264. * Returns:
  265. * 0 if the inode free state matches the lookup context
  266. * -ENOENT if the inode is free and we are not allocating
  267. * -EFSCORRUPTED if there is any state mismatch at all
  268. */
  269. static int
  270. xfs_iget_check_free_state(
  271. struct xfs_inode *ip,
  272. int flags)
  273. {
  274. if (flags & XFS_IGET_CREATE) {
  275. /* should be a free inode */
  276. if (VFS_I(ip)->i_mode != 0) {
  277. xfs_warn(ip->i_mount,
  278. "Corruption detected! Free inode 0x%llx not marked free! (mode 0x%x)",
  279. ip->i_ino, VFS_I(ip)->i_mode);
  280. return -EFSCORRUPTED;
  281. }
  282. if (ip->i_d.di_nblocks != 0) {
  283. xfs_warn(ip->i_mount,
  284. "Corruption detected! Free inode 0x%llx has blocks allocated!",
  285. ip->i_ino);
  286. return -EFSCORRUPTED;
  287. }
  288. return 0;
  289. }
  290. /* should be an allocated inode */
  291. if (VFS_I(ip)->i_mode == 0)
  292. return -ENOENT;
  293. return 0;
  294. }
  295. /*
  296. * Check the validity of the inode we just found it the cache
  297. */
  298. static int
  299. xfs_iget_cache_hit(
  300. struct xfs_perag *pag,
  301. struct xfs_inode *ip,
  302. xfs_ino_t ino,
  303. int flags,
  304. int lock_flags) __releases(RCU)
  305. {
  306. struct inode *inode = VFS_I(ip);
  307. struct xfs_mount *mp = ip->i_mount;
  308. int error;
  309. /*
  310. * check for re-use of an inode within an RCU grace period due to the
  311. * radix tree nodes not being updated yet. We monitor for this by
  312. * setting the inode number to zero before freeing the inode structure.
  313. * If the inode has been reallocated and set up, then the inode number
  314. * will not match, so check for that, too.
  315. */
  316. spin_lock(&ip->i_flags_lock);
  317. if (ip->i_ino != ino) {
  318. trace_xfs_iget_skip(ip);
  319. XFS_STATS_INC(mp, xs_ig_frecycle);
  320. error = -EAGAIN;
  321. goto out_error;
  322. }
  323. /*
  324. * If we are racing with another cache hit that is currently
  325. * instantiating this inode or currently recycling it out of
  326. * reclaimabe state, wait for the initialisation to complete
  327. * before continuing.
  328. *
  329. * XXX(hch): eventually we should do something equivalent to
  330. * wait_on_inode to wait for these flags to be cleared
  331. * instead of polling for it.
  332. */
  333. if (ip->i_flags & (XFS_INEW|XFS_IRECLAIM)) {
  334. trace_xfs_iget_skip(ip);
  335. XFS_STATS_INC(mp, xs_ig_frecycle);
  336. error = -EAGAIN;
  337. goto out_error;
  338. }
  339. /*
  340. * Check the inode free state is valid. This also detects lookup
  341. * racing with unlinks.
  342. */
  343. error = xfs_iget_check_free_state(ip, flags);
  344. if (error)
  345. goto out_error;
  346. /*
  347. * If IRECLAIMABLE is set, we've torn down the VFS inode already.
  348. * Need to carefully get it back into useable state.
  349. */
  350. if (ip->i_flags & XFS_IRECLAIMABLE) {
  351. trace_xfs_iget_reclaim(ip);
  352. if (flags & XFS_IGET_INCORE) {
  353. error = -EAGAIN;
  354. goto out_error;
  355. }
  356. /*
  357. * We need to set XFS_IRECLAIM to prevent xfs_reclaim_inode
  358. * from stomping over us while we recycle the inode. We can't
  359. * clear the radix tree reclaimable tag yet as it requires
  360. * pag_ici_lock to be held exclusive.
  361. */
  362. ip->i_flags |= XFS_IRECLAIM;
  363. spin_unlock(&ip->i_flags_lock);
  364. rcu_read_unlock();
  365. error = xfs_reinit_inode(mp, inode);
  366. if (error) {
  367. bool wake;
  368. /*
  369. * Re-initializing the inode failed, and we are in deep
  370. * trouble. Try to re-add it to the reclaim list.
  371. */
  372. rcu_read_lock();
  373. spin_lock(&ip->i_flags_lock);
  374. wake = !!__xfs_iflags_test(ip, XFS_INEW);
  375. ip->i_flags &= ~(XFS_INEW | XFS_IRECLAIM);
  376. if (wake)
  377. wake_up_bit(&ip->i_flags, __XFS_INEW_BIT);
  378. ASSERT(ip->i_flags & XFS_IRECLAIMABLE);
  379. trace_xfs_iget_reclaim_fail(ip);
  380. goto out_error;
  381. }
  382. spin_lock(&pag->pag_ici_lock);
  383. spin_lock(&ip->i_flags_lock);
  384. /*
  385. * Clear the per-lifetime state in the inode as we are now
  386. * effectively a new inode and need to return to the initial
  387. * state before reuse occurs.
  388. */
  389. ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS;
  390. ip->i_flags |= XFS_INEW;
  391. xfs_inode_clear_reclaim_tag(pag, ip->i_ino);
  392. inode->i_state = I_NEW;
  393. ASSERT(!rwsem_is_locked(&inode->i_rwsem));
  394. init_rwsem(&inode->i_rwsem);
  395. spin_unlock(&ip->i_flags_lock);
  396. spin_unlock(&pag->pag_ici_lock);
  397. } else {
  398. /* If the VFS inode is being torn down, pause and try again. */
  399. if (!igrab(inode)) {
  400. trace_xfs_iget_skip(ip);
  401. error = -EAGAIN;
  402. goto out_error;
  403. }
  404. /* We've got a live one. */
  405. spin_unlock(&ip->i_flags_lock);
  406. rcu_read_unlock();
  407. trace_xfs_iget_hit(ip);
  408. }
  409. if (lock_flags != 0)
  410. xfs_ilock(ip, lock_flags);
  411. if (!(flags & XFS_IGET_INCORE))
  412. xfs_iflags_clear(ip, XFS_ISTALE | XFS_IDONTCACHE);
  413. XFS_STATS_INC(mp, xs_ig_found);
  414. return 0;
  415. out_error:
  416. spin_unlock(&ip->i_flags_lock);
  417. rcu_read_unlock();
  418. return error;
  419. }
  420. static int
  421. xfs_iget_cache_miss(
  422. struct xfs_mount *mp,
  423. struct xfs_perag *pag,
  424. xfs_trans_t *tp,
  425. xfs_ino_t ino,
  426. struct xfs_inode **ipp,
  427. int flags,
  428. int lock_flags)
  429. {
  430. struct xfs_inode *ip;
  431. int error;
  432. xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ino);
  433. int iflags;
  434. ip = xfs_inode_alloc(mp, ino);
  435. if (!ip)
  436. return -ENOMEM;
  437. error = xfs_iread(mp, tp, ip, flags);
  438. if (error)
  439. goto out_destroy;
  440. if (!xfs_inode_verify_forks(ip)) {
  441. error = -EFSCORRUPTED;
  442. goto out_destroy;
  443. }
  444. trace_xfs_iget_miss(ip);
  445. /*
  446. * Check the inode free state is valid. This also detects lookup
  447. * racing with unlinks.
  448. */
  449. error = xfs_iget_check_free_state(ip, flags);
  450. if (error)
  451. goto out_destroy;
  452. /*
  453. * Preload the radix tree so we can insert safely under the
  454. * write spinlock. Note that we cannot sleep inside the preload
  455. * region. Since we can be called from transaction context, don't
  456. * recurse into the file system.
  457. */
  458. if (radix_tree_preload(GFP_NOFS)) {
  459. error = -EAGAIN;
  460. goto out_destroy;
  461. }
  462. /*
  463. * Because the inode hasn't been added to the radix-tree yet it can't
  464. * be found by another thread, so we can do the non-sleeping lock here.
  465. */
  466. if (lock_flags) {
  467. if (!xfs_ilock_nowait(ip, lock_flags))
  468. BUG();
  469. }
  470. /*
  471. * These values must be set before inserting the inode into the radix
  472. * tree as the moment it is inserted a concurrent lookup (allowed by the
  473. * RCU locking mechanism) can find it and that lookup must see that this
  474. * is an inode currently under construction (i.e. that XFS_INEW is set).
  475. * The ip->i_flags_lock that protects the XFS_INEW flag forms the
  476. * memory barrier that ensures this detection works correctly at lookup
  477. * time.
  478. */
  479. iflags = XFS_INEW;
  480. if (flags & XFS_IGET_DONTCACHE)
  481. iflags |= XFS_IDONTCACHE;
  482. ip->i_udquot = NULL;
  483. ip->i_gdquot = NULL;
  484. ip->i_pdquot = NULL;
  485. xfs_iflags_set(ip, iflags);
  486. /* insert the new inode */
  487. spin_lock(&pag->pag_ici_lock);
  488. error = radix_tree_insert(&pag->pag_ici_root, agino, ip);
  489. if (unlikely(error)) {
  490. WARN_ON(error != -EEXIST);
  491. XFS_STATS_INC(mp, xs_ig_dup);
  492. error = -EAGAIN;
  493. goto out_preload_end;
  494. }
  495. spin_unlock(&pag->pag_ici_lock);
  496. radix_tree_preload_end();
  497. *ipp = ip;
  498. return 0;
  499. out_preload_end:
  500. spin_unlock(&pag->pag_ici_lock);
  501. radix_tree_preload_end();
  502. if (lock_flags)
  503. xfs_iunlock(ip, lock_flags);
  504. out_destroy:
  505. __destroy_inode(VFS_I(ip));
  506. xfs_inode_free(ip);
  507. return error;
  508. }
  509. /*
  510. * Look up an inode by number in the given file system.
  511. * The inode is looked up in the cache held in each AG.
  512. * If the inode is found in the cache, initialise the vfs inode
  513. * if necessary.
  514. *
  515. * If it is not in core, read it in from the file system's device,
  516. * add it to the cache and initialise the vfs inode.
  517. *
  518. * The inode is locked according to the value of the lock_flags parameter.
  519. * This flag parameter indicates how and if the inode's IO lock and inode lock
  520. * should be taken.
  521. *
  522. * mp -- the mount point structure for the current file system. It points
  523. * to the inode hash table.
  524. * tp -- a pointer to the current transaction if there is one. This is
  525. * simply passed through to the xfs_iread() call.
  526. * ino -- the number of the inode desired. This is the unique identifier
  527. * within the file system for the inode being requested.
  528. * lock_flags -- flags indicating how to lock the inode. See the comment
  529. * for xfs_ilock() for a list of valid values.
  530. */
  531. int
  532. xfs_iget(
  533. xfs_mount_t *mp,
  534. xfs_trans_t *tp,
  535. xfs_ino_t ino,
  536. uint flags,
  537. uint lock_flags,
  538. xfs_inode_t **ipp)
  539. {
  540. xfs_inode_t *ip;
  541. int error;
  542. xfs_perag_t *pag;
  543. xfs_agino_t agino;
  544. /*
  545. * xfs_reclaim_inode() uses the ILOCK to ensure an inode
  546. * doesn't get freed while it's being referenced during a
  547. * radix tree traversal here. It assumes this function
  548. * aqcuires only the ILOCK (and therefore it has no need to
  549. * involve the IOLOCK in this synchronization).
  550. */
  551. ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0);
  552. /* reject inode numbers outside existing AGs */
  553. if (!ino || XFS_INO_TO_AGNO(mp, ino) >= mp->m_sb.sb_agcount)
  554. return -EINVAL;
  555. XFS_STATS_INC(mp, xs_ig_attempts);
  556. /* get the perag structure and ensure that it's inode capable */
  557. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ino));
  558. agino = XFS_INO_TO_AGINO(mp, ino);
  559. again:
  560. error = 0;
  561. rcu_read_lock();
  562. ip = radix_tree_lookup(&pag->pag_ici_root, agino);
  563. if (ip) {
  564. error = xfs_iget_cache_hit(pag, ip, ino, flags, lock_flags);
  565. if (error)
  566. goto out_error_or_again;
  567. } else {
  568. rcu_read_unlock();
  569. if (flags & XFS_IGET_INCORE) {
  570. error = -ENODATA;
  571. goto out_error_or_again;
  572. }
  573. XFS_STATS_INC(mp, xs_ig_missed);
  574. error = xfs_iget_cache_miss(mp, pag, tp, ino, &ip,
  575. flags, lock_flags);
  576. if (error)
  577. goto out_error_or_again;
  578. }
  579. xfs_perag_put(pag);
  580. *ipp = ip;
  581. /*
  582. * If we have a real type for an on-disk inode, we can setup the inode
  583. * now. If it's a new inode being created, xfs_ialloc will handle it.
  584. */
  585. if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0)
  586. xfs_setup_existing_inode(ip);
  587. return 0;
  588. out_error_or_again:
  589. if (!(flags & XFS_IGET_INCORE) && error == -EAGAIN) {
  590. delay(1);
  591. goto again;
  592. }
  593. xfs_perag_put(pag);
  594. return error;
  595. }
  596. /*
  597. * "Is this a cached inode that's also allocated?"
  598. *
  599. * Look up an inode by number in the given file system. If the inode is
  600. * in cache and isn't in purgatory, return 1 if the inode is allocated
  601. * and 0 if it is not. For all other cases (not in cache, being torn
  602. * down, etc.), return a negative error code.
  603. *
  604. * The caller has to prevent inode allocation and freeing activity,
  605. * presumably by locking the AGI buffer. This is to ensure that an
  606. * inode cannot transition from allocated to freed until the caller is
  607. * ready to allow that. If the inode is in an intermediate state (new,
  608. * reclaimable, or being reclaimed), -EAGAIN will be returned; if the
  609. * inode is not in the cache, -ENOENT will be returned. The caller must
  610. * deal with these scenarios appropriately.
  611. *
  612. * This is a specialized use case for the online scrubber; if you're
  613. * reading this, you probably want xfs_iget.
  614. */
  615. int
  616. xfs_icache_inode_is_allocated(
  617. struct xfs_mount *mp,
  618. struct xfs_trans *tp,
  619. xfs_ino_t ino,
  620. bool *inuse)
  621. {
  622. struct xfs_inode *ip;
  623. int error;
  624. error = xfs_iget(mp, tp, ino, XFS_IGET_INCORE, 0, &ip);
  625. if (error)
  626. return error;
  627. *inuse = !!(VFS_I(ip)->i_mode);
  628. xfs_irele(ip);
  629. return 0;
  630. }
  631. /*
  632. * The inode lookup is done in batches to keep the amount of lock traffic and
  633. * radix tree lookups to a minimum. The batch size is a trade off between
  634. * lookup reduction and stack usage. This is in the reclaim path, so we can't
  635. * be too greedy.
  636. */
  637. #define XFS_LOOKUP_BATCH 32
  638. STATIC int
  639. xfs_inode_ag_walk_grab(
  640. struct xfs_inode *ip,
  641. int flags)
  642. {
  643. struct inode *inode = VFS_I(ip);
  644. bool newinos = !!(flags & XFS_AGITER_INEW_WAIT);
  645. ASSERT(rcu_read_lock_held());
  646. /*
  647. * check for stale RCU freed inode
  648. *
  649. * If the inode has been reallocated, it doesn't matter if it's not in
  650. * the AG we are walking - we are walking for writeback, so if it
  651. * passes all the "valid inode" checks and is dirty, then we'll write
  652. * it back anyway. If it has been reallocated and still being
  653. * initialised, the XFS_INEW check below will catch it.
  654. */
  655. spin_lock(&ip->i_flags_lock);
  656. if (!ip->i_ino)
  657. goto out_unlock_noent;
  658. /* avoid new or reclaimable inodes. Leave for reclaim code to flush */
  659. if ((!newinos && __xfs_iflags_test(ip, XFS_INEW)) ||
  660. __xfs_iflags_test(ip, XFS_IRECLAIMABLE | XFS_IRECLAIM))
  661. goto out_unlock_noent;
  662. spin_unlock(&ip->i_flags_lock);
  663. /* nothing to sync during shutdown */
  664. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  665. return -EFSCORRUPTED;
  666. /* If we can't grab the inode, it must on it's way to reclaim. */
  667. if (!igrab(inode))
  668. return -ENOENT;
  669. /* inode is valid */
  670. return 0;
  671. out_unlock_noent:
  672. spin_unlock(&ip->i_flags_lock);
  673. return -ENOENT;
  674. }
  675. STATIC int
  676. xfs_inode_ag_walk(
  677. struct xfs_mount *mp,
  678. struct xfs_perag *pag,
  679. int (*execute)(struct xfs_inode *ip, int flags,
  680. void *args),
  681. int flags,
  682. void *args,
  683. int tag,
  684. int iter_flags)
  685. {
  686. uint32_t first_index;
  687. int last_error = 0;
  688. int skipped;
  689. int done;
  690. int nr_found;
  691. restart:
  692. done = 0;
  693. skipped = 0;
  694. first_index = 0;
  695. nr_found = 0;
  696. do {
  697. struct xfs_inode *batch[XFS_LOOKUP_BATCH];
  698. int error = 0;
  699. int i;
  700. rcu_read_lock();
  701. if (tag == -1)
  702. nr_found = radix_tree_gang_lookup(&pag->pag_ici_root,
  703. (void **)batch, first_index,
  704. XFS_LOOKUP_BATCH);
  705. else
  706. nr_found = radix_tree_gang_lookup_tag(
  707. &pag->pag_ici_root,
  708. (void **) batch, first_index,
  709. XFS_LOOKUP_BATCH, tag);
  710. if (!nr_found) {
  711. rcu_read_unlock();
  712. break;
  713. }
  714. /*
  715. * Grab the inodes before we drop the lock. if we found
  716. * nothing, nr == 0 and the loop will be skipped.
  717. */
  718. for (i = 0; i < nr_found; i++) {
  719. struct xfs_inode *ip = batch[i];
  720. if (done || xfs_inode_ag_walk_grab(ip, iter_flags))
  721. batch[i] = NULL;
  722. /*
  723. * Update the index for the next lookup. Catch
  724. * overflows into the next AG range which can occur if
  725. * we have inodes in the last block of the AG and we
  726. * are currently pointing to the last inode.
  727. *
  728. * Because we may see inodes that are from the wrong AG
  729. * due to RCU freeing and reallocation, only update the
  730. * index if it lies in this AG. It was a race that lead
  731. * us to see this inode, so another lookup from the
  732. * same index will not find it again.
  733. */
  734. if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno)
  735. continue;
  736. first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
  737. if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
  738. done = 1;
  739. }
  740. /* unlock now we've grabbed the inodes. */
  741. rcu_read_unlock();
  742. for (i = 0; i < nr_found; i++) {
  743. if (!batch[i])
  744. continue;
  745. if ((iter_flags & XFS_AGITER_INEW_WAIT) &&
  746. xfs_iflags_test(batch[i], XFS_INEW))
  747. xfs_inew_wait(batch[i]);
  748. error = execute(batch[i], flags, args);
  749. xfs_irele(batch[i]);
  750. if (error == -EAGAIN) {
  751. skipped++;
  752. continue;
  753. }
  754. if (error && last_error != -EFSCORRUPTED)
  755. last_error = error;
  756. }
  757. /* bail out if the filesystem is corrupted. */
  758. if (error == -EFSCORRUPTED)
  759. break;
  760. cond_resched();
  761. } while (nr_found && !done);
  762. if (skipped) {
  763. delay(1);
  764. goto restart;
  765. }
  766. return last_error;
  767. }
  768. /*
  769. * Background scanning to trim post-EOF preallocated space. This is queued
  770. * based on the 'speculative_prealloc_lifetime' tunable (5m by default).
  771. */
  772. void
  773. xfs_queue_eofblocks(
  774. struct xfs_mount *mp)
  775. {
  776. rcu_read_lock();
  777. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG))
  778. queue_delayed_work(mp->m_eofblocks_workqueue,
  779. &mp->m_eofblocks_work,
  780. msecs_to_jiffies(xfs_eofb_secs * 1000));
  781. rcu_read_unlock();
  782. }
  783. void
  784. xfs_eofblocks_worker(
  785. struct work_struct *work)
  786. {
  787. struct xfs_mount *mp = container_of(to_delayed_work(work),
  788. struct xfs_mount, m_eofblocks_work);
  789. if (!sb_start_write_trylock(mp->m_super))
  790. return;
  791. xfs_icache_free_eofblocks(mp, NULL);
  792. sb_end_write(mp->m_super);
  793. xfs_queue_eofblocks(mp);
  794. }
  795. /*
  796. * Background scanning to trim preallocated CoW space. This is queued
  797. * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default).
  798. * (We'll just piggyback on the post-EOF prealloc space workqueue.)
  799. */
  800. void
  801. xfs_queue_cowblocks(
  802. struct xfs_mount *mp)
  803. {
  804. rcu_read_lock();
  805. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_COWBLOCKS_TAG))
  806. queue_delayed_work(mp->m_eofblocks_workqueue,
  807. &mp->m_cowblocks_work,
  808. msecs_to_jiffies(xfs_cowb_secs * 1000));
  809. rcu_read_unlock();
  810. }
  811. void
  812. xfs_cowblocks_worker(
  813. struct work_struct *work)
  814. {
  815. struct xfs_mount *mp = container_of(to_delayed_work(work),
  816. struct xfs_mount, m_cowblocks_work);
  817. if (!sb_start_write_trylock(mp->m_super))
  818. return;
  819. xfs_icache_free_cowblocks(mp, NULL);
  820. sb_end_write(mp->m_super);
  821. xfs_queue_cowblocks(mp);
  822. }
  823. int
  824. xfs_inode_ag_iterator_flags(
  825. struct xfs_mount *mp,
  826. int (*execute)(struct xfs_inode *ip, int flags,
  827. void *args),
  828. int flags,
  829. void *args,
  830. int iter_flags)
  831. {
  832. struct xfs_perag *pag;
  833. int error = 0;
  834. int last_error = 0;
  835. xfs_agnumber_t ag;
  836. ag = 0;
  837. while ((pag = xfs_perag_get(mp, ag))) {
  838. ag = pag->pag_agno + 1;
  839. error = xfs_inode_ag_walk(mp, pag, execute, flags, args, -1,
  840. iter_flags);
  841. xfs_perag_put(pag);
  842. if (error) {
  843. last_error = error;
  844. if (error == -EFSCORRUPTED)
  845. break;
  846. }
  847. }
  848. return last_error;
  849. }
  850. int
  851. xfs_inode_ag_iterator(
  852. struct xfs_mount *mp,
  853. int (*execute)(struct xfs_inode *ip, int flags,
  854. void *args),
  855. int flags,
  856. void *args)
  857. {
  858. return xfs_inode_ag_iterator_flags(mp, execute, flags, args, 0);
  859. }
  860. int
  861. xfs_inode_ag_iterator_tag(
  862. struct xfs_mount *mp,
  863. int (*execute)(struct xfs_inode *ip, int flags,
  864. void *args),
  865. int flags,
  866. void *args,
  867. int tag)
  868. {
  869. struct xfs_perag *pag;
  870. int error = 0;
  871. int last_error = 0;
  872. xfs_agnumber_t ag;
  873. ag = 0;
  874. while ((pag = xfs_perag_get_tag(mp, ag, tag))) {
  875. ag = pag->pag_agno + 1;
  876. error = xfs_inode_ag_walk(mp, pag, execute, flags, args, tag,
  877. 0);
  878. xfs_perag_put(pag);
  879. if (error) {
  880. last_error = error;
  881. if (error == -EFSCORRUPTED)
  882. break;
  883. }
  884. }
  885. return last_error;
  886. }
  887. /*
  888. * Grab the inode for reclaim exclusively.
  889. * Return 0 if we grabbed it, non-zero otherwise.
  890. */
  891. STATIC int
  892. xfs_reclaim_inode_grab(
  893. struct xfs_inode *ip,
  894. int flags)
  895. {
  896. ASSERT(rcu_read_lock_held());
  897. /* quick check for stale RCU freed inode */
  898. if (!ip->i_ino)
  899. return 1;
  900. /*
  901. * If we are asked for non-blocking operation, do unlocked checks to
  902. * see if the inode already is being flushed or in reclaim to avoid
  903. * lock traffic.
  904. */
  905. if ((flags & SYNC_TRYLOCK) &&
  906. __xfs_iflags_test(ip, XFS_IFLOCK | XFS_IRECLAIM))
  907. return 1;
  908. /*
  909. * The radix tree lock here protects a thread in xfs_iget from racing
  910. * with us starting reclaim on the inode. Once we have the
  911. * XFS_IRECLAIM flag set it will not touch us.
  912. *
  913. * Due to RCU lookup, we may find inodes that have been freed and only
  914. * have XFS_IRECLAIM set. Indeed, we may see reallocated inodes that
  915. * aren't candidates for reclaim at all, so we must check the
  916. * XFS_IRECLAIMABLE is set first before proceeding to reclaim.
  917. */
  918. spin_lock(&ip->i_flags_lock);
  919. if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) ||
  920. __xfs_iflags_test(ip, XFS_IRECLAIM)) {
  921. /* not a reclaim candidate. */
  922. spin_unlock(&ip->i_flags_lock);
  923. return 1;
  924. }
  925. __xfs_iflags_set(ip, XFS_IRECLAIM);
  926. spin_unlock(&ip->i_flags_lock);
  927. return 0;
  928. }
  929. /*
  930. * Inodes in different states need to be treated differently. The following
  931. * table lists the inode states and the reclaim actions necessary:
  932. *
  933. * inode state iflush ret required action
  934. * --------------- ---------- ---------------
  935. * bad - reclaim
  936. * shutdown EIO unpin and reclaim
  937. * clean, unpinned 0 reclaim
  938. * stale, unpinned 0 reclaim
  939. * clean, pinned(*) 0 requeue
  940. * stale, pinned EAGAIN requeue
  941. * dirty, async - requeue
  942. * dirty, sync 0 reclaim
  943. *
  944. * (*) dgc: I don't think the clean, pinned state is possible but it gets
  945. * handled anyway given the order of checks implemented.
  946. *
  947. * Also, because we get the flush lock first, we know that any inode that has
  948. * been flushed delwri has had the flush completed by the time we check that
  949. * the inode is clean.
  950. *
  951. * Note that because the inode is flushed delayed write by AIL pushing, the
  952. * flush lock may already be held here and waiting on it can result in very
  953. * long latencies. Hence for sync reclaims, where we wait on the flush lock,
  954. * the caller should push the AIL first before trying to reclaim inodes to
  955. * minimise the amount of time spent waiting. For background relaim, we only
  956. * bother to reclaim clean inodes anyway.
  957. *
  958. * Hence the order of actions after gaining the locks should be:
  959. * bad => reclaim
  960. * shutdown => unpin and reclaim
  961. * pinned, async => requeue
  962. * pinned, sync => unpin
  963. * stale => reclaim
  964. * clean => reclaim
  965. * dirty, async => requeue
  966. * dirty, sync => flush, wait and reclaim
  967. */
  968. STATIC int
  969. xfs_reclaim_inode(
  970. struct xfs_inode *ip,
  971. struct xfs_perag *pag,
  972. int sync_mode)
  973. {
  974. struct xfs_buf *bp = NULL;
  975. xfs_ino_t ino = ip->i_ino; /* for radix_tree_delete */
  976. int error;
  977. restart:
  978. error = 0;
  979. xfs_ilock(ip, XFS_ILOCK_EXCL);
  980. if (!xfs_iflock_nowait(ip)) {
  981. if (!(sync_mode & SYNC_WAIT))
  982. goto out;
  983. xfs_iflock(ip);
  984. }
  985. if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
  986. xfs_iunpin_wait(ip);
  987. /* xfs_iflush_abort() drops the flush lock */
  988. xfs_iflush_abort(ip, false);
  989. goto reclaim;
  990. }
  991. if (xfs_ipincount(ip)) {
  992. if (!(sync_mode & SYNC_WAIT))
  993. goto out_ifunlock;
  994. xfs_iunpin_wait(ip);
  995. }
  996. if (xfs_inode_clean(ip)) {
  997. xfs_ifunlock(ip);
  998. goto reclaim;
  999. }
  1000. /*
  1001. * Never flush out dirty data during non-blocking reclaim, as it would
  1002. * just contend with AIL pushing trying to do the same job.
  1003. */
  1004. if (!(sync_mode & SYNC_WAIT))
  1005. goto out_ifunlock;
  1006. /*
  1007. * Now we have an inode that needs flushing.
  1008. *
  1009. * Note that xfs_iflush will never block on the inode buffer lock, as
  1010. * xfs_ifree_cluster() can lock the inode buffer before it locks the
  1011. * ip->i_lock, and we are doing the exact opposite here. As a result,
  1012. * doing a blocking xfs_imap_to_bp() to get the cluster buffer would
  1013. * result in an ABBA deadlock with xfs_ifree_cluster().
  1014. *
  1015. * As xfs_ifree_cluser() must gather all inodes that are active in the
  1016. * cache to mark them stale, if we hit this case we don't actually want
  1017. * to do IO here - we want the inode marked stale so we can simply
  1018. * reclaim it. Hence if we get an EAGAIN error here, just unlock the
  1019. * inode, back off and try again. Hopefully the next pass through will
  1020. * see the stale flag set on the inode.
  1021. */
  1022. error = xfs_iflush(ip, &bp);
  1023. if (error == -EAGAIN) {
  1024. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1025. /* backoff longer than in xfs_ifree_cluster */
  1026. delay(2);
  1027. goto restart;
  1028. }
  1029. if (!error) {
  1030. error = xfs_bwrite(bp);
  1031. xfs_buf_relse(bp);
  1032. }
  1033. reclaim:
  1034. ASSERT(!xfs_isiflocked(ip));
  1035. /*
  1036. * Because we use RCU freeing we need to ensure the inode always appears
  1037. * to be reclaimed with an invalid inode number when in the free state.
  1038. * We do this as early as possible under the ILOCK so that
  1039. * xfs_iflush_cluster() and xfs_ifree_cluster() can be guaranteed to
  1040. * detect races with us here. By doing this, we guarantee that once
  1041. * xfs_iflush_cluster() or xfs_ifree_cluster() has locked XFS_ILOCK that
  1042. * it will see either a valid inode that will serialise correctly, or it
  1043. * will see an invalid inode that it can skip.
  1044. */
  1045. spin_lock(&ip->i_flags_lock);
  1046. ip->i_flags = XFS_IRECLAIM;
  1047. ip->i_ino = 0;
  1048. spin_unlock(&ip->i_flags_lock);
  1049. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1050. XFS_STATS_INC(ip->i_mount, xs_ig_reclaims);
  1051. /*
  1052. * Remove the inode from the per-AG radix tree.
  1053. *
  1054. * Because radix_tree_delete won't complain even if the item was never
  1055. * added to the tree assert that it's been there before to catch
  1056. * problems with the inode life time early on.
  1057. */
  1058. spin_lock(&pag->pag_ici_lock);
  1059. if (!radix_tree_delete(&pag->pag_ici_root,
  1060. XFS_INO_TO_AGINO(ip->i_mount, ino)))
  1061. ASSERT(0);
  1062. xfs_perag_clear_reclaim_tag(pag);
  1063. spin_unlock(&pag->pag_ici_lock);
  1064. /*
  1065. * Here we do an (almost) spurious inode lock in order to coordinate
  1066. * with inode cache radix tree lookups. This is because the lookup
  1067. * can reference the inodes in the cache without taking references.
  1068. *
  1069. * We make that OK here by ensuring that we wait until the inode is
  1070. * unlocked after the lookup before we go ahead and free it.
  1071. */
  1072. xfs_ilock(ip, XFS_ILOCK_EXCL);
  1073. xfs_qm_dqdetach(ip);
  1074. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1075. ASSERT(xfs_inode_clean(ip));
  1076. __xfs_inode_free(ip);
  1077. return error;
  1078. out_ifunlock:
  1079. xfs_ifunlock(ip);
  1080. out:
  1081. xfs_iflags_clear(ip, XFS_IRECLAIM);
  1082. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1083. /*
  1084. * We could return -EAGAIN here to make reclaim rescan the inode tree in
  1085. * a short while. However, this just burns CPU time scanning the tree
  1086. * waiting for IO to complete and the reclaim work never goes back to
  1087. * the idle state. Instead, return 0 to let the next scheduled
  1088. * background reclaim attempt to reclaim the inode again.
  1089. */
  1090. return 0;
  1091. }
  1092. /*
  1093. * Walk the AGs and reclaim the inodes in them. Even if the filesystem is
  1094. * corrupted, we still want to try to reclaim all the inodes. If we don't,
  1095. * then a shut down during filesystem unmount reclaim walk leak all the
  1096. * unreclaimed inodes.
  1097. */
  1098. STATIC int
  1099. xfs_reclaim_inodes_ag(
  1100. struct xfs_mount *mp,
  1101. int flags,
  1102. int *nr_to_scan)
  1103. {
  1104. struct xfs_perag *pag;
  1105. int error = 0;
  1106. int last_error = 0;
  1107. xfs_agnumber_t ag;
  1108. int trylock = flags & SYNC_TRYLOCK;
  1109. int skipped;
  1110. restart:
  1111. ag = 0;
  1112. skipped = 0;
  1113. while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
  1114. unsigned long first_index = 0;
  1115. int done = 0;
  1116. int nr_found = 0;
  1117. ag = pag->pag_agno + 1;
  1118. if (trylock) {
  1119. if (!mutex_trylock(&pag->pag_ici_reclaim_lock)) {
  1120. skipped++;
  1121. xfs_perag_put(pag);
  1122. continue;
  1123. }
  1124. first_index = pag->pag_ici_reclaim_cursor;
  1125. } else
  1126. mutex_lock(&pag->pag_ici_reclaim_lock);
  1127. do {
  1128. struct xfs_inode *batch[XFS_LOOKUP_BATCH];
  1129. int i;
  1130. rcu_read_lock();
  1131. nr_found = radix_tree_gang_lookup_tag(
  1132. &pag->pag_ici_root,
  1133. (void **)batch, first_index,
  1134. XFS_LOOKUP_BATCH,
  1135. XFS_ICI_RECLAIM_TAG);
  1136. if (!nr_found) {
  1137. done = 1;
  1138. rcu_read_unlock();
  1139. break;
  1140. }
  1141. /*
  1142. * Grab the inodes before we drop the lock. if we found
  1143. * nothing, nr == 0 and the loop will be skipped.
  1144. */
  1145. for (i = 0; i < nr_found; i++) {
  1146. struct xfs_inode *ip = batch[i];
  1147. if (done || xfs_reclaim_inode_grab(ip, flags))
  1148. batch[i] = NULL;
  1149. /*
  1150. * Update the index for the next lookup. Catch
  1151. * overflows into the next AG range which can
  1152. * occur if we have inodes in the last block of
  1153. * the AG and we are currently pointing to the
  1154. * last inode.
  1155. *
  1156. * Because we may see inodes that are from the
  1157. * wrong AG due to RCU freeing and
  1158. * reallocation, only update the index if it
  1159. * lies in this AG. It was a race that lead us
  1160. * to see this inode, so another lookup from
  1161. * the same index will not find it again.
  1162. */
  1163. if (XFS_INO_TO_AGNO(mp, ip->i_ino) !=
  1164. pag->pag_agno)
  1165. continue;
  1166. first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
  1167. if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
  1168. done = 1;
  1169. }
  1170. /* unlock now we've grabbed the inodes. */
  1171. rcu_read_unlock();
  1172. for (i = 0; i < nr_found; i++) {
  1173. if (!batch[i])
  1174. continue;
  1175. error = xfs_reclaim_inode(batch[i], pag, flags);
  1176. if (error && last_error != -EFSCORRUPTED)
  1177. last_error = error;
  1178. }
  1179. *nr_to_scan -= XFS_LOOKUP_BATCH;
  1180. cond_resched();
  1181. } while (nr_found && !done && *nr_to_scan > 0);
  1182. if (trylock && !done)
  1183. pag->pag_ici_reclaim_cursor = first_index;
  1184. else
  1185. pag->pag_ici_reclaim_cursor = 0;
  1186. mutex_unlock(&pag->pag_ici_reclaim_lock);
  1187. xfs_perag_put(pag);
  1188. }
  1189. /*
  1190. * if we skipped any AG, and we still have scan count remaining, do
  1191. * another pass this time using blocking reclaim semantics (i.e
  1192. * waiting on the reclaim locks and ignoring the reclaim cursors). This
  1193. * ensure that when we get more reclaimers than AGs we block rather
  1194. * than spin trying to execute reclaim.
  1195. */
  1196. if (skipped && (flags & SYNC_WAIT) && *nr_to_scan > 0) {
  1197. trylock = 0;
  1198. goto restart;
  1199. }
  1200. return last_error;
  1201. }
  1202. int
  1203. xfs_reclaim_inodes(
  1204. xfs_mount_t *mp,
  1205. int mode)
  1206. {
  1207. int nr_to_scan = INT_MAX;
  1208. return xfs_reclaim_inodes_ag(mp, mode, &nr_to_scan);
  1209. }
  1210. /*
  1211. * Scan a certain number of inodes for reclaim.
  1212. *
  1213. * When called we make sure that there is a background (fast) inode reclaim in
  1214. * progress, while we will throttle the speed of reclaim via doing synchronous
  1215. * reclaim of inodes. That means if we come across dirty inodes, we wait for
  1216. * them to be cleaned, which we hope will not be very long due to the
  1217. * background walker having already kicked the IO off on those dirty inodes.
  1218. */
  1219. long
  1220. xfs_reclaim_inodes_nr(
  1221. struct xfs_mount *mp,
  1222. int nr_to_scan)
  1223. {
  1224. /* kick background reclaimer and push the AIL */
  1225. xfs_reclaim_work_queue(mp);
  1226. xfs_ail_push_all(mp->m_ail);
  1227. return xfs_reclaim_inodes_ag(mp, SYNC_TRYLOCK | SYNC_WAIT, &nr_to_scan);
  1228. }
  1229. /*
  1230. * Return the number of reclaimable inodes in the filesystem for
  1231. * the shrinker to determine how much to reclaim.
  1232. */
  1233. int
  1234. xfs_reclaim_inodes_count(
  1235. struct xfs_mount *mp)
  1236. {
  1237. struct xfs_perag *pag;
  1238. xfs_agnumber_t ag = 0;
  1239. int reclaimable = 0;
  1240. while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
  1241. ag = pag->pag_agno + 1;
  1242. reclaimable += pag->pag_ici_reclaimable;
  1243. xfs_perag_put(pag);
  1244. }
  1245. return reclaimable;
  1246. }
  1247. STATIC int
  1248. xfs_inode_match_id(
  1249. struct xfs_inode *ip,
  1250. struct xfs_eofblocks *eofb)
  1251. {
  1252. if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
  1253. !uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
  1254. return 0;
  1255. if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
  1256. !gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
  1257. return 0;
  1258. if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
  1259. xfs_get_projid(ip) != eofb->eof_prid)
  1260. return 0;
  1261. return 1;
  1262. }
  1263. /*
  1264. * A union-based inode filtering algorithm. Process the inode if any of the
  1265. * criteria match. This is for global/internal scans only.
  1266. */
  1267. STATIC int
  1268. xfs_inode_match_id_union(
  1269. struct xfs_inode *ip,
  1270. struct xfs_eofblocks *eofb)
  1271. {
  1272. if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
  1273. uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
  1274. return 1;
  1275. if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
  1276. gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
  1277. return 1;
  1278. if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
  1279. xfs_get_projid(ip) == eofb->eof_prid)
  1280. return 1;
  1281. return 0;
  1282. }
  1283. STATIC int
  1284. xfs_inode_free_eofblocks(
  1285. struct xfs_inode *ip,
  1286. int flags,
  1287. void *args)
  1288. {
  1289. int ret = 0;
  1290. struct xfs_eofblocks *eofb = args;
  1291. int match;
  1292. if (!xfs_can_free_eofblocks(ip, false)) {
  1293. /* inode could be preallocated or append-only */
  1294. trace_xfs_inode_free_eofblocks_invalid(ip);
  1295. xfs_inode_clear_eofblocks_tag(ip);
  1296. return 0;
  1297. }
  1298. /*
  1299. * If the mapping is dirty the operation can block and wait for some
  1300. * time. Unless we are waiting, skip it.
  1301. */
  1302. if (!(flags & SYNC_WAIT) &&
  1303. mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY))
  1304. return 0;
  1305. if (eofb) {
  1306. if (eofb->eof_flags & XFS_EOF_FLAGS_UNION)
  1307. match = xfs_inode_match_id_union(ip, eofb);
  1308. else
  1309. match = xfs_inode_match_id(ip, eofb);
  1310. if (!match)
  1311. return 0;
  1312. /* skip the inode if the file size is too small */
  1313. if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE &&
  1314. XFS_ISIZE(ip) < eofb->eof_min_file_size)
  1315. return 0;
  1316. }
  1317. /*
  1318. * If the caller is waiting, return -EAGAIN to keep the background
  1319. * scanner moving and revisit the inode in a subsequent pass.
  1320. */
  1321. if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
  1322. if (flags & SYNC_WAIT)
  1323. ret = -EAGAIN;
  1324. return ret;
  1325. }
  1326. ret = xfs_free_eofblocks(ip);
  1327. xfs_iunlock(ip, XFS_IOLOCK_EXCL);
  1328. return ret;
  1329. }
  1330. static int
  1331. __xfs_icache_free_eofblocks(
  1332. struct xfs_mount *mp,
  1333. struct xfs_eofblocks *eofb,
  1334. int (*execute)(struct xfs_inode *ip, int flags,
  1335. void *args),
  1336. int tag)
  1337. {
  1338. int flags = SYNC_TRYLOCK;
  1339. if (eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC))
  1340. flags = SYNC_WAIT;
  1341. return xfs_inode_ag_iterator_tag(mp, execute, flags,
  1342. eofb, tag);
  1343. }
  1344. int
  1345. xfs_icache_free_eofblocks(
  1346. struct xfs_mount *mp,
  1347. struct xfs_eofblocks *eofb)
  1348. {
  1349. return __xfs_icache_free_eofblocks(mp, eofb, xfs_inode_free_eofblocks,
  1350. XFS_ICI_EOFBLOCKS_TAG);
  1351. }
  1352. /*
  1353. * Run eofblocks scans on the quotas applicable to the inode. For inodes with
  1354. * multiple quotas, we don't know exactly which quota caused an allocation
  1355. * failure. We make a best effort by including each quota under low free space
  1356. * conditions (less than 1% free space) in the scan.
  1357. */
  1358. static int
  1359. __xfs_inode_free_quota_eofblocks(
  1360. struct xfs_inode *ip,
  1361. int (*execute)(struct xfs_mount *mp,
  1362. struct xfs_eofblocks *eofb))
  1363. {
  1364. int scan = 0;
  1365. struct xfs_eofblocks eofb = {0};
  1366. struct xfs_dquot *dq;
  1367. /*
  1368. * Run a sync scan to increase effectiveness and use the union filter to
  1369. * cover all applicable quotas in a single scan.
  1370. */
  1371. eofb.eof_flags = XFS_EOF_FLAGS_UNION|XFS_EOF_FLAGS_SYNC;
  1372. if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) {
  1373. dq = xfs_inode_dquot(ip, XFS_DQ_USER);
  1374. if (dq && xfs_dquot_lowsp(dq)) {
  1375. eofb.eof_uid = VFS_I(ip)->i_uid;
  1376. eofb.eof_flags |= XFS_EOF_FLAGS_UID;
  1377. scan = 1;
  1378. }
  1379. }
  1380. if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) {
  1381. dq = xfs_inode_dquot(ip, XFS_DQ_GROUP);
  1382. if (dq && xfs_dquot_lowsp(dq)) {
  1383. eofb.eof_gid = VFS_I(ip)->i_gid;
  1384. eofb.eof_flags |= XFS_EOF_FLAGS_GID;
  1385. scan = 1;
  1386. }
  1387. }
  1388. if (scan)
  1389. execute(ip->i_mount, &eofb);
  1390. return scan;
  1391. }
  1392. int
  1393. xfs_inode_free_quota_eofblocks(
  1394. struct xfs_inode *ip)
  1395. {
  1396. return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_eofblocks);
  1397. }
  1398. static inline unsigned long
  1399. xfs_iflag_for_tag(
  1400. int tag)
  1401. {
  1402. switch (tag) {
  1403. case XFS_ICI_EOFBLOCKS_TAG:
  1404. return XFS_IEOFBLOCKS;
  1405. case XFS_ICI_COWBLOCKS_TAG:
  1406. return XFS_ICOWBLOCKS;
  1407. default:
  1408. ASSERT(0);
  1409. return 0;
  1410. }
  1411. }
  1412. static void
  1413. __xfs_inode_set_blocks_tag(
  1414. xfs_inode_t *ip,
  1415. void (*execute)(struct xfs_mount *mp),
  1416. void (*set_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
  1417. int error, unsigned long caller_ip),
  1418. int tag)
  1419. {
  1420. struct xfs_mount *mp = ip->i_mount;
  1421. struct xfs_perag *pag;
  1422. int tagged;
  1423. /*
  1424. * Don't bother locking the AG and looking up in the radix trees
  1425. * if we already know that we have the tag set.
  1426. */
  1427. if (ip->i_flags & xfs_iflag_for_tag(tag))
  1428. return;
  1429. spin_lock(&ip->i_flags_lock);
  1430. ip->i_flags |= xfs_iflag_for_tag(tag);
  1431. spin_unlock(&ip->i_flags_lock);
  1432. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  1433. spin_lock(&pag->pag_ici_lock);
  1434. tagged = radix_tree_tagged(&pag->pag_ici_root, tag);
  1435. radix_tree_tag_set(&pag->pag_ici_root,
  1436. XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
  1437. if (!tagged) {
  1438. /* propagate the eofblocks tag up into the perag radix tree */
  1439. spin_lock(&ip->i_mount->m_perag_lock);
  1440. radix_tree_tag_set(&ip->i_mount->m_perag_tree,
  1441. XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
  1442. tag);
  1443. spin_unlock(&ip->i_mount->m_perag_lock);
  1444. /* kick off background trimming */
  1445. execute(ip->i_mount);
  1446. set_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
  1447. }
  1448. spin_unlock(&pag->pag_ici_lock);
  1449. xfs_perag_put(pag);
  1450. }
  1451. void
  1452. xfs_inode_set_eofblocks_tag(
  1453. xfs_inode_t *ip)
  1454. {
  1455. trace_xfs_inode_set_eofblocks_tag(ip);
  1456. return __xfs_inode_set_blocks_tag(ip, xfs_queue_eofblocks,
  1457. trace_xfs_perag_set_eofblocks,
  1458. XFS_ICI_EOFBLOCKS_TAG);
  1459. }
  1460. static void
  1461. __xfs_inode_clear_blocks_tag(
  1462. xfs_inode_t *ip,
  1463. void (*clear_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
  1464. int error, unsigned long caller_ip),
  1465. int tag)
  1466. {
  1467. struct xfs_mount *mp = ip->i_mount;
  1468. struct xfs_perag *pag;
  1469. spin_lock(&ip->i_flags_lock);
  1470. ip->i_flags &= ~xfs_iflag_for_tag(tag);
  1471. spin_unlock(&ip->i_flags_lock);
  1472. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  1473. spin_lock(&pag->pag_ici_lock);
  1474. radix_tree_tag_clear(&pag->pag_ici_root,
  1475. XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
  1476. if (!radix_tree_tagged(&pag->pag_ici_root, tag)) {
  1477. /* clear the eofblocks tag from the perag radix tree */
  1478. spin_lock(&ip->i_mount->m_perag_lock);
  1479. radix_tree_tag_clear(&ip->i_mount->m_perag_tree,
  1480. XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
  1481. tag);
  1482. spin_unlock(&ip->i_mount->m_perag_lock);
  1483. clear_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
  1484. }
  1485. spin_unlock(&pag->pag_ici_lock);
  1486. xfs_perag_put(pag);
  1487. }
  1488. void
  1489. xfs_inode_clear_eofblocks_tag(
  1490. xfs_inode_t *ip)
  1491. {
  1492. trace_xfs_inode_clear_eofblocks_tag(ip);
  1493. return __xfs_inode_clear_blocks_tag(ip,
  1494. trace_xfs_perag_clear_eofblocks, XFS_ICI_EOFBLOCKS_TAG);
  1495. }
  1496. /*
  1497. * Set ourselves up to free CoW blocks from this file. If it's already clean
  1498. * then we can bail out quickly, but otherwise we must back off if the file
  1499. * is undergoing some kind of write.
  1500. */
  1501. static bool
  1502. xfs_prep_free_cowblocks(
  1503. struct xfs_inode *ip)
  1504. {
  1505. /*
  1506. * Just clear the tag if we have an empty cow fork or none at all. It's
  1507. * possible the inode was fully unshared since it was originally tagged.
  1508. */
  1509. if (!xfs_inode_has_cow_data(ip)) {
  1510. trace_xfs_inode_free_cowblocks_invalid(ip);
  1511. xfs_inode_clear_cowblocks_tag(ip);
  1512. return false;
  1513. }
  1514. /*
  1515. * If the mapping is dirty or under writeback we cannot touch the
  1516. * CoW fork. Leave it alone if we're in the midst of a directio.
  1517. */
  1518. if ((VFS_I(ip)->i_state & I_DIRTY_PAGES) ||
  1519. mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY) ||
  1520. mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_WRITEBACK) ||
  1521. atomic_read(&VFS_I(ip)->i_dio_count))
  1522. return false;
  1523. return true;
  1524. }
  1525. /*
  1526. * Automatic CoW Reservation Freeing
  1527. *
  1528. * These functions automatically garbage collect leftover CoW reservations
  1529. * that were made on behalf of a cowextsize hint when we start to run out
  1530. * of quota or when the reservations sit around for too long. If the file
  1531. * has dirty pages or is undergoing writeback, its CoW reservations will
  1532. * be retained.
  1533. *
  1534. * The actual garbage collection piggybacks off the same code that runs
  1535. * the speculative EOF preallocation garbage collector.
  1536. */
  1537. STATIC int
  1538. xfs_inode_free_cowblocks(
  1539. struct xfs_inode *ip,
  1540. int flags,
  1541. void *args)
  1542. {
  1543. struct xfs_eofblocks *eofb = args;
  1544. int match;
  1545. int ret = 0;
  1546. if (!xfs_prep_free_cowblocks(ip))
  1547. return 0;
  1548. if (eofb) {
  1549. if (eofb->eof_flags & XFS_EOF_FLAGS_UNION)
  1550. match = xfs_inode_match_id_union(ip, eofb);
  1551. else
  1552. match = xfs_inode_match_id(ip, eofb);
  1553. if (!match)
  1554. return 0;
  1555. /* skip the inode if the file size is too small */
  1556. if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE &&
  1557. XFS_ISIZE(ip) < eofb->eof_min_file_size)
  1558. return 0;
  1559. }
  1560. /* Free the CoW blocks */
  1561. xfs_ilock(ip, XFS_IOLOCK_EXCL);
  1562. xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
  1563. /*
  1564. * Check again, nobody else should be able to dirty blocks or change
  1565. * the reflink iflag now that we have the first two locks held.
  1566. */
  1567. if (xfs_prep_free_cowblocks(ip))
  1568. ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false);
  1569. xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
  1570. xfs_iunlock(ip, XFS_IOLOCK_EXCL);
  1571. return ret;
  1572. }
  1573. int
  1574. xfs_icache_free_cowblocks(
  1575. struct xfs_mount *mp,
  1576. struct xfs_eofblocks *eofb)
  1577. {
  1578. return __xfs_icache_free_eofblocks(mp, eofb, xfs_inode_free_cowblocks,
  1579. XFS_ICI_COWBLOCKS_TAG);
  1580. }
  1581. int
  1582. xfs_inode_free_quota_cowblocks(
  1583. struct xfs_inode *ip)
  1584. {
  1585. return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_cowblocks);
  1586. }
  1587. void
  1588. xfs_inode_set_cowblocks_tag(
  1589. xfs_inode_t *ip)
  1590. {
  1591. trace_xfs_inode_set_cowblocks_tag(ip);
  1592. return __xfs_inode_set_blocks_tag(ip, xfs_queue_cowblocks,
  1593. trace_xfs_perag_set_cowblocks,
  1594. XFS_ICI_COWBLOCKS_TAG);
  1595. }
  1596. void
  1597. xfs_inode_clear_cowblocks_tag(
  1598. xfs_inode_t *ip)
  1599. {
  1600. trace_xfs_inode_clear_cowblocks_tag(ip);
  1601. return __xfs_inode_clear_blocks_tag(ip,
  1602. trace_xfs_perag_clear_cowblocks, XFS_ICI_COWBLOCKS_TAG);
  1603. }
  1604. /* Disable post-EOF and CoW block auto-reclamation. */
  1605. void
  1606. xfs_icache_disable_reclaim(
  1607. struct xfs_mount *mp)
  1608. {
  1609. cancel_delayed_work_sync(&mp->m_eofblocks_work);
  1610. cancel_delayed_work_sync(&mp->m_cowblocks_work);
  1611. }
  1612. /* Enable post-EOF and CoW block auto-reclamation. */
  1613. void
  1614. xfs_icache_enable_reclaim(
  1615. struct xfs_mount *mp)
  1616. {
  1617. xfs_queue_eofblocks(mp);
  1618. xfs_queue_cowblocks(mp);
  1619. }