xfs_trans_ail.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
  4. * Copyright (c) 2008 Dave Chinner
  5. * All Rights Reserved.
  6. */
  7. #include "xfs.h"
  8. #include "xfs_fs.h"
  9. #include "xfs_format.h"
  10. #include "xfs_log_format.h"
  11. #include "xfs_trans_resv.h"
  12. #include "xfs_mount.h"
  13. #include "xfs_trans.h"
  14. #include "xfs_trans_priv.h"
  15. #include "xfs_trace.h"
  16. #include "xfs_errortag.h"
  17. #include "xfs_error.h"
  18. #include "xfs_log.h"
  19. #ifdef DEBUG
  20. /*
  21. * Check that the list is sorted as it should be.
  22. *
  23. * Called with the ail lock held, but we don't want to assert fail with it
  24. * held otherwise we'll lock everything up and won't be able to debug the
  25. * cause. Hence we sample and check the state under the AIL lock and return if
  26. * everything is fine, otherwise we drop the lock and run the ASSERT checks.
  27. * Asserts may not be fatal, so pick the lock back up and continue onwards.
  28. */
  29. STATIC void
  30. xfs_ail_check(
  31. struct xfs_ail *ailp,
  32. struct xfs_log_item *lip)
  33. {
  34. struct xfs_log_item *prev_lip;
  35. struct xfs_log_item *next_lip;
  36. xfs_lsn_t prev_lsn = NULLCOMMITLSN;
  37. xfs_lsn_t next_lsn = NULLCOMMITLSN;
  38. xfs_lsn_t lsn;
  39. bool in_ail;
  40. if (list_empty(&ailp->ail_head))
  41. return;
  42. /*
  43. * Sample then check the next and previous entries are valid.
  44. */
  45. in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags);
  46. prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail);
  47. if (&prev_lip->li_ail != &ailp->ail_head)
  48. prev_lsn = prev_lip->li_lsn;
  49. next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail);
  50. if (&next_lip->li_ail != &ailp->ail_head)
  51. next_lsn = next_lip->li_lsn;
  52. lsn = lip->li_lsn;
  53. if (in_ail &&
  54. (prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0) &&
  55. (next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0))
  56. return;
  57. spin_unlock(&ailp->ail_lock);
  58. ASSERT(in_ail);
  59. ASSERT(prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0);
  60. ASSERT(next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0);
  61. spin_lock(&ailp->ail_lock);
  62. }
  63. #else /* !DEBUG */
  64. #define xfs_ail_check(a,l)
  65. #endif /* DEBUG */
  66. /*
  67. * Return a pointer to the last item in the AIL. If the AIL is empty, then
  68. * return NULL.
  69. */
  70. static xfs_log_item_t *
  71. xfs_ail_max(
  72. struct xfs_ail *ailp)
  73. {
  74. if (list_empty(&ailp->ail_head))
  75. return NULL;
  76. return list_entry(ailp->ail_head.prev, xfs_log_item_t, li_ail);
  77. }
  78. /*
  79. * Return a pointer to the item which follows the given item in the AIL. If
  80. * the given item is the last item in the list, then return NULL.
  81. */
  82. static xfs_log_item_t *
  83. xfs_ail_next(
  84. struct xfs_ail *ailp,
  85. xfs_log_item_t *lip)
  86. {
  87. if (lip->li_ail.next == &ailp->ail_head)
  88. return NULL;
  89. return list_first_entry(&lip->li_ail, xfs_log_item_t, li_ail);
  90. }
  91. /*
  92. * This is called by the log manager code to determine the LSN of the tail of
  93. * the log. This is exactly the LSN of the first item in the AIL. If the AIL
  94. * is empty, then this function returns 0.
  95. *
  96. * We need the AIL lock in order to get a coherent read of the lsn of the last
  97. * item in the AIL.
  98. */
  99. xfs_lsn_t
  100. xfs_ail_min_lsn(
  101. struct xfs_ail *ailp)
  102. {
  103. xfs_lsn_t lsn = 0;
  104. xfs_log_item_t *lip;
  105. spin_lock(&ailp->ail_lock);
  106. lip = xfs_ail_min(ailp);
  107. if (lip)
  108. lsn = lip->li_lsn;
  109. spin_unlock(&ailp->ail_lock);
  110. return lsn;
  111. }
  112. /*
  113. * Return the maximum lsn held in the AIL, or zero if the AIL is empty.
  114. */
  115. static xfs_lsn_t
  116. xfs_ail_max_lsn(
  117. struct xfs_ail *ailp)
  118. {
  119. xfs_lsn_t lsn = 0;
  120. xfs_log_item_t *lip;
  121. spin_lock(&ailp->ail_lock);
  122. lip = xfs_ail_max(ailp);
  123. if (lip)
  124. lsn = lip->li_lsn;
  125. spin_unlock(&ailp->ail_lock);
  126. return lsn;
  127. }
  128. /*
  129. * The cursor keeps track of where our current traversal is up to by tracking
  130. * the next item in the list for us. However, for this to be safe, removing an
  131. * object from the AIL needs to invalidate any cursor that points to it. hence
  132. * the traversal cursor needs to be linked to the struct xfs_ail so that
  133. * deletion can search all the active cursors for invalidation.
  134. */
  135. STATIC void
  136. xfs_trans_ail_cursor_init(
  137. struct xfs_ail *ailp,
  138. struct xfs_ail_cursor *cur)
  139. {
  140. cur->item = NULL;
  141. list_add_tail(&cur->list, &ailp->ail_cursors);
  142. }
  143. /*
  144. * Get the next item in the traversal and advance the cursor. If the cursor
  145. * was invalidated (indicated by a lip of 1), restart the traversal.
  146. */
  147. struct xfs_log_item *
  148. xfs_trans_ail_cursor_next(
  149. struct xfs_ail *ailp,
  150. struct xfs_ail_cursor *cur)
  151. {
  152. struct xfs_log_item *lip = cur->item;
  153. if ((uintptr_t)lip & 1)
  154. lip = xfs_ail_min(ailp);
  155. if (lip)
  156. cur->item = xfs_ail_next(ailp, lip);
  157. return lip;
  158. }
  159. /*
  160. * When the traversal is complete, we need to remove the cursor from the list
  161. * of traversing cursors.
  162. */
  163. void
  164. xfs_trans_ail_cursor_done(
  165. struct xfs_ail_cursor *cur)
  166. {
  167. cur->item = NULL;
  168. list_del_init(&cur->list);
  169. }
  170. /*
  171. * Invalidate any cursor that is pointing to this item. This is called when an
  172. * item is removed from the AIL. Any cursor pointing to this object is now
  173. * invalid and the traversal needs to be terminated so it doesn't reference a
  174. * freed object. We set the low bit of the cursor item pointer so we can
  175. * distinguish between an invalidation and the end of the list when getting the
  176. * next item from the cursor.
  177. */
  178. STATIC void
  179. xfs_trans_ail_cursor_clear(
  180. struct xfs_ail *ailp,
  181. struct xfs_log_item *lip)
  182. {
  183. struct xfs_ail_cursor *cur;
  184. list_for_each_entry(cur, &ailp->ail_cursors, list) {
  185. if (cur->item == lip)
  186. cur->item = (struct xfs_log_item *)
  187. ((uintptr_t)cur->item | 1);
  188. }
  189. }
  190. /*
  191. * Find the first item in the AIL with the given @lsn by searching in ascending
  192. * LSN order and initialise the cursor to point to the next item for a
  193. * ascending traversal. Pass a @lsn of zero to initialise the cursor to the
  194. * first item in the AIL. Returns NULL if the list is empty.
  195. */
  196. xfs_log_item_t *
  197. xfs_trans_ail_cursor_first(
  198. struct xfs_ail *ailp,
  199. struct xfs_ail_cursor *cur,
  200. xfs_lsn_t lsn)
  201. {
  202. xfs_log_item_t *lip;
  203. xfs_trans_ail_cursor_init(ailp, cur);
  204. if (lsn == 0) {
  205. lip = xfs_ail_min(ailp);
  206. goto out;
  207. }
  208. list_for_each_entry(lip, &ailp->ail_head, li_ail) {
  209. if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)
  210. goto out;
  211. }
  212. return NULL;
  213. out:
  214. if (lip)
  215. cur->item = xfs_ail_next(ailp, lip);
  216. return lip;
  217. }
  218. static struct xfs_log_item *
  219. __xfs_trans_ail_cursor_last(
  220. struct xfs_ail *ailp,
  221. xfs_lsn_t lsn)
  222. {
  223. xfs_log_item_t *lip;
  224. list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) {
  225. if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)
  226. return lip;
  227. }
  228. return NULL;
  229. }
  230. /*
  231. * Find the last item in the AIL with the given @lsn by searching in descending
  232. * LSN order and initialise the cursor to point to that item. If there is no
  233. * item with the value of @lsn, then it sets the cursor to the last item with an
  234. * LSN lower than @lsn. Returns NULL if the list is empty.
  235. */
  236. struct xfs_log_item *
  237. xfs_trans_ail_cursor_last(
  238. struct xfs_ail *ailp,
  239. struct xfs_ail_cursor *cur,
  240. xfs_lsn_t lsn)
  241. {
  242. xfs_trans_ail_cursor_init(ailp, cur);
  243. cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);
  244. return cur->item;
  245. }
  246. /*
  247. * Splice the log item list into the AIL at the given LSN. We splice to the
  248. * tail of the given LSN to maintain insert order for push traversals. The
  249. * cursor is optional, allowing repeated updates to the same LSN to avoid
  250. * repeated traversals. This should not be called with an empty list.
  251. */
  252. static void
  253. xfs_ail_splice(
  254. struct xfs_ail *ailp,
  255. struct xfs_ail_cursor *cur,
  256. struct list_head *list,
  257. xfs_lsn_t lsn)
  258. {
  259. struct xfs_log_item *lip;
  260. ASSERT(!list_empty(list));
  261. /*
  262. * Use the cursor to determine the insertion point if one is
  263. * provided. If not, or if the one we got is not valid,
  264. * find the place in the AIL where the items belong.
  265. */
  266. lip = cur ? cur->item : NULL;
  267. if (!lip || (uintptr_t)lip & 1)
  268. lip = __xfs_trans_ail_cursor_last(ailp, lsn);
  269. /*
  270. * If a cursor is provided, we know we're processing the AIL
  271. * in lsn order, and future items to be spliced in will
  272. * follow the last one being inserted now. Update the
  273. * cursor to point to that last item, now while we have a
  274. * reliable pointer to it.
  275. */
  276. if (cur)
  277. cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);
  278. /*
  279. * Finally perform the splice. Unless the AIL was empty,
  280. * lip points to the item in the AIL _after_ which the new
  281. * items should go. If lip is null the AIL was empty, so
  282. * the new items go at the head of the AIL.
  283. */
  284. if (lip)
  285. list_splice(list, &lip->li_ail);
  286. else
  287. list_splice(list, &ailp->ail_head);
  288. }
  289. /*
  290. * Delete the given item from the AIL. Return a pointer to the item.
  291. */
  292. static void
  293. xfs_ail_delete(
  294. struct xfs_ail *ailp,
  295. xfs_log_item_t *lip)
  296. {
  297. xfs_ail_check(ailp, lip);
  298. list_del(&lip->li_ail);
  299. xfs_trans_ail_cursor_clear(ailp, lip);
  300. }
  301. static inline uint
  302. xfsaild_push_item(
  303. struct xfs_ail *ailp,
  304. struct xfs_log_item *lip)
  305. {
  306. /*
  307. * If log item pinning is enabled, skip the push and track the item as
  308. * pinned. This can help induce head-behind-tail conditions.
  309. */
  310. if (XFS_TEST_ERROR(false, ailp->ail_mount, XFS_ERRTAG_LOG_ITEM_PIN))
  311. return XFS_ITEM_PINNED;
  312. return lip->li_ops->iop_push(lip, &ailp->ail_buf_list);
  313. }
  314. static long
  315. xfsaild_push(
  316. struct xfs_ail *ailp)
  317. {
  318. xfs_mount_t *mp = ailp->ail_mount;
  319. struct xfs_ail_cursor cur;
  320. xfs_log_item_t *lip;
  321. xfs_lsn_t lsn;
  322. xfs_lsn_t target;
  323. long tout;
  324. int stuck = 0;
  325. int flushing = 0;
  326. int count = 0;
  327. /*
  328. * If we encountered pinned items or did not finish writing out all
  329. * buffers the last time we ran, force the log first and wait for it
  330. * before pushing again.
  331. */
  332. if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 &&
  333. (!list_empty_careful(&ailp->ail_buf_list) ||
  334. xfs_ail_min_lsn(ailp))) {
  335. ailp->ail_log_flush = 0;
  336. XFS_STATS_INC(mp, xs_push_ail_flush);
  337. xfs_log_force(mp, XFS_LOG_SYNC);
  338. }
  339. spin_lock(&ailp->ail_lock);
  340. /* barrier matches the ail_target update in xfs_ail_push() */
  341. smp_rmb();
  342. target = ailp->ail_target;
  343. ailp->ail_target_prev = target;
  344. lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn);
  345. if (!lip) {
  346. /*
  347. * If the AIL is empty or our push has reached the end we are
  348. * done now.
  349. */
  350. xfs_trans_ail_cursor_done(&cur);
  351. spin_unlock(&ailp->ail_lock);
  352. goto out_done;
  353. }
  354. XFS_STATS_INC(mp, xs_push_ail);
  355. lsn = lip->li_lsn;
  356. while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) {
  357. int lock_result;
  358. /*
  359. * Note that iop_push may unlock and reacquire the AIL lock. We
  360. * rely on the AIL cursor implementation to be able to deal with
  361. * the dropped lock.
  362. */
  363. lock_result = xfsaild_push_item(ailp, lip);
  364. switch (lock_result) {
  365. case XFS_ITEM_SUCCESS:
  366. XFS_STATS_INC(mp, xs_push_ail_success);
  367. trace_xfs_ail_push(lip);
  368. ailp->ail_last_pushed_lsn = lsn;
  369. break;
  370. case XFS_ITEM_FLUSHING:
  371. /*
  372. * The item or its backing buffer is already beeing
  373. * flushed. The typical reason for that is that an
  374. * inode buffer is locked because we already pushed the
  375. * updates to it as part of inode clustering.
  376. *
  377. * We do not want to to stop flushing just because lots
  378. * of items are already beeing flushed, but we need to
  379. * re-try the flushing relatively soon if most of the
  380. * AIL is beeing flushed.
  381. */
  382. XFS_STATS_INC(mp, xs_push_ail_flushing);
  383. trace_xfs_ail_flushing(lip);
  384. flushing++;
  385. ailp->ail_last_pushed_lsn = lsn;
  386. break;
  387. case XFS_ITEM_PINNED:
  388. XFS_STATS_INC(mp, xs_push_ail_pinned);
  389. trace_xfs_ail_pinned(lip);
  390. stuck++;
  391. ailp->ail_log_flush++;
  392. break;
  393. case XFS_ITEM_LOCKED:
  394. XFS_STATS_INC(mp, xs_push_ail_locked);
  395. trace_xfs_ail_locked(lip);
  396. stuck++;
  397. break;
  398. default:
  399. ASSERT(0);
  400. break;
  401. }
  402. count++;
  403. /*
  404. * Are there too many items we can't do anything with?
  405. *
  406. * If we we are skipping too many items because we can't flush
  407. * them or they are already being flushed, we back off and
  408. * given them time to complete whatever operation is being
  409. * done. i.e. remove pressure from the AIL while we can't make
  410. * progress so traversals don't slow down further inserts and
  411. * removals to/from the AIL.
  412. *
  413. * The value of 100 is an arbitrary magic number based on
  414. * observation.
  415. */
  416. if (stuck > 100)
  417. break;
  418. lip = xfs_trans_ail_cursor_next(ailp, &cur);
  419. if (lip == NULL)
  420. break;
  421. lsn = lip->li_lsn;
  422. }
  423. xfs_trans_ail_cursor_done(&cur);
  424. spin_unlock(&ailp->ail_lock);
  425. if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list))
  426. ailp->ail_log_flush++;
  427. if (!count || XFS_LSN_CMP(lsn, target) >= 0) {
  428. out_done:
  429. /*
  430. * We reached the target or the AIL is empty, so wait a bit
  431. * longer for I/O to complete and remove pushed items from the
  432. * AIL before we start the next scan from the start of the AIL.
  433. */
  434. tout = 50;
  435. ailp->ail_last_pushed_lsn = 0;
  436. } else if (((stuck + flushing) * 100) / count > 90) {
  437. /*
  438. * Either there is a lot of contention on the AIL or we are
  439. * stuck due to operations in progress. "Stuck" in this case
  440. * is defined as >90% of the items we tried to push were stuck.
  441. *
  442. * Backoff a bit more to allow some I/O to complete before
  443. * restarting from the start of the AIL. This prevents us from
  444. * spinning on the same items, and if they are pinned will all
  445. * the restart to issue a log force to unpin the stuck items.
  446. */
  447. tout = 20;
  448. ailp->ail_last_pushed_lsn = 0;
  449. } else {
  450. /*
  451. * Assume we have more work to do in a short while.
  452. */
  453. tout = 10;
  454. }
  455. return tout;
  456. }
  457. static int
  458. xfsaild(
  459. void *data)
  460. {
  461. struct xfs_ail *ailp = data;
  462. long tout = 0; /* milliseconds */
  463. unsigned int noreclaim_flag;
  464. noreclaim_flag = memalloc_noreclaim_save();
  465. set_freezable();
  466. while (1) {
  467. if (tout && tout <= 20)
  468. set_current_state(TASK_KILLABLE);
  469. else
  470. set_current_state(TASK_INTERRUPTIBLE);
  471. /*
  472. * Check kthread_should_stop() after we set the task state to
  473. * guarantee that we either see the stop bit and exit or the
  474. * task state is reset to runnable such that it's not scheduled
  475. * out indefinitely and detects the stop bit at next iteration.
  476. * A memory barrier is included in above task state set to
  477. * serialize again kthread_stop().
  478. */
  479. if (kthread_should_stop()) {
  480. __set_current_state(TASK_RUNNING);
  481. /*
  482. * The caller forces out the AIL before stopping the
  483. * thread in the common case, which means the delwri
  484. * queue is drained. In the shutdown case, the queue may
  485. * still hold relogged buffers that haven't been
  486. * submitted because they were pinned since added to the
  487. * queue.
  488. *
  489. * Log I/O error processing stales the underlying buffer
  490. * and clears the delwri state, expecting the buf to be
  491. * removed on the next submission attempt. That won't
  492. * happen if we're shutting down, so this is the last
  493. * opportunity to release such buffers from the queue.
  494. */
  495. ASSERT(list_empty(&ailp->ail_buf_list) ||
  496. XFS_FORCED_SHUTDOWN(ailp->ail_mount));
  497. xfs_buf_delwri_cancel(&ailp->ail_buf_list);
  498. break;
  499. }
  500. spin_lock(&ailp->ail_lock);
  501. /*
  502. * Idle if the AIL is empty and we are not racing with a target
  503. * update. We check the AIL after we set the task to a sleep
  504. * state to guarantee that we either catch an ail_target update
  505. * or that a wake_up resets the state to TASK_RUNNING.
  506. * Otherwise, we run the risk of sleeping indefinitely.
  507. *
  508. * The barrier matches the ail_target update in xfs_ail_push().
  509. */
  510. smp_rmb();
  511. if (!xfs_ail_min(ailp) &&
  512. ailp->ail_target == ailp->ail_target_prev) {
  513. spin_unlock(&ailp->ail_lock);
  514. freezable_schedule();
  515. tout = 0;
  516. continue;
  517. }
  518. spin_unlock(&ailp->ail_lock);
  519. if (tout)
  520. freezable_schedule_timeout(msecs_to_jiffies(tout));
  521. __set_current_state(TASK_RUNNING);
  522. try_to_freeze();
  523. tout = xfsaild_push(ailp);
  524. }
  525. memalloc_noreclaim_restore(noreclaim_flag);
  526. return 0;
  527. }
  528. /*
  529. * This routine is called to move the tail of the AIL forward. It does this by
  530. * trying to flush items in the AIL whose lsns are below the given
  531. * threshold_lsn.
  532. *
  533. * The push is run asynchronously in a workqueue, which means the caller needs
  534. * to handle waiting on the async flush for space to become available.
  535. * We don't want to interrupt any push that is in progress, hence we only queue
  536. * work if we set the pushing bit approriately.
  537. *
  538. * We do this unlocked - we only need to know whether there is anything in the
  539. * AIL at the time we are called. We don't need to access the contents of
  540. * any of the objects, so the lock is not needed.
  541. */
  542. void
  543. xfs_ail_push(
  544. struct xfs_ail *ailp,
  545. xfs_lsn_t threshold_lsn)
  546. {
  547. xfs_log_item_t *lip;
  548. lip = xfs_ail_min(ailp);
  549. if (!lip || XFS_FORCED_SHUTDOWN(ailp->ail_mount) ||
  550. XFS_LSN_CMP(threshold_lsn, ailp->ail_target) <= 0)
  551. return;
  552. /*
  553. * Ensure that the new target is noticed in push code before it clears
  554. * the XFS_AIL_PUSHING_BIT.
  555. */
  556. smp_wmb();
  557. xfs_trans_ail_copy_lsn(ailp, &ailp->ail_target, &threshold_lsn);
  558. smp_wmb();
  559. wake_up_process(ailp->ail_task);
  560. }
  561. /*
  562. * Push out all items in the AIL immediately
  563. */
  564. void
  565. xfs_ail_push_all(
  566. struct xfs_ail *ailp)
  567. {
  568. xfs_lsn_t threshold_lsn = xfs_ail_max_lsn(ailp);
  569. if (threshold_lsn)
  570. xfs_ail_push(ailp, threshold_lsn);
  571. }
  572. /*
  573. * Push out all items in the AIL immediately and wait until the AIL is empty.
  574. */
  575. void
  576. xfs_ail_push_all_sync(
  577. struct xfs_ail *ailp)
  578. {
  579. struct xfs_log_item *lip;
  580. DEFINE_WAIT(wait);
  581. spin_lock(&ailp->ail_lock);
  582. while ((lip = xfs_ail_max(ailp)) != NULL) {
  583. prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE);
  584. ailp->ail_target = lip->li_lsn;
  585. wake_up_process(ailp->ail_task);
  586. spin_unlock(&ailp->ail_lock);
  587. schedule();
  588. spin_lock(&ailp->ail_lock);
  589. }
  590. spin_unlock(&ailp->ail_lock);
  591. finish_wait(&ailp->ail_empty, &wait);
  592. }
  593. /*
  594. * xfs_trans_ail_update - bulk AIL insertion operation.
  595. *
  596. * @xfs_trans_ail_update takes an array of log items that all need to be
  597. * positioned at the same LSN in the AIL. If an item is not in the AIL, it will
  598. * be added. Otherwise, it will be repositioned by removing it and re-adding
  599. * it to the AIL. If we move the first item in the AIL, update the log tail to
  600. * match the new minimum LSN in the AIL.
  601. *
  602. * This function takes the AIL lock once to execute the update operations on
  603. * all the items in the array, and as such should not be called with the AIL
  604. * lock held. As a result, once we have the AIL lock, we need to check each log
  605. * item LSN to confirm it needs to be moved forward in the AIL.
  606. *
  607. * To optimise the insert operation, we delete all the items from the AIL in
  608. * the first pass, moving them into a temporary list, then splice the temporary
  609. * list into the correct position in the AIL. This avoids needing to do an
  610. * insert operation on every item.
  611. *
  612. * This function must be called with the AIL lock held. The lock is dropped
  613. * before returning.
  614. */
  615. void
  616. xfs_trans_ail_update_bulk(
  617. struct xfs_ail *ailp,
  618. struct xfs_ail_cursor *cur,
  619. struct xfs_log_item **log_items,
  620. int nr_items,
  621. xfs_lsn_t lsn) __releases(ailp->ail_lock)
  622. {
  623. xfs_log_item_t *mlip;
  624. int mlip_changed = 0;
  625. int i;
  626. LIST_HEAD(tmp);
  627. ASSERT(nr_items > 0); /* Not required, but true. */
  628. mlip = xfs_ail_min(ailp);
  629. for (i = 0; i < nr_items; i++) {
  630. struct xfs_log_item *lip = log_items[i];
  631. if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) {
  632. /* check if we really need to move the item */
  633. if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)
  634. continue;
  635. trace_xfs_ail_move(lip, lip->li_lsn, lsn);
  636. xfs_ail_delete(ailp, lip);
  637. if (mlip == lip)
  638. mlip_changed = 1;
  639. } else {
  640. trace_xfs_ail_insert(lip, 0, lsn);
  641. }
  642. lip->li_lsn = lsn;
  643. list_add(&lip->li_ail, &tmp);
  644. }
  645. if (!list_empty(&tmp))
  646. xfs_ail_splice(ailp, cur, &tmp, lsn);
  647. if (mlip_changed) {
  648. if (!XFS_FORCED_SHUTDOWN(ailp->ail_mount))
  649. xlog_assign_tail_lsn_locked(ailp->ail_mount);
  650. spin_unlock(&ailp->ail_lock);
  651. xfs_log_space_wake(ailp->ail_mount);
  652. } else {
  653. spin_unlock(&ailp->ail_lock);
  654. }
  655. }
  656. bool
  657. xfs_ail_delete_one(
  658. struct xfs_ail *ailp,
  659. struct xfs_log_item *lip)
  660. {
  661. struct xfs_log_item *mlip = xfs_ail_min(ailp);
  662. trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
  663. xfs_ail_delete(ailp, lip);
  664. xfs_clear_li_failed(lip);
  665. clear_bit(XFS_LI_IN_AIL, &lip->li_flags);
  666. lip->li_lsn = 0;
  667. return mlip == lip;
  668. }
  669. /**
  670. * Remove a log items from the AIL
  671. *
  672. * @xfs_trans_ail_delete_bulk takes an array of log items that all need to
  673. * removed from the AIL. The caller is already holding the AIL lock, and done
  674. * all the checks necessary to ensure the items passed in via @log_items are
  675. * ready for deletion. This includes checking that the items are in the AIL.
  676. *
  677. * For each log item to be removed, unlink it from the AIL, clear the IN_AIL
  678. * flag from the item and reset the item's lsn to 0. If we remove the first
  679. * item in the AIL, update the log tail to match the new minimum LSN in the
  680. * AIL.
  681. *
  682. * This function will not drop the AIL lock until all items are removed from
  683. * the AIL to minimise the amount of lock traffic on the AIL. This does not
  684. * greatly increase the AIL hold time, but does significantly reduce the amount
  685. * of traffic on the lock, especially during IO completion.
  686. *
  687. * This function must be called with the AIL lock held. The lock is dropped
  688. * before returning.
  689. */
  690. void
  691. xfs_trans_ail_delete(
  692. struct xfs_ail *ailp,
  693. struct xfs_log_item *lip,
  694. int shutdown_type) __releases(ailp->ail_lock)
  695. {
  696. struct xfs_mount *mp = ailp->ail_mount;
  697. bool mlip_changed;
  698. if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) {
  699. spin_unlock(&ailp->ail_lock);
  700. if (!XFS_FORCED_SHUTDOWN(mp)) {
  701. xfs_alert_tag(mp, XFS_PTAG_AILDELETE,
  702. "%s: attempting to delete a log item that is not in the AIL",
  703. __func__);
  704. xfs_force_shutdown(mp, shutdown_type);
  705. }
  706. return;
  707. }
  708. mlip_changed = xfs_ail_delete_one(ailp, lip);
  709. if (mlip_changed) {
  710. if (!XFS_FORCED_SHUTDOWN(mp))
  711. xlog_assign_tail_lsn_locked(mp);
  712. if (list_empty(&ailp->ail_head))
  713. wake_up_all(&ailp->ail_empty);
  714. }
  715. spin_unlock(&ailp->ail_lock);
  716. if (mlip_changed)
  717. xfs_log_space_wake(ailp->ail_mount);
  718. }
  719. int
  720. xfs_trans_ail_init(
  721. xfs_mount_t *mp)
  722. {
  723. struct xfs_ail *ailp;
  724. ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL);
  725. if (!ailp)
  726. return -ENOMEM;
  727. ailp->ail_mount = mp;
  728. INIT_LIST_HEAD(&ailp->ail_head);
  729. INIT_LIST_HEAD(&ailp->ail_cursors);
  730. spin_lock_init(&ailp->ail_lock);
  731. INIT_LIST_HEAD(&ailp->ail_buf_list);
  732. init_waitqueue_head(&ailp->ail_empty);
  733. ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s",
  734. ailp->ail_mount->m_fsname);
  735. if (IS_ERR(ailp->ail_task))
  736. goto out_free_ailp;
  737. mp->m_ail = ailp;
  738. return 0;
  739. out_free_ailp:
  740. kmem_free(ailp);
  741. return -ENOMEM;
  742. }
  743. void
  744. xfs_trans_ail_destroy(
  745. xfs_mount_t *mp)
  746. {
  747. struct xfs_ail *ailp = mp->m_ail;
  748. kthread_stop(ailp->ail_task);
  749. kmem_free(ailp);
  750. }