dma-fence.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626
  1. /*
  2. * Fence mechanism for dma-buf and to allow for asynchronous dma access
  3. *
  4. * Copyright (C) 2012 Canonical Ltd
  5. * Copyright (C) 2012 Texas Instruments
  6. *
  7. * Authors:
  8. * Rob Clark <robdclark@gmail.com>
  9. * Maarten Lankhorst <maarten.lankhorst@canonical.com>
  10. *
  11. * This program is free software; you can redistribute it and/or modify it
  12. * under the terms of the GNU General Public License version 2 as published by
  13. * the Free Software Foundation.
  14. *
  15. * This program is distributed in the hope that it will be useful, but WITHOUT
  16. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  18. * more details.
  19. */
  20. #include <linux/slab.h>
  21. #include <linux/export.h>
  22. #include <linux/atomic.h>
  23. #include <linux/dma-fence.h>
  24. #include <linux/sched/signal.h>
  25. #define CREATE_TRACE_POINTS
  26. #include <trace/events/dma_fence.h>
  27. EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
  28. EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal);
  29. /*
  30. * fence context counter: each execution context should have its own
  31. * fence context, this allows checking if fences belong to the same
  32. * context or not. One device can have multiple separate contexts,
  33. * and they're used if some engine can run independently of another.
  34. */
  35. static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(0);
  36. /**
  37. * DOC: DMA fences overview
  38. *
  39. * DMA fences, represented by &struct dma_fence, are the kernel internal
  40. * synchronization primitive for DMA operations like GPU rendering, video
  41. * encoding/decoding, or displaying buffers on a screen.
  42. *
  43. * A fence is initialized using dma_fence_init() and completed using
  44. * dma_fence_signal(). Fences are associated with a context, allocated through
  45. * dma_fence_context_alloc(), and all fences on the same context are
  46. * fully ordered.
  47. *
  48. * Since the purposes of fences is to facilitate cross-device and
  49. * cross-application synchronization, there's multiple ways to use one:
  50. *
  51. * - Individual fences can be exposed as a &sync_file, accessed as a file
  52. * descriptor from userspace, created by calling sync_file_create(). This is
  53. * called explicit fencing, since userspace passes around explicit
  54. * synchronization points.
  55. *
  56. * - Some subsystems also have their own explicit fencing primitives, like
  57. * &drm_syncobj. Compared to &sync_file, a &drm_syncobj allows the underlying
  58. * fence to be updated.
  59. *
  60. * - Then there's also implicit fencing, where the synchronization points are
  61. * implicitly passed around as part of shared &dma_buf instances. Such
  62. * implicit fences are stored in &struct reservation_object through the
  63. * &dma_buf.resv pointer.
  64. */
  65. /**
  66. * dma_fence_context_alloc - allocate an array of fence contexts
  67. * @num: amount of contexts to allocate
  68. *
  69. * This function will return the first index of the number of fence contexts
  70. * allocated. The fence context is used for setting &dma_fence.context to a
  71. * unique number by passing the context to dma_fence_init().
  72. */
  73. u64 dma_fence_context_alloc(unsigned num)
  74. {
  75. WARN_ON(!num);
  76. return atomic64_add_return(num, &dma_fence_context_counter) - num;
  77. }
  78. EXPORT_SYMBOL(dma_fence_context_alloc);
  79. /**
  80. * dma_fence_signal_locked - signal completion of a fence
  81. * @fence: the fence to signal
  82. *
  83. * Signal completion for software callbacks on a fence, this will unblock
  84. * dma_fence_wait() calls and run all the callbacks added with
  85. * dma_fence_add_callback(). Can be called multiple times, but since a fence
  86. * can only go from the unsignaled to the signaled state and not back, it will
  87. * only be effective the first time.
  88. *
  89. * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock
  90. * held.
  91. *
  92. * Returns 0 on success and a negative error value when @fence has been
  93. * signalled already.
  94. */
  95. int dma_fence_signal_locked(struct dma_fence *fence)
  96. {
  97. struct dma_fence_cb *cur, *tmp;
  98. int ret = 0;
  99. lockdep_assert_held(fence->lock);
  100. if (WARN_ON(!fence))
  101. return -EINVAL;
  102. if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
  103. ret = -EINVAL;
  104. /*
  105. * we might have raced with the unlocked dma_fence_signal,
  106. * still run through all callbacks
  107. */
  108. } else {
  109. fence->timestamp = ktime_get();
  110. set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
  111. trace_dma_fence_signaled(fence);
  112. }
  113. list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
  114. list_del_init(&cur->node);
  115. cur->func(fence, cur);
  116. }
  117. return ret;
  118. }
  119. EXPORT_SYMBOL(dma_fence_signal_locked);
  120. /**
  121. * dma_fence_signal - signal completion of a fence
  122. * @fence: the fence to signal
  123. *
  124. * Signal completion for software callbacks on a fence, this will unblock
  125. * dma_fence_wait() calls and run all the callbacks added with
  126. * dma_fence_add_callback(). Can be called multiple times, but since a fence
  127. * can only go from the unsignaled to the signaled state and not back, it will
  128. * only be effective the first time.
  129. *
  130. * Returns 0 on success and a negative error value when @fence has been
  131. * signalled already.
  132. */
  133. int dma_fence_signal(struct dma_fence *fence)
  134. {
  135. unsigned long flags;
  136. if (!fence)
  137. return -EINVAL;
  138. if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  139. return -EINVAL;
  140. fence->timestamp = ktime_get();
  141. set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
  142. trace_dma_fence_signaled(fence);
  143. if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {
  144. struct dma_fence_cb *cur, *tmp;
  145. spin_lock_irqsave(fence->lock, flags);
  146. list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
  147. list_del_init(&cur->node);
  148. cur->func(fence, cur);
  149. }
  150. spin_unlock_irqrestore(fence->lock, flags);
  151. }
  152. return 0;
  153. }
  154. EXPORT_SYMBOL(dma_fence_signal);
  155. /**
  156. * dma_fence_wait_timeout - sleep until the fence gets signaled
  157. * or until timeout elapses
  158. * @fence: the fence to wait on
  159. * @intr: if true, do an interruptible wait
  160. * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
  161. *
  162. * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
  163. * remaining timeout in jiffies on success. Other error values may be
  164. * returned on custom implementations.
  165. *
  166. * Performs a synchronous wait on this fence. It is assumed the caller
  167. * directly or indirectly (buf-mgr between reservation and committing)
  168. * holds a reference to the fence, otherwise the fence might be
  169. * freed before return, resulting in undefined behavior.
  170. *
  171. * See also dma_fence_wait() and dma_fence_wait_any_timeout().
  172. */
  173. signed long
  174. dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
  175. {
  176. signed long ret;
  177. if (WARN_ON(timeout < 0))
  178. return -EINVAL;
  179. trace_dma_fence_wait_start(fence);
  180. if (fence->ops->wait)
  181. ret = fence->ops->wait(fence, intr, timeout);
  182. else
  183. ret = dma_fence_default_wait(fence, intr, timeout);
  184. trace_dma_fence_wait_end(fence);
  185. return ret;
  186. }
  187. EXPORT_SYMBOL(dma_fence_wait_timeout);
  188. /**
  189. * dma_fence_release - default relese function for fences
  190. * @kref: &dma_fence.recfount
  191. *
  192. * This is the default release functions for &dma_fence. Drivers shouldn't call
  193. * this directly, but instead call dma_fence_put().
  194. */
  195. void dma_fence_release(struct kref *kref)
  196. {
  197. struct dma_fence *fence =
  198. container_of(kref, struct dma_fence, refcount);
  199. trace_dma_fence_destroy(fence);
  200. /* Failed to signal before release, could be a refcounting issue */
  201. WARN_ON(!list_empty(&fence->cb_list));
  202. if (fence->ops->release)
  203. fence->ops->release(fence);
  204. else
  205. dma_fence_free(fence);
  206. }
  207. EXPORT_SYMBOL(dma_fence_release);
  208. /**
  209. * dma_fence_free - default release function for &dma_fence.
  210. * @fence: fence to release
  211. *
  212. * This is the default implementation for &dma_fence_ops.release. It calls
  213. * kfree_rcu() on @fence.
  214. */
  215. void dma_fence_free(struct dma_fence *fence)
  216. {
  217. kfree_rcu(fence, rcu);
  218. }
  219. EXPORT_SYMBOL(dma_fence_free);
  220. static bool __dma_fence_enable_signaling(struct dma_fence *fence)
  221. {
  222. bool was_set;
  223. lockdep_assert_held(fence->lock);
  224. was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
  225. &fence->flags);
  226. if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  227. return false;
  228. if (!was_set && fence->ops->enable_signaling) {
  229. trace_dma_fence_enable_signal(fence);
  230. if (!fence->ops->enable_signaling(fence)) {
  231. dma_fence_signal_locked(fence);
  232. return false;
  233. }
  234. }
  235. return true;
  236. }
  237. /**
  238. * dma_fence_enable_sw_signaling - enable signaling on fence
  239. * @fence: the fence to enable
  240. *
  241. * This will request for sw signaling to be enabled, to make the fence
  242. * complete as soon as possible. This calls &dma_fence_ops.enable_signaling
  243. * internally.
  244. */
  245. void dma_fence_enable_sw_signaling(struct dma_fence *fence)
  246. {
  247. unsigned long flags;
  248. if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  249. return;
  250. spin_lock_irqsave(fence->lock, flags);
  251. __dma_fence_enable_signaling(fence);
  252. spin_unlock_irqrestore(fence->lock, flags);
  253. }
  254. EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
  255. /**
  256. * dma_fence_add_callback - add a callback to be called when the fence
  257. * is signaled
  258. * @fence: the fence to wait on
  259. * @cb: the callback to register
  260. * @func: the function to call
  261. *
  262. * @cb will be initialized by dma_fence_add_callback(), no initialization
  263. * by the caller is required. Any number of callbacks can be registered
  264. * to a fence, but a callback can only be registered to one fence at a time.
  265. *
  266. * Note that the callback can be called from an atomic context. If
  267. * fence is already signaled, this function will return -ENOENT (and
  268. * *not* call the callback).
  269. *
  270. * Add a software callback to the fence. Same restrictions apply to
  271. * refcount as it does to dma_fence_wait(), however the caller doesn't need to
  272. * keep a refcount to fence afterward dma_fence_add_callback() has returned:
  273. * when software access is enabled, the creator of the fence is required to keep
  274. * the fence alive until after it signals with dma_fence_signal(). The callback
  275. * itself can be called from irq context.
  276. *
  277. * Returns 0 in case of success, -ENOENT if the fence is already signaled
  278. * and -EINVAL in case of error.
  279. */
  280. int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
  281. dma_fence_func_t func)
  282. {
  283. unsigned long flags;
  284. int ret = 0;
  285. if (WARN_ON(!fence || !func))
  286. return -EINVAL;
  287. if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
  288. INIT_LIST_HEAD(&cb->node);
  289. return -ENOENT;
  290. }
  291. spin_lock_irqsave(fence->lock, flags);
  292. if (__dma_fence_enable_signaling(fence)) {
  293. cb->func = func;
  294. list_add_tail(&cb->node, &fence->cb_list);
  295. } else {
  296. INIT_LIST_HEAD(&cb->node);
  297. ret = -ENOENT;
  298. }
  299. spin_unlock_irqrestore(fence->lock, flags);
  300. return ret;
  301. }
  302. EXPORT_SYMBOL(dma_fence_add_callback);
  303. /**
  304. * dma_fence_get_status - returns the status upon completion
  305. * @fence: the dma_fence to query
  306. *
  307. * This wraps dma_fence_get_status_locked() to return the error status
  308. * condition on a signaled fence. See dma_fence_get_status_locked() for more
  309. * details.
  310. *
  311. * Returns 0 if the fence has not yet been signaled, 1 if the fence has
  312. * been signaled without an error condition, or a negative error code
  313. * if the fence has been completed in err.
  314. */
  315. int dma_fence_get_status(struct dma_fence *fence)
  316. {
  317. unsigned long flags;
  318. int status;
  319. spin_lock_irqsave(fence->lock, flags);
  320. status = dma_fence_get_status_locked(fence);
  321. spin_unlock_irqrestore(fence->lock, flags);
  322. return status;
  323. }
  324. EXPORT_SYMBOL(dma_fence_get_status);
  325. /**
  326. * dma_fence_remove_callback - remove a callback from the signaling list
  327. * @fence: the fence to wait on
  328. * @cb: the callback to remove
  329. *
  330. * Remove a previously queued callback from the fence. This function returns
  331. * true if the callback is successfully removed, or false if the fence has
  332. * already been signaled.
  333. *
  334. * *WARNING*:
  335. * Cancelling a callback should only be done if you really know what you're
  336. * doing, since deadlocks and race conditions could occur all too easily. For
  337. * this reason, it should only ever be done on hardware lockup recovery,
  338. * with a reference held to the fence.
  339. *
  340. * Behaviour is undefined if @cb has not been added to @fence using
  341. * dma_fence_add_callback() beforehand.
  342. */
  343. bool
  344. dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
  345. {
  346. unsigned long flags;
  347. bool ret;
  348. spin_lock_irqsave(fence->lock, flags);
  349. ret = !list_empty(&cb->node);
  350. if (ret)
  351. list_del_init(&cb->node);
  352. spin_unlock_irqrestore(fence->lock, flags);
  353. return ret;
  354. }
  355. EXPORT_SYMBOL(dma_fence_remove_callback);
  356. struct default_wait_cb {
  357. struct dma_fence_cb base;
  358. struct task_struct *task;
  359. };
  360. static void
  361. dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
  362. {
  363. struct default_wait_cb *wait =
  364. container_of(cb, struct default_wait_cb, base);
  365. wake_up_state(wait->task, TASK_NORMAL);
  366. }
  367. /**
  368. * dma_fence_default_wait - default sleep until the fence gets signaled
  369. * or until timeout elapses
  370. * @fence: the fence to wait on
  371. * @intr: if true, do an interruptible wait
  372. * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
  373. *
  374. * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
  375. * remaining timeout in jiffies on success. If timeout is zero the value one is
  376. * returned if the fence is already signaled for consistency with other
  377. * functions taking a jiffies timeout.
  378. */
  379. signed long
  380. dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
  381. {
  382. struct default_wait_cb cb;
  383. unsigned long flags;
  384. signed long ret = timeout ? timeout : 1;
  385. if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  386. return ret;
  387. spin_lock_irqsave(fence->lock, flags);
  388. if (intr && signal_pending(current)) {
  389. ret = -ERESTARTSYS;
  390. goto out;
  391. }
  392. if (!__dma_fence_enable_signaling(fence))
  393. goto out;
  394. if (!timeout) {
  395. ret = 0;
  396. goto out;
  397. }
  398. cb.base.func = dma_fence_default_wait_cb;
  399. cb.task = current;
  400. list_add(&cb.base.node, &fence->cb_list);
  401. while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
  402. if (intr)
  403. __set_current_state(TASK_INTERRUPTIBLE);
  404. else
  405. __set_current_state(TASK_UNINTERRUPTIBLE);
  406. spin_unlock_irqrestore(fence->lock, flags);
  407. ret = schedule_timeout(ret);
  408. spin_lock_irqsave(fence->lock, flags);
  409. if (ret > 0 && intr && signal_pending(current))
  410. ret = -ERESTARTSYS;
  411. }
  412. if (!list_empty(&cb.base.node))
  413. list_del(&cb.base.node);
  414. __set_current_state(TASK_RUNNING);
  415. out:
  416. spin_unlock_irqrestore(fence->lock, flags);
  417. return ret;
  418. }
  419. EXPORT_SYMBOL(dma_fence_default_wait);
  420. static bool
  421. dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
  422. uint32_t *idx)
  423. {
  424. int i;
  425. for (i = 0; i < count; ++i) {
  426. struct dma_fence *fence = fences[i];
  427. if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
  428. if (idx)
  429. *idx = i;
  430. return true;
  431. }
  432. }
  433. return false;
  434. }
  435. /**
  436. * dma_fence_wait_any_timeout - sleep until any fence gets signaled
  437. * or until timeout elapses
  438. * @fences: array of fences to wait on
  439. * @count: number of fences to wait on
  440. * @intr: if true, do an interruptible wait
  441. * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
  442. * @idx: used to store the first signaled fence index, meaningful only on
  443. * positive return
  444. *
  445. * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
  446. * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
  447. * on success.
  448. *
  449. * Synchronous waits for the first fence in the array to be signaled. The
  450. * caller needs to hold a reference to all fences in the array, otherwise a
  451. * fence might be freed before return, resulting in undefined behavior.
  452. *
  453. * See also dma_fence_wait() and dma_fence_wait_timeout().
  454. */
  455. signed long
  456. dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
  457. bool intr, signed long timeout, uint32_t *idx)
  458. {
  459. struct default_wait_cb *cb;
  460. signed long ret = timeout;
  461. unsigned i;
  462. if (WARN_ON(!fences || !count || timeout < 0))
  463. return -EINVAL;
  464. if (timeout == 0) {
  465. for (i = 0; i < count; ++i)
  466. if (dma_fence_is_signaled(fences[i])) {
  467. if (idx)
  468. *idx = i;
  469. return 1;
  470. }
  471. return 0;
  472. }
  473. cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
  474. if (cb == NULL) {
  475. ret = -ENOMEM;
  476. goto err_free_cb;
  477. }
  478. for (i = 0; i < count; ++i) {
  479. struct dma_fence *fence = fences[i];
  480. cb[i].task = current;
  481. if (dma_fence_add_callback(fence, &cb[i].base,
  482. dma_fence_default_wait_cb)) {
  483. /* This fence is already signaled */
  484. if (idx)
  485. *idx = i;
  486. goto fence_rm_cb;
  487. }
  488. }
  489. while (ret > 0) {
  490. if (intr)
  491. set_current_state(TASK_INTERRUPTIBLE);
  492. else
  493. set_current_state(TASK_UNINTERRUPTIBLE);
  494. if (dma_fence_test_signaled_any(fences, count, idx))
  495. break;
  496. ret = schedule_timeout(ret);
  497. if (ret > 0 && intr && signal_pending(current))
  498. ret = -ERESTARTSYS;
  499. }
  500. __set_current_state(TASK_RUNNING);
  501. fence_rm_cb:
  502. while (i-- > 0)
  503. dma_fence_remove_callback(fences[i], &cb[i].base);
  504. err_free_cb:
  505. kfree(cb);
  506. return ret;
  507. }
  508. EXPORT_SYMBOL(dma_fence_wait_any_timeout);
  509. /**
  510. * dma_fence_init - Initialize a custom fence.
  511. * @fence: the fence to initialize
  512. * @ops: the dma_fence_ops for operations on this fence
  513. * @lock: the irqsafe spinlock to use for locking this fence
  514. * @context: the execution context this fence is run on
  515. * @seqno: a linear increasing sequence number for this context
  516. *
  517. * Initializes an allocated fence, the caller doesn't have to keep its
  518. * refcount after committing with this fence, but it will need to hold a
  519. * refcount again if &dma_fence_ops.enable_signaling gets called.
  520. *
  521. * context and seqno are used for easy comparison between fences, allowing
  522. * to check which fence is later by simply using dma_fence_later().
  523. */
  524. void
  525. dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
  526. spinlock_t *lock, u64 context, unsigned seqno)
  527. {
  528. BUG_ON(!lock);
  529. BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
  530. kref_init(&fence->refcount);
  531. fence->ops = ops;
  532. INIT_LIST_HEAD(&fence->cb_list);
  533. fence->lock = lock;
  534. fence->context = context;
  535. fence->seqno = seqno;
  536. fence->flags = 0UL;
  537. fence->error = 0;
  538. trace_dma_fence_init(fence);
  539. }
  540. EXPORT_SYMBOL(dma_fence_init);