submit.c 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
  3. #include <linux/init.h>
  4. #include <linux/kernel.h>
  5. #include <linux/module.h>
  6. #include <linux/pci.h>
  7. #include <uapi/linux/idxd.h>
  8. #include "idxd.h"
  9. #include "registers.h"
  10. static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu)
  11. {
  12. struct idxd_desc *desc;
  13. struct idxd_device *idxd = wq->idxd;
  14. desc = wq->descs[idx];
  15. memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
  16. memset(desc->completion, 0, idxd->data->compl_size);
  17. desc->cpu = cpu;
  18. if (device_pasid_enabled(idxd))
  19. desc->hw->pasid = idxd->pasid;
  20. return desc;
  21. }
  22. struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype)
  23. {
  24. int cpu, idx;
  25. struct idxd_device *idxd = wq->idxd;
  26. DEFINE_SBQ_WAIT(wait);
  27. struct sbq_wait_state *ws;
  28. struct sbitmap_queue *sbq;
  29. if (idxd->state != IDXD_DEV_ENABLED)
  30. return ERR_PTR(-EIO);
  31. sbq = &wq->sbq;
  32. idx = sbitmap_queue_get(sbq, &cpu);
  33. if (idx < 0) {
  34. if (optype == IDXD_OP_NONBLOCK)
  35. return ERR_PTR(-EAGAIN);
  36. } else {
  37. return __get_desc(wq, idx, cpu);
  38. }
  39. ws = &sbq->ws[0];
  40. for (;;) {
  41. sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_INTERRUPTIBLE);
  42. if (signal_pending_state(TASK_INTERRUPTIBLE, current))
  43. break;
  44. idx = sbitmap_queue_get(sbq, &cpu);
  45. if (idx >= 0)
  46. break;
  47. schedule();
  48. }
  49. sbitmap_finish_wait(sbq, ws, &wait);
  50. if (idx < 0)
  51. return ERR_PTR(-EAGAIN);
  52. return __get_desc(wq, idx, cpu);
  53. }
  54. EXPORT_SYMBOL_NS_GPL(idxd_alloc_desc, IDXD);
  55. void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc)
  56. {
  57. int cpu = desc->cpu;
  58. desc->cpu = -1;
  59. sbitmap_queue_clear(&wq->sbq, desc->id, cpu);
  60. }
  61. EXPORT_SYMBOL_NS_GPL(idxd_free_desc, IDXD);
  62. static struct idxd_desc *list_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,
  63. struct idxd_desc *desc)
  64. {
  65. struct idxd_desc *d, *n;
  66. lockdep_assert_held(&ie->list_lock);
  67. list_for_each_entry_safe(d, n, &ie->work_list, list) {
  68. if (d == desc) {
  69. list_del(&d->list);
  70. return d;
  71. }
  72. }
  73. /*
  74. * At this point, the desc needs to be aborted is held by the completion
  75. * handler where it has taken it off the pending list but has not added to the
  76. * work list. It will be cleaned up by the interrupt handler when it sees the
  77. * IDXD_COMP_DESC_ABORT for completion status.
  78. */
  79. return NULL;
  80. }
  81. static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,
  82. struct idxd_desc *desc)
  83. {
  84. struct idxd_desc *d, *t, *found = NULL;
  85. struct llist_node *head;
  86. LIST_HEAD(flist);
  87. desc->completion->status = IDXD_COMP_DESC_ABORT;
  88. /*
  89. * Grab the list lock so it will block the irq thread handler. This allows the
  90. * abort code to locate the descriptor need to be aborted.
  91. */
  92. spin_lock(&ie->list_lock);
  93. head = llist_del_all(&ie->pending_llist);
  94. if (head) {
  95. llist_for_each_entry_safe(d, t, head, llnode) {
  96. if (d == desc) {
  97. found = desc;
  98. continue;
  99. }
  100. if (d->completion->status)
  101. list_add_tail(&d->list, &flist);
  102. else
  103. list_add_tail(&d->list, &ie->work_list);
  104. }
  105. }
  106. if (!found)
  107. found = list_abort_desc(wq, ie, desc);
  108. spin_unlock(&ie->list_lock);
  109. if (found)
  110. idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, false,
  111. NULL, NULL);
  112. /*
  113. * completing the descriptor will return desc to allocator and
  114. * the desc can be acquired by a different process and the
  115. * desc->list can be modified. Delete desc from list so the
  116. * list traversing does not get corrupted by the other process.
  117. */
  118. list_for_each_entry_safe(d, t, &flist, list) {
  119. list_del_init(&d->list);
  120. idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true,
  121. NULL, NULL);
  122. }
  123. }
  124. /*
  125. * ENQCMDS typically fail when the WQ is inactive or busy. On host submission, the driver
  126. * has better control of number of descriptors being submitted to a shared wq by limiting
  127. * the number of driver allocated descriptors to the wq size. However, when the swq is
  128. * exported to a guest kernel, it may be shared with multiple guest kernels. This means
  129. * the likelihood of getting busy returned on the swq when submitting goes significantly up.
  130. * Having a tunable retry mechanism allows the driver to keep trying for a bit before giving
  131. * up. The sysfs knob can be tuned by the system administrator.
  132. */
  133. int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc)
  134. {
  135. unsigned int retries = wq->enqcmds_retries;
  136. int rc;
  137. do {
  138. rc = enqcmds(portal, desc);
  139. if (rc == 0)
  140. break;
  141. cpu_relax();
  142. } while (retries--);
  143. return rc;
  144. }
  145. int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
  146. {
  147. struct idxd_device *idxd = wq->idxd;
  148. struct idxd_irq_entry *ie = NULL;
  149. u32 desc_flags = desc->hw->flags;
  150. void __iomem *portal;
  151. int rc;
  152. if (idxd->state != IDXD_DEV_ENABLED)
  153. return -EIO;
  154. if (!percpu_ref_tryget_live(&wq->wq_active)) {
  155. wait_for_completion(&wq->wq_resurrect);
  156. if (!percpu_ref_tryget_live(&wq->wq_active))
  157. return -ENXIO;
  158. }
  159. portal = idxd_wq_portal_addr(wq);
  160. /*
  161. * Pending the descriptor to the lockless list for the irq_entry
  162. * that we designated the descriptor to.
  163. */
  164. if (desc_flags & IDXD_OP_FLAG_RCI) {
  165. ie = &wq->ie;
  166. desc->hw->int_handle = ie->int_handle;
  167. llist_add(&desc->llnode, &ie->pending_llist);
  168. }
  169. /*
  170. * The wmb() flushes writes to coherent DMA data before
  171. * possibly triggering a DMA read. The wmb() is necessary
  172. * even on UP because the recipient is a device.
  173. */
  174. wmb();
  175. if (wq_dedicated(wq)) {
  176. iosubmit_cmds512(portal, desc->hw, 1);
  177. } else {
  178. rc = idxd_enqcmds(wq, portal, desc->hw);
  179. if (rc < 0) {
  180. percpu_ref_put(&wq->wq_active);
  181. /* abort operation frees the descriptor */
  182. if (ie)
  183. llist_abort_desc(wq, ie, desc);
  184. return rc;
  185. }
  186. }
  187. percpu_ref_put(&wq->wq_active);
  188. return 0;
  189. }
  190. EXPORT_SYMBOL_NS_GPL(idxd_submit_desc, IDXD);