cdma.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547
  1. /*
  2. * Tegra host1x Command DMA
  3. *
  4. * Copyright (c) 2010-2013, NVIDIA Corporation.
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms and conditions of the GNU General Public License,
  8. * version 2, as published by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope it will be useful, but WITHOUT
  11. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  13. * more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. */
  18. #include <asm/cacheflush.h>
  19. #include <linux/device.h>
  20. #include <linux/dma-mapping.h>
  21. #include <linux/host1x.h>
  22. #include <linux/interrupt.h>
  23. #include <linux/kernel.h>
  24. #include <linux/kfifo.h>
  25. #include <linux/slab.h>
  26. #include <trace/events/host1x.h>
  27. #include "cdma.h"
  28. #include "channel.h"
  29. #include "dev.h"
  30. #include "debug.h"
  31. #include "job.h"
  32. /*
  33. * push_buffer
  34. *
  35. * The push buffer is a circular array of words to be fetched by command DMA.
  36. * Note that it works slightly differently to the sync queue; fence == pos
  37. * means that the push buffer is full, not empty.
  38. */
  39. #define HOST1X_PUSHBUFFER_SLOTS 512
  40. /*
  41. * Clean up push buffer resources
  42. */
  43. static void host1x_pushbuffer_destroy(struct push_buffer *pb)
  44. {
  45. struct host1x_cdma *cdma = pb_to_cdma(pb);
  46. struct host1x *host1x = cdma_to_host1x(cdma);
  47. if (!pb->mapped)
  48. return;
  49. if (host1x->domain) {
  50. iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
  51. free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
  52. }
  53. dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
  54. pb->mapped = NULL;
  55. pb->phys = 0;
  56. }
  57. /*
  58. * Init push buffer resources
  59. */
  60. static int host1x_pushbuffer_init(struct push_buffer *pb)
  61. {
  62. struct host1x_cdma *cdma = pb_to_cdma(pb);
  63. struct host1x *host1x = cdma_to_host1x(cdma);
  64. struct iova *alloc;
  65. u32 size;
  66. int err;
  67. pb->mapped = NULL;
  68. pb->phys = 0;
  69. pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;
  70. size = pb->size + 4;
  71. /* initialize buffer pointers */
  72. pb->fence = pb->size - 8;
  73. pb->pos = 0;
  74. if (host1x->domain) {
  75. unsigned long shift;
  76. size = iova_align(&host1x->iova, size);
  77. pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
  78. GFP_KERNEL);
  79. if (!pb->mapped)
  80. return -ENOMEM;
  81. shift = iova_shift(&host1x->iova);
  82. alloc = alloc_iova(&host1x->iova, size >> shift,
  83. host1x->iova_end >> shift, true);
  84. if (!alloc) {
  85. err = -ENOMEM;
  86. goto iommu_free_mem;
  87. }
  88. pb->dma = iova_dma_addr(&host1x->iova, alloc);
  89. err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
  90. IOMMU_READ);
  91. if (err)
  92. goto iommu_free_iova;
  93. } else {
  94. pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
  95. GFP_KERNEL);
  96. if (!pb->mapped)
  97. return -ENOMEM;
  98. pb->dma = pb->phys;
  99. }
  100. pb->alloc_size = size;
  101. host1x_hw_pushbuffer_init(host1x, pb);
  102. return 0;
  103. iommu_free_iova:
  104. __free_iova(&host1x->iova, alloc);
  105. iommu_free_mem:
  106. dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
  107. return err;
  108. }
  109. /*
  110. * Push two words to the push buffer
  111. * Caller must ensure push buffer is not full
  112. */
  113. static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
  114. {
  115. u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
  116. WARN_ON(pb->pos == pb->fence);
  117. *(p++) = op1;
  118. *(p++) = op2;
  119. pb->pos = (pb->pos + 8) & (pb->size - 1);
  120. }
  121. /*
  122. * Pop a number of two word slots from the push buffer
  123. * Caller must ensure push buffer is not empty
  124. */
  125. static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
  126. {
  127. /* Advance the next write position */
  128. pb->fence = (pb->fence + slots * 8) & (pb->size - 1);
  129. }
  130. /*
  131. * Return the number of two word slots free in the push buffer
  132. */
  133. static u32 host1x_pushbuffer_space(struct push_buffer *pb)
  134. {
  135. return ((pb->fence - pb->pos) & (pb->size - 1)) / 8;
  136. }
  137. /*
  138. * Sleep (if necessary) until the requested event happens
  139. * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
  140. * - Returns 1
  141. * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
  142. * - Return the amount of space (> 0)
  143. * Must be called with the cdma lock held.
  144. */
  145. unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
  146. enum cdma_event event)
  147. {
  148. for (;;) {
  149. struct push_buffer *pb = &cdma->push_buffer;
  150. unsigned int space;
  151. switch (event) {
  152. case CDMA_EVENT_SYNC_QUEUE_EMPTY:
  153. space = list_empty(&cdma->sync_queue) ? 1 : 0;
  154. break;
  155. case CDMA_EVENT_PUSH_BUFFER_SPACE:
  156. space = host1x_pushbuffer_space(pb);
  157. break;
  158. default:
  159. WARN_ON(1);
  160. return -EINVAL;
  161. }
  162. if (space)
  163. return space;
  164. trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
  165. event);
  166. /* If somebody has managed to already start waiting, yield */
  167. if (cdma->event != CDMA_EVENT_NONE) {
  168. mutex_unlock(&cdma->lock);
  169. schedule();
  170. mutex_lock(&cdma->lock);
  171. continue;
  172. }
  173. cdma->event = event;
  174. mutex_unlock(&cdma->lock);
  175. down(&cdma->sem);
  176. mutex_lock(&cdma->lock);
  177. }
  178. return 0;
  179. }
  180. /*
  181. * Start timer that tracks the time spent by the job.
  182. * Must be called with the cdma lock held.
  183. */
  184. static void cdma_start_timer_locked(struct host1x_cdma *cdma,
  185. struct host1x_job *job)
  186. {
  187. struct host1x *host = cdma_to_host1x(cdma);
  188. if (cdma->timeout.client) {
  189. /* timer already started */
  190. return;
  191. }
  192. cdma->timeout.client = job->client;
  193. cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
  194. cdma->timeout.syncpt_val = job->syncpt_end;
  195. cdma->timeout.start_ktime = ktime_get();
  196. schedule_delayed_work(&cdma->timeout.wq,
  197. msecs_to_jiffies(job->timeout));
  198. }
  199. /*
  200. * Stop timer when a buffer submission completes.
  201. * Must be called with the cdma lock held.
  202. */
  203. static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
  204. {
  205. cancel_delayed_work(&cdma->timeout.wq);
  206. cdma->timeout.client = NULL;
  207. }
  208. /*
  209. * For all sync queue entries that have already finished according to the
  210. * current sync point registers:
  211. * - unpin & unref their mems
  212. * - pop their push buffer slots
  213. * - remove them from the sync queue
  214. * This is normally called from the host code's worker thread, but can be
  215. * called manually if necessary.
  216. * Must be called with the cdma lock held.
  217. */
  218. static void update_cdma_locked(struct host1x_cdma *cdma)
  219. {
  220. bool signal = false;
  221. struct host1x *host1x = cdma_to_host1x(cdma);
  222. struct host1x_job *job, *n;
  223. /* If CDMA is stopped, queue is cleared and we can return */
  224. if (!cdma->running)
  225. return;
  226. /*
  227. * Walk the sync queue, reading the sync point registers as necessary,
  228. * to consume as many sync queue entries as possible without blocking
  229. */
  230. list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
  231. struct host1x_syncpt *sp =
  232. host1x_syncpt_get(host1x, job->syncpt_id);
  233. /* Check whether this syncpt has completed, and bail if not */
  234. if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
  235. /* Start timer on next pending syncpt */
  236. if (job->timeout)
  237. cdma_start_timer_locked(cdma, job);
  238. break;
  239. }
  240. /* Cancel timeout, when a buffer completes */
  241. if (cdma->timeout.client)
  242. stop_cdma_timer_locked(cdma);
  243. /* Unpin the memory */
  244. host1x_job_unpin(job);
  245. /* Pop push buffer slots */
  246. if (job->num_slots) {
  247. struct push_buffer *pb = &cdma->push_buffer;
  248. host1x_pushbuffer_pop(pb, job->num_slots);
  249. if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
  250. signal = true;
  251. }
  252. list_del(&job->list);
  253. host1x_job_put(job);
  254. }
  255. if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
  256. list_empty(&cdma->sync_queue))
  257. signal = true;
  258. if (signal) {
  259. cdma->event = CDMA_EVENT_NONE;
  260. up(&cdma->sem);
  261. }
  262. }
  263. void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
  264. struct device *dev)
  265. {
  266. struct host1x *host1x = cdma_to_host1x(cdma);
  267. u32 restart_addr, syncpt_incrs, syncpt_val;
  268. struct host1x_job *job = NULL;
  269. syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
  270. dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
  271. __func__, syncpt_val);
  272. /*
  273. * Move the sync_queue read pointer to the first entry that hasn't
  274. * completed based on the current HW syncpt value. It's likely there
  275. * won't be any (i.e. we're still at the head), but covers the case
  276. * where a syncpt incr happens just prior/during the teardown.
  277. */
  278. dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
  279. __func__);
  280. list_for_each_entry(job, &cdma->sync_queue, list) {
  281. if (syncpt_val < job->syncpt_end)
  282. break;
  283. host1x_job_dump(dev, job);
  284. }
  285. /*
  286. * Walk the sync_queue, first incrementing with the CPU syncpts that
  287. * are partially executed (the first buffer) or fully skipped while
  288. * still in the current context (slots are also NOP-ed).
  289. *
  290. * At the point contexts are interleaved, syncpt increments must be
  291. * done inline with the pushbuffer from a GATHER buffer to maintain
  292. * the order (slots are modified to be a GATHER of syncpt incrs).
  293. *
  294. * Note: save in restart_addr the location where the timed out buffer
  295. * started in the PB, so we can start the refetch from there (with the
  296. * modified NOP-ed PB slots). This lets things appear to have completed
  297. * properly for this buffer and resources are freed.
  298. */
  299. dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n",
  300. __func__);
  301. if (!list_empty(&cdma->sync_queue))
  302. restart_addr = job->first_get;
  303. else
  304. restart_addr = cdma->last_pos;
  305. /* do CPU increments as long as this context continues */
  306. list_for_each_entry_from(job, &cdma->sync_queue, list) {
  307. /* different context, gets us out of this loop */
  308. if (job->client != cdma->timeout.client)
  309. break;
  310. /* won't need a timeout when replayed */
  311. job->timeout = 0;
  312. syncpt_incrs = job->syncpt_end - syncpt_val;
  313. dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
  314. host1x_job_dump(dev, job);
  315. /* safe to use CPU to incr syncpts */
  316. host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
  317. syncpt_incrs, job->syncpt_end,
  318. job->num_slots);
  319. syncpt_val += syncpt_incrs;
  320. }
  321. /*
  322. * The following sumbits from the same client may be dependent on the
  323. * failed submit and therefore they may fail. Force a small timeout
  324. * to make the queue cleanup faster.
  325. */
  326. list_for_each_entry_from(job, &cdma->sync_queue, list)
  327. if (job->client == cdma->timeout.client)
  328. job->timeout = min_t(unsigned int, job->timeout, 500);
  329. dev_dbg(dev, "%s: finished sync_queue modification\n", __func__);
  330. /* roll back DMAGET and start up channel again */
  331. host1x_hw_cdma_resume(host1x, cdma, restart_addr);
  332. }
  333. /*
  334. * Create a cdma
  335. */
  336. int host1x_cdma_init(struct host1x_cdma *cdma)
  337. {
  338. int err;
  339. mutex_init(&cdma->lock);
  340. sema_init(&cdma->sem, 0);
  341. INIT_LIST_HEAD(&cdma->sync_queue);
  342. cdma->event = CDMA_EVENT_NONE;
  343. cdma->running = false;
  344. cdma->torndown = false;
  345. err = host1x_pushbuffer_init(&cdma->push_buffer);
  346. if (err)
  347. return err;
  348. return 0;
  349. }
  350. /*
  351. * Destroy a cdma
  352. */
  353. int host1x_cdma_deinit(struct host1x_cdma *cdma)
  354. {
  355. struct push_buffer *pb = &cdma->push_buffer;
  356. struct host1x *host1x = cdma_to_host1x(cdma);
  357. if (cdma->running) {
  358. pr_warn("%s: CDMA still running\n", __func__);
  359. return -EBUSY;
  360. }
  361. host1x_pushbuffer_destroy(pb);
  362. host1x_hw_cdma_timeout_destroy(host1x, cdma);
  363. return 0;
  364. }
  365. /*
  366. * Begin a cdma submit
  367. */
  368. int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
  369. {
  370. struct host1x *host1x = cdma_to_host1x(cdma);
  371. mutex_lock(&cdma->lock);
  372. if (job->timeout) {
  373. /* init state on first submit with timeout value */
  374. if (!cdma->timeout.initialized) {
  375. int err;
  376. err = host1x_hw_cdma_timeout_init(host1x, cdma,
  377. job->syncpt_id);
  378. if (err) {
  379. mutex_unlock(&cdma->lock);
  380. return err;
  381. }
  382. }
  383. }
  384. if (!cdma->running)
  385. host1x_hw_cdma_start(host1x, cdma);
  386. cdma->slots_free = 0;
  387. cdma->slots_used = 0;
  388. cdma->first_get = cdma->push_buffer.pos;
  389. trace_host1x_cdma_begin(dev_name(job->channel->dev));
  390. return 0;
  391. }
  392. /*
  393. * Push two words into a push buffer slot
  394. * Blocks as necessary if the push buffer is full.
  395. */
  396. void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
  397. {
  398. struct host1x *host1x = cdma_to_host1x(cdma);
  399. struct push_buffer *pb = &cdma->push_buffer;
  400. u32 slots_free = cdma->slots_free;
  401. if (host1x_debug_trace_cmdbuf)
  402. trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
  403. op1, op2);
  404. if (slots_free == 0) {
  405. host1x_hw_cdma_flush(host1x, cdma);
  406. slots_free = host1x_cdma_wait_locked(cdma,
  407. CDMA_EVENT_PUSH_BUFFER_SPACE);
  408. }
  409. cdma->slots_free = slots_free - 1;
  410. cdma->slots_used++;
  411. host1x_pushbuffer_push(pb, op1, op2);
  412. }
  413. /*
  414. * End a cdma submit
  415. * Kick off DMA, add job to the sync queue, and a number of slots to be freed
  416. * from the pushbuffer. The handles for a submit must all be pinned at the same
  417. * time, but they can be unpinned in smaller chunks.
  418. */
  419. void host1x_cdma_end(struct host1x_cdma *cdma,
  420. struct host1x_job *job)
  421. {
  422. struct host1x *host1x = cdma_to_host1x(cdma);
  423. bool idle = list_empty(&cdma->sync_queue);
  424. host1x_hw_cdma_flush(host1x, cdma);
  425. job->first_get = cdma->first_get;
  426. job->num_slots = cdma->slots_used;
  427. host1x_job_get(job);
  428. list_add_tail(&job->list, &cdma->sync_queue);
  429. /* start timer on idle -> active transitions */
  430. if (job->timeout && idle)
  431. cdma_start_timer_locked(cdma, job);
  432. trace_host1x_cdma_end(dev_name(job->channel->dev));
  433. mutex_unlock(&cdma->lock);
  434. }
  435. /*
  436. * Update cdma state according to current sync point values
  437. */
  438. void host1x_cdma_update(struct host1x_cdma *cdma)
  439. {
  440. mutex_lock(&cdma->lock);
  441. update_cdma_locked(cdma);
  442. mutex_unlock(&cdma->lock);
  443. }