mali_timeline.c 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374
  1. /*
  2. * This confidential and proprietary software may be used only as
  3. * authorised by a licensing agreement from ARM Limited
  4. * (C) COPYRIGHT 2013 ARM Limited
  5. * ALL RIGHTS RESERVED
  6. * The entire notice above must be reproduced on all authorised
  7. * copies and copies may only be made to the extent permitted
  8. * by a licensing agreement from ARM Limited.
  9. */
  10. #include "mali_timeline.h"
  11. #include "mali_kernel_common.h"
  12. #include "mali_osk_mali.h"
  13. #include "mali_scheduler.h"
  14. #include "mali_soft_job.h"
  15. #include "mali_timeline_fence_wait.h"
  16. #include "mali_timeline_sync_fence.h"
  17. #define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid()))
  18. static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
  19. struct mali_timeline_waiter *waiter);
  20. #if defined(CONFIG_SYNC)
  21. /* Callback that is called when a sync fence a tracker is waiting on is signaled. */
  22. static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter)
  23. {
  24. struct mali_timeline_system *system;
  25. struct mali_timeline_waiter *waiter;
  26. struct mali_timeline_tracker *tracker;
  27. mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  28. u32 tid = _mali_osk_get_tid();
  29. mali_bool is_aborting = MALI_FALSE;
  30. int fence_status = sync_fence->status;
  31. MALI_DEBUG_ASSERT_POINTER(sync_fence);
  32. MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter);
  33. tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter);
  34. MALI_DEBUG_ASSERT_POINTER(tracker);
  35. system = tracker->system;
  36. MALI_DEBUG_ASSERT_POINTER(system);
  37. MALI_DEBUG_ASSERT_POINTER(system->session);
  38. mali_spinlock_reentrant_wait(system->spinlock, tid);
  39. is_aborting = system->session->is_aborting;
  40. if (!is_aborting && (0 > fence_status)) {
  41. MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status));
  42. tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
  43. }
  44. waiter = tracker->waiter_sync;
  45. MALI_DEBUG_ASSERT_POINTER(waiter);
  46. tracker->sync_fence = NULL;
  47. schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
  48. /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */
  49. if (is_aborting) {
  50. _mali_osk_wait_queue_wake_up(system->wait_queue);
  51. }
  52. mali_spinlock_reentrant_signal(system->spinlock, tid);
  53. sync_fence_put(sync_fence);
  54. if (!is_aborting) {
  55. mali_scheduler_schedule_from_mask(schedule_mask, MALI_TRUE);
  56. }
  57. }
  58. #endif /* defined(CONFIG_SYNC) */
  59. static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker)
  60. {
  61. MALI_DEBUG_ASSERT_POINTER(tracker);
  62. MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type);
  63. return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job);
  64. }
  65. static void mali_timeline_timer_callback(void *data)
  66. {
  67. struct mali_timeline_system *system;
  68. struct mali_timeline_tracker *tracker;
  69. struct mali_timeline *timeline;
  70. mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  71. u32 tid = _mali_osk_get_tid();
  72. timeline = (struct mali_timeline *) data;
  73. MALI_DEBUG_ASSERT_POINTER(timeline);
  74. system = timeline->system;
  75. MALI_DEBUG_ASSERT_POINTER(system);
  76. mali_spinlock_reentrant_wait(system->spinlock, tid);
  77. if (!system->timer_enabled) {
  78. mali_spinlock_reentrant_signal(system->spinlock, tid);
  79. return;
  80. }
  81. tracker = timeline->tracker_tail;
  82. timeline->timer_active = MALI_FALSE;
  83. if (NULL != tracker && MALI_TRUE == tracker->timer_active) {
  84. /* This is likely the delayed work that has been schedule out before cancelled. */
  85. if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) {
  86. mali_spinlock_reentrant_signal(system->spinlock, tid);
  87. return;
  88. }
  89. schedule_mask = mali_timeline_tracker_time_out(tracker);
  90. tracker->timer_active = MALI_FALSE;
  91. } else {
  92. MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n"));
  93. }
  94. mali_spinlock_reentrant_signal(system->spinlock, tid);
  95. mali_scheduler_schedule_from_mask(schedule_mask, MALI_FALSE);
  96. }
  97. void mali_timeline_system_stop_timer(struct mali_timeline_system *system)
  98. {
  99. u32 i;
  100. u32 tid = _mali_osk_get_tid();
  101. MALI_DEBUG_ASSERT_POINTER(system);
  102. mali_spinlock_reentrant_wait(system->spinlock, tid);
  103. system->timer_enabled = MALI_FALSE;
  104. mali_spinlock_reentrant_signal(system->spinlock, tid);
  105. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  106. struct mali_timeline *timeline = system->timelines[i];
  107. MALI_DEBUG_ASSERT_POINTER(timeline);
  108. if (NULL != timeline->delayed_work) {
  109. _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
  110. timeline->timer_active = MALI_FALSE;
  111. }
  112. }
  113. }
  114. static void mali_timeline_destroy(struct mali_timeline *timeline)
  115. {
  116. MALI_DEBUG_ASSERT_POINTER(timeline);
  117. if (NULL != timeline) {
  118. /* Assert that the timeline object has been properly cleaned up before destroying it. */
  119. MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
  120. MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
  121. MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
  122. MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
  123. MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
  124. MALI_DEBUG_ASSERT(NULL != timeline->system);
  125. MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id);
  126. #if defined(CONFIG_SYNC)
  127. if (NULL != timeline->sync_tl) {
  128. sync_timeline_destroy(timeline->sync_tl);
  129. }
  130. #endif /* defined(CONFIG_SYNC) */
  131. if (NULL != timeline->delayed_work) {
  132. _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
  133. _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work);
  134. }
  135. _mali_osk_free(timeline);
  136. }
  137. }
  138. static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id)
  139. {
  140. struct mali_timeline *timeline;
  141. MALI_DEBUG_ASSERT_POINTER(system);
  142. MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX);
  143. timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline));
  144. if (NULL == timeline) {
  145. return NULL;
  146. }
  147. /* Initially the timeline is empty. */
  148. #if defined(MALI_TIMELINE_DEBUG_START_POINT)
  149. /* Start the timeline a bit before wrapping when debugging. */
  150. timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128;
  151. #else
  152. timeline->point_next = 1;
  153. #endif
  154. timeline->point_oldest = timeline->point_next;
  155. /* The tracker and waiter lists will initially be empty. */
  156. timeline->system = system;
  157. timeline->id = id;
  158. timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline);
  159. if (NULL == timeline->delayed_work) {
  160. mali_timeline_destroy(timeline);
  161. return NULL;
  162. }
  163. timeline->timer_active = MALI_FALSE;
  164. #if defined(CONFIG_SYNC)
  165. {
  166. char timeline_name[32];
  167. switch (id) {
  168. case MALI_TIMELINE_GP:
  169. _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid());
  170. break;
  171. case MALI_TIMELINE_PP:
  172. _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid());
  173. break;
  174. case MALI_TIMELINE_SOFT:
  175. _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid());
  176. break;
  177. default:
  178. MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id));
  179. mali_timeline_destroy(timeline);
  180. return NULL;
  181. }
  182. timeline->sync_tl = mali_sync_timeline_create(timeline_name);
  183. if (NULL == timeline->sync_tl) {
  184. mali_timeline_destroy(timeline);
  185. return NULL;
  186. }
  187. }
  188. #endif /* defined(CONFIG_SYNC) */
  189. return timeline;
  190. }
  191. static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker)
  192. {
  193. MALI_DEBUG_ASSERT_POINTER(timeline);
  194. MALI_DEBUG_ASSERT_POINTER(tracker);
  195. if (mali_timeline_is_full(timeline)) {
  196. /* Don't add tracker if timeline is full. */
  197. tracker->point = MALI_TIMELINE_NO_POINT;
  198. return;
  199. }
  200. tracker->timeline = timeline;
  201. tracker->point = timeline->point_next;
  202. /* Find next available point. */
  203. timeline->point_next++;
  204. if (MALI_TIMELINE_NO_POINT == timeline->point_next) {
  205. timeline->point_next++;
  206. }
  207. MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
  208. /* Add tracker as new head on timeline's tracker list. */
  209. if (NULL == timeline->tracker_head) {
  210. /* Tracker list is empty. */
  211. MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
  212. timeline->tracker_tail = tracker;
  213. MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
  214. MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev);
  215. } else {
  216. MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
  217. tracker->timeline_prev = timeline->tracker_head;
  218. timeline->tracker_head->timeline_next = tracker;
  219. MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
  220. }
  221. timeline->tracker_head = tracker;
  222. MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
  223. MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev);
  224. }
  225. /* Inserting the waiter object into the given timeline */
  226. static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new)
  227. {
  228. struct mali_timeline_waiter *waiter_prev;
  229. struct mali_timeline_waiter *waiter_next;
  230. /* Waiter time must be between timeline head and tail, and there must
  231. * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */
  232. MALI_DEBUG_ASSERT(( waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN);
  233. MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN);
  234. /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/
  235. waiter_prev = timeline->waiter_head; /* Insert new after waiter_prev */
  236. waiter_next = NULL; /* Insert new before waiter_next */
  237. /* Iterating backwards from head (newest) to tail (oldest) until we
  238. * find the correct spot to insert the new waiter */
  239. while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) {
  240. waiter_next = waiter_prev;
  241. waiter_prev = waiter_prev->timeline_prev;
  242. }
  243. if (NULL == waiter_prev && NULL == waiter_next) {
  244. /* list is empty */
  245. timeline->waiter_head = waiter_new;
  246. timeline->waiter_tail = waiter_new;
  247. } else if (NULL == waiter_next) {
  248. /* insert at head */
  249. waiter_new->timeline_prev = timeline->waiter_head;
  250. timeline->waiter_head->timeline_next = waiter_new;
  251. timeline->waiter_head = waiter_new;
  252. } else if (NULL == waiter_prev) {
  253. /* insert at tail */
  254. waiter_new->timeline_next = timeline->waiter_tail;
  255. timeline->waiter_tail->timeline_prev = waiter_new;
  256. timeline->waiter_tail = waiter_new;
  257. } else {
  258. /* insert between */
  259. waiter_new->timeline_next = waiter_next;
  260. waiter_new->timeline_prev = waiter_prev;
  261. waiter_next->timeline_prev = waiter_new;
  262. waiter_prev->timeline_next = waiter_new;
  263. }
  264. }
  265. static void mali_timeline_update_delayed_work(struct mali_timeline *timeline)
  266. {
  267. struct mali_timeline_system *system;
  268. struct mali_timeline_tracker *oldest_tracker;
  269. MALI_DEBUG_ASSERT_POINTER(timeline);
  270. MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
  271. system = timeline->system;
  272. MALI_DEBUG_ASSERT_POINTER(system);
  273. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  274. /* Timer is disabled, early out. */
  275. if (!system->timer_enabled) return;
  276. oldest_tracker = timeline->tracker_tail;
  277. if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) {
  278. if (MALI_FALSE == oldest_tracker->timer_active) {
  279. if (MALI_TRUE == timeline->timer_active) {
  280. _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
  281. }
  282. _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ);
  283. oldest_tracker->timer_active = MALI_TRUE;
  284. timeline->timer_active = MALI_TRUE;
  285. }
  286. } else if (MALI_TRUE == timeline->timer_active) {
  287. _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
  288. timeline->timer_active = MALI_FALSE;
  289. }
  290. }
  291. static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline)
  292. {
  293. mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  294. MALI_DEBUG_ASSERT_POINTER(timeline);
  295. MALI_DEBUG_CODE({
  296. struct mali_timeline_system *system = timeline->system;
  297. MALI_DEBUG_ASSERT_POINTER(system);
  298. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  299. });
  300. if (NULL != timeline->tracker_tail) {
  301. /* Set oldest point to oldest tracker's point */
  302. timeline->point_oldest = timeline->tracker_tail->point;
  303. } else {
  304. /* No trackers, mark point list as empty */
  305. timeline->point_oldest = timeline->point_next;
  306. }
  307. /* Release all waiters no longer on the timeline's point list.
  308. * Releasing a waiter can trigger this function to be called again, so
  309. * we do not store any pointers on stack. */
  310. while (NULL != timeline->waiter_tail) {
  311. u32 waiter_time_relative;
  312. u32 time_head_relative;
  313. struct mali_timeline_waiter *waiter = timeline->waiter_tail;
  314. time_head_relative = timeline->point_next - timeline->point_oldest;
  315. waiter_time_relative = waiter->point - timeline->point_oldest;
  316. if (waiter_time_relative < time_head_relative) {
  317. /* This and all following waiters are on the point list, so we are done. */
  318. break;
  319. }
  320. /* Remove waiter from timeline's waiter list. */
  321. if (NULL != waiter->timeline_next) {
  322. waiter->timeline_next->timeline_prev = NULL;
  323. } else {
  324. /* This was the last waiter */
  325. timeline->waiter_head = NULL;
  326. }
  327. timeline->waiter_tail = waiter->timeline_next;
  328. /* Release waiter. This could activate a tracker, if this was
  329. * the last waiter for the tracker. */
  330. schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
  331. }
  332. return schedule_mask;
  333. }
  334. void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker,
  335. mali_timeline_tracker_type type,
  336. struct mali_timeline_fence *fence,
  337. void *job)
  338. {
  339. MALI_DEBUG_ASSERT_POINTER(tracker);
  340. MALI_DEBUG_ASSERT_POINTER(job);
  341. MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type);
  342. /* Zero out all tracker members. */
  343. _mali_osk_memset(tracker, 0, sizeof(*tracker));
  344. tracker->type = type;
  345. tracker->job = job;
  346. tracker->trigger_ref_count = 1; /* Prevents any callback from trigging while adding it */
  347. tracker->os_tick_create = _mali_osk_time_tickcount();
  348. MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC);
  349. tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE;
  350. /* Copy fence. */
  351. if (NULL != fence) {
  352. _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence));
  353. }
  354. }
  355. mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker)
  356. {
  357. struct mali_timeline *timeline;
  358. struct mali_timeline_system *system;
  359. struct mali_timeline_tracker *tracker_next, *tracker_prev;
  360. mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  361. u32 tid = _mali_osk_get_tid();
  362. /* Upon entry a group lock will be held, but not a scheduler lock. */
  363. MALI_DEBUG_ASSERT_POINTER(tracker);
  364. MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
  365. /* Tracker should have been triggered */
  366. MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count);
  367. /* All waiters should have been released at this point */
  368. MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
  369. MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
  370. MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job));
  371. timeline = tracker->timeline;
  372. if (NULL == timeline) {
  373. /* Tracker was not on a timeline, there is nothing to release. */
  374. return MALI_SCHEDULER_MASK_EMPTY;
  375. }
  376. system = timeline->system;
  377. MALI_DEBUG_ASSERT_POINTER(system);
  378. mali_spinlock_reentrant_wait(system->spinlock, tid);
  379. /* Tracker should still be on timeline */
  380. MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
  381. MALI_DEBUG_ASSERT( mali_timeline_is_point_on(timeline, tracker->point));
  382. /* Tracker is no longer valid. */
  383. MALI_DEBUG_CODE(tracker->magic = 0);
  384. tracker_next = tracker->timeline_next;
  385. tracker_prev = tracker->timeline_prev;
  386. tracker->timeline_next = NULL;
  387. tracker->timeline_prev = NULL;
  388. /* Removing tracker from timeline's tracker list */
  389. if (NULL == tracker_next) {
  390. /* This tracker was the head */
  391. timeline->tracker_head = tracker_prev;
  392. } else {
  393. tracker_next->timeline_prev = tracker_prev;
  394. }
  395. if (NULL == tracker_prev) {
  396. /* This tracker was the tail */
  397. timeline->tracker_tail = tracker_next;
  398. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  399. /* Update the timeline's oldest time and release any waiters */
  400. schedule_mask |= mali_timeline_update_oldest_point(timeline);
  401. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  402. } else {
  403. tracker_prev->timeline_next = tracker_next;
  404. }
  405. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  406. /* Update delayed work only when it is the soft job timeline */
  407. if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
  408. mali_timeline_update_delayed_work(tracker->timeline);
  409. }
  410. mali_spinlock_reentrant_signal(system->spinlock, tid);
  411. return schedule_mask;
  412. }
  413. void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system,
  414. struct mali_timeline_waiter *tail,
  415. struct mali_timeline_waiter *head)
  416. {
  417. MALI_DEBUG_ASSERT_POINTER(system);
  418. MALI_DEBUG_ASSERT_POINTER(head);
  419. MALI_DEBUG_ASSERT_POINTER(tail);
  420. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  421. head->tracker_next = system->waiter_empty_list;
  422. system->waiter_empty_list = tail;
  423. }
  424. static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker)
  425. {
  426. mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  427. struct mali_timeline_system *system;
  428. struct mali_timeline *timeline;
  429. u32 tid = _mali_osk_get_tid();
  430. MALI_DEBUG_ASSERT_POINTER(tracker);
  431. MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
  432. system = tracker->system;
  433. MALI_DEBUG_ASSERT_POINTER(system);
  434. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  435. tracker->os_tick_activate = _mali_osk_time_tickcount();
  436. if (NULL != tracker->waiter_head) {
  437. mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head);
  438. tracker->waiter_head = NULL;
  439. tracker->waiter_tail = NULL;
  440. }
  441. switch (tracker->type) {
  442. case MALI_TIMELINE_TRACKER_GP:
  443. schedule_mask = mali_gp_scheduler_activate_job((struct mali_gp_job *) tracker->job);
  444. break;
  445. case MALI_TIMELINE_TRACKER_PP:
  446. schedule_mask = mali_pp_scheduler_activate_job((struct mali_pp_job *) tracker->job);
  447. break;
  448. case MALI_TIMELINE_TRACKER_SOFT:
  449. timeline = tracker->timeline;
  450. MALI_DEBUG_ASSERT_POINTER(timeline);
  451. mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job);
  452. /* Start a soft timer to make sure the soft job be released in a limited time */
  453. mali_spinlock_reentrant_wait(system->spinlock, tid);
  454. mali_timeline_update_delayed_work(timeline);
  455. mali_spinlock_reentrant_signal(system->spinlock, tid);
  456. break;
  457. case MALI_TIMELINE_TRACKER_WAIT:
  458. mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job);
  459. break;
  460. case MALI_TIMELINE_TRACKER_SYNC:
  461. #if defined(CONFIG_SYNC)
  462. mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job);
  463. #else
  464. MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type));
  465. #endif /* defined(CONFIG_SYNC) */
  466. break;
  467. default:
  468. MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type));
  469. break;
  470. }
  471. return schedule_mask;
  472. }
  473. void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker)
  474. {
  475. u32 tid = _mali_osk_get_tid();
  476. MALI_DEBUG_ASSERT_POINTER(tracker);
  477. MALI_DEBUG_ASSERT_POINTER(system);
  478. mali_spinlock_reentrant_wait(system->spinlock, tid);
  479. MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
  480. tracker->trigger_ref_count++;
  481. mali_spinlock_reentrant_signal(system->spinlock, tid);
  482. }
  483. mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error)
  484. {
  485. u32 tid = _mali_osk_get_tid();
  486. mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  487. MALI_DEBUG_ASSERT_POINTER(tracker);
  488. MALI_DEBUG_ASSERT_POINTER(system);
  489. mali_spinlock_reentrant_wait(system->spinlock, tid);
  490. MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
  491. tracker->trigger_ref_count--;
  492. tracker->activation_error |= activation_error;
  493. if (0 == tracker->trigger_ref_count) {
  494. schedule_mask |= mali_timeline_tracker_activate(tracker);
  495. tracker = NULL;
  496. }
  497. mali_spinlock_reentrant_signal(system->spinlock, tid);
  498. return schedule_mask;
  499. }
  500. void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence)
  501. {
  502. u32 i;
  503. MALI_DEBUG_ASSERT_POINTER(fence);
  504. MALI_DEBUG_ASSERT_POINTER(uk_fence);
  505. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  506. fence->points[i] = uk_fence->points[i];
  507. }
  508. fence->sync_fd = uk_fence->sync_fd;
  509. }
  510. struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session)
  511. {
  512. u32 i;
  513. struct mali_timeline_system *system;
  514. MALI_DEBUG_ASSERT_POINTER(session);
  515. MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n"));
  516. system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system));
  517. if (NULL == system) {
  518. return NULL;
  519. }
  520. system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
  521. if (NULL == system->spinlock) {
  522. mali_timeline_system_destroy(system);
  523. return NULL;
  524. }
  525. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  526. system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i);
  527. if (NULL == system->timelines[i]) {
  528. mali_timeline_system_destroy(system);
  529. return NULL;
  530. }
  531. }
  532. #if defined(CONFIG_SYNC)
  533. system->signaled_sync_tl = mali_sync_timeline_create("mali-always-signaled");
  534. if (NULL == system->signaled_sync_tl) {
  535. mali_timeline_system_destroy(system);
  536. return NULL;
  537. }
  538. #endif /* defined(CONFIG_SYNC) */
  539. system->waiter_empty_list = NULL;
  540. system->session = session;
  541. system->timer_enabled = MALI_TRUE;
  542. system->wait_queue = _mali_osk_wait_queue_init();
  543. if (NULL == system->wait_queue) {
  544. mali_timeline_system_destroy(system);
  545. return NULL;
  546. }
  547. return system;
  548. }
  549. #if defined(CONFIG_SYNC)
  550. /**
  551. * Check if there are any trackers left on timeline.
  552. *
  553. * Used as a wait queue conditional.
  554. *
  555. * @param data Timeline.
  556. * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not.
  557. */
  558. static mali_bool mali_timeline_has_no_trackers(void *data)
  559. {
  560. struct mali_timeline *timeline = (struct mali_timeline *) data;
  561. MALI_DEBUG_ASSERT_POINTER(timeline);
  562. return mali_timeline_is_empty(timeline);
  563. }
  564. /**
  565. * Cancel sync fence waiters waited upon by trackers on all timelines.
  566. *
  567. * Will return after all timelines have no trackers left.
  568. *
  569. * @param system Timeline system.
  570. */
  571. static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system)
  572. {
  573. u32 i;
  574. u32 tid = _mali_osk_get_tid();
  575. struct mali_timeline_tracker *tracker, *tracker_next;
  576. _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list);
  577. MALI_DEBUG_ASSERT_POINTER(system);
  578. MALI_DEBUG_ASSERT_POINTER(system->session);
  579. MALI_DEBUG_ASSERT(system->session->is_aborting);
  580. mali_spinlock_reentrant_wait(system->spinlock, tid);
  581. /* Cancel sync fence waiters. */
  582. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  583. struct mali_timeline *timeline = system->timelines[i];
  584. MALI_DEBUG_ASSERT_POINTER(timeline);
  585. tracker_next = timeline->tracker_tail;
  586. while (NULL != tracker_next) {
  587. tracker = tracker_next;
  588. tracker_next = tracker->timeline_next;
  589. if (NULL == tracker->sync_fence) continue;
  590. MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker));
  591. /* Cancel sync fence waiter. */
  592. if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
  593. /* Callback was not called, move tracker to local list. */
  594. _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list);
  595. }
  596. }
  597. }
  598. mali_spinlock_reentrant_signal(system->spinlock, tid);
  599. /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */
  600. _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) {
  601. mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter);
  602. }
  603. /* Sleep until all sync fence callbacks are done and all timelines are empty. */
  604. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  605. struct mali_timeline *timeline = system->timelines[i];
  606. MALI_DEBUG_ASSERT_POINTER(timeline);
  607. _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
  608. }
  609. }
  610. #endif /* defined(CONFIG_SYNC) */
  611. void mali_timeline_system_abort(struct mali_timeline_system *system)
  612. {
  613. MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid(););
  614. MALI_DEBUG_ASSERT_POINTER(system);
  615. MALI_DEBUG_ASSERT_POINTER(system->session);
  616. MALI_DEBUG_ASSERT(system->session->is_aborting);
  617. MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session));
  618. #if defined(CONFIG_SYNC)
  619. mali_timeline_cancel_sync_fence_waiters(system);
  620. #endif /* defined(CONFIG_SYNC) */
  621. /* Should not be any waiters or trackers left at this point. */
  622. MALI_DEBUG_CODE( {
  623. u32 i;
  624. mali_spinlock_reentrant_wait(system->spinlock, tid);
  625. for (i = 0; i < MALI_TIMELINE_MAX; ++i)
  626. {
  627. struct mali_timeline *timeline = system->timelines[i];
  628. MALI_DEBUG_ASSERT_POINTER(timeline);
  629. MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
  630. MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
  631. MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
  632. MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
  633. MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
  634. }
  635. mali_spinlock_reentrant_signal(system->spinlock, tid);
  636. });
  637. }
  638. void mali_timeline_system_destroy(struct mali_timeline_system *system)
  639. {
  640. u32 i;
  641. struct mali_timeline_waiter *waiter, *next;
  642. MALI_DEBUG_ASSERT_POINTER(system);
  643. MALI_DEBUG_ASSERT_POINTER(system->session);
  644. MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n"));
  645. if (NULL != system) {
  646. /* There should be no waiters left on this queue. */
  647. if (NULL != system->wait_queue) {
  648. _mali_osk_wait_queue_term(system->wait_queue);
  649. system->wait_queue = NULL;
  650. }
  651. /* Free all waiters in empty list */
  652. waiter = system->waiter_empty_list;
  653. while (NULL != waiter) {
  654. next = waiter->tracker_next;
  655. _mali_osk_free(waiter);
  656. waiter = next;
  657. }
  658. #if defined(CONFIG_SYNC)
  659. if (NULL != system->signaled_sync_tl) {
  660. sync_timeline_destroy(system->signaled_sync_tl);
  661. }
  662. #endif /* defined(CONFIG_SYNC) */
  663. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  664. if (NULL != system->timelines[i]) {
  665. mali_timeline_destroy(system->timelines[i]);
  666. }
  667. }
  668. if (NULL != system->spinlock) {
  669. mali_spinlock_reentrant_term(system->spinlock);
  670. }
  671. _mali_osk_free(system);
  672. }
  673. }
  674. /**
  675. * Find how many waiters are needed for a given fence.
  676. *
  677. * @param fence The fence to check.
  678. * @return Number of waiters needed for fence.
  679. */
  680. static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence)
  681. {
  682. u32 i, num_waiters = 0;
  683. MALI_DEBUG_ASSERT_POINTER(fence);
  684. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  685. if (MALI_TIMELINE_NO_POINT != fence->points[i]) {
  686. ++num_waiters;
  687. }
  688. }
  689. #if defined(CONFIG_SYNC)
  690. if (-1 != fence->sync_fd) ++num_waiters;
  691. #endif /* defined(CONFIG_SYNC) */
  692. return num_waiters;
  693. }
  694. static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system)
  695. {
  696. struct mali_timeline_waiter *waiter;
  697. MALI_DEBUG_ASSERT_POINTER(system);
  698. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  699. waiter = system->waiter_empty_list;
  700. if (NULL != waiter) {
  701. /* Remove waiter from empty list and zero it */
  702. system->waiter_empty_list = waiter->tracker_next;
  703. _mali_osk_memset(waiter, 0, sizeof(*waiter));
  704. }
  705. /* Return NULL if list was empty. */
  706. return waiter;
  707. }
  708. static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system,
  709. struct mali_timeline_waiter **tail,
  710. struct mali_timeline_waiter **head,
  711. int max_num_waiters)
  712. {
  713. u32 i, tid = _mali_osk_get_tid();
  714. mali_bool do_alloc;
  715. struct mali_timeline_waiter *waiter;
  716. MALI_DEBUG_ASSERT_POINTER(system);
  717. MALI_DEBUG_ASSERT_POINTER(tail);
  718. MALI_DEBUG_ASSERT_POINTER(head);
  719. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  720. *head = *tail = NULL;
  721. do_alloc = MALI_FALSE;
  722. i = 0;
  723. while (i < max_num_waiters) {
  724. if (MALI_FALSE == do_alloc) {
  725. waiter = mali_timeline_system_get_zeroed_waiter(system);
  726. if (NULL == waiter) {
  727. do_alloc = MALI_TRUE;
  728. mali_spinlock_reentrant_signal(system->spinlock, tid);
  729. continue;
  730. }
  731. } else {
  732. waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter));
  733. if (NULL == waiter) break;
  734. }
  735. ++i;
  736. if (NULL == *tail) {
  737. *tail = waiter;
  738. *head = waiter;
  739. } else {
  740. (*head)->tracker_next = waiter;
  741. *head = waiter;
  742. }
  743. }
  744. if (MALI_TRUE == do_alloc) {
  745. mali_spinlock_reentrant_wait(system->spinlock, tid);
  746. }
  747. }
  748. /**
  749. * Create waiters for the given tracker. The tracker is activated when all waiters are release.
  750. *
  751. * @note Tracker can potentially be activated before this function returns.
  752. *
  753. * @param system Timeline system.
  754. * @param tracker Tracker we will create waiters for.
  755. * @param waiter_tail List of pre-allocated waiters.
  756. * @param waiter_head List of pre-allocated waiters.
  757. */
  758. static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system,
  759. struct mali_timeline_tracker *tracker,
  760. struct mali_timeline_waiter *waiter_tail,
  761. struct mali_timeline_waiter *waiter_head)
  762. {
  763. int i;
  764. u32 tid = _mali_osk_get_tid();
  765. mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  766. #if defined(CONFIG_SYNC)
  767. struct sync_fence *sync_fence = NULL;
  768. #endif /* defined(CONFIG_SYNC) */
  769. MALI_DEBUG_ASSERT_POINTER(system);
  770. MALI_DEBUG_ASSERT_POINTER(tracker);
  771. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  772. MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
  773. MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
  774. MALI_DEBUG_ASSERT(NULL != tracker->job);
  775. /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter
  776. * into the timelines sorted list of waiters */
  777. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  778. mali_timeline_point point;
  779. struct mali_timeline *timeline;
  780. struct mali_timeline_waiter *waiter;
  781. /* Get point on current timeline from tracker's fence. */
  782. point = tracker->fence.points[i];
  783. if (likely(MALI_TIMELINE_NO_POINT == point)) {
  784. /* Fence contains no point on this timeline so we don't need a waiter. */
  785. continue;
  786. }
  787. timeline = system->timelines[i];
  788. MALI_DEBUG_ASSERT_POINTER(timeline);
  789. if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
  790. MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n",
  791. point, timeline->point_oldest, timeline->point_next));
  792. continue;
  793. }
  794. if (likely(mali_timeline_is_point_released(timeline, point))) {
  795. /* Tracker representing the point has been released so we don't need a
  796. * waiter. */
  797. continue;
  798. }
  799. /* The point is on timeline. */
  800. MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point));
  801. /* Get a new zeroed waiter object. */
  802. if (likely(NULL != waiter_tail)) {
  803. waiter = waiter_tail;
  804. waiter_tail = waiter_tail->tracker_next;
  805. } else {
  806. MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
  807. continue;
  808. }
  809. /* Yanking the trigger ref count of the tracker. */
  810. tracker->trigger_ref_count++;
  811. waiter->point = point;
  812. waiter->tracker = tracker;
  813. /* Insert waiter on tracker's singly-linked waiter list. */
  814. if (NULL == tracker->waiter_head) {
  815. /* list is empty */
  816. MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
  817. tracker->waiter_tail = waiter;
  818. } else {
  819. tracker->waiter_head->tracker_next = waiter;
  820. }
  821. tracker->waiter_head = waiter;
  822. /* Add waiter to timeline. */
  823. mali_timeline_insert_waiter(timeline, waiter);
  824. }
  825. #if defined(CONFIG_SYNC)
  826. if (-1 != tracker->fence.sync_fd) {
  827. int ret;
  828. struct mali_timeline_waiter *waiter;
  829. sync_fence = sync_fence_fdget(tracker->fence.sync_fd);
  830. if (unlikely(NULL == sync_fence)) {
  831. MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd));
  832. goto exit;
  833. }
  834. /* Check if we have a zeroed waiter object available. */
  835. if (unlikely(NULL == waiter_tail)) {
  836. MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
  837. goto exit;
  838. }
  839. /* Start asynchronous wait that will release waiter when the fence is signaled. */
  840. sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
  841. ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
  842. if (1 == ret) {
  843. /* Fence already signaled, no waiter needed. */
  844. goto exit;
  845. } else if (0 != ret) {
  846. MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret));
  847. tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
  848. goto exit;
  849. }
  850. /* Grab new zeroed waiter object. */
  851. waiter = waiter_tail;
  852. waiter_tail = waiter_tail->tracker_next;
  853. /* Increase the trigger ref count of the tracker. */
  854. tracker->trigger_ref_count++;
  855. waiter->point = MALI_TIMELINE_NO_POINT;
  856. waiter->tracker = tracker;
  857. /* Insert waiter on tracker's singly-linked waiter list. */
  858. if (NULL == tracker->waiter_head) {
  859. /* list is empty */
  860. MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
  861. tracker->waiter_tail = waiter;
  862. } else {
  863. tracker->waiter_head->tracker_next = waiter;
  864. }
  865. tracker->waiter_head = waiter;
  866. /* Also store waiter in separate field for easy access by sync callback. */
  867. tracker->waiter_sync = waiter;
  868. /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */
  869. tracker->sync_fence = sync_fence;
  870. sync_fence = NULL;
  871. }
  872. exit:
  873. #endif /* defined(CONFIG_SYNC) */
  874. if (NULL != waiter_tail) {
  875. mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head);
  876. }
  877. /* Release the initial trigger ref count. */
  878. tracker->trigger_ref_count--;
  879. /* If there were no waiters added to this tracker we activate immediately. */
  880. if (0 == tracker->trigger_ref_count) {
  881. schedule_mask |= mali_timeline_tracker_activate(tracker);
  882. }
  883. mali_spinlock_reentrant_signal(system->spinlock, tid);
  884. #if defined(CONFIG_SYNC)
  885. if (NULL != sync_fence) {
  886. sync_fence_put(sync_fence);
  887. }
  888. #endif /* defined(CONFIG_SYNC) */
  889. mali_scheduler_schedule_from_mask(schedule_mask, MALI_FALSE);
  890. }
  891. mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system,
  892. struct mali_timeline_tracker *tracker,
  893. enum mali_timeline_id timeline_id)
  894. {
  895. int num_waiters = 0;
  896. struct mali_timeline_waiter *waiter_tail, *waiter_head;
  897. u32 tid = _mali_osk_get_tid();
  898. mali_timeline_point point = MALI_TIMELINE_NO_POINT;
  899. MALI_DEBUG_ASSERT_POINTER(system);
  900. MALI_DEBUG_ASSERT_POINTER(system->session);
  901. MALI_DEBUG_ASSERT_POINTER(tracker);
  902. MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting);
  903. MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type);
  904. MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
  905. MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id));
  906. MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
  907. tracker->system = system;
  908. mali_spinlock_reentrant_wait(system->spinlock, tid);
  909. num_waiters = mali_timeline_fence_num_waiters(&tracker->fence);
  910. /* Allocate waiters. */
  911. mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters);
  912. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  913. /* Add tracker to timeline. This will allocate a point for the tracker on the timeline. If
  914. * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the
  915. * point will be MALI_TIMELINE_NO_POINT.
  916. *
  917. * NOTE: the tracker can fail to be added if the timeline is full. If this happens, the
  918. * point will be MALI_TIMELINE_NO_POINT. */
  919. MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE);
  920. if (likely(timeline_id < MALI_TIMELINE_MAX)) {
  921. struct mali_timeline *timeline = system->timelines[timeline_id];
  922. mali_timeline_insert_tracker(timeline, tracker);
  923. MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
  924. }
  925. point = tracker->point;
  926. /* Create waiters for tracker based on supplied fence. Each waiter will increase the
  927. * trigger ref count. */
  928. mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head);
  929. tracker = NULL;
  930. /* At this point the tracker object might have been freed so we should no longer
  931. * access it. */
  932. /* The tracker will always be activated after calling add_tracker, even if NO_POINT is
  933. * returned. */
  934. return point;
  935. }
  936. static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
  937. struct mali_timeline_waiter *waiter)
  938. {
  939. struct mali_timeline_tracker *tracker;
  940. mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  941. MALI_DEBUG_ASSERT_POINTER(system);
  942. MALI_DEBUG_ASSERT_POINTER(waiter);
  943. MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
  944. tracker = waiter->tracker;
  945. MALI_DEBUG_ASSERT_POINTER(tracker);
  946. /* At this point the waiter has been removed from the timeline's waiter list, but it is
  947. * still on the tracker's waiter list. All of the tracker's waiters will be released when
  948. * the tracker is activated. */
  949. waiter->point = MALI_TIMELINE_NO_POINT;
  950. waiter->tracker = NULL;
  951. tracker->trigger_ref_count--;
  952. if (0 == tracker->trigger_ref_count) {
  953. /* This was the last waiter; activate tracker */
  954. schedule_mask |= mali_timeline_tracker_activate(tracker);
  955. tracker = NULL;
  956. }
  957. return schedule_mask;
  958. }
  959. mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system,
  960. enum mali_timeline_id timeline_id)
  961. {
  962. mali_timeline_point point;
  963. struct mali_timeline *timeline;
  964. u32 tid = _mali_osk_get_tid();
  965. MALI_DEBUG_ASSERT_POINTER(system);
  966. if (MALI_TIMELINE_MAX <= timeline_id) {
  967. return MALI_TIMELINE_NO_POINT;
  968. }
  969. mali_spinlock_reentrant_wait(system->spinlock, tid);
  970. timeline = system->timelines[timeline_id];
  971. MALI_DEBUG_ASSERT_POINTER(timeline);
  972. point = MALI_TIMELINE_NO_POINT;
  973. if (timeline->point_oldest != timeline->point_next) {
  974. point = timeline->point_next - 1;
  975. if (MALI_TIMELINE_NO_POINT == point) point--;
  976. }
  977. mali_spinlock_reentrant_signal(system->spinlock, tid);
  978. return point;
  979. }
  980. #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
  981. static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id)
  982. {
  983. struct mali_timeline *timeline;
  984. struct mali_timeline_system *system;
  985. MALI_DEBUG_ASSERT_POINTER(tracker);
  986. MALI_DEBUG_ASSERT_POINTER(tracker->timeline);
  987. timeline = tracker->timeline;
  988. MALI_DEBUG_ASSERT_POINTER(timeline->system);
  989. system = timeline->system;
  990. if (MALI_TIMELINE_MAX > id) {
  991. return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]);
  992. } else {
  993. MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id);
  994. return MALI_FALSE;
  995. }
  996. }
  997. static const char *timeline_id_to_string(enum mali_timeline_id id)
  998. {
  999. switch (id) {
  1000. case MALI_TIMELINE_GP:
  1001. return " GP";
  1002. case MALI_TIMELINE_PP:
  1003. return " PP";
  1004. case MALI_TIMELINE_SOFT:
  1005. return "SOFT";
  1006. default:
  1007. return "NONE";
  1008. }
  1009. }
  1010. static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type)
  1011. {
  1012. switch (type) {
  1013. case MALI_TIMELINE_TRACKER_GP:
  1014. return " GP";
  1015. case MALI_TIMELINE_TRACKER_PP:
  1016. return " PP";
  1017. case MALI_TIMELINE_TRACKER_SOFT:
  1018. return "SOFT";
  1019. case MALI_TIMELINE_TRACKER_WAIT:
  1020. return "WAIT";
  1021. case MALI_TIMELINE_TRACKER_SYNC:
  1022. return "SYNC";
  1023. default:
  1024. return "INVALID";
  1025. }
  1026. }
  1027. mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker)
  1028. {
  1029. struct mali_timeline *timeline = NULL;
  1030. MALI_DEBUG_ASSERT_POINTER(tracker);
  1031. timeline = tracker->timeline;
  1032. if (0 != tracker->trigger_ref_count) {
  1033. return MALI_TIMELINE_TS_WAITING;
  1034. }
  1035. if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) {
  1036. return MALI_TIMELINE_TS_ACTIVE;
  1037. }
  1038. if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) {
  1039. return MALI_TIMELINE_TS_INIT;
  1040. }
  1041. return MALI_TIMELINE_TS_FINISH;
  1042. }
  1043. void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker)
  1044. {
  1045. const char *tracker_state = "IWAF";
  1046. MALI_DEBUG_ASSERT_POINTER(tracker);
  1047. if (0 != tracker->trigger_ref_count) {
  1048. MALI_PRINTF(("TL: %s %u %c - ref_wait:%u [%s%u,%s%u,%s%u,%d] (0x%08X)\n",
  1049. timeline_tracker_type_to_string(tracker->type), tracker->point,
  1050. *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)),
  1051. tracker->trigger_ref_count,
  1052. is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "W" : " ", tracker->fence.points[0],
  1053. is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "W" : " ", tracker->fence.points[1],
  1054. is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "W" : " ", tracker->fence.points[2],
  1055. tracker->fence.sync_fd, tracker->job));
  1056. } else {
  1057. MALI_PRINTF(("TL: %s %u %c (0x%08X)\n",
  1058. timeline_tracker_type_to_string(tracker->type), tracker->point,
  1059. *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)),
  1060. tracker->job));
  1061. }
  1062. }
  1063. void mali_timeline_debug_print_timeline(struct mali_timeline *timeline)
  1064. {
  1065. struct mali_timeline_tracker *tracker = NULL;
  1066. int i_max = 30;
  1067. MALI_DEBUG_ASSERT_POINTER(timeline);
  1068. tracker = timeline->tracker_tail;
  1069. while (NULL != tracker && 0 < --i_max) {
  1070. mali_timeline_debug_print_tracker(tracker);
  1071. tracker = tracker->timeline_next;
  1072. }
  1073. if (0 == i_max) {
  1074. MALI_PRINTF(("TL: Too many trackers in list to print\n"));
  1075. }
  1076. }
  1077. void mali_timeline_debug_print_system(struct mali_timeline_system *system)
  1078. {
  1079. int i;
  1080. int num_printed = 0;
  1081. MALI_DEBUG_ASSERT_POINTER(system);
  1082. /* Print all timelines */
  1083. for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
  1084. struct mali_timeline *timeline = system->timelines[i];
  1085. MALI_DEBUG_ASSERT_POINTER(timeline);
  1086. if (NULL == timeline->tracker_head) continue;
  1087. MALI_PRINTF(("TL: Timeline %s:\n",
  1088. timeline_id_to_string((enum mali_timeline_id)i)));
  1089. mali_timeline_debug_print_timeline(timeline);
  1090. num_printed++;
  1091. }
  1092. if (0 == num_printed) {
  1093. MALI_PRINTF(("TL: All timelines empty\n"));
  1094. }
  1095. }
  1096. #endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */