runtime.c 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721
  1. /*
  2. * drivers/base/power/runtime.c - Helper functions for device runtime PM
  3. *
  4. * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
  5. * Copyright (C) 2010 Alan Stern <stern@rowland.harvard.edu>
  6. *
  7. * This file is released under the GPLv2.
  8. */
  9. #include <linux/sched/mm.h>
  10. #include <linux/export.h>
  11. #include <linux/pm_runtime.h>
  12. #include <linux/pm_wakeirq.h>
  13. #include <trace/events/rpm.h>
  14. #include "../base.h"
  15. #include "power.h"
  16. typedef int (*pm_callback_t)(struct device *);
  17. static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)
  18. {
  19. pm_callback_t cb;
  20. const struct dev_pm_ops *ops;
  21. if (dev->pm_domain)
  22. ops = &dev->pm_domain->ops;
  23. else if (dev->type && dev->type->pm)
  24. ops = dev->type->pm;
  25. else if (dev->class && dev->class->pm)
  26. ops = dev->class->pm;
  27. else if (dev->bus && dev->bus->pm)
  28. ops = dev->bus->pm;
  29. else
  30. ops = NULL;
  31. if (ops)
  32. cb = *(pm_callback_t *)((void *)ops + cb_offset);
  33. else
  34. cb = NULL;
  35. if (!cb && dev->driver && dev->driver->pm)
  36. cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset);
  37. return cb;
  38. }
  39. #define RPM_GET_CALLBACK(dev, callback) \
  40. __rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
  41. static int rpm_resume(struct device *dev, int rpmflags);
  42. static int rpm_suspend(struct device *dev, int rpmflags);
  43. /**
  44. * update_pm_runtime_accounting - Update the time accounting of power states
  45. * @dev: Device to update the accounting for
  46. *
  47. * In order to be able to have time accounting of the various power states
  48. * (as used by programs such as PowerTOP to show the effectiveness of runtime
  49. * PM), we need to track the time spent in each state.
  50. * update_pm_runtime_accounting must be called each time before the
  51. * runtime_status field is updated, to account the time in the old state
  52. * correctly.
  53. */
  54. void update_pm_runtime_accounting(struct device *dev)
  55. {
  56. unsigned long now = jiffies;
  57. unsigned long delta;
  58. delta = now - dev->power.accounting_timestamp;
  59. dev->power.accounting_timestamp = now;
  60. if (dev->power.disable_depth > 0)
  61. return;
  62. if (dev->power.runtime_status == RPM_SUSPENDED)
  63. dev->power.suspended_jiffies += delta;
  64. else
  65. dev->power.active_jiffies += delta;
  66. }
  67. static void __update_runtime_status(struct device *dev, enum rpm_status status)
  68. {
  69. update_pm_runtime_accounting(dev);
  70. dev->power.runtime_status = status;
  71. }
  72. /**
  73. * pm_runtime_deactivate_timer - Deactivate given device's suspend timer.
  74. * @dev: Device to handle.
  75. */
  76. static void pm_runtime_deactivate_timer(struct device *dev)
  77. {
  78. if (dev->power.timer_expires > 0) {
  79. del_timer(&dev->power.suspend_timer);
  80. dev->power.timer_expires = 0;
  81. }
  82. }
  83. /**
  84. * pm_runtime_cancel_pending - Deactivate suspend timer and cancel requests.
  85. * @dev: Device to handle.
  86. */
  87. static void pm_runtime_cancel_pending(struct device *dev)
  88. {
  89. pm_runtime_deactivate_timer(dev);
  90. /*
  91. * In case there's a request pending, make sure its work function will
  92. * return without doing anything.
  93. */
  94. dev->power.request = RPM_REQ_NONE;
  95. }
  96. /*
  97. * pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time.
  98. * @dev: Device to handle.
  99. *
  100. * Compute the autosuspend-delay expiration time based on the device's
  101. * power.last_busy time. If the delay has already expired or is disabled
  102. * (negative) or the power.use_autosuspend flag isn't set, return 0.
  103. * Otherwise return the expiration time in jiffies (adjusted to be nonzero).
  104. *
  105. * This function may be called either with or without dev->power.lock held.
  106. * Either way it can be racy, since power.last_busy may be updated at any time.
  107. */
  108. unsigned long pm_runtime_autosuspend_expiration(struct device *dev)
  109. {
  110. int autosuspend_delay;
  111. long elapsed;
  112. unsigned long last_busy;
  113. unsigned long expires = 0;
  114. if (!dev->power.use_autosuspend)
  115. goto out;
  116. autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);
  117. if (autosuspend_delay < 0)
  118. goto out;
  119. last_busy = READ_ONCE(dev->power.last_busy);
  120. elapsed = jiffies - last_busy;
  121. if (elapsed < 0)
  122. goto out; /* jiffies has wrapped around. */
  123. /*
  124. * If the autosuspend_delay is >= 1 second, align the timer by rounding
  125. * up to the nearest second.
  126. */
  127. expires = last_busy + msecs_to_jiffies(autosuspend_delay);
  128. if (autosuspend_delay >= 1000)
  129. expires = round_jiffies(expires);
  130. expires += !expires;
  131. if (elapsed >= expires - last_busy)
  132. expires = 0; /* Already expired. */
  133. out:
  134. return expires;
  135. }
  136. EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration);
  137. static int dev_memalloc_noio(struct device *dev, void *data)
  138. {
  139. return dev->power.memalloc_noio;
  140. }
  141. /*
  142. * pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag.
  143. * @dev: Device to handle.
  144. * @enable: True for setting the flag and False for clearing the flag.
  145. *
  146. * Set the flag for all devices in the path from the device to the
  147. * root device in the device tree if @enable is true, otherwise clear
  148. * the flag for devices in the path whose siblings don't set the flag.
  149. *
  150. * The function should only be called by block device, or network
  151. * device driver for solving the deadlock problem during runtime
  152. * resume/suspend:
  153. *
  154. * If memory allocation with GFP_KERNEL is called inside runtime
  155. * resume/suspend callback of any one of its ancestors(or the
  156. * block device itself), the deadlock may be triggered inside the
  157. * memory allocation since it might not complete until the block
  158. * device becomes active and the involed page I/O finishes. The
  159. * situation is pointed out first by Alan Stern. Network device
  160. * are involved in iSCSI kind of situation.
  161. *
  162. * The lock of dev_hotplug_mutex is held in the function for handling
  163. * hotplug race because pm_runtime_set_memalloc_noio() may be called
  164. * in async probe().
  165. *
  166. * The function should be called between device_add() and device_del()
  167. * on the affected device(block/network device).
  168. */
  169. void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)
  170. {
  171. static DEFINE_MUTEX(dev_hotplug_mutex);
  172. mutex_lock(&dev_hotplug_mutex);
  173. for (;;) {
  174. bool enabled;
  175. /* hold power lock since bitfield is not SMP-safe. */
  176. spin_lock_irq(&dev->power.lock);
  177. enabled = dev->power.memalloc_noio;
  178. dev->power.memalloc_noio = enable;
  179. spin_unlock_irq(&dev->power.lock);
  180. /*
  181. * not need to enable ancestors any more if the device
  182. * has been enabled.
  183. */
  184. if (enabled && enable)
  185. break;
  186. dev = dev->parent;
  187. /*
  188. * clear flag of the parent device only if all the
  189. * children don't set the flag because ancestor's
  190. * flag was set by any one of the descendants.
  191. */
  192. if (!dev || (!enable &&
  193. device_for_each_child(dev, NULL,
  194. dev_memalloc_noio)))
  195. break;
  196. }
  197. mutex_unlock(&dev_hotplug_mutex);
  198. }
  199. EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio);
  200. /**
  201. * rpm_check_suspend_allowed - Test whether a device may be suspended.
  202. * @dev: Device to test.
  203. */
  204. static int rpm_check_suspend_allowed(struct device *dev)
  205. {
  206. int retval = 0;
  207. if (dev->power.runtime_error)
  208. retval = -EINVAL;
  209. else if (dev->power.disable_depth > 0)
  210. retval = -EACCES;
  211. else if (atomic_read(&dev->power.usage_count) > 0)
  212. retval = -EAGAIN;
  213. else if (!dev->power.ignore_children &&
  214. atomic_read(&dev->power.child_count))
  215. retval = -EBUSY;
  216. /* Pending resume requests take precedence over suspends. */
  217. else if ((dev->power.deferred_resume
  218. && dev->power.runtime_status == RPM_SUSPENDING)
  219. || (dev->power.request_pending
  220. && dev->power.request == RPM_REQ_RESUME))
  221. retval = -EAGAIN;
  222. else if (__dev_pm_qos_read_value(dev) == 0)
  223. retval = -EPERM;
  224. else if (dev->power.runtime_status == RPM_SUSPENDED)
  225. retval = 1;
  226. return retval;
  227. }
  228. static int rpm_get_suppliers(struct device *dev)
  229. {
  230. struct device_link *link;
  231. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
  232. int retval;
  233. if (!(link->flags & DL_FLAG_PM_RUNTIME) ||
  234. READ_ONCE(link->status) == DL_STATE_SUPPLIER_UNBIND)
  235. continue;
  236. retval = pm_runtime_get_sync(link->supplier);
  237. /* Ignore suppliers with disabled runtime PM. */
  238. if (retval < 0 && retval != -EACCES) {
  239. pm_runtime_put_noidle(link->supplier);
  240. return retval;
  241. }
  242. refcount_inc(&link->rpm_active);
  243. }
  244. return 0;
  245. }
  246. static void rpm_put_suppliers(struct device *dev)
  247. {
  248. struct device_link *link;
  249. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
  250. if (READ_ONCE(link->status) == DL_STATE_SUPPLIER_UNBIND)
  251. continue;
  252. while (refcount_dec_not_one(&link->rpm_active))
  253. pm_runtime_put(link->supplier);
  254. }
  255. }
  256. /**
  257. * __rpm_callback - Run a given runtime PM callback for a given device.
  258. * @cb: Runtime PM callback to run.
  259. * @dev: Device to run the callback for.
  260. */
  261. static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
  262. __releases(&dev->power.lock) __acquires(&dev->power.lock)
  263. {
  264. int retval, idx;
  265. bool use_links = dev->power.links_count > 0;
  266. if (dev->power.irq_safe) {
  267. spin_unlock(&dev->power.lock);
  268. } else {
  269. spin_unlock_irq(&dev->power.lock);
  270. /*
  271. * Resume suppliers if necessary.
  272. *
  273. * The device's runtime PM status cannot change until this
  274. * routine returns, so it is safe to read the status outside of
  275. * the lock.
  276. */
  277. if (use_links && dev->power.runtime_status == RPM_RESUMING) {
  278. idx = device_links_read_lock();
  279. retval = rpm_get_suppliers(dev);
  280. if (retval)
  281. goto fail;
  282. device_links_read_unlock(idx);
  283. }
  284. }
  285. retval = cb(dev);
  286. if (dev->power.irq_safe) {
  287. spin_lock(&dev->power.lock);
  288. } else {
  289. /*
  290. * If the device is suspending and the callback has returned
  291. * success, drop the usage counters of the suppliers that have
  292. * been reference counted on its resume.
  293. *
  294. * Do that if resume fails too.
  295. */
  296. if (use_links
  297. && ((dev->power.runtime_status == RPM_SUSPENDING && !retval)
  298. || (dev->power.runtime_status == RPM_RESUMING && retval))) {
  299. idx = device_links_read_lock();
  300. fail:
  301. rpm_put_suppliers(dev);
  302. device_links_read_unlock(idx);
  303. }
  304. spin_lock_irq(&dev->power.lock);
  305. }
  306. return retval;
  307. }
  308. /**
  309. * rpm_idle - Notify device bus type if the device can be suspended.
  310. * @dev: Device to notify the bus type about.
  311. * @rpmflags: Flag bits.
  312. *
  313. * Check if the device's runtime PM status allows it to be suspended. If
  314. * another idle notification has been started earlier, return immediately. If
  315. * the RPM_ASYNC flag is set then queue an idle-notification request; otherwise
  316. * run the ->runtime_idle() callback directly. If the ->runtime_idle callback
  317. * doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag.
  318. *
  319. * This function must be called under dev->power.lock with interrupts disabled.
  320. */
  321. static int rpm_idle(struct device *dev, int rpmflags)
  322. {
  323. int (*callback)(struct device *);
  324. int retval;
  325. trace_rpm_idle_rcuidle(dev, rpmflags);
  326. retval = rpm_check_suspend_allowed(dev);
  327. if (retval < 0)
  328. ; /* Conditions are wrong. */
  329. /* Idle notifications are allowed only in the RPM_ACTIVE state. */
  330. else if (dev->power.runtime_status != RPM_ACTIVE)
  331. retval = -EAGAIN;
  332. /*
  333. * Any pending request other than an idle notification takes
  334. * precedence over us, except that the timer may be running.
  335. */
  336. else if (dev->power.request_pending &&
  337. dev->power.request > RPM_REQ_IDLE)
  338. retval = -EAGAIN;
  339. /* Act as though RPM_NOWAIT is always set. */
  340. else if (dev->power.idle_notification)
  341. retval = -EINPROGRESS;
  342. if (retval)
  343. goto out;
  344. /* Pending requests need to be canceled. */
  345. dev->power.request = RPM_REQ_NONE;
  346. if (dev->power.no_callbacks)
  347. goto out;
  348. /* Carry out an asynchronous or a synchronous idle notification. */
  349. if (rpmflags & RPM_ASYNC) {
  350. dev->power.request = RPM_REQ_IDLE;
  351. if (!dev->power.request_pending) {
  352. dev->power.request_pending = true;
  353. queue_work(pm_wq, &dev->power.work);
  354. }
  355. trace_rpm_return_int_rcuidle(dev, _THIS_IP_, 0);
  356. return 0;
  357. }
  358. dev->power.idle_notification = true;
  359. callback = RPM_GET_CALLBACK(dev, runtime_idle);
  360. if (callback)
  361. retval = __rpm_callback(callback, dev);
  362. dev->power.idle_notification = false;
  363. wake_up_all(&dev->power.wait_queue);
  364. out:
  365. trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
  366. return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);
  367. }
  368. /**
  369. * rpm_callback - Run a given runtime PM callback for a given device.
  370. * @cb: Runtime PM callback to run.
  371. * @dev: Device to run the callback for.
  372. */
  373. static int rpm_callback(int (*cb)(struct device *), struct device *dev)
  374. {
  375. int retval;
  376. if (!cb)
  377. return -ENOSYS;
  378. if (dev->power.memalloc_noio) {
  379. unsigned int noio_flag;
  380. /*
  381. * Deadlock might be caused if memory allocation with
  382. * GFP_KERNEL happens inside runtime_suspend and
  383. * runtime_resume callbacks of one block device's
  384. * ancestor or the block device itself. Network
  385. * device might be thought as part of iSCSI block
  386. * device, so network device and its ancestor should
  387. * be marked as memalloc_noio too.
  388. */
  389. noio_flag = memalloc_noio_save();
  390. retval = __rpm_callback(cb, dev);
  391. memalloc_noio_restore(noio_flag);
  392. } else {
  393. retval = __rpm_callback(cb, dev);
  394. }
  395. dev->power.runtime_error = retval;
  396. return retval != -EACCES ? retval : -EIO;
  397. }
  398. /**
  399. * rpm_suspend - Carry out runtime suspend of given device.
  400. * @dev: Device to suspend.
  401. * @rpmflags: Flag bits.
  402. *
  403. * Check if the device's runtime PM status allows it to be suspended.
  404. * Cancel a pending idle notification, autosuspend or suspend. If
  405. * another suspend has been started earlier, either return immediately
  406. * or wait for it to finish, depending on the RPM_NOWAIT and RPM_ASYNC
  407. * flags. If the RPM_ASYNC flag is set then queue a suspend request;
  408. * otherwise run the ->runtime_suspend() callback directly. When
  409. * ->runtime_suspend succeeded, if a deferred resume was requested while
  410. * the callback was running then carry it out, otherwise send an idle
  411. * notification for its parent (if the suspend succeeded and both
  412. * ignore_children of parent->power and irq_safe of dev->power are not set).
  413. * If ->runtime_suspend failed with -EAGAIN or -EBUSY, and if the RPM_AUTO
  414. * flag is set and the next autosuspend-delay expiration time is in the
  415. * future, schedule another autosuspend attempt.
  416. *
  417. * This function must be called under dev->power.lock with interrupts disabled.
  418. */
  419. static int rpm_suspend(struct device *dev, int rpmflags)
  420. __releases(&dev->power.lock) __acquires(&dev->power.lock)
  421. {
  422. int (*callback)(struct device *);
  423. struct device *parent = NULL;
  424. int retval;
  425. trace_rpm_suspend_rcuidle(dev, rpmflags);
  426. repeat:
  427. retval = rpm_check_suspend_allowed(dev);
  428. if (retval < 0)
  429. ; /* Conditions are wrong. */
  430. /* Synchronous suspends are not allowed in the RPM_RESUMING state. */
  431. else if (dev->power.runtime_status == RPM_RESUMING &&
  432. !(rpmflags & RPM_ASYNC))
  433. retval = -EAGAIN;
  434. if (retval)
  435. goto out;
  436. /* If the autosuspend_delay time hasn't expired yet, reschedule. */
  437. if ((rpmflags & RPM_AUTO)
  438. && dev->power.runtime_status != RPM_SUSPENDING) {
  439. unsigned long expires = pm_runtime_autosuspend_expiration(dev);
  440. if (expires != 0) {
  441. /* Pending requests need to be canceled. */
  442. dev->power.request = RPM_REQ_NONE;
  443. /*
  444. * Optimization: If the timer is already running and is
  445. * set to expire at or before the autosuspend delay,
  446. * avoid the overhead of resetting it. Just let it
  447. * expire; pm_suspend_timer_fn() will take care of the
  448. * rest.
  449. */
  450. if (!(dev->power.timer_expires && time_before_eq(
  451. dev->power.timer_expires, expires))) {
  452. dev->power.timer_expires = expires;
  453. mod_timer(&dev->power.suspend_timer, expires);
  454. }
  455. dev->power.timer_autosuspends = 1;
  456. goto out;
  457. }
  458. }
  459. /* Other scheduled or pending requests need to be canceled. */
  460. pm_runtime_cancel_pending(dev);
  461. if (dev->power.runtime_status == RPM_SUSPENDING) {
  462. DEFINE_WAIT(wait);
  463. if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
  464. retval = -EINPROGRESS;
  465. goto out;
  466. }
  467. if (dev->power.irq_safe) {
  468. spin_unlock(&dev->power.lock);
  469. cpu_relax();
  470. spin_lock(&dev->power.lock);
  471. goto repeat;
  472. }
  473. /* Wait for the other suspend running in parallel with us. */
  474. for (;;) {
  475. prepare_to_wait(&dev->power.wait_queue, &wait,
  476. TASK_UNINTERRUPTIBLE);
  477. if (dev->power.runtime_status != RPM_SUSPENDING)
  478. break;
  479. spin_unlock_irq(&dev->power.lock);
  480. schedule();
  481. spin_lock_irq(&dev->power.lock);
  482. }
  483. finish_wait(&dev->power.wait_queue, &wait);
  484. goto repeat;
  485. }
  486. if (dev->power.no_callbacks)
  487. goto no_callback; /* Assume success. */
  488. /* Carry out an asynchronous or a synchronous suspend. */
  489. if (rpmflags & RPM_ASYNC) {
  490. dev->power.request = (rpmflags & RPM_AUTO) ?
  491. RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND;
  492. if (!dev->power.request_pending) {
  493. dev->power.request_pending = true;
  494. queue_work(pm_wq, &dev->power.work);
  495. }
  496. goto out;
  497. }
  498. __update_runtime_status(dev, RPM_SUSPENDING);
  499. callback = RPM_GET_CALLBACK(dev, runtime_suspend);
  500. dev_pm_enable_wake_irq_check(dev, true);
  501. retval = rpm_callback(callback, dev);
  502. if (retval)
  503. goto fail;
  504. no_callback:
  505. __update_runtime_status(dev, RPM_SUSPENDED);
  506. pm_runtime_deactivate_timer(dev);
  507. if (dev->parent) {
  508. parent = dev->parent;
  509. atomic_add_unless(&parent->power.child_count, -1, 0);
  510. }
  511. wake_up_all(&dev->power.wait_queue);
  512. if (dev->power.deferred_resume) {
  513. dev->power.deferred_resume = false;
  514. rpm_resume(dev, 0);
  515. retval = -EAGAIN;
  516. goto out;
  517. }
  518. /* Maybe the parent is now able to suspend. */
  519. if (parent && !parent->power.ignore_children && !dev->power.irq_safe) {
  520. spin_unlock(&dev->power.lock);
  521. spin_lock(&parent->power.lock);
  522. rpm_idle(parent, RPM_ASYNC);
  523. spin_unlock(&parent->power.lock);
  524. spin_lock(&dev->power.lock);
  525. }
  526. out:
  527. trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
  528. return retval;
  529. fail:
  530. dev_pm_disable_wake_irq_check(dev);
  531. __update_runtime_status(dev, RPM_ACTIVE);
  532. dev->power.deferred_resume = false;
  533. wake_up_all(&dev->power.wait_queue);
  534. if (retval == -EAGAIN || retval == -EBUSY) {
  535. dev->power.runtime_error = 0;
  536. /*
  537. * If the callback routine failed an autosuspend, and
  538. * if the last_busy time has been updated so that there
  539. * is a new autosuspend expiration time, automatically
  540. * reschedule another autosuspend.
  541. */
  542. if ((rpmflags & RPM_AUTO) &&
  543. pm_runtime_autosuspend_expiration(dev) != 0)
  544. goto repeat;
  545. } else {
  546. pm_runtime_cancel_pending(dev);
  547. }
  548. goto out;
  549. }
  550. /**
  551. * rpm_resume - Carry out runtime resume of given device.
  552. * @dev: Device to resume.
  553. * @rpmflags: Flag bits.
  554. *
  555. * Check if the device's runtime PM status allows it to be resumed. Cancel
  556. * any scheduled or pending requests. If another resume has been started
  557. * earlier, either return immediately or wait for it to finish, depending on the
  558. * RPM_NOWAIT and RPM_ASYNC flags. Similarly, if there's a suspend running in
  559. * parallel with this function, either tell the other process to resume after
  560. * suspending (deferred_resume) or wait for it to finish. If the RPM_ASYNC
  561. * flag is set then queue a resume request; otherwise run the
  562. * ->runtime_resume() callback directly. Queue an idle notification for the
  563. * device if the resume succeeded.
  564. *
  565. * This function must be called under dev->power.lock with interrupts disabled.
  566. */
  567. static int rpm_resume(struct device *dev, int rpmflags)
  568. __releases(&dev->power.lock) __acquires(&dev->power.lock)
  569. {
  570. int (*callback)(struct device *);
  571. struct device *parent = NULL;
  572. int retval = 0;
  573. trace_rpm_resume_rcuidle(dev, rpmflags);
  574. repeat:
  575. if (dev->power.runtime_error)
  576. retval = -EINVAL;
  577. else if (dev->power.disable_depth == 1 && dev->power.is_suspended
  578. && dev->power.runtime_status == RPM_ACTIVE)
  579. retval = 1;
  580. else if (dev->power.disable_depth > 0)
  581. retval = -EACCES;
  582. if (retval)
  583. goto out;
  584. /*
  585. * Other scheduled or pending requests need to be canceled. Small
  586. * optimization: If an autosuspend timer is running, leave it running
  587. * rather than cancelling it now only to restart it again in the near
  588. * future.
  589. */
  590. dev->power.request = RPM_REQ_NONE;
  591. if (!dev->power.timer_autosuspends)
  592. pm_runtime_deactivate_timer(dev);
  593. if (dev->power.runtime_status == RPM_ACTIVE) {
  594. retval = 1;
  595. goto out;
  596. }
  597. if (dev->power.runtime_status == RPM_RESUMING
  598. || dev->power.runtime_status == RPM_SUSPENDING) {
  599. DEFINE_WAIT(wait);
  600. if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
  601. if (dev->power.runtime_status == RPM_SUSPENDING)
  602. dev->power.deferred_resume = true;
  603. else
  604. retval = -EINPROGRESS;
  605. goto out;
  606. }
  607. if (dev->power.irq_safe) {
  608. spin_unlock(&dev->power.lock);
  609. cpu_relax();
  610. spin_lock(&dev->power.lock);
  611. goto repeat;
  612. }
  613. /* Wait for the operation carried out in parallel with us. */
  614. for (;;) {
  615. prepare_to_wait(&dev->power.wait_queue, &wait,
  616. TASK_UNINTERRUPTIBLE);
  617. if (dev->power.runtime_status != RPM_RESUMING
  618. && dev->power.runtime_status != RPM_SUSPENDING)
  619. break;
  620. spin_unlock_irq(&dev->power.lock);
  621. schedule();
  622. spin_lock_irq(&dev->power.lock);
  623. }
  624. finish_wait(&dev->power.wait_queue, &wait);
  625. goto repeat;
  626. }
  627. /*
  628. * See if we can skip waking up the parent. This is safe only if
  629. * power.no_callbacks is set, because otherwise we don't know whether
  630. * the resume will actually succeed.
  631. */
  632. if (dev->power.no_callbacks && !parent && dev->parent) {
  633. spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);
  634. if (dev->parent->power.disable_depth > 0
  635. || dev->parent->power.ignore_children
  636. || dev->parent->power.runtime_status == RPM_ACTIVE) {
  637. atomic_inc(&dev->parent->power.child_count);
  638. spin_unlock(&dev->parent->power.lock);
  639. retval = 1;
  640. goto no_callback; /* Assume success. */
  641. }
  642. spin_unlock(&dev->parent->power.lock);
  643. }
  644. /* Carry out an asynchronous or a synchronous resume. */
  645. if (rpmflags & RPM_ASYNC) {
  646. dev->power.request = RPM_REQ_RESUME;
  647. if (!dev->power.request_pending) {
  648. dev->power.request_pending = true;
  649. queue_work(pm_wq, &dev->power.work);
  650. }
  651. retval = 0;
  652. goto out;
  653. }
  654. if (!parent && dev->parent) {
  655. /*
  656. * Increment the parent's usage counter and resume it if
  657. * necessary. Not needed if dev is irq-safe; then the
  658. * parent is permanently resumed.
  659. */
  660. parent = dev->parent;
  661. if (dev->power.irq_safe)
  662. goto skip_parent;
  663. spin_unlock(&dev->power.lock);
  664. pm_runtime_get_noresume(parent);
  665. spin_lock(&parent->power.lock);
  666. /*
  667. * Resume the parent if it has runtime PM enabled and not been
  668. * set to ignore its children.
  669. */
  670. if (!parent->power.disable_depth
  671. && !parent->power.ignore_children) {
  672. rpm_resume(parent, 0);
  673. if (parent->power.runtime_status != RPM_ACTIVE)
  674. retval = -EBUSY;
  675. }
  676. spin_unlock(&parent->power.lock);
  677. spin_lock(&dev->power.lock);
  678. if (retval)
  679. goto out;
  680. goto repeat;
  681. }
  682. skip_parent:
  683. if (dev->power.no_callbacks)
  684. goto no_callback; /* Assume success. */
  685. __update_runtime_status(dev, RPM_RESUMING);
  686. callback = RPM_GET_CALLBACK(dev, runtime_resume);
  687. dev_pm_disable_wake_irq_check(dev);
  688. retval = rpm_callback(callback, dev);
  689. if (retval) {
  690. __update_runtime_status(dev, RPM_SUSPENDED);
  691. pm_runtime_cancel_pending(dev);
  692. dev_pm_enable_wake_irq_check(dev, false);
  693. } else {
  694. no_callback:
  695. __update_runtime_status(dev, RPM_ACTIVE);
  696. pm_runtime_mark_last_busy(dev);
  697. if (parent)
  698. atomic_inc(&parent->power.child_count);
  699. }
  700. wake_up_all(&dev->power.wait_queue);
  701. if (retval >= 0)
  702. rpm_idle(dev, RPM_ASYNC);
  703. out:
  704. if (parent && !dev->power.irq_safe) {
  705. spin_unlock_irq(&dev->power.lock);
  706. pm_runtime_put(parent);
  707. spin_lock_irq(&dev->power.lock);
  708. }
  709. trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
  710. return retval;
  711. }
  712. /**
  713. * pm_runtime_work - Universal runtime PM work function.
  714. * @work: Work structure used for scheduling the execution of this function.
  715. *
  716. * Use @work to get the device object the work is to be done for, determine what
  717. * is to be done and execute the appropriate runtime PM function.
  718. */
  719. static void pm_runtime_work(struct work_struct *work)
  720. {
  721. struct device *dev = container_of(work, struct device, power.work);
  722. enum rpm_request req;
  723. spin_lock_irq(&dev->power.lock);
  724. if (!dev->power.request_pending)
  725. goto out;
  726. req = dev->power.request;
  727. dev->power.request = RPM_REQ_NONE;
  728. dev->power.request_pending = false;
  729. switch (req) {
  730. case RPM_REQ_NONE:
  731. break;
  732. case RPM_REQ_IDLE:
  733. rpm_idle(dev, RPM_NOWAIT);
  734. break;
  735. case RPM_REQ_SUSPEND:
  736. rpm_suspend(dev, RPM_NOWAIT);
  737. break;
  738. case RPM_REQ_AUTOSUSPEND:
  739. rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);
  740. break;
  741. case RPM_REQ_RESUME:
  742. rpm_resume(dev, RPM_NOWAIT);
  743. break;
  744. }
  745. out:
  746. spin_unlock_irq(&dev->power.lock);
  747. }
  748. /**
  749. * pm_suspend_timer_fn - Timer function for pm_schedule_suspend().
  750. * @data: Device pointer passed by pm_schedule_suspend().
  751. *
  752. * Check if the time is right and queue a suspend request.
  753. */
  754. static void pm_suspend_timer_fn(struct timer_list *t)
  755. {
  756. struct device *dev = from_timer(dev, t, power.suspend_timer);
  757. unsigned long flags;
  758. unsigned long expires;
  759. spin_lock_irqsave(&dev->power.lock, flags);
  760. expires = dev->power.timer_expires;
  761. /* If 'expire' is after 'jiffies' we've been called too early. */
  762. if (expires > 0 && !time_after(expires, jiffies)) {
  763. dev->power.timer_expires = 0;
  764. rpm_suspend(dev, dev->power.timer_autosuspends ?
  765. (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC);
  766. }
  767. spin_unlock_irqrestore(&dev->power.lock, flags);
  768. }
  769. /**
  770. * pm_schedule_suspend - Set up a timer to submit a suspend request in future.
  771. * @dev: Device to suspend.
  772. * @delay: Time to wait before submitting a suspend request, in milliseconds.
  773. */
  774. int pm_schedule_suspend(struct device *dev, unsigned int delay)
  775. {
  776. unsigned long flags;
  777. int retval;
  778. spin_lock_irqsave(&dev->power.lock, flags);
  779. if (!delay) {
  780. retval = rpm_suspend(dev, RPM_ASYNC);
  781. goto out;
  782. }
  783. retval = rpm_check_suspend_allowed(dev);
  784. if (retval)
  785. goto out;
  786. /* Other scheduled or pending requests need to be canceled. */
  787. pm_runtime_cancel_pending(dev);
  788. dev->power.timer_expires = jiffies + msecs_to_jiffies(delay);
  789. dev->power.timer_expires += !dev->power.timer_expires;
  790. dev->power.timer_autosuspends = 0;
  791. mod_timer(&dev->power.suspend_timer, dev->power.timer_expires);
  792. out:
  793. spin_unlock_irqrestore(&dev->power.lock, flags);
  794. return retval;
  795. }
  796. EXPORT_SYMBOL_GPL(pm_schedule_suspend);
  797. /**
  798. * __pm_runtime_idle - Entry point for runtime idle operations.
  799. * @dev: Device to send idle notification for.
  800. * @rpmflags: Flag bits.
  801. *
  802. * If the RPM_GET_PUT flag is set, decrement the device's usage count and
  803. * return immediately if it is larger than zero. Then carry out an idle
  804. * notification, either synchronous or asynchronous.
  805. *
  806. * This routine may be called in atomic context if the RPM_ASYNC flag is set,
  807. * or if pm_runtime_irq_safe() has been called.
  808. */
  809. int __pm_runtime_idle(struct device *dev, int rpmflags)
  810. {
  811. unsigned long flags;
  812. int retval;
  813. if (rpmflags & RPM_GET_PUT) {
  814. if (!atomic_dec_and_test(&dev->power.usage_count))
  815. return 0;
  816. }
  817. might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
  818. spin_lock_irqsave(&dev->power.lock, flags);
  819. retval = rpm_idle(dev, rpmflags);
  820. spin_unlock_irqrestore(&dev->power.lock, flags);
  821. return retval;
  822. }
  823. EXPORT_SYMBOL_GPL(__pm_runtime_idle);
  824. /**
  825. * __pm_runtime_suspend - Entry point for runtime put/suspend operations.
  826. * @dev: Device to suspend.
  827. * @rpmflags: Flag bits.
  828. *
  829. * If the RPM_GET_PUT flag is set, decrement the device's usage count and
  830. * return immediately if it is larger than zero. Then carry out a suspend,
  831. * either synchronous or asynchronous.
  832. *
  833. * This routine may be called in atomic context if the RPM_ASYNC flag is set,
  834. * or if pm_runtime_irq_safe() has been called.
  835. */
  836. int __pm_runtime_suspend(struct device *dev, int rpmflags)
  837. {
  838. unsigned long flags;
  839. int retval;
  840. if (rpmflags & RPM_GET_PUT) {
  841. if (!atomic_dec_and_test(&dev->power.usage_count))
  842. return 0;
  843. }
  844. might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
  845. spin_lock_irqsave(&dev->power.lock, flags);
  846. retval = rpm_suspend(dev, rpmflags);
  847. spin_unlock_irqrestore(&dev->power.lock, flags);
  848. return retval;
  849. }
  850. EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
  851. /**
  852. * __pm_runtime_resume - Entry point for runtime resume operations.
  853. * @dev: Device to resume.
  854. * @rpmflags: Flag bits.
  855. *
  856. * If the RPM_GET_PUT flag is set, increment the device's usage count. Then
  857. * carry out a resume, either synchronous or asynchronous.
  858. *
  859. * This routine may be called in atomic context if the RPM_ASYNC flag is set,
  860. * or if pm_runtime_irq_safe() has been called.
  861. */
  862. int __pm_runtime_resume(struct device *dev, int rpmflags)
  863. {
  864. unsigned long flags;
  865. int retval;
  866. might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&
  867. dev->power.runtime_status != RPM_ACTIVE);
  868. if (rpmflags & RPM_GET_PUT)
  869. atomic_inc(&dev->power.usage_count);
  870. spin_lock_irqsave(&dev->power.lock, flags);
  871. retval = rpm_resume(dev, rpmflags);
  872. spin_unlock_irqrestore(&dev->power.lock, flags);
  873. return retval;
  874. }
  875. EXPORT_SYMBOL_GPL(__pm_runtime_resume);
  876. /**
  877. * pm_runtime_get_if_in_use - Conditionally bump up the device's usage counter.
  878. * @dev: Device to handle.
  879. *
  880. * Return -EINVAL if runtime PM is disabled for the device.
  881. *
  882. * If that's not the case and if the device's runtime PM status is RPM_ACTIVE
  883. * and the runtime PM usage counter is nonzero, increment the counter and
  884. * return 1. Otherwise return 0 without changing the counter.
  885. */
  886. int pm_runtime_get_if_in_use(struct device *dev)
  887. {
  888. unsigned long flags;
  889. int retval;
  890. spin_lock_irqsave(&dev->power.lock, flags);
  891. retval = dev->power.disable_depth > 0 ? -EINVAL :
  892. dev->power.runtime_status == RPM_ACTIVE
  893. && atomic_inc_not_zero(&dev->power.usage_count);
  894. spin_unlock_irqrestore(&dev->power.lock, flags);
  895. return retval;
  896. }
  897. EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use);
  898. /**
  899. * __pm_runtime_set_status - Set runtime PM status of a device.
  900. * @dev: Device to handle.
  901. * @status: New runtime PM status of the device.
  902. *
  903. * If runtime PM of the device is disabled or its power.runtime_error field is
  904. * different from zero, the status may be changed either to RPM_ACTIVE, or to
  905. * RPM_SUSPENDED, as long as that reflects the actual state of the device.
  906. * However, if the device has a parent and the parent is not active, and the
  907. * parent's power.ignore_children flag is unset, the device's status cannot be
  908. * set to RPM_ACTIVE, so -EBUSY is returned in that case.
  909. *
  910. * If successful, __pm_runtime_set_status() clears the power.runtime_error field
  911. * and the device parent's counter of unsuspended children is modified to
  912. * reflect the new status. If the new status is RPM_SUSPENDED, an idle
  913. * notification request for the parent is submitted.
  914. */
  915. int __pm_runtime_set_status(struct device *dev, unsigned int status)
  916. {
  917. struct device *parent = dev->parent;
  918. unsigned long flags;
  919. bool notify_parent = false;
  920. int error = 0;
  921. if (status != RPM_ACTIVE && status != RPM_SUSPENDED)
  922. return -EINVAL;
  923. spin_lock_irqsave(&dev->power.lock, flags);
  924. if (!dev->power.runtime_error && !dev->power.disable_depth) {
  925. error = -EAGAIN;
  926. goto out;
  927. }
  928. if (dev->power.runtime_status == status || !parent)
  929. goto out_set;
  930. if (status == RPM_SUSPENDED) {
  931. atomic_add_unless(&parent->power.child_count, -1, 0);
  932. notify_parent = !parent->power.ignore_children;
  933. } else {
  934. spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);
  935. /*
  936. * It is invalid to put an active child under a parent that is
  937. * not active, has runtime PM enabled and the
  938. * 'power.ignore_children' flag unset.
  939. */
  940. if (!parent->power.disable_depth
  941. && !parent->power.ignore_children
  942. && parent->power.runtime_status != RPM_ACTIVE) {
  943. dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",
  944. dev_name(dev),
  945. dev_name(parent));
  946. error = -EBUSY;
  947. } else if (dev->power.runtime_status == RPM_SUSPENDED) {
  948. atomic_inc(&parent->power.child_count);
  949. }
  950. spin_unlock(&parent->power.lock);
  951. if (error)
  952. goto out;
  953. }
  954. out_set:
  955. __update_runtime_status(dev, status);
  956. dev->power.runtime_error = 0;
  957. out:
  958. spin_unlock_irqrestore(&dev->power.lock, flags);
  959. if (notify_parent)
  960. pm_request_idle(parent);
  961. return error;
  962. }
  963. EXPORT_SYMBOL_GPL(__pm_runtime_set_status);
  964. /**
  965. * __pm_runtime_barrier - Cancel pending requests and wait for completions.
  966. * @dev: Device to handle.
  967. *
  968. * Flush all pending requests for the device from pm_wq and wait for all
  969. * runtime PM operations involving the device in progress to complete.
  970. *
  971. * Should be called under dev->power.lock with interrupts disabled.
  972. */
  973. static void __pm_runtime_barrier(struct device *dev)
  974. {
  975. pm_runtime_deactivate_timer(dev);
  976. if (dev->power.request_pending) {
  977. dev->power.request = RPM_REQ_NONE;
  978. spin_unlock_irq(&dev->power.lock);
  979. cancel_work_sync(&dev->power.work);
  980. spin_lock_irq(&dev->power.lock);
  981. dev->power.request_pending = false;
  982. }
  983. if (dev->power.runtime_status == RPM_SUSPENDING
  984. || dev->power.runtime_status == RPM_RESUMING
  985. || dev->power.idle_notification) {
  986. DEFINE_WAIT(wait);
  987. /* Suspend, wake-up or idle notification in progress. */
  988. for (;;) {
  989. prepare_to_wait(&dev->power.wait_queue, &wait,
  990. TASK_UNINTERRUPTIBLE);
  991. if (dev->power.runtime_status != RPM_SUSPENDING
  992. && dev->power.runtime_status != RPM_RESUMING
  993. && !dev->power.idle_notification)
  994. break;
  995. spin_unlock_irq(&dev->power.lock);
  996. schedule();
  997. spin_lock_irq(&dev->power.lock);
  998. }
  999. finish_wait(&dev->power.wait_queue, &wait);
  1000. }
  1001. }
  1002. /**
  1003. * pm_runtime_barrier - Flush pending requests and wait for completions.
  1004. * @dev: Device to handle.
  1005. *
  1006. * Prevent the device from being suspended by incrementing its usage counter and
  1007. * if there's a pending resume request for the device, wake the device up.
  1008. * Next, make sure that all pending requests for the device have been flushed
  1009. * from pm_wq and wait for all runtime PM operations involving the device in
  1010. * progress to complete.
  1011. *
  1012. * Return value:
  1013. * 1, if there was a resume request pending and the device had to be woken up,
  1014. * 0, otherwise
  1015. */
  1016. int pm_runtime_barrier(struct device *dev)
  1017. {
  1018. int retval = 0;
  1019. pm_runtime_get_noresume(dev);
  1020. spin_lock_irq(&dev->power.lock);
  1021. if (dev->power.request_pending
  1022. && dev->power.request == RPM_REQ_RESUME) {
  1023. rpm_resume(dev, 0);
  1024. retval = 1;
  1025. }
  1026. __pm_runtime_barrier(dev);
  1027. spin_unlock_irq(&dev->power.lock);
  1028. pm_runtime_put_noidle(dev);
  1029. return retval;
  1030. }
  1031. EXPORT_SYMBOL_GPL(pm_runtime_barrier);
  1032. /**
  1033. * __pm_runtime_disable - Disable runtime PM of a device.
  1034. * @dev: Device to handle.
  1035. * @check_resume: If set, check if there's a resume request for the device.
  1036. *
  1037. * Increment power.disable_depth for the device and if it was zero previously,
  1038. * cancel all pending runtime PM requests for the device and wait for all
  1039. * operations in progress to complete. The device can be either active or
  1040. * suspended after its runtime PM has been disabled.
  1041. *
  1042. * If @check_resume is set and there's a resume request pending when
  1043. * __pm_runtime_disable() is called and power.disable_depth is zero, the
  1044. * function will wake up the device before disabling its runtime PM.
  1045. */
  1046. void __pm_runtime_disable(struct device *dev, bool check_resume)
  1047. {
  1048. spin_lock_irq(&dev->power.lock);
  1049. if (dev->power.disable_depth > 0) {
  1050. dev->power.disable_depth++;
  1051. goto out;
  1052. }
  1053. /*
  1054. * Wake up the device if there's a resume request pending, because that
  1055. * means there probably is some I/O to process and disabling runtime PM
  1056. * shouldn't prevent the device from processing the I/O.
  1057. */
  1058. if (check_resume && dev->power.request_pending
  1059. && dev->power.request == RPM_REQ_RESUME) {
  1060. /*
  1061. * Prevent suspends and idle notifications from being carried
  1062. * out after we have woken up the device.
  1063. */
  1064. pm_runtime_get_noresume(dev);
  1065. rpm_resume(dev, 0);
  1066. pm_runtime_put_noidle(dev);
  1067. }
  1068. if (!dev->power.disable_depth++)
  1069. __pm_runtime_barrier(dev);
  1070. out:
  1071. spin_unlock_irq(&dev->power.lock);
  1072. }
  1073. EXPORT_SYMBOL_GPL(__pm_runtime_disable);
  1074. /**
  1075. * pm_runtime_enable - Enable runtime PM of a device.
  1076. * @dev: Device to handle.
  1077. */
  1078. void pm_runtime_enable(struct device *dev)
  1079. {
  1080. unsigned long flags;
  1081. spin_lock_irqsave(&dev->power.lock, flags);
  1082. if (dev->power.disable_depth > 0)
  1083. dev->power.disable_depth--;
  1084. else
  1085. dev_warn(dev, "Unbalanced %s!\n", __func__);
  1086. WARN(!dev->power.disable_depth &&
  1087. dev->power.runtime_status == RPM_SUSPENDED &&
  1088. !dev->power.ignore_children &&
  1089. atomic_read(&dev->power.child_count) > 0,
  1090. "Enabling runtime PM for inactive device (%s) with active children\n",
  1091. dev_name(dev));
  1092. spin_unlock_irqrestore(&dev->power.lock, flags);
  1093. }
  1094. EXPORT_SYMBOL_GPL(pm_runtime_enable);
  1095. /**
  1096. * pm_runtime_forbid - Block runtime PM of a device.
  1097. * @dev: Device to handle.
  1098. *
  1099. * Increase the device's usage count and clear its power.runtime_auto flag,
  1100. * so that it cannot be suspended at run time until pm_runtime_allow() is called
  1101. * for it.
  1102. */
  1103. void pm_runtime_forbid(struct device *dev)
  1104. {
  1105. spin_lock_irq(&dev->power.lock);
  1106. if (!dev->power.runtime_auto)
  1107. goto out;
  1108. dev->power.runtime_auto = false;
  1109. atomic_inc(&dev->power.usage_count);
  1110. rpm_resume(dev, 0);
  1111. out:
  1112. spin_unlock_irq(&dev->power.lock);
  1113. }
  1114. EXPORT_SYMBOL_GPL(pm_runtime_forbid);
  1115. /**
  1116. * pm_runtime_allow - Unblock runtime PM of a device.
  1117. * @dev: Device to handle.
  1118. *
  1119. * Decrease the device's usage count and set its power.runtime_auto flag.
  1120. */
  1121. void pm_runtime_allow(struct device *dev)
  1122. {
  1123. spin_lock_irq(&dev->power.lock);
  1124. if (dev->power.runtime_auto)
  1125. goto out;
  1126. dev->power.runtime_auto = true;
  1127. if (atomic_dec_and_test(&dev->power.usage_count))
  1128. rpm_idle(dev, RPM_AUTO | RPM_ASYNC);
  1129. out:
  1130. spin_unlock_irq(&dev->power.lock);
  1131. }
  1132. EXPORT_SYMBOL_GPL(pm_runtime_allow);
  1133. /**
  1134. * pm_runtime_no_callbacks - Ignore runtime PM callbacks for a device.
  1135. * @dev: Device to handle.
  1136. *
  1137. * Set the power.no_callbacks flag, which tells the PM core that this
  1138. * device is power-managed through its parent and has no runtime PM
  1139. * callbacks of its own. The runtime sysfs attributes will be removed.
  1140. */
  1141. void pm_runtime_no_callbacks(struct device *dev)
  1142. {
  1143. spin_lock_irq(&dev->power.lock);
  1144. dev->power.no_callbacks = 1;
  1145. spin_unlock_irq(&dev->power.lock);
  1146. if (device_is_registered(dev))
  1147. rpm_sysfs_remove(dev);
  1148. }
  1149. EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks);
  1150. /**
  1151. * pm_runtime_irq_safe - Leave interrupts disabled during callbacks.
  1152. * @dev: Device to handle
  1153. *
  1154. * Set the power.irq_safe flag, which tells the PM core that the
  1155. * ->runtime_suspend() and ->runtime_resume() callbacks for this device should
  1156. * always be invoked with the spinlock held and interrupts disabled. It also
  1157. * causes the parent's usage counter to be permanently incremented, preventing
  1158. * the parent from runtime suspending -- otherwise an irq-safe child might have
  1159. * to wait for a non-irq-safe parent.
  1160. */
  1161. void pm_runtime_irq_safe(struct device *dev)
  1162. {
  1163. if (dev->parent)
  1164. pm_runtime_get_sync(dev->parent);
  1165. spin_lock_irq(&dev->power.lock);
  1166. dev->power.irq_safe = 1;
  1167. spin_unlock_irq(&dev->power.lock);
  1168. }
  1169. EXPORT_SYMBOL_GPL(pm_runtime_irq_safe);
  1170. /**
  1171. * update_autosuspend - Handle a change to a device's autosuspend settings.
  1172. * @dev: Device to handle.
  1173. * @old_delay: The former autosuspend_delay value.
  1174. * @old_use: The former use_autosuspend value.
  1175. *
  1176. * Prevent runtime suspend if the new delay is negative and use_autosuspend is
  1177. * set; otherwise allow it. Send an idle notification if suspends are allowed.
  1178. *
  1179. * This function must be called under dev->power.lock with interrupts disabled.
  1180. */
  1181. static void update_autosuspend(struct device *dev, int old_delay, int old_use)
  1182. {
  1183. int delay = dev->power.autosuspend_delay;
  1184. /* Should runtime suspend be prevented now? */
  1185. if (dev->power.use_autosuspend && delay < 0) {
  1186. /* If it used to be allowed then prevent it. */
  1187. if (!old_use || old_delay >= 0) {
  1188. atomic_inc(&dev->power.usage_count);
  1189. rpm_resume(dev, 0);
  1190. }
  1191. }
  1192. /* Runtime suspend should be allowed now. */
  1193. else {
  1194. /* If it used to be prevented then allow it. */
  1195. if (old_use && old_delay < 0)
  1196. atomic_dec(&dev->power.usage_count);
  1197. /* Maybe we can autosuspend now. */
  1198. rpm_idle(dev, RPM_AUTO);
  1199. }
  1200. }
  1201. /**
  1202. * pm_runtime_set_autosuspend_delay - Set a device's autosuspend_delay value.
  1203. * @dev: Device to handle.
  1204. * @delay: Value of the new delay in milliseconds.
  1205. *
  1206. * Set the device's power.autosuspend_delay value. If it changes to negative
  1207. * and the power.use_autosuspend flag is set, prevent runtime suspends. If it
  1208. * changes the other way, allow runtime suspends.
  1209. */
  1210. void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)
  1211. {
  1212. int old_delay, old_use;
  1213. spin_lock_irq(&dev->power.lock);
  1214. old_delay = dev->power.autosuspend_delay;
  1215. old_use = dev->power.use_autosuspend;
  1216. dev->power.autosuspend_delay = delay;
  1217. update_autosuspend(dev, old_delay, old_use);
  1218. spin_unlock_irq(&dev->power.lock);
  1219. }
  1220. EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay);
  1221. /**
  1222. * __pm_runtime_use_autosuspend - Set a device's use_autosuspend flag.
  1223. * @dev: Device to handle.
  1224. * @use: New value for use_autosuspend.
  1225. *
  1226. * Set the device's power.use_autosuspend flag, and allow or prevent runtime
  1227. * suspends as needed.
  1228. */
  1229. void __pm_runtime_use_autosuspend(struct device *dev, bool use)
  1230. {
  1231. int old_delay, old_use;
  1232. spin_lock_irq(&dev->power.lock);
  1233. old_delay = dev->power.autosuspend_delay;
  1234. old_use = dev->power.use_autosuspend;
  1235. dev->power.use_autosuspend = use;
  1236. update_autosuspend(dev, old_delay, old_use);
  1237. spin_unlock_irq(&dev->power.lock);
  1238. }
  1239. EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend);
  1240. /**
  1241. * pm_runtime_init - Initialize runtime PM fields in given device object.
  1242. * @dev: Device object to initialize.
  1243. */
  1244. void pm_runtime_init(struct device *dev)
  1245. {
  1246. dev->power.runtime_status = RPM_SUSPENDED;
  1247. dev->power.idle_notification = false;
  1248. dev->power.disable_depth = 1;
  1249. atomic_set(&dev->power.usage_count, 0);
  1250. dev->power.runtime_error = 0;
  1251. atomic_set(&dev->power.child_count, 0);
  1252. pm_suspend_ignore_children(dev, false);
  1253. dev->power.runtime_auto = true;
  1254. dev->power.request_pending = false;
  1255. dev->power.request = RPM_REQ_NONE;
  1256. dev->power.deferred_resume = false;
  1257. dev->power.accounting_timestamp = jiffies;
  1258. INIT_WORK(&dev->power.work, pm_runtime_work);
  1259. dev->power.timer_expires = 0;
  1260. timer_setup(&dev->power.suspend_timer, pm_suspend_timer_fn, 0);
  1261. init_waitqueue_head(&dev->power.wait_queue);
  1262. }
  1263. /**
  1264. * pm_runtime_reinit - Re-initialize runtime PM fields in given device object.
  1265. * @dev: Device object to re-initialize.
  1266. */
  1267. void pm_runtime_reinit(struct device *dev)
  1268. {
  1269. if (!pm_runtime_enabled(dev)) {
  1270. if (dev->power.runtime_status == RPM_ACTIVE)
  1271. pm_runtime_set_suspended(dev);
  1272. if (dev->power.irq_safe) {
  1273. spin_lock_irq(&dev->power.lock);
  1274. dev->power.irq_safe = 0;
  1275. spin_unlock_irq(&dev->power.lock);
  1276. if (dev->parent)
  1277. pm_runtime_put(dev->parent);
  1278. }
  1279. }
  1280. }
  1281. /**
  1282. * pm_runtime_remove - Prepare for removing a device from device hierarchy.
  1283. * @dev: Device object being removed from device hierarchy.
  1284. */
  1285. void pm_runtime_remove(struct device *dev)
  1286. {
  1287. __pm_runtime_disable(dev, false);
  1288. pm_runtime_reinit(dev);
  1289. }
  1290. /**
  1291. * pm_runtime_clean_up_links - Prepare links to consumers for driver removal.
  1292. * @dev: Device whose driver is going to be removed.
  1293. *
  1294. * Check links from this device to any consumers and if any of them have active
  1295. * runtime PM references to the device, drop the usage counter of the device
  1296. * (as many times as needed).
  1297. *
  1298. * Links with the DL_FLAG_MANAGED flag unset are ignored.
  1299. *
  1300. * Since the device is guaranteed to be runtime-active at the point this is
  1301. * called, nothing else needs to be done here.
  1302. *
  1303. * Moreover, this is called after device_links_busy() has returned 'false', so
  1304. * the status of each link is guaranteed to be DL_STATE_SUPPLIER_UNBIND and
  1305. * therefore rpm_active can't be manipulated concurrently.
  1306. */
  1307. void pm_runtime_clean_up_links(struct device *dev)
  1308. {
  1309. struct device_link *link;
  1310. int idx;
  1311. idx = device_links_read_lock();
  1312. list_for_each_entry_rcu(link, &dev->links.consumers, s_node) {
  1313. if (!(link->flags & DL_FLAG_MANAGED))
  1314. continue;
  1315. while (refcount_dec_not_one(&link->rpm_active))
  1316. pm_runtime_put_noidle(dev);
  1317. }
  1318. device_links_read_unlock(idx);
  1319. }
  1320. /**
  1321. * pm_runtime_get_suppliers - Resume and reference-count supplier devices.
  1322. * @dev: Consumer device.
  1323. */
  1324. void pm_runtime_get_suppliers(struct device *dev)
  1325. {
  1326. struct device_link *link;
  1327. int idx;
  1328. idx = device_links_read_lock();
  1329. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
  1330. if (link->flags & DL_FLAG_PM_RUNTIME) {
  1331. link->supplier_preactivated = true;
  1332. pm_runtime_get_sync(link->supplier);
  1333. refcount_inc(&link->rpm_active);
  1334. }
  1335. device_links_read_unlock(idx);
  1336. }
  1337. /**
  1338. * pm_runtime_put_suppliers - Drop references to supplier devices.
  1339. * @dev: Consumer device.
  1340. */
  1341. void pm_runtime_put_suppliers(struct device *dev)
  1342. {
  1343. struct device_link *link;
  1344. unsigned long flags;
  1345. bool put;
  1346. int idx;
  1347. idx = device_links_read_lock();
  1348. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
  1349. if (link->supplier_preactivated) {
  1350. link->supplier_preactivated = false;
  1351. spin_lock_irqsave(&dev->power.lock, flags);
  1352. put = pm_runtime_status_suspended(dev) &&
  1353. refcount_dec_not_one(&link->rpm_active);
  1354. spin_unlock_irqrestore(&dev->power.lock, flags);
  1355. if (put)
  1356. pm_runtime_put(link->supplier);
  1357. }
  1358. device_links_read_unlock(idx);
  1359. }
  1360. void pm_runtime_new_link(struct device *dev)
  1361. {
  1362. spin_lock_irq(&dev->power.lock);
  1363. dev->power.links_count++;
  1364. spin_unlock_irq(&dev->power.lock);
  1365. }
  1366. void pm_runtime_drop_link(struct device *dev)
  1367. {
  1368. spin_lock_irq(&dev->power.lock);
  1369. WARN_ON(dev->power.links_count == 0);
  1370. dev->power.links_count--;
  1371. spin_unlock_irq(&dev->power.lock);
  1372. }
  1373. static bool pm_runtime_need_not_resume(struct device *dev)
  1374. {
  1375. return atomic_read(&dev->power.usage_count) <= 1 &&
  1376. (atomic_read(&dev->power.child_count) == 0 ||
  1377. dev->power.ignore_children);
  1378. }
  1379. /**
  1380. * pm_runtime_force_suspend - Force a device into suspend state if needed.
  1381. * @dev: Device to suspend.
  1382. *
  1383. * Disable runtime PM so we safely can check the device's runtime PM status and
  1384. * if it is active, invoke its ->runtime_suspend callback to suspend it and
  1385. * change its runtime PM status field to RPM_SUSPENDED. Also, if the device's
  1386. * usage and children counters don't indicate that the device was in use before
  1387. * the system-wide transition under way, decrement its parent's children counter
  1388. * (if there is a parent). Keep runtime PM disabled to preserve the state
  1389. * unless we encounter errors.
  1390. *
  1391. * Typically this function may be invoked from a system suspend callback to make
  1392. * sure the device is put into low power state and it should only be used during
  1393. * system-wide PM transitions to sleep states. It assumes that the analogous
  1394. * pm_runtime_force_resume() will be used to resume the device.
  1395. */
  1396. int pm_runtime_force_suspend(struct device *dev)
  1397. {
  1398. int (*callback)(struct device *);
  1399. int ret;
  1400. pm_runtime_disable(dev);
  1401. if (pm_runtime_status_suspended(dev))
  1402. return 0;
  1403. callback = RPM_GET_CALLBACK(dev, runtime_suspend);
  1404. ret = callback ? callback(dev) : 0;
  1405. if (ret)
  1406. goto err;
  1407. /*
  1408. * If the device can stay in suspend after the system-wide transition
  1409. * to the working state that will follow, drop the children counter of
  1410. * its parent, but set its status to RPM_SUSPENDED anyway in case this
  1411. * function will be called again for it in the meantime.
  1412. */
  1413. if (pm_runtime_need_not_resume(dev))
  1414. pm_runtime_set_suspended(dev);
  1415. else
  1416. __update_runtime_status(dev, RPM_SUSPENDED);
  1417. return 0;
  1418. err:
  1419. pm_runtime_enable(dev);
  1420. return ret;
  1421. }
  1422. EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
  1423. /**
  1424. * pm_runtime_force_resume - Force a device into resume state if needed.
  1425. * @dev: Device to resume.
  1426. *
  1427. * Prior invoking this function we expect the user to have brought the device
  1428. * into low power state by a call to pm_runtime_force_suspend(). Here we reverse
  1429. * those actions and bring the device into full power, if it is expected to be
  1430. * used on system resume. In the other case, we defer the resume to be managed
  1431. * via runtime PM.
  1432. *
  1433. * Typically this function may be invoked from a system resume callback.
  1434. */
  1435. int pm_runtime_force_resume(struct device *dev)
  1436. {
  1437. int (*callback)(struct device *);
  1438. int ret = 0;
  1439. if (!pm_runtime_status_suspended(dev) || pm_runtime_need_not_resume(dev))
  1440. goto out;
  1441. /*
  1442. * The value of the parent's children counter is correct already, so
  1443. * just update the status of the device.
  1444. */
  1445. __update_runtime_status(dev, RPM_ACTIVE);
  1446. callback = RPM_GET_CALLBACK(dev, runtime_resume);
  1447. ret = callback ? callback(dev) : 0;
  1448. if (ret) {
  1449. pm_runtime_set_suspended(dev);
  1450. goto out;
  1451. }
  1452. pm_runtime_mark_last_busy(dev);
  1453. out:
  1454. pm_runtime_enable(dev);
  1455. return ret;
  1456. }
  1457. EXPORT_SYMBOL_GPL(pm_runtime_force_resume);