libahci.c 68 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625
  1. /*
  2. * libahci.c - Common AHCI SATA low-level routines
  3. *
  4. * Maintained by: Tejun Heo <tj@kernel.org>
  5. * Please ALWAYS copy linux-ide@vger.kernel.org
  6. * on emails.
  7. *
  8. * Copyright 2004-2005 Red Hat, Inc.
  9. *
  10. *
  11. * This program is free software; you can redistribute it and/or modify
  12. * it under the terms of the GNU General Public License as published by
  13. * the Free Software Foundation; either version 2, or (at your option)
  14. * any later version.
  15. *
  16. * This program is distributed in the hope that it will be useful,
  17. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  19. * GNU General Public License for more details.
  20. *
  21. * You should have received a copy of the GNU General Public License
  22. * along with this program; see the file COPYING. If not, write to
  23. * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  24. *
  25. *
  26. * libata documentation is available via 'make {ps|pdf}docs',
  27. * as Documentation/driver-api/libata.rst
  28. *
  29. * AHCI hardware documentation:
  30. * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
  31. * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
  32. *
  33. */
  34. #include <linux/kernel.h>
  35. #include <linux/gfp.h>
  36. #include <linux/module.h>
  37. #include <linux/nospec.h>
  38. #include <linux/blkdev.h>
  39. #include <linux/delay.h>
  40. #include <linux/interrupt.h>
  41. #include <linux/dma-mapping.h>
  42. #include <linux/device.h>
  43. #include <scsi/scsi_host.h>
  44. #include <scsi/scsi_cmnd.h>
  45. #include <linux/libata.h>
  46. #include <linux/pci.h>
  47. #include "ahci.h"
  48. #include "libata.h"
  49. static int ahci_skip_host_reset;
  50. int ahci_ignore_sss;
  51. EXPORT_SYMBOL_GPL(ahci_ignore_sss);
  52. module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
  53. MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
  54. module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
  55. MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
  56. static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
  57. unsigned hints);
  58. static ssize_t ahci_led_show(struct ata_port *ap, char *buf);
  59. static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
  60. size_t size);
  61. static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
  62. ssize_t size);
  63. static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
  64. static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
  65. static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
  66. static int ahci_port_start(struct ata_port *ap);
  67. static void ahci_port_stop(struct ata_port *ap);
  68. static enum ata_completion_errors ahci_qc_prep(struct ata_queued_cmd *qc);
  69. static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
  70. static void ahci_freeze(struct ata_port *ap);
  71. static void ahci_thaw(struct ata_port *ap);
  72. static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep);
  73. static void ahci_enable_fbs(struct ata_port *ap);
  74. static void ahci_disable_fbs(struct ata_port *ap);
  75. static void ahci_pmp_attach(struct ata_port *ap);
  76. static void ahci_pmp_detach(struct ata_port *ap);
  77. static int ahci_softreset(struct ata_link *link, unsigned int *class,
  78. unsigned long deadline);
  79. static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
  80. unsigned long deadline);
  81. static int ahci_hardreset(struct ata_link *link, unsigned int *class,
  82. unsigned long deadline);
  83. static void ahci_postreset(struct ata_link *link, unsigned int *class);
  84. static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
  85. static void ahci_dev_config(struct ata_device *dev);
  86. #ifdef CONFIG_PM
  87. static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
  88. #endif
  89. static ssize_t ahci_activity_show(struct ata_device *dev, char *buf);
  90. static ssize_t ahci_activity_store(struct ata_device *dev,
  91. enum sw_activity val);
  92. static void ahci_init_sw_activity(struct ata_link *link);
  93. static ssize_t ahci_show_host_caps(struct device *dev,
  94. struct device_attribute *attr, char *buf);
  95. static ssize_t ahci_show_host_cap2(struct device *dev,
  96. struct device_attribute *attr, char *buf);
  97. static ssize_t ahci_show_host_version(struct device *dev,
  98. struct device_attribute *attr, char *buf);
  99. static ssize_t ahci_show_port_cmd(struct device *dev,
  100. struct device_attribute *attr, char *buf);
  101. static ssize_t ahci_read_em_buffer(struct device *dev,
  102. struct device_attribute *attr, char *buf);
  103. static ssize_t ahci_store_em_buffer(struct device *dev,
  104. struct device_attribute *attr,
  105. const char *buf, size_t size);
  106. static ssize_t ahci_show_em_supported(struct device *dev,
  107. struct device_attribute *attr, char *buf);
  108. static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance);
  109. static DEVICE_ATTR(ahci_host_caps, S_IRUGO, ahci_show_host_caps, NULL);
  110. static DEVICE_ATTR(ahci_host_cap2, S_IRUGO, ahci_show_host_cap2, NULL);
  111. static DEVICE_ATTR(ahci_host_version, S_IRUGO, ahci_show_host_version, NULL);
  112. static DEVICE_ATTR(ahci_port_cmd, S_IRUGO, ahci_show_port_cmd, NULL);
  113. static DEVICE_ATTR(em_buffer, S_IWUSR | S_IRUGO,
  114. ahci_read_em_buffer, ahci_store_em_buffer);
  115. static DEVICE_ATTR(em_message_supported, S_IRUGO, ahci_show_em_supported, NULL);
  116. struct device_attribute *ahci_shost_attrs[] = {
  117. &dev_attr_link_power_management_policy,
  118. &dev_attr_em_message_type,
  119. &dev_attr_em_message,
  120. &dev_attr_ahci_host_caps,
  121. &dev_attr_ahci_host_cap2,
  122. &dev_attr_ahci_host_version,
  123. &dev_attr_ahci_port_cmd,
  124. &dev_attr_em_buffer,
  125. &dev_attr_em_message_supported,
  126. NULL
  127. };
  128. EXPORT_SYMBOL_GPL(ahci_shost_attrs);
  129. struct device_attribute *ahci_sdev_attrs[] = {
  130. &dev_attr_sw_activity,
  131. &dev_attr_unload_heads,
  132. &dev_attr_ncq_prio_enable,
  133. NULL
  134. };
  135. EXPORT_SYMBOL_GPL(ahci_sdev_attrs);
  136. struct ata_port_operations ahci_ops = {
  137. .inherits = &sata_pmp_port_ops,
  138. .qc_defer = ahci_pmp_qc_defer,
  139. .qc_prep = ahci_qc_prep,
  140. .qc_issue = ahci_qc_issue,
  141. .qc_fill_rtf = ahci_qc_fill_rtf,
  142. .freeze = ahci_freeze,
  143. .thaw = ahci_thaw,
  144. .softreset = ahci_softreset,
  145. .hardreset = ahci_hardreset,
  146. .postreset = ahci_postreset,
  147. .pmp_softreset = ahci_softreset,
  148. .error_handler = ahci_error_handler,
  149. .post_internal_cmd = ahci_post_internal_cmd,
  150. .dev_config = ahci_dev_config,
  151. .scr_read = ahci_scr_read,
  152. .scr_write = ahci_scr_write,
  153. .pmp_attach = ahci_pmp_attach,
  154. .pmp_detach = ahci_pmp_detach,
  155. .set_lpm = ahci_set_lpm,
  156. .em_show = ahci_led_show,
  157. .em_store = ahci_led_store,
  158. .sw_activity_show = ahci_activity_show,
  159. .sw_activity_store = ahci_activity_store,
  160. .transmit_led_message = ahci_transmit_led_message,
  161. #ifdef CONFIG_PM
  162. .port_suspend = ahci_port_suspend,
  163. .port_resume = ahci_port_resume,
  164. #endif
  165. .port_start = ahci_port_start,
  166. .port_stop = ahci_port_stop,
  167. };
  168. EXPORT_SYMBOL_GPL(ahci_ops);
  169. struct ata_port_operations ahci_pmp_retry_srst_ops = {
  170. .inherits = &ahci_ops,
  171. .softreset = ahci_pmp_retry_softreset,
  172. };
  173. EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
  174. static bool ahci_em_messages __read_mostly = true;
  175. module_param(ahci_em_messages, bool, 0444);
  176. /* add other LED protocol types when they become supported */
  177. MODULE_PARM_DESC(ahci_em_messages,
  178. "AHCI Enclosure Management Message control (0 = off, 1 = on)");
  179. /* device sleep idle timeout in ms */
  180. static int devslp_idle_timeout __read_mostly = 1000;
  181. module_param(devslp_idle_timeout, int, 0644);
  182. MODULE_PARM_DESC(devslp_idle_timeout, "device sleep idle timeout");
  183. static void ahci_enable_ahci(void __iomem *mmio)
  184. {
  185. int i;
  186. u32 tmp;
  187. /* turn on AHCI_EN */
  188. tmp = readl(mmio + HOST_CTL);
  189. if (tmp & HOST_AHCI_EN)
  190. return;
  191. /* Some controllers need AHCI_EN to be written multiple times.
  192. * Try a few times before giving up.
  193. */
  194. for (i = 0; i < 5; i++) {
  195. tmp |= HOST_AHCI_EN;
  196. writel(tmp, mmio + HOST_CTL);
  197. tmp = readl(mmio + HOST_CTL); /* flush && sanity check */
  198. if (tmp & HOST_AHCI_EN)
  199. return;
  200. msleep(10);
  201. }
  202. WARN_ON(1);
  203. }
  204. /**
  205. * ahci_rpm_get_port - Make sure the port is powered on
  206. * @ap: Port to power on
  207. *
  208. * Whenever there is need to access the AHCI host registers outside of
  209. * normal execution paths, call this function to make sure the host is
  210. * actually powered on.
  211. */
  212. static int ahci_rpm_get_port(struct ata_port *ap)
  213. {
  214. return pm_runtime_get_sync(ap->dev);
  215. }
  216. /**
  217. * ahci_rpm_put_port - Undoes ahci_rpm_get_port()
  218. * @ap: Port to power down
  219. *
  220. * Undoes ahci_rpm_get_port() and possibly powers down the AHCI host
  221. * if it has no more active users.
  222. */
  223. static void ahci_rpm_put_port(struct ata_port *ap)
  224. {
  225. pm_runtime_put(ap->dev);
  226. }
  227. static ssize_t ahci_show_host_caps(struct device *dev,
  228. struct device_attribute *attr, char *buf)
  229. {
  230. struct Scsi_Host *shost = class_to_shost(dev);
  231. struct ata_port *ap = ata_shost_to_port(shost);
  232. struct ahci_host_priv *hpriv = ap->host->private_data;
  233. return sprintf(buf, "%x\n", hpriv->cap);
  234. }
  235. static ssize_t ahci_show_host_cap2(struct device *dev,
  236. struct device_attribute *attr, char *buf)
  237. {
  238. struct Scsi_Host *shost = class_to_shost(dev);
  239. struct ata_port *ap = ata_shost_to_port(shost);
  240. struct ahci_host_priv *hpriv = ap->host->private_data;
  241. return sprintf(buf, "%x\n", hpriv->cap2);
  242. }
  243. static ssize_t ahci_show_host_version(struct device *dev,
  244. struct device_attribute *attr, char *buf)
  245. {
  246. struct Scsi_Host *shost = class_to_shost(dev);
  247. struct ata_port *ap = ata_shost_to_port(shost);
  248. struct ahci_host_priv *hpriv = ap->host->private_data;
  249. return sprintf(buf, "%x\n", hpriv->version);
  250. }
  251. static ssize_t ahci_show_port_cmd(struct device *dev,
  252. struct device_attribute *attr, char *buf)
  253. {
  254. struct Scsi_Host *shost = class_to_shost(dev);
  255. struct ata_port *ap = ata_shost_to_port(shost);
  256. void __iomem *port_mmio = ahci_port_base(ap);
  257. ssize_t ret;
  258. ahci_rpm_get_port(ap);
  259. ret = sprintf(buf, "%x\n", readl(port_mmio + PORT_CMD));
  260. ahci_rpm_put_port(ap);
  261. return ret;
  262. }
  263. static ssize_t ahci_read_em_buffer(struct device *dev,
  264. struct device_attribute *attr, char *buf)
  265. {
  266. struct Scsi_Host *shost = class_to_shost(dev);
  267. struct ata_port *ap = ata_shost_to_port(shost);
  268. struct ahci_host_priv *hpriv = ap->host->private_data;
  269. void __iomem *mmio = hpriv->mmio;
  270. void __iomem *em_mmio = mmio + hpriv->em_loc;
  271. u32 em_ctl, msg;
  272. unsigned long flags;
  273. size_t count;
  274. int i;
  275. ahci_rpm_get_port(ap);
  276. spin_lock_irqsave(ap->lock, flags);
  277. em_ctl = readl(mmio + HOST_EM_CTL);
  278. if (!(ap->flags & ATA_FLAG_EM) || em_ctl & EM_CTL_XMT ||
  279. !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {
  280. spin_unlock_irqrestore(ap->lock, flags);
  281. ahci_rpm_put_port(ap);
  282. return -EINVAL;
  283. }
  284. if (!(em_ctl & EM_CTL_MR)) {
  285. spin_unlock_irqrestore(ap->lock, flags);
  286. ahci_rpm_put_port(ap);
  287. return -EAGAIN;
  288. }
  289. if (!(em_ctl & EM_CTL_SMB))
  290. em_mmio += hpriv->em_buf_sz;
  291. count = hpriv->em_buf_sz;
  292. /* the count should not be larger than PAGE_SIZE */
  293. if (count > PAGE_SIZE) {
  294. if (printk_ratelimit())
  295. ata_port_warn(ap,
  296. "EM read buffer size too large: "
  297. "buffer size %u, page size %lu\n",
  298. hpriv->em_buf_sz, PAGE_SIZE);
  299. count = PAGE_SIZE;
  300. }
  301. for (i = 0; i < count; i += 4) {
  302. msg = readl(em_mmio + i);
  303. buf[i] = msg & 0xff;
  304. buf[i + 1] = (msg >> 8) & 0xff;
  305. buf[i + 2] = (msg >> 16) & 0xff;
  306. buf[i + 3] = (msg >> 24) & 0xff;
  307. }
  308. spin_unlock_irqrestore(ap->lock, flags);
  309. ahci_rpm_put_port(ap);
  310. return i;
  311. }
  312. static ssize_t ahci_store_em_buffer(struct device *dev,
  313. struct device_attribute *attr,
  314. const char *buf, size_t size)
  315. {
  316. struct Scsi_Host *shost = class_to_shost(dev);
  317. struct ata_port *ap = ata_shost_to_port(shost);
  318. struct ahci_host_priv *hpriv = ap->host->private_data;
  319. void __iomem *mmio = hpriv->mmio;
  320. void __iomem *em_mmio = mmio + hpriv->em_loc;
  321. const unsigned char *msg_buf = buf;
  322. u32 em_ctl, msg;
  323. unsigned long flags;
  324. int i;
  325. /* check size validity */
  326. if (!(ap->flags & ATA_FLAG_EM) ||
  327. !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) ||
  328. size % 4 || size > hpriv->em_buf_sz)
  329. return -EINVAL;
  330. ahci_rpm_get_port(ap);
  331. spin_lock_irqsave(ap->lock, flags);
  332. em_ctl = readl(mmio + HOST_EM_CTL);
  333. if (em_ctl & EM_CTL_TM) {
  334. spin_unlock_irqrestore(ap->lock, flags);
  335. ahci_rpm_put_port(ap);
  336. return -EBUSY;
  337. }
  338. for (i = 0; i < size; i += 4) {
  339. msg = msg_buf[i] | msg_buf[i + 1] << 8 |
  340. msg_buf[i + 2] << 16 | msg_buf[i + 3] << 24;
  341. writel(msg, em_mmio + i);
  342. }
  343. writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
  344. spin_unlock_irqrestore(ap->lock, flags);
  345. ahci_rpm_put_port(ap);
  346. return size;
  347. }
  348. static ssize_t ahci_show_em_supported(struct device *dev,
  349. struct device_attribute *attr, char *buf)
  350. {
  351. struct Scsi_Host *shost = class_to_shost(dev);
  352. struct ata_port *ap = ata_shost_to_port(shost);
  353. struct ahci_host_priv *hpriv = ap->host->private_data;
  354. void __iomem *mmio = hpriv->mmio;
  355. u32 em_ctl;
  356. ahci_rpm_get_port(ap);
  357. em_ctl = readl(mmio + HOST_EM_CTL);
  358. ahci_rpm_put_port(ap);
  359. return sprintf(buf, "%s%s%s%s\n",
  360. em_ctl & EM_CTL_LED ? "led " : "",
  361. em_ctl & EM_CTL_SAFTE ? "saf-te " : "",
  362. em_ctl & EM_CTL_SES ? "ses-2 " : "",
  363. em_ctl & EM_CTL_SGPIO ? "sgpio " : "");
  364. }
  365. /**
  366. * ahci_save_initial_config - Save and fixup initial config values
  367. * @dev: target AHCI device
  368. * @hpriv: host private area to store config values
  369. *
  370. * Some registers containing configuration info might be setup by
  371. * BIOS and might be cleared on reset. This function saves the
  372. * initial values of those registers into @hpriv such that they
  373. * can be restored after controller reset.
  374. *
  375. * If inconsistent, config values are fixed up by this function.
  376. *
  377. * If it is not set already this function sets hpriv->start_engine to
  378. * ahci_start_engine.
  379. *
  380. * LOCKING:
  381. * None.
  382. */
  383. void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv)
  384. {
  385. void __iomem *mmio = hpriv->mmio;
  386. u32 cap, cap2, vers, port_map;
  387. int i;
  388. /* make sure AHCI mode is enabled before accessing CAP */
  389. ahci_enable_ahci(mmio);
  390. /* Values prefixed with saved_ are written back to host after
  391. * reset. Values without are used for driver operation.
  392. */
  393. hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
  394. hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
  395. /* CAP2 register is only defined for AHCI 1.2 and later */
  396. vers = readl(mmio + HOST_VERSION);
  397. if ((vers >> 16) > 1 ||
  398. ((vers >> 16) == 1 && (vers & 0xFFFF) >= 0x200))
  399. hpriv->saved_cap2 = cap2 = readl(mmio + HOST_CAP2);
  400. else
  401. hpriv->saved_cap2 = cap2 = 0;
  402. /* some chips have errata preventing 64bit use */
  403. if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
  404. dev_info(dev, "controller can't do 64bit DMA, forcing 32bit\n");
  405. cap &= ~HOST_CAP_64;
  406. }
  407. if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
  408. dev_info(dev, "controller can't do NCQ, turning off CAP_NCQ\n");
  409. cap &= ~HOST_CAP_NCQ;
  410. }
  411. if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
  412. dev_info(dev, "controller can do NCQ, turning on CAP_NCQ\n");
  413. cap |= HOST_CAP_NCQ;
  414. }
  415. if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
  416. dev_info(dev, "controller can't do PMP, turning off CAP_PMP\n");
  417. cap &= ~HOST_CAP_PMP;
  418. }
  419. if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {
  420. dev_info(dev,
  421. "controller can't do SNTF, turning off CAP_SNTF\n");
  422. cap &= ~HOST_CAP_SNTF;
  423. }
  424. if ((cap2 & HOST_CAP2_SDS) && (hpriv->flags & AHCI_HFLAG_NO_DEVSLP)) {
  425. dev_info(dev,
  426. "controller can't do DEVSLP, turning off\n");
  427. cap2 &= ~HOST_CAP2_SDS;
  428. cap2 &= ~HOST_CAP2_SADM;
  429. }
  430. if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {
  431. dev_info(dev, "controller can do FBS, turning on CAP_FBS\n");
  432. cap |= HOST_CAP_FBS;
  433. }
  434. if ((cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_NO_FBS)) {
  435. dev_info(dev, "controller can't do FBS, turning off CAP_FBS\n");
  436. cap &= ~HOST_CAP_FBS;
  437. }
  438. if (!(cap & HOST_CAP_ALPM) && (hpriv->flags & AHCI_HFLAG_YES_ALPM)) {
  439. dev_info(dev, "controller can do ALPM, turning on CAP_ALPM\n");
  440. cap |= HOST_CAP_ALPM;
  441. }
  442. if (hpriv->force_port_map && port_map != hpriv->force_port_map) {
  443. dev_info(dev, "forcing port_map 0x%x -> 0x%x\n",
  444. port_map, hpriv->force_port_map);
  445. port_map = hpriv->force_port_map;
  446. hpriv->saved_port_map = port_map;
  447. }
  448. if (hpriv->mask_port_map) {
  449. dev_warn(dev, "masking port_map 0x%x -> 0x%x\n",
  450. port_map,
  451. port_map & hpriv->mask_port_map);
  452. port_map &= hpriv->mask_port_map;
  453. }
  454. /* cross check port_map and cap.n_ports */
  455. if (port_map) {
  456. int map_ports = 0;
  457. for (i = 0; i < AHCI_MAX_PORTS; i++)
  458. if (port_map & (1 << i))
  459. map_ports++;
  460. /* If PI has more ports than n_ports, whine, clear
  461. * port_map and let it be generated from n_ports.
  462. */
  463. if (map_ports > ahci_nr_ports(cap)) {
  464. dev_warn(dev,
  465. "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n",
  466. port_map, ahci_nr_ports(cap));
  467. port_map = 0;
  468. }
  469. }
  470. /* fabricate port_map from cap.nr_ports for < AHCI 1.3 */
  471. if (!port_map && vers < 0x10300) {
  472. port_map = (1 << ahci_nr_ports(cap)) - 1;
  473. dev_warn(dev, "forcing PORTS_IMPL to 0x%x\n", port_map);
  474. /* write the fixed up value to the PI register */
  475. hpriv->saved_port_map = port_map;
  476. }
  477. /* record values to use during operation */
  478. hpriv->cap = cap;
  479. hpriv->cap2 = cap2;
  480. hpriv->version = readl(mmio + HOST_VERSION);
  481. hpriv->port_map = port_map;
  482. if (!hpriv->start_engine)
  483. hpriv->start_engine = ahci_start_engine;
  484. if (!hpriv->stop_engine)
  485. hpriv->stop_engine = ahci_stop_engine;
  486. if (!hpriv->irq_handler)
  487. hpriv->irq_handler = ahci_single_level_irq_intr;
  488. }
  489. EXPORT_SYMBOL_GPL(ahci_save_initial_config);
  490. /**
  491. * ahci_restore_initial_config - Restore initial config
  492. * @host: target ATA host
  493. *
  494. * Restore initial config stored by ahci_save_initial_config().
  495. *
  496. * LOCKING:
  497. * None.
  498. */
  499. static void ahci_restore_initial_config(struct ata_host *host)
  500. {
  501. struct ahci_host_priv *hpriv = host->private_data;
  502. void __iomem *mmio = hpriv->mmio;
  503. writel(hpriv->saved_cap, mmio + HOST_CAP);
  504. if (hpriv->saved_cap2)
  505. writel(hpriv->saved_cap2, mmio + HOST_CAP2);
  506. writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
  507. (void) readl(mmio + HOST_PORTS_IMPL); /* flush */
  508. }
  509. static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
  510. {
  511. static const int offset[] = {
  512. [SCR_STATUS] = PORT_SCR_STAT,
  513. [SCR_CONTROL] = PORT_SCR_CTL,
  514. [SCR_ERROR] = PORT_SCR_ERR,
  515. [SCR_ACTIVE] = PORT_SCR_ACT,
  516. [SCR_NOTIFICATION] = PORT_SCR_NTF,
  517. };
  518. struct ahci_host_priv *hpriv = ap->host->private_data;
  519. if (sc_reg < ARRAY_SIZE(offset) &&
  520. (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
  521. return offset[sc_reg];
  522. return 0;
  523. }
  524. static int ahci_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
  525. {
  526. void __iomem *port_mmio = ahci_port_base(link->ap);
  527. int offset = ahci_scr_offset(link->ap, sc_reg);
  528. if (offset) {
  529. *val = readl(port_mmio + offset);
  530. return 0;
  531. }
  532. return -EINVAL;
  533. }
  534. static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
  535. {
  536. void __iomem *port_mmio = ahci_port_base(link->ap);
  537. int offset = ahci_scr_offset(link->ap, sc_reg);
  538. if (offset) {
  539. writel(val, port_mmio + offset);
  540. return 0;
  541. }
  542. return -EINVAL;
  543. }
  544. void ahci_start_engine(struct ata_port *ap)
  545. {
  546. void __iomem *port_mmio = ahci_port_base(ap);
  547. u32 tmp;
  548. /* start DMA */
  549. tmp = readl(port_mmio + PORT_CMD);
  550. tmp |= PORT_CMD_START;
  551. writel(tmp, port_mmio + PORT_CMD);
  552. readl(port_mmio + PORT_CMD); /* flush */
  553. }
  554. EXPORT_SYMBOL_GPL(ahci_start_engine);
  555. int ahci_stop_engine(struct ata_port *ap)
  556. {
  557. void __iomem *port_mmio = ahci_port_base(ap);
  558. struct ahci_host_priv *hpriv = ap->host->private_data;
  559. u32 tmp;
  560. /*
  561. * On some controllers, stopping a port's DMA engine while the port
  562. * is in ALPM state (partial or slumber) results in failures on
  563. * subsequent DMA engine starts. For those controllers, put the
  564. * port back in active state before stopping its DMA engine.
  565. */
  566. if ((hpriv->flags & AHCI_HFLAG_WAKE_BEFORE_STOP) &&
  567. (ap->link.lpm_policy > ATA_LPM_MAX_POWER) &&
  568. ahci_set_lpm(&ap->link, ATA_LPM_MAX_POWER, ATA_LPM_WAKE_ONLY)) {
  569. dev_err(ap->host->dev, "Failed to wake up port before engine stop\n");
  570. return -EIO;
  571. }
  572. tmp = readl(port_mmio + PORT_CMD);
  573. /* check if the HBA is idle */
  574. if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
  575. return 0;
  576. /*
  577. * Don't try to issue commands but return with ENODEV if the
  578. * AHCI controller not available anymore (e.g. due to PCIe hot
  579. * unplugging). Otherwise a 500ms delay for each port is added.
  580. */
  581. if (tmp == 0xffffffff) {
  582. dev_err(ap->host->dev, "AHCI controller unavailable!\n");
  583. return -ENODEV;
  584. }
  585. /* setting HBA to idle */
  586. tmp &= ~PORT_CMD_START;
  587. writel(tmp, port_mmio + PORT_CMD);
  588. /* wait for engine to stop. This could be as long as 500 msec */
  589. tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
  590. PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
  591. if (tmp & PORT_CMD_LIST_ON)
  592. return -EIO;
  593. return 0;
  594. }
  595. EXPORT_SYMBOL_GPL(ahci_stop_engine);
  596. void ahci_start_fis_rx(struct ata_port *ap)
  597. {
  598. void __iomem *port_mmio = ahci_port_base(ap);
  599. struct ahci_host_priv *hpriv = ap->host->private_data;
  600. struct ahci_port_priv *pp = ap->private_data;
  601. u32 tmp;
  602. /* set FIS registers */
  603. if (hpriv->cap & HOST_CAP_64)
  604. writel((pp->cmd_slot_dma >> 16) >> 16,
  605. port_mmio + PORT_LST_ADDR_HI);
  606. writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
  607. if (hpriv->cap & HOST_CAP_64)
  608. writel((pp->rx_fis_dma >> 16) >> 16,
  609. port_mmio + PORT_FIS_ADDR_HI);
  610. writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
  611. /* enable FIS reception */
  612. tmp = readl(port_mmio + PORT_CMD);
  613. tmp |= PORT_CMD_FIS_RX;
  614. writel(tmp, port_mmio + PORT_CMD);
  615. /* flush */
  616. readl(port_mmio + PORT_CMD);
  617. }
  618. EXPORT_SYMBOL_GPL(ahci_start_fis_rx);
  619. static int ahci_stop_fis_rx(struct ata_port *ap)
  620. {
  621. void __iomem *port_mmio = ahci_port_base(ap);
  622. u32 tmp;
  623. /* disable FIS reception */
  624. tmp = readl(port_mmio + PORT_CMD);
  625. tmp &= ~PORT_CMD_FIS_RX;
  626. writel(tmp, port_mmio + PORT_CMD);
  627. /* wait for completion, spec says 500ms, give it 1000 */
  628. tmp = ata_wait_register(ap, port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
  629. PORT_CMD_FIS_ON, 10, 1000);
  630. if (tmp & PORT_CMD_FIS_ON)
  631. return -EBUSY;
  632. return 0;
  633. }
  634. static void ahci_power_up(struct ata_port *ap)
  635. {
  636. struct ahci_host_priv *hpriv = ap->host->private_data;
  637. void __iomem *port_mmio = ahci_port_base(ap);
  638. u32 cmd;
  639. cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
  640. /* spin up device */
  641. if (hpriv->cap & HOST_CAP_SSS) {
  642. cmd |= PORT_CMD_SPIN_UP;
  643. writel(cmd, port_mmio + PORT_CMD);
  644. }
  645. /* wake up link */
  646. writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
  647. }
  648. static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
  649. unsigned int hints)
  650. {
  651. struct ata_port *ap = link->ap;
  652. struct ahci_host_priv *hpriv = ap->host->private_data;
  653. struct ahci_port_priv *pp = ap->private_data;
  654. void __iomem *port_mmio = ahci_port_base(ap);
  655. if (policy != ATA_LPM_MAX_POWER) {
  656. /* wakeup flag only applies to the max power policy */
  657. hints &= ~ATA_LPM_WAKE_ONLY;
  658. /*
  659. * Disable interrupts on Phy Ready. This keeps us from
  660. * getting woken up due to spurious phy ready
  661. * interrupts.
  662. */
  663. pp->intr_mask &= ~PORT_IRQ_PHYRDY;
  664. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  665. sata_link_scr_lpm(link, policy, false);
  666. }
  667. if (hpriv->cap & HOST_CAP_ALPM) {
  668. u32 cmd = readl(port_mmio + PORT_CMD);
  669. if (policy == ATA_LPM_MAX_POWER || !(hints & ATA_LPM_HIPM)) {
  670. if (!(hints & ATA_LPM_WAKE_ONLY))
  671. cmd &= ~(PORT_CMD_ASP | PORT_CMD_ALPE);
  672. cmd |= PORT_CMD_ICC_ACTIVE;
  673. writel(cmd, port_mmio + PORT_CMD);
  674. readl(port_mmio + PORT_CMD);
  675. /* wait 10ms to be sure we've come out of LPM state */
  676. ata_msleep(ap, 10);
  677. if (hints & ATA_LPM_WAKE_ONLY)
  678. return 0;
  679. } else {
  680. cmd |= PORT_CMD_ALPE;
  681. if (policy == ATA_LPM_MIN_POWER)
  682. cmd |= PORT_CMD_ASP;
  683. else if (policy == ATA_LPM_MIN_POWER_WITH_PARTIAL)
  684. cmd &= ~PORT_CMD_ASP;
  685. /* write out new cmd value */
  686. writel(cmd, port_mmio + PORT_CMD);
  687. }
  688. }
  689. /* set aggressive device sleep */
  690. if ((hpriv->cap2 & HOST_CAP2_SDS) &&
  691. (hpriv->cap2 & HOST_CAP2_SADM) &&
  692. (link->device->flags & ATA_DFLAG_DEVSLP)) {
  693. if (policy == ATA_LPM_MIN_POWER ||
  694. policy == ATA_LPM_MIN_POWER_WITH_PARTIAL)
  695. ahci_set_aggressive_devslp(ap, true);
  696. else
  697. ahci_set_aggressive_devslp(ap, false);
  698. }
  699. if (policy == ATA_LPM_MAX_POWER) {
  700. sata_link_scr_lpm(link, policy, false);
  701. /* turn PHYRDY IRQ back on */
  702. pp->intr_mask |= PORT_IRQ_PHYRDY;
  703. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  704. }
  705. return 0;
  706. }
  707. #ifdef CONFIG_PM
  708. static void ahci_power_down(struct ata_port *ap)
  709. {
  710. struct ahci_host_priv *hpriv = ap->host->private_data;
  711. void __iomem *port_mmio = ahci_port_base(ap);
  712. u32 cmd, scontrol;
  713. if (!(hpriv->cap & HOST_CAP_SSS))
  714. return;
  715. /* put device into listen mode, first set PxSCTL.DET to 0 */
  716. scontrol = readl(port_mmio + PORT_SCR_CTL);
  717. scontrol &= ~0xf;
  718. writel(scontrol, port_mmio + PORT_SCR_CTL);
  719. /* then set PxCMD.SUD to 0 */
  720. cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
  721. cmd &= ~PORT_CMD_SPIN_UP;
  722. writel(cmd, port_mmio + PORT_CMD);
  723. }
  724. #endif
  725. static void ahci_start_port(struct ata_port *ap)
  726. {
  727. struct ahci_host_priv *hpriv = ap->host->private_data;
  728. struct ahci_port_priv *pp = ap->private_data;
  729. struct ata_link *link;
  730. struct ahci_em_priv *emp;
  731. ssize_t rc;
  732. int i;
  733. /* enable FIS reception */
  734. ahci_start_fis_rx(ap);
  735. /* enable DMA */
  736. if (!(hpriv->flags & AHCI_HFLAG_DELAY_ENGINE))
  737. hpriv->start_engine(ap);
  738. /* turn on LEDs */
  739. if (ap->flags & ATA_FLAG_EM) {
  740. ata_for_each_link(link, ap, EDGE) {
  741. emp = &pp->em_priv[link->pmp];
  742. /* EM Transmit bit maybe busy during init */
  743. for (i = 0; i < EM_MAX_RETRY; i++) {
  744. rc = ap->ops->transmit_led_message(ap,
  745. emp->led_state,
  746. 4);
  747. /*
  748. * If busy, give a breather but do not
  749. * release EH ownership by using msleep()
  750. * instead of ata_msleep(). EM Transmit
  751. * bit is busy for the whole host and
  752. * releasing ownership will cause other
  753. * ports to fail the same way.
  754. */
  755. if (rc == -EBUSY)
  756. msleep(1);
  757. else
  758. break;
  759. }
  760. }
  761. }
  762. if (ap->flags & ATA_FLAG_SW_ACTIVITY)
  763. ata_for_each_link(link, ap, EDGE)
  764. ahci_init_sw_activity(link);
  765. }
  766. static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
  767. {
  768. int rc;
  769. struct ahci_host_priv *hpriv = ap->host->private_data;
  770. /* disable DMA */
  771. rc = hpriv->stop_engine(ap);
  772. if (rc) {
  773. *emsg = "failed to stop engine";
  774. return rc;
  775. }
  776. /* disable FIS reception */
  777. rc = ahci_stop_fis_rx(ap);
  778. if (rc) {
  779. *emsg = "failed stop FIS RX";
  780. return rc;
  781. }
  782. return 0;
  783. }
  784. int ahci_reset_controller(struct ata_host *host)
  785. {
  786. struct ahci_host_priv *hpriv = host->private_data;
  787. void __iomem *mmio = hpriv->mmio;
  788. u32 tmp;
  789. /* we must be in AHCI mode, before using anything
  790. * AHCI-specific, such as HOST_RESET.
  791. */
  792. ahci_enable_ahci(mmio);
  793. /* global controller reset */
  794. if (!ahci_skip_host_reset) {
  795. tmp = readl(mmio + HOST_CTL);
  796. if ((tmp & HOST_RESET) == 0) {
  797. writel(tmp | HOST_RESET, mmio + HOST_CTL);
  798. readl(mmio + HOST_CTL); /* flush */
  799. }
  800. /*
  801. * to perform host reset, OS should set HOST_RESET
  802. * and poll until this bit is read to be "0".
  803. * reset must complete within 1 second, or
  804. * the hardware should be considered fried.
  805. */
  806. tmp = ata_wait_register(NULL, mmio + HOST_CTL, HOST_RESET,
  807. HOST_RESET, 10, 1000);
  808. if (tmp & HOST_RESET) {
  809. dev_err(host->dev, "controller reset failed (0x%x)\n",
  810. tmp);
  811. return -EIO;
  812. }
  813. /* turn on AHCI mode */
  814. ahci_enable_ahci(mmio);
  815. /* Some registers might be cleared on reset. Restore
  816. * initial values.
  817. */
  818. if (!(hpriv->flags & AHCI_HFLAG_NO_WRITE_TO_RO))
  819. ahci_restore_initial_config(host);
  820. } else
  821. dev_info(host->dev, "skipping global host reset\n");
  822. return 0;
  823. }
  824. EXPORT_SYMBOL_GPL(ahci_reset_controller);
  825. static void ahci_sw_activity(struct ata_link *link)
  826. {
  827. struct ata_port *ap = link->ap;
  828. struct ahci_port_priv *pp = ap->private_data;
  829. struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
  830. if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
  831. return;
  832. emp->activity++;
  833. if (!timer_pending(&emp->timer))
  834. mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));
  835. }
  836. static void ahci_sw_activity_blink(struct timer_list *t)
  837. {
  838. struct ahci_em_priv *emp = from_timer(emp, t, timer);
  839. struct ata_link *link = emp->link;
  840. struct ata_port *ap = link->ap;
  841. unsigned long led_message = emp->led_state;
  842. u32 activity_led_state;
  843. unsigned long flags;
  844. led_message &= EM_MSG_LED_VALUE;
  845. led_message |= ap->port_no | (link->pmp << 8);
  846. /* check to see if we've had activity. If so,
  847. * toggle state of LED and reset timer. If not,
  848. * turn LED to desired idle state.
  849. */
  850. spin_lock_irqsave(ap->lock, flags);
  851. if (emp->saved_activity != emp->activity) {
  852. emp->saved_activity = emp->activity;
  853. /* get the current LED state */
  854. activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
  855. if (activity_led_state)
  856. activity_led_state = 0;
  857. else
  858. activity_led_state = 1;
  859. /* clear old state */
  860. led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
  861. /* toggle state */
  862. led_message |= (activity_led_state << 16);
  863. mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));
  864. } else {
  865. /* switch to idle */
  866. led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
  867. if (emp->blink_policy == BLINK_OFF)
  868. led_message |= (1 << 16);
  869. }
  870. spin_unlock_irqrestore(ap->lock, flags);
  871. ap->ops->transmit_led_message(ap, led_message, 4);
  872. }
  873. static void ahci_init_sw_activity(struct ata_link *link)
  874. {
  875. struct ata_port *ap = link->ap;
  876. struct ahci_port_priv *pp = ap->private_data;
  877. struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
  878. /* init activity stats, setup timer */
  879. emp->saved_activity = emp->activity = 0;
  880. emp->link = link;
  881. timer_setup(&emp->timer, ahci_sw_activity_blink, 0);
  882. /* check our blink policy and set flag for link if it's enabled */
  883. if (emp->blink_policy)
  884. link->flags |= ATA_LFLAG_SW_ACTIVITY;
  885. }
  886. int ahci_reset_em(struct ata_host *host)
  887. {
  888. struct ahci_host_priv *hpriv = host->private_data;
  889. void __iomem *mmio = hpriv->mmio;
  890. u32 em_ctl;
  891. em_ctl = readl(mmio + HOST_EM_CTL);
  892. if ((em_ctl & EM_CTL_TM) || (em_ctl & EM_CTL_RST))
  893. return -EINVAL;
  894. writel(em_ctl | EM_CTL_RST, mmio + HOST_EM_CTL);
  895. return 0;
  896. }
  897. EXPORT_SYMBOL_GPL(ahci_reset_em);
  898. static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
  899. ssize_t size)
  900. {
  901. struct ahci_host_priv *hpriv = ap->host->private_data;
  902. struct ahci_port_priv *pp = ap->private_data;
  903. void __iomem *mmio = hpriv->mmio;
  904. u32 em_ctl;
  905. u32 message[] = {0, 0};
  906. unsigned long flags;
  907. int pmp;
  908. struct ahci_em_priv *emp;
  909. /* get the slot number from the message */
  910. pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
  911. if (pmp < EM_MAX_SLOTS)
  912. emp = &pp->em_priv[pmp];
  913. else
  914. return -EINVAL;
  915. ahci_rpm_get_port(ap);
  916. spin_lock_irqsave(ap->lock, flags);
  917. /*
  918. * if we are still busy transmitting a previous message,
  919. * do not allow
  920. */
  921. em_ctl = readl(mmio + HOST_EM_CTL);
  922. if (em_ctl & EM_CTL_TM) {
  923. spin_unlock_irqrestore(ap->lock, flags);
  924. ahci_rpm_put_port(ap);
  925. return -EBUSY;
  926. }
  927. if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {
  928. /*
  929. * create message header - this is all zero except for
  930. * the message size, which is 4 bytes.
  931. */
  932. message[0] |= (4 << 8);
  933. /* ignore 0:4 of byte zero, fill in port info yourself */
  934. message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);
  935. /* write message to EM_LOC */
  936. writel(message[0], mmio + hpriv->em_loc);
  937. writel(message[1], mmio + hpriv->em_loc+4);
  938. /*
  939. * tell hardware to transmit the message
  940. */
  941. writel(em_ctl | EM_CTL_TM, mmio + HOST_EM_CTL);
  942. }
  943. /* save off new led state for port/slot */
  944. emp->led_state = state;
  945. spin_unlock_irqrestore(ap->lock, flags);
  946. ahci_rpm_put_port(ap);
  947. return size;
  948. }
  949. static ssize_t ahci_led_show(struct ata_port *ap, char *buf)
  950. {
  951. struct ahci_port_priv *pp = ap->private_data;
  952. struct ata_link *link;
  953. struct ahci_em_priv *emp;
  954. int rc = 0;
  955. ata_for_each_link(link, ap, EDGE) {
  956. emp = &pp->em_priv[link->pmp];
  957. rc += sprintf(buf, "%lx\n", emp->led_state);
  958. }
  959. return rc;
  960. }
  961. static ssize_t ahci_led_store(struct ata_port *ap, const char *buf,
  962. size_t size)
  963. {
  964. unsigned int state;
  965. int pmp;
  966. struct ahci_port_priv *pp = ap->private_data;
  967. struct ahci_em_priv *emp;
  968. if (kstrtouint(buf, 0, &state) < 0)
  969. return -EINVAL;
  970. /* get the slot number from the message */
  971. pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
  972. if (pmp < EM_MAX_SLOTS) {
  973. pmp = array_index_nospec(pmp, EM_MAX_SLOTS);
  974. emp = &pp->em_priv[pmp];
  975. } else {
  976. return -EINVAL;
  977. }
  978. /* mask off the activity bits if we are in sw_activity
  979. * mode, user should turn off sw_activity before setting
  980. * activity led through em_message
  981. */
  982. if (emp->blink_policy)
  983. state &= ~EM_MSG_LED_VALUE_ACTIVITY;
  984. return ap->ops->transmit_led_message(ap, state, size);
  985. }
  986. static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
  987. {
  988. struct ata_link *link = dev->link;
  989. struct ata_port *ap = link->ap;
  990. struct ahci_port_priv *pp = ap->private_data;
  991. struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
  992. u32 port_led_state = emp->led_state;
  993. /* save the desired Activity LED behavior */
  994. if (val == OFF) {
  995. /* clear LFLAG */
  996. link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
  997. /* set the LED to OFF */
  998. port_led_state &= EM_MSG_LED_VALUE_OFF;
  999. port_led_state |= (ap->port_no | (link->pmp << 8));
  1000. ap->ops->transmit_led_message(ap, port_led_state, 4);
  1001. } else {
  1002. link->flags |= ATA_LFLAG_SW_ACTIVITY;
  1003. if (val == BLINK_OFF) {
  1004. /* set LED to ON for idle */
  1005. port_led_state &= EM_MSG_LED_VALUE_OFF;
  1006. port_led_state |= (ap->port_no | (link->pmp << 8));
  1007. port_led_state |= EM_MSG_LED_VALUE_ON; /* check this */
  1008. ap->ops->transmit_led_message(ap, port_led_state, 4);
  1009. }
  1010. }
  1011. emp->blink_policy = val;
  1012. return 0;
  1013. }
  1014. static ssize_t ahci_activity_show(struct ata_device *dev, char *buf)
  1015. {
  1016. struct ata_link *link = dev->link;
  1017. struct ata_port *ap = link->ap;
  1018. struct ahci_port_priv *pp = ap->private_data;
  1019. struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
  1020. /* display the saved value of activity behavior for this
  1021. * disk.
  1022. */
  1023. return sprintf(buf, "%d\n", emp->blink_policy);
  1024. }
  1025. static void ahci_port_init(struct device *dev, struct ata_port *ap,
  1026. int port_no, void __iomem *mmio,
  1027. void __iomem *port_mmio)
  1028. {
  1029. struct ahci_host_priv *hpriv = ap->host->private_data;
  1030. const char *emsg = NULL;
  1031. int rc;
  1032. u32 tmp;
  1033. /* make sure port is not active */
  1034. rc = ahci_deinit_port(ap, &emsg);
  1035. if (rc)
  1036. dev_warn(dev, "%s (%d)\n", emsg, rc);
  1037. /* clear SError */
  1038. tmp = readl(port_mmio + PORT_SCR_ERR);
  1039. VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
  1040. writel(tmp, port_mmio + PORT_SCR_ERR);
  1041. /* clear port IRQ */
  1042. tmp = readl(port_mmio + PORT_IRQ_STAT);
  1043. VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
  1044. if (tmp)
  1045. writel(tmp, port_mmio + PORT_IRQ_STAT);
  1046. writel(1 << port_no, mmio + HOST_IRQ_STAT);
  1047. /* mark esata ports */
  1048. tmp = readl(port_mmio + PORT_CMD);
  1049. if ((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS))
  1050. ap->pflags |= ATA_PFLAG_EXTERNAL;
  1051. }
  1052. void ahci_init_controller(struct ata_host *host)
  1053. {
  1054. struct ahci_host_priv *hpriv = host->private_data;
  1055. void __iomem *mmio = hpriv->mmio;
  1056. int i;
  1057. void __iomem *port_mmio;
  1058. u32 tmp;
  1059. for (i = 0; i < host->n_ports; i++) {
  1060. struct ata_port *ap = host->ports[i];
  1061. port_mmio = ahci_port_base(ap);
  1062. if (ata_port_is_dummy(ap))
  1063. continue;
  1064. ahci_port_init(host->dev, ap, i, mmio, port_mmio);
  1065. }
  1066. tmp = readl(mmio + HOST_CTL);
  1067. VPRINTK("HOST_CTL 0x%x\n", tmp);
  1068. writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
  1069. tmp = readl(mmio + HOST_CTL);
  1070. VPRINTK("HOST_CTL 0x%x\n", tmp);
  1071. }
  1072. EXPORT_SYMBOL_GPL(ahci_init_controller);
  1073. static void ahci_dev_config(struct ata_device *dev)
  1074. {
  1075. struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
  1076. if (hpriv->flags & AHCI_HFLAG_SECT255) {
  1077. dev->max_sectors = 255;
  1078. ata_dev_info(dev,
  1079. "SB600 AHCI: limiting to 255 sectors per cmd\n");
  1080. }
  1081. }
  1082. unsigned int ahci_dev_classify(struct ata_port *ap)
  1083. {
  1084. void __iomem *port_mmio = ahci_port_base(ap);
  1085. struct ata_taskfile tf;
  1086. u32 tmp;
  1087. tmp = readl(port_mmio + PORT_SIG);
  1088. tf.lbah = (tmp >> 24) & 0xff;
  1089. tf.lbam = (tmp >> 16) & 0xff;
  1090. tf.lbal = (tmp >> 8) & 0xff;
  1091. tf.nsect = (tmp) & 0xff;
  1092. return ata_dev_classify(&tf);
  1093. }
  1094. EXPORT_SYMBOL_GPL(ahci_dev_classify);
  1095. void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
  1096. u32 opts)
  1097. {
  1098. dma_addr_t cmd_tbl_dma;
  1099. cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
  1100. pp->cmd_slot[tag].opts = cpu_to_le32(opts);
  1101. pp->cmd_slot[tag].status = 0;
  1102. pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
  1103. pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
  1104. }
  1105. EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot);
  1106. int ahci_kick_engine(struct ata_port *ap)
  1107. {
  1108. void __iomem *port_mmio = ahci_port_base(ap);
  1109. struct ahci_host_priv *hpriv = ap->host->private_data;
  1110. u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
  1111. u32 tmp;
  1112. int busy, rc;
  1113. /* stop engine */
  1114. rc = hpriv->stop_engine(ap);
  1115. if (rc)
  1116. goto out_restart;
  1117. /* need to do CLO?
  1118. * always do CLO if PMP is attached (AHCI-1.3 9.2)
  1119. */
  1120. busy = status & (ATA_BUSY | ATA_DRQ);
  1121. if (!busy && !sata_pmp_attached(ap)) {
  1122. rc = 0;
  1123. goto out_restart;
  1124. }
  1125. if (!(hpriv->cap & HOST_CAP_CLO)) {
  1126. rc = -EOPNOTSUPP;
  1127. goto out_restart;
  1128. }
  1129. /* perform CLO */
  1130. tmp = readl(port_mmio + PORT_CMD);
  1131. tmp |= PORT_CMD_CLO;
  1132. writel(tmp, port_mmio + PORT_CMD);
  1133. rc = 0;
  1134. tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
  1135. PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
  1136. if (tmp & PORT_CMD_CLO)
  1137. rc = -EIO;
  1138. /* restart engine */
  1139. out_restart:
  1140. hpriv->start_engine(ap);
  1141. return rc;
  1142. }
  1143. EXPORT_SYMBOL_GPL(ahci_kick_engine);
  1144. static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
  1145. struct ata_taskfile *tf, int is_cmd, u16 flags,
  1146. unsigned long timeout_msec)
  1147. {
  1148. const u32 cmd_fis_len = 5; /* five dwords */
  1149. struct ahci_port_priv *pp = ap->private_data;
  1150. void __iomem *port_mmio = ahci_port_base(ap);
  1151. u8 *fis = pp->cmd_tbl;
  1152. u32 tmp;
  1153. /* prep the command */
  1154. ata_tf_to_fis(tf, pmp, is_cmd, fis);
  1155. ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
  1156. /* set port value for softreset of Port Multiplier */
  1157. if (pp->fbs_enabled && pp->fbs_last_dev != pmp) {
  1158. tmp = readl(port_mmio + PORT_FBS);
  1159. tmp &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
  1160. tmp |= pmp << PORT_FBS_DEV_OFFSET;
  1161. writel(tmp, port_mmio + PORT_FBS);
  1162. pp->fbs_last_dev = pmp;
  1163. }
  1164. /* issue & wait */
  1165. writel(1, port_mmio + PORT_CMD_ISSUE);
  1166. if (timeout_msec) {
  1167. tmp = ata_wait_register(ap, port_mmio + PORT_CMD_ISSUE,
  1168. 0x1, 0x1, 1, timeout_msec);
  1169. if (tmp & 0x1) {
  1170. ahci_kick_engine(ap);
  1171. return -EBUSY;
  1172. }
  1173. } else
  1174. readl(port_mmio + PORT_CMD_ISSUE); /* flush */
  1175. return 0;
  1176. }
  1177. int ahci_do_softreset(struct ata_link *link, unsigned int *class,
  1178. int pmp, unsigned long deadline,
  1179. int (*check_ready)(struct ata_link *link))
  1180. {
  1181. struct ata_port *ap = link->ap;
  1182. struct ahci_host_priv *hpriv = ap->host->private_data;
  1183. struct ahci_port_priv *pp = ap->private_data;
  1184. const char *reason = NULL;
  1185. unsigned long now, msecs;
  1186. struct ata_taskfile tf;
  1187. bool fbs_disabled = false;
  1188. int rc;
  1189. DPRINTK("ENTER\n");
  1190. /* prepare for SRST (AHCI-1.1 10.4.1) */
  1191. rc = ahci_kick_engine(ap);
  1192. if (rc && rc != -EOPNOTSUPP)
  1193. ata_link_warn(link, "failed to reset engine (errno=%d)\n", rc);
  1194. /*
  1195. * According to AHCI-1.2 9.3.9: if FBS is enable, software shall
  1196. * clear PxFBS.EN to '0' prior to issuing software reset to devices
  1197. * that is attached to port multiplier.
  1198. */
  1199. if (!ata_is_host_link(link) && pp->fbs_enabled) {
  1200. ahci_disable_fbs(ap);
  1201. fbs_disabled = true;
  1202. }
  1203. ata_tf_init(link->device, &tf);
  1204. /* issue the first H2D Register FIS */
  1205. msecs = 0;
  1206. now = jiffies;
  1207. if (time_after(deadline, now))
  1208. msecs = jiffies_to_msecs(deadline - now);
  1209. tf.ctl |= ATA_SRST;
  1210. if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
  1211. AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
  1212. rc = -EIO;
  1213. reason = "1st FIS failed";
  1214. goto fail;
  1215. }
  1216. /* spec says at least 5us, but be generous and sleep for 1ms */
  1217. ata_msleep(ap, 1);
  1218. /* issue the second H2D Register FIS */
  1219. tf.ctl &= ~ATA_SRST;
  1220. ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
  1221. /* wait for link to become ready */
  1222. rc = ata_wait_after_reset(link, deadline, check_ready);
  1223. if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {
  1224. /*
  1225. * Workaround for cases where link online status can't
  1226. * be trusted. Treat device readiness timeout as link
  1227. * offline.
  1228. */
  1229. ata_link_info(link, "device not ready, treating as offline\n");
  1230. *class = ATA_DEV_NONE;
  1231. } else if (rc) {
  1232. /* link occupied, -ENODEV too is an error */
  1233. reason = "device not ready";
  1234. goto fail;
  1235. } else
  1236. *class = ahci_dev_classify(ap);
  1237. /* re-enable FBS if disabled before */
  1238. if (fbs_disabled)
  1239. ahci_enable_fbs(ap);
  1240. DPRINTK("EXIT, class=%u\n", *class);
  1241. return 0;
  1242. fail:
  1243. ata_link_err(link, "softreset failed (%s)\n", reason);
  1244. return rc;
  1245. }
  1246. int ahci_check_ready(struct ata_link *link)
  1247. {
  1248. void __iomem *port_mmio = ahci_port_base(link->ap);
  1249. u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
  1250. return ata_check_ready(status);
  1251. }
  1252. EXPORT_SYMBOL_GPL(ahci_check_ready);
  1253. static int ahci_softreset(struct ata_link *link, unsigned int *class,
  1254. unsigned long deadline)
  1255. {
  1256. int pmp = sata_srst_pmp(link);
  1257. DPRINTK("ENTER\n");
  1258. return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
  1259. }
  1260. EXPORT_SYMBOL_GPL(ahci_do_softreset);
  1261. static int ahci_bad_pmp_check_ready(struct ata_link *link)
  1262. {
  1263. void __iomem *port_mmio = ahci_port_base(link->ap);
  1264. u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
  1265. u32 irq_status = readl(port_mmio + PORT_IRQ_STAT);
  1266. /*
  1267. * There is no need to check TFDATA if BAD PMP is found due to HW bug,
  1268. * which can save timeout delay.
  1269. */
  1270. if (irq_status & PORT_IRQ_BAD_PMP)
  1271. return -EIO;
  1272. return ata_check_ready(status);
  1273. }
  1274. static int ahci_pmp_retry_softreset(struct ata_link *link, unsigned int *class,
  1275. unsigned long deadline)
  1276. {
  1277. struct ata_port *ap = link->ap;
  1278. void __iomem *port_mmio = ahci_port_base(ap);
  1279. int pmp = sata_srst_pmp(link);
  1280. int rc;
  1281. u32 irq_sts;
  1282. DPRINTK("ENTER\n");
  1283. rc = ahci_do_softreset(link, class, pmp, deadline,
  1284. ahci_bad_pmp_check_ready);
  1285. /*
  1286. * Soft reset fails with IPMS set when PMP is enabled but
  1287. * SATA HDD/ODD is connected to SATA port, do soft reset
  1288. * again to port 0.
  1289. */
  1290. if (rc == -EIO) {
  1291. irq_sts = readl(port_mmio + PORT_IRQ_STAT);
  1292. if (irq_sts & PORT_IRQ_BAD_PMP) {
  1293. ata_link_warn(link,
  1294. "applying PMP SRST workaround "
  1295. "and retrying\n");
  1296. rc = ahci_do_softreset(link, class, 0, deadline,
  1297. ahci_check_ready);
  1298. }
  1299. }
  1300. return rc;
  1301. }
  1302. int ahci_do_hardreset(struct ata_link *link, unsigned int *class,
  1303. unsigned long deadline, bool *online)
  1304. {
  1305. const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
  1306. struct ata_port *ap = link->ap;
  1307. struct ahci_port_priv *pp = ap->private_data;
  1308. struct ahci_host_priv *hpriv = ap->host->private_data;
  1309. u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
  1310. struct ata_taskfile tf;
  1311. int rc;
  1312. DPRINTK("ENTER\n");
  1313. hpriv->stop_engine(ap);
  1314. /* clear D2H reception area to properly wait for D2H FIS */
  1315. ata_tf_init(link->device, &tf);
  1316. tf.command = ATA_BUSY;
  1317. ata_tf_to_fis(&tf, 0, 0, d2h_fis);
  1318. rc = sata_link_hardreset(link, timing, deadline, online,
  1319. ahci_check_ready);
  1320. hpriv->start_engine(ap);
  1321. if (*online)
  1322. *class = ahci_dev_classify(ap);
  1323. DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
  1324. return rc;
  1325. }
  1326. EXPORT_SYMBOL_GPL(ahci_do_hardreset);
  1327. static int ahci_hardreset(struct ata_link *link, unsigned int *class,
  1328. unsigned long deadline)
  1329. {
  1330. bool online;
  1331. return ahci_do_hardreset(link, class, deadline, &online);
  1332. }
  1333. static void ahci_postreset(struct ata_link *link, unsigned int *class)
  1334. {
  1335. struct ata_port *ap = link->ap;
  1336. void __iomem *port_mmio = ahci_port_base(ap);
  1337. u32 new_tmp, tmp;
  1338. ata_std_postreset(link, class);
  1339. /* Make sure port's ATAPI bit is set appropriately */
  1340. new_tmp = tmp = readl(port_mmio + PORT_CMD);
  1341. if (*class == ATA_DEV_ATAPI)
  1342. new_tmp |= PORT_CMD_ATAPI;
  1343. else
  1344. new_tmp &= ~PORT_CMD_ATAPI;
  1345. if (new_tmp != tmp) {
  1346. writel(new_tmp, port_mmio + PORT_CMD);
  1347. readl(port_mmio + PORT_CMD); /* flush */
  1348. }
  1349. }
  1350. static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
  1351. {
  1352. struct scatterlist *sg;
  1353. struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
  1354. unsigned int si;
  1355. VPRINTK("ENTER\n");
  1356. /*
  1357. * Next, the S/G list.
  1358. */
  1359. for_each_sg(qc->sg, sg, qc->n_elem, si) {
  1360. dma_addr_t addr = sg_dma_address(sg);
  1361. u32 sg_len = sg_dma_len(sg);
  1362. ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
  1363. ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
  1364. ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
  1365. }
  1366. return si;
  1367. }
  1368. static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
  1369. {
  1370. struct ata_port *ap = qc->ap;
  1371. struct ahci_port_priv *pp = ap->private_data;
  1372. if (!sata_pmp_attached(ap) || pp->fbs_enabled)
  1373. return ata_std_qc_defer(qc);
  1374. else
  1375. return sata_pmp_qc_defer_cmd_switch(qc);
  1376. }
  1377. static enum ata_completion_errors ahci_qc_prep(struct ata_queued_cmd *qc)
  1378. {
  1379. struct ata_port *ap = qc->ap;
  1380. struct ahci_port_priv *pp = ap->private_data;
  1381. int is_atapi = ata_is_atapi(qc->tf.protocol);
  1382. void *cmd_tbl;
  1383. u32 opts;
  1384. const u32 cmd_fis_len = 5; /* five dwords */
  1385. unsigned int n_elem;
  1386. /*
  1387. * Fill in command table information. First, the header,
  1388. * a SATA Register - Host to Device command FIS.
  1389. */
  1390. cmd_tbl = pp->cmd_tbl + qc->hw_tag * AHCI_CMD_TBL_SZ;
  1391. ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
  1392. if (is_atapi) {
  1393. memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
  1394. memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
  1395. }
  1396. n_elem = 0;
  1397. if (qc->flags & ATA_QCFLAG_DMAMAP)
  1398. n_elem = ahci_fill_sg(qc, cmd_tbl);
  1399. /*
  1400. * Fill in command slot information.
  1401. */
  1402. opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
  1403. if (qc->tf.flags & ATA_TFLAG_WRITE)
  1404. opts |= AHCI_CMD_WRITE;
  1405. if (is_atapi)
  1406. opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
  1407. ahci_fill_cmd_slot(pp, qc->hw_tag, opts);
  1408. return AC_ERR_OK;
  1409. }
  1410. static void ahci_fbs_dec_intr(struct ata_port *ap)
  1411. {
  1412. struct ahci_port_priv *pp = ap->private_data;
  1413. void __iomem *port_mmio = ahci_port_base(ap);
  1414. u32 fbs = readl(port_mmio + PORT_FBS);
  1415. int retries = 3;
  1416. DPRINTK("ENTER\n");
  1417. BUG_ON(!pp->fbs_enabled);
  1418. /* time to wait for DEC is not specified by AHCI spec,
  1419. * add a retry loop for safety.
  1420. */
  1421. writel(fbs | PORT_FBS_DEC, port_mmio + PORT_FBS);
  1422. fbs = readl(port_mmio + PORT_FBS);
  1423. while ((fbs & PORT_FBS_DEC) && retries--) {
  1424. udelay(1);
  1425. fbs = readl(port_mmio + PORT_FBS);
  1426. }
  1427. if (fbs & PORT_FBS_DEC)
  1428. dev_err(ap->host->dev, "failed to clear device error\n");
  1429. }
  1430. static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
  1431. {
  1432. struct ahci_host_priv *hpriv = ap->host->private_data;
  1433. struct ahci_port_priv *pp = ap->private_data;
  1434. struct ata_eh_info *host_ehi = &ap->link.eh_info;
  1435. struct ata_link *link = NULL;
  1436. struct ata_queued_cmd *active_qc;
  1437. struct ata_eh_info *active_ehi;
  1438. bool fbs_need_dec = false;
  1439. u32 serror;
  1440. /* determine active link with error */
  1441. if (pp->fbs_enabled) {
  1442. void __iomem *port_mmio = ahci_port_base(ap);
  1443. u32 fbs = readl(port_mmio + PORT_FBS);
  1444. int pmp = fbs >> PORT_FBS_DWE_OFFSET;
  1445. if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links)) {
  1446. link = &ap->pmp_link[pmp];
  1447. fbs_need_dec = true;
  1448. }
  1449. } else
  1450. ata_for_each_link(link, ap, EDGE)
  1451. if (ata_link_active(link))
  1452. break;
  1453. if (!link)
  1454. link = &ap->link;
  1455. active_qc = ata_qc_from_tag(ap, link->active_tag);
  1456. active_ehi = &link->eh_info;
  1457. /* record irq stat */
  1458. ata_ehi_clear_desc(host_ehi);
  1459. ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
  1460. /* AHCI needs SError cleared; otherwise, it might lock up */
  1461. ahci_scr_read(&ap->link, SCR_ERROR, &serror);
  1462. ahci_scr_write(&ap->link, SCR_ERROR, serror);
  1463. host_ehi->serror |= serror;
  1464. /* some controllers set IRQ_IF_ERR on device errors, ignore it */
  1465. if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
  1466. irq_stat &= ~PORT_IRQ_IF_ERR;
  1467. if (irq_stat & PORT_IRQ_TF_ERR) {
  1468. /* If qc is active, charge it; otherwise, the active
  1469. * link. There's no active qc on NCQ errors. It will
  1470. * be determined by EH by reading log page 10h.
  1471. */
  1472. if (active_qc)
  1473. active_qc->err_mask |= AC_ERR_DEV;
  1474. else
  1475. active_ehi->err_mask |= AC_ERR_DEV;
  1476. if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
  1477. host_ehi->serror &= ~SERR_INTERNAL;
  1478. }
  1479. if (irq_stat & PORT_IRQ_UNK_FIS) {
  1480. u32 *unk = pp->rx_fis + RX_FIS_UNK;
  1481. active_ehi->err_mask |= AC_ERR_HSM;
  1482. active_ehi->action |= ATA_EH_RESET;
  1483. ata_ehi_push_desc(active_ehi,
  1484. "unknown FIS %08x %08x %08x %08x" ,
  1485. unk[0], unk[1], unk[2], unk[3]);
  1486. }
  1487. if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
  1488. active_ehi->err_mask |= AC_ERR_HSM;
  1489. active_ehi->action |= ATA_EH_RESET;
  1490. ata_ehi_push_desc(active_ehi, "incorrect PMP");
  1491. }
  1492. if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
  1493. host_ehi->err_mask |= AC_ERR_HOST_BUS;
  1494. host_ehi->action |= ATA_EH_RESET;
  1495. ata_ehi_push_desc(host_ehi, "host bus error");
  1496. }
  1497. if (irq_stat & PORT_IRQ_IF_ERR) {
  1498. if (fbs_need_dec)
  1499. active_ehi->err_mask |= AC_ERR_DEV;
  1500. else {
  1501. host_ehi->err_mask |= AC_ERR_ATA_BUS;
  1502. host_ehi->action |= ATA_EH_RESET;
  1503. }
  1504. ata_ehi_push_desc(host_ehi, "interface fatal error");
  1505. }
  1506. if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
  1507. ata_ehi_hotplugged(host_ehi);
  1508. ata_ehi_push_desc(host_ehi, "%s",
  1509. irq_stat & PORT_IRQ_CONNECT ?
  1510. "connection status changed" : "PHY RDY changed");
  1511. }
  1512. /* okay, let's hand over to EH */
  1513. if (irq_stat & PORT_IRQ_FREEZE)
  1514. ata_port_freeze(ap);
  1515. else if (fbs_need_dec) {
  1516. ata_link_abort(link);
  1517. ahci_fbs_dec_intr(ap);
  1518. } else
  1519. ata_port_abort(ap);
  1520. }
  1521. static void ahci_handle_port_interrupt(struct ata_port *ap,
  1522. void __iomem *port_mmio, u32 status)
  1523. {
  1524. struct ata_eh_info *ehi = &ap->link.eh_info;
  1525. struct ahci_port_priv *pp = ap->private_data;
  1526. struct ahci_host_priv *hpriv = ap->host->private_data;
  1527. int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
  1528. u32 qc_active = 0;
  1529. int rc;
  1530. /* ignore BAD_PMP while resetting */
  1531. if (unlikely(resetting))
  1532. status &= ~PORT_IRQ_BAD_PMP;
  1533. if (sata_lpm_ignore_phy_events(&ap->link)) {
  1534. status &= ~PORT_IRQ_PHYRDY;
  1535. ahci_scr_write(&ap->link, SCR_ERROR, SERR_PHYRDY_CHG);
  1536. }
  1537. if (unlikely(status & PORT_IRQ_ERROR)) {
  1538. ahci_error_intr(ap, status);
  1539. return;
  1540. }
  1541. if (status & PORT_IRQ_SDB_FIS) {
  1542. /* If SNotification is available, leave notification
  1543. * handling to sata_async_notification(). If not,
  1544. * emulate it by snooping SDB FIS RX area.
  1545. *
  1546. * Snooping FIS RX area is probably cheaper than
  1547. * poking SNotification but some constrollers which
  1548. * implement SNotification, ICH9 for example, don't
  1549. * store AN SDB FIS into receive area.
  1550. */
  1551. if (hpriv->cap & HOST_CAP_SNTF)
  1552. sata_async_notification(ap);
  1553. else {
  1554. /* If the 'N' bit in word 0 of the FIS is set,
  1555. * we just received asynchronous notification.
  1556. * Tell libata about it.
  1557. *
  1558. * Lack of SNotification should not appear in
  1559. * ahci 1.2, so the workaround is unnecessary
  1560. * when FBS is enabled.
  1561. */
  1562. if (pp->fbs_enabled)
  1563. WARN_ON_ONCE(1);
  1564. else {
  1565. const __le32 *f = pp->rx_fis + RX_FIS_SDB;
  1566. u32 f0 = le32_to_cpu(f[0]);
  1567. if (f0 & (1 << 15))
  1568. sata_async_notification(ap);
  1569. }
  1570. }
  1571. }
  1572. /* pp->active_link is not reliable once FBS is enabled, both
  1573. * PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
  1574. * NCQ and non-NCQ commands may be in flight at the same time.
  1575. */
  1576. if (pp->fbs_enabled) {
  1577. if (ap->qc_active) {
  1578. qc_active = readl(port_mmio + PORT_SCR_ACT);
  1579. qc_active |= readl(port_mmio + PORT_CMD_ISSUE);
  1580. }
  1581. } else {
  1582. /* pp->active_link is valid iff any command is in flight */
  1583. if (ap->qc_active && pp->active_link->sactive)
  1584. qc_active = readl(port_mmio + PORT_SCR_ACT);
  1585. else
  1586. qc_active = readl(port_mmio + PORT_CMD_ISSUE);
  1587. }
  1588. rc = ata_qc_complete_multiple(ap, qc_active);
  1589. /* while resetting, invalid completions are expected */
  1590. if (unlikely(rc < 0 && !resetting)) {
  1591. ehi->err_mask |= AC_ERR_HSM;
  1592. ehi->action |= ATA_EH_RESET;
  1593. ata_port_freeze(ap);
  1594. }
  1595. }
  1596. static void ahci_port_intr(struct ata_port *ap)
  1597. {
  1598. void __iomem *port_mmio = ahci_port_base(ap);
  1599. u32 status;
  1600. status = readl(port_mmio + PORT_IRQ_STAT);
  1601. writel(status, port_mmio + PORT_IRQ_STAT);
  1602. ahci_handle_port_interrupt(ap, port_mmio, status);
  1603. }
  1604. static irqreturn_t ahci_multi_irqs_intr_hard(int irq, void *dev_instance)
  1605. {
  1606. struct ata_port *ap = dev_instance;
  1607. void __iomem *port_mmio = ahci_port_base(ap);
  1608. u32 status;
  1609. VPRINTK("ENTER\n");
  1610. status = readl(port_mmio + PORT_IRQ_STAT);
  1611. writel(status, port_mmio + PORT_IRQ_STAT);
  1612. spin_lock(ap->lock);
  1613. ahci_handle_port_interrupt(ap, port_mmio, status);
  1614. spin_unlock(ap->lock);
  1615. VPRINTK("EXIT\n");
  1616. return IRQ_HANDLED;
  1617. }
  1618. u32 ahci_handle_port_intr(struct ata_host *host, u32 irq_masked)
  1619. {
  1620. unsigned int i, handled = 0;
  1621. for (i = 0; i < host->n_ports; i++) {
  1622. struct ata_port *ap;
  1623. if (!(irq_masked & (1 << i)))
  1624. continue;
  1625. ap = host->ports[i];
  1626. if (ap) {
  1627. ahci_port_intr(ap);
  1628. VPRINTK("port %u\n", i);
  1629. } else {
  1630. VPRINTK("port %u (no irq)\n", i);
  1631. if (ata_ratelimit())
  1632. dev_warn(host->dev,
  1633. "interrupt on disabled port %u\n", i);
  1634. }
  1635. handled = 1;
  1636. }
  1637. return handled;
  1638. }
  1639. EXPORT_SYMBOL_GPL(ahci_handle_port_intr);
  1640. static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance)
  1641. {
  1642. struct ata_host *host = dev_instance;
  1643. struct ahci_host_priv *hpriv;
  1644. unsigned int rc = 0;
  1645. void __iomem *mmio;
  1646. u32 irq_stat, irq_masked;
  1647. VPRINTK("ENTER\n");
  1648. hpriv = host->private_data;
  1649. mmio = hpriv->mmio;
  1650. /* sigh. 0xffffffff is a valid return from h/w */
  1651. irq_stat = readl(mmio + HOST_IRQ_STAT);
  1652. if (!irq_stat)
  1653. return IRQ_NONE;
  1654. irq_masked = irq_stat & hpriv->port_map;
  1655. spin_lock(&host->lock);
  1656. rc = ahci_handle_port_intr(host, irq_masked);
  1657. /* HOST_IRQ_STAT behaves as level triggered latch meaning that
  1658. * it should be cleared after all the port events are cleared;
  1659. * otherwise, it will raise a spurious interrupt after each
  1660. * valid one. Please read section 10.6.2 of ahci 1.1 for more
  1661. * information.
  1662. *
  1663. * Also, use the unmasked value to clear interrupt as spurious
  1664. * pending event on a dummy port might cause screaming IRQ.
  1665. */
  1666. writel(irq_stat, mmio + HOST_IRQ_STAT);
  1667. spin_unlock(&host->lock);
  1668. VPRINTK("EXIT\n");
  1669. return IRQ_RETVAL(rc);
  1670. }
  1671. unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
  1672. {
  1673. struct ata_port *ap = qc->ap;
  1674. void __iomem *port_mmio = ahci_port_base(ap);
  1675. struct ahci_port_priv *pp = ap->private_data;
  1676. /* Keep track of the currently active link. It will be used
  1677. * in completion path to determine whether NCQ phase is in
  1678. * progress.
  1679. */
  1680. pp->active_link = qc->dev->link;
  1681. if (ata_is_ncq(qc->tf.protocol))
  1682. writel(1 << qc->hw_tag, port_mmio + PORT_SCR_ACT);
  1683. if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
  1684. u32 fbs = readl(port_mmio + PORT_FBS);
  1685. fbs &= ~(PORT_FBS_DEV_MASK | PORT_FBS_DEC);
  1686. fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
  1687. writel(fbs, port_mmio + PORT_FBS);
  1688. pp->fbs_last_dev = qc->dev->link->pmp;
  1689. }
  1690. writel(1 << qc->hw_tag, port_mmio + PORT_CMD_ISSUE);
  1691. ahci_sw_activity(qc->dev->link);
  1692. return 0;
  1693. }
  1694. EXPORT_SYMBOL_GPL(ahci_qc_issue);
  1695. static bool ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
  1696. {
  1697. struct ahci_port_priv *pp = qc->ap->private_data;
  1698. u8 *rx_fis = pp->rx_fis;
  1699. if (pp->fbs_enabled)
  1700. rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
  1701. /*
  1702. * After a successful execution of an ATA PIO data-in command,
  1703. * the device doesn't send D2H Reg FIS to update the TF and
  1704. * the host should take TF and E_Status from the preceding PIO
  1705. * Setup FIS.
  1706. */
  1707. if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&
  1708. !(qc->flags & ATA_QCFLAG_FAILED)) {
  1709. ata_tf_from_fis(rx_fis + RX_FIS_PIO_SETUP, &qc->result_tf);
  1710. qc->result_tf.command = (rx_fis + RX_FIS_PIO_SETUP)[15];
  1711. } else
  1712. ata_tf_from_fis(rx_fis + RX_FIS_D2H_REG, &qc->result_tf);
  1713. return true;
  1714. }
  1715. static void ahci_freeze(struct ata_port *ap)
  1716. {
  1717. void __iomem *port_mmio = ahci_port_base(ap);
  1718. /* turn IRQ off */
  1719. writel(0, port_mmio + PORT_IRQ_MASK);
  1720. }
  1721. static void ahci_thaw(struct ata_port *ap)
  1722. {
  1723. struct ahci_host_priv *hpriv = ap->host->private_data;
  1724. void __iomem *mmio = hpriv->mmio;
  1725. void __iomem *port_mmio = ahci_port_base(ap);
  1726. u32 tmp;
  1727. struct ahci_port_priv *pp = ap->private_data;
  1728. /* clear IRQ */
  1729. tmp = readl(port_mmio + PORT_IRQ_STAT);
  1730. writel(tmp, port_mmio + PORT_IRQ_STAT);
  1731. writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
  1732. /* turn IRQ back on */
  1733. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  1734. }
  1735. void ahci_error_handler(struct ata_port *ap)
  1736. {
  1737. struct ahci_host_priv *hpriv = ap->host->private_data;
  1738. if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
  1739. /* restart engine */
  1740. hpriv->stop_engine(ap);
  1741. hpriv->start_engine(ap);
  1742. }
  1743. sata_pmp_error_handler(ap);
  1744. if (!ata_dev_enabled(ap->link.device))
  1745. hpriv->stop_engine(ap);
  1746. }
  1747. EXPORT_SYMBOL_GPL(ahci_error_handler);
  1748. static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
  1749. {
  1750. struct ata_port *ap = qc->ap;
  1751. /* make DMA engine forget about the failed command */
  1752. if (qc->flags & ATA_QCFLAG_FAILED)
  1753. ahci_kick_engine(ap);
  1754. }
  1755. static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep)
  1756. {
  1757. struct ahci_host_priv *hpriv = ap->host->private_data;
  1758. void __iomem *port_mmio = ahci_port_base(ap);
  1759. struct ata_device *dev = ap->link.device;
  1760. u32 devslp, dm, dito, mdat, deto, dito_conf;
  1761. int rc;
  1762. unsigned int err_mask;
  1763. devslp = readl(port_mmio + PORT_DEVSLP);
  1764. if (!(devslp & PORT_DEVSLP_DSP)) {
  1765. dev_info(ap->host->dev, "port does not support device sleep\n");
  1766. return;
  1767. }
  1768. /* disable device sleep */
  1769. if (!sleep) {
  1770. if (devslp & PORT_DEVSLP_ADSE) {
  1771. writel(devslp & ~PORT_DEVSLP_ADSE,
  1772. port_mmio + PORT_DEVSLP);
  1773. err_mask = ata_dev_set_feature(dev,
  1774. SETFEATURES_SATA_DISABLE,
  1775. SATA_DEVSLP);
  1776. if (err_mask && err_mask != AC_ERR_DEV)
  1777. ata_dev_warn(dev, "failed to disable DEVSLP\n");
  1778. }
  1779. return;
  1780. }
  1781. dm = (devslp & PORT_DEVSLP_DM_MASK) >> PORT_DEVSLP_DM_OFFSET;
  1782. dito = devslp_idle_timeout / (dm + 1);
  1783. if (dito > 0x3ff)
  1784. dito = 0x3ff;
  1785. dito_conf = (devslp >> PORT_DEVSLP_DITO_OFFSET) & 0x3FF;
  1786. /* device sleep was already enabled and same dito */
  1787. if ((devslp & PORT_DEVSLP_ADSE) && (dito_conf == dito))
  1788. return;
  1789. /* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
  1790. rc = hpriv->stop_engine(ap);
  1791. if (rc)
  1792. return;
  1793. /* Use the nominal value 10 ms if the read MDAT is zero,
  1794. * the nominal value of DETO is 20 ms.
  1795. */
  1796. if (dev->devslp_timing[ATA_LOG_DEVSLP_VALID] &
  1797. ATA_LOG_DEVSLP_VALID_MASK) {
  1798. mdat = dev->devslp_timing[ATA_LOG_DEVSLP_MDAT] &
  1799. ATA_LOG_DEVSLP_MDAT_MASK;
  1800. if (!mdat)
  1801. mdat = 10;
  1802. deto = dev->devslp_timing[ATA_LOG_DEVSLP_DETO];
  1803. if (!deto)
  1804. deto = 20;
  1805. } else {
  1806. mdat = 10;
  1807. deto = 20;
  1808. }
  1809. /* Make dito, mdat, deto bits to 0s */
  1810. devslp &= ~GENMASK_ULL(24, 2);
  1811. devslp |= ((dito << PORT_DEVSLP_DITO_OFFSET) |
  1812. (mdat << PORT_DEVSLP_MDAT_OFFSET) |
  1813. (deto << PORT_DEVSLP_DETO_OFFSET) |
  1814. PORT_DEVSLP_ADSE);
  1815. writel(devslp, port_mmio + PORT_DEVSLP);
  1816. hpriv->start_engine(ap);
  1817. /* enable device sleep feature for the drive */
  1818. err_mask = ata_dev_set_feature(dev,
  1819. SETFEATURES_SATA_ENABLE,
  1820. SATA_DEVSLP);
  1821. if (err_mask && err_mask != AC_ERR_DEV)
  1822. ata_dev_warn(dev, "failed to enable DEVSLP\n");
  1823. }
  1824. static void ahci_enable_fbs(struct ata_port *ap)
  1825. {
  1826. struct ahci_host_priv *hpriv = ap->host->private_data;
  1827. struct ahci_port_priv *pp = ap->private_data;
  1828. void __iomem *port_mmio = ahci_port_base(ap);
  1829. u32 fbs;
  1830. int rc;
  1831. if (!pp->fbs_supported)
  1832. return;
  1833. fbs = readl(port_mmio + PORT_FBS);
  1834. if (fbs & PORT_FBS_EN) {
  1835. pp->fbs_enabled = true;
  1836. pp->fbs_last_dev = -1; /* initialization */
  1837. return;
  1838. }
  1839. rc = hpriv->stop_engine(ap);
  1840. if (rc)
  1841. return;
  1842. writel(fbs | PORT_FBS_EN, port_mmio + PORT_FBS);
  1843. fbs = readl(port_mmio + PORT_FBS);
  1844. if (fbs & PORT_FBS_EN) {
  1845. dev_info(ap->host->dev, "FBS is enabled\n");
  1846. pp->fbs_enabled = true;
  1847. pp->fbs_last_dev = -1; /* initialization */
  1848. } else
  1849. dev_err(ap->host->dev, "Failed to enable FBS\n");
  1850. hpriv->start_engine(ap);
  1851. }
  1852. static void ahci_disable_fbs(struct ata_port *ap)
  1853. {
  1854. struct ahci_host_priv *hpriv = ap->host->private_data;
  1855. struct ahci_port_priv *pp = ap->private_data;
  1856. void __iomem *port_mmio = ahci_port_base(ap);
  1857. u32 fbs;
  1858. int rc;
  1859. if (!pp->fbs_supported)
  1860. return;
  1861. fbs = readl(port_mmio + PORT_FBS);
  1862. if ((fbs & PORT_FBS_EN) == 0) {
  1863. pp->fbs_enabled = false;
  1864. return;
  1865. }
  1866. rc = hpriv->stop_engine(ap);
  1867. if (rc)
  1868. return;
  1869. writel(fbs & ~PORT_FBS_EN, port_mmio + PORT_FBS);
  1870. fbs = readl(port_mmio + PORT_FBS);
  1871. if (fbs & PORT_FBS_EN)
  1872. dev_err(ap->host->dev, "Failed to disable FBS\n");
  1873. else {
  1874. dev_info(ap->host->dev, "FBS is disabled\n");
  1875. pp->fbs_enabled = false;
  1876. }
  1877. hpriv->start_engine(ap);
  1878. }
  1879. static void ahci_pmp_attach(struct ata_port *ap)
  1880. {
  1881. void __iomem *port_mmio = ahci_port_base(ap);
  1882. struct ahci_port_priv *pp = ap->private_data;
  1883. u32 cmd;
  1884. cmd = readl(port_mmio + PORT_CMD);
  1885. cmd |= PORT_CMD_PMP;
  1886. writel(cmd, port_mmio + PORT_CMD);
  1887. ahci_enable_fbs(ap);
  1888. pp->intr_mask |= PORT_IRQ_BAD_PMP;
  1889. /*
  1890. * We must not change the port interrupt mask register if the
  1891. * port is marked frozen, the value in pp->intr_mask will be
  1892. * restored later when the port is thawed.
  1893. *
  1894. * Note that during initialization, the port is marked as
  1895. * frozen since the irq handler is not yet registered.
  1896. */
  1897. if (!(ap->pflags & ATA_PFLAG_FROZEN))
  1898. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  1899. }
  1900. static void ahci_pmp_detach(struct ata_port *ap)
  1901. {
  1902. void __iomem *port_mmio = ahci_port_base(ap);
  1903. struct ahci_port_priv *pp = ap->private_data;
  1904. u32 cmd;
  1905. ahci_disable_fbs(ap);
  1906. cmd = readl(port_mmio + PORT_CMD);
  1907. cmd &= ~PORT_CMD_PMP;
  1908. writel(cmd, port_mmio + PORT_CMD);
  1909. pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
  1910. /* see comment above in ahci_pmp_attach() */
  1911. if (!(ap->pflags & ATA_PFLAG_FROZEN))
  1912. writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
  1913. }
  1914. int ahci_port_resume(struct ata_port *ap)
  1915. {
  1916. ahci_rpm_get_port(ap);
  1917. ahci_power_up(ap);
  1918. ahci_start_port(ap);
  1919. if (sata_pmp_attached(ap))
  1920. ahci_pmp_attach(ap);
  1921. else
  1922. ahci_pmp_detach(ap);
  1923. return 0;
  1924. }
  1925. EXPORT_SYMBOL_GPL(ahci_port_resume);
  1926. #ifdef CONFIG_PM
  1927. static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
  1928. {
  1929. const char *emsg = NULL;
  1930. int rc;
  1931. rc = ahci_deinit_port(ap, &emsg);
  1932. if (rc == 0)
  1933. ahci_power_down(ap);
  1934. else {
  1935. ata_port_err(ap, "%s (%d)\n", emsg, rc);
  1936. ata_port_freeze(ap);
  1937. }
  1938. ahci_rpm_put_port(ap);
  1939. return rc;
  1940. }
  1941. #endif
  1942. static int ahci_port_start(struct ata_port *ap)
  1943. {
  1944. struct ahci_host_priv *hpriv = ap->host->private_data;
  1945. struct device *dev = ap->host->dev;
  1946. struct ahci_port_priv *pp;
  1947. void *mem;
  1948. dma_addr_t mem_dma;
  1949. size_t dma_sz, rx_fis_sz;
  1950. pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
  1951. if (!pp)
  1952. return -ENOMEM;
  1953. if (ap->host->n_ports > 1) {
  1954. pp->irq_desc = devm_kzalloc(dev, 8, GFP_KERNEL);
  1955. if (!pp->irq_desc) {
  1956. devm_kfree(dev, pp);
  1957. return -ENOMEM;
  1958. }
  1959. snprintf(pp->irq_desc, 8,
  1960. "%s%d", dev_driver_string(dev), ap->port_no);
  1961. }
  1962. /* check FBS capability */
  1963. if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
  1964. void __iomem *port_mmio = ahci_port_base(ap);
  1965. u32 cmd = readl(port_mmio + PORT_CMD);
  1966. if (cmd & PORT_CMD_FBSCP)
  1967. pp->fbs_supported = true;
  1968. else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {
  1969. dev_info(dev, "port %d can do FBS, forcing FBSCP\n",
  1970. ap->port_no);
  1971. pp->fbs_supported = true;
  1972. } else
  1973. dev_warn(dev, "port %d is not capable of FBS\n",
  1974. ap->port_no);
  1975. }
  1976. if (pp->fbs_supported) {
  1977. dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
  1978. rx_fis_sz = AHCI_RX_FIS_SZ * 16;
  1979. } else {
  1980. dma_sz = AHCI_PORT_PRIV_DMA_SZ;
  1981. rx_fis_sz = AHCI_RX_FIS_SZ;
  1982. }
  1983. mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
  1984. if (!mem)
  1985. return -ENOMEM;
  1986. memset(mem, 0, dma_sz);
  1987. /*
  1988. * First item in chunk of DMA memory: 32-slot command table,
  1989. * 32 bytes each in size
  1990. */
  1991. pp->cmd_slot = mem;
  1992. pp->cmd_slot_dma = mem_dma;
  1993. mem += AHCI_CMD_SLOT_SZ;
  1994. mem_dma += AHCI_CMD_SLOT_SZ;
  1995. /*
  1996. * Second item: Received-FIS area
  1997. */
  1998. pp->rx_fis = mem;
  1999. pp->rx_fis_dma = mem_dma;
  2000. mem += rx_fis_sz;
  2001. mem_dma += rx_fis_sz;
  2002. /*
  2003. * Third item: data area for storing a single command
  2004. * and its scatter-gather table
  2005. */
  2006. pp->cmd_tbl = mem;
  2007. pp->cmd_tbl_dma = mem_dma;
  2008. /*
  2009. * Save off initial list of interrupts to be enabled.
  2010. * This could be changed later
  2011. */
  2012. pp->intr_mask = DEF_PORT_IRQ;
  2013. /*
  2014. * Switch to per-port locking in case each port has its own MSI vector.
  2015. */
  2016. if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
  2017. spin_lock_init(&pp->lock);
  2018. ap->lock = &pp->lock;
  2019. }
  2020. ap->private_data = pp;
  2021. /* engage engines, captain */
  2022. return ahci_port_resume(ap);
  2023. }
  2024. static void ahci_port_stop(struct ata_port *ap)
  2025. {
  2026. const char *emsg = NULL;
  2027. struct ahci_host_priv *hpriv = ap->host->private_data;
  2028. void __iomem *host_mmio = hpriv->mmio;
  2029. int rc;
  2030. /* de-initialize port */
  2031. rc = ahci_deinit_port(ap, &emsg);
  2032. if (rc)
  2033. ata_port_warn(ap, "%s (%d)\n", emsg, rc);
  2034. /*
  2035. * Clear GHC.IS to prevent stuck INTx after disabling MSI and
  2036. * re-enabling INTx.
  2037. */
  2038. writel(1 << ap->port_no, host_mmio + HOST_IRQ_STAT);
  2039. ahci_rpm_put_port(ap);
  2040. }
  2041. void ahci_print_info(struct ata_host *host, const char *scc_s)
  2042. {
  2043. struct ahci_host_priv *hpriv = host->private_data;
  2044. u32 vers, cap, cap2, impl, speed;
  2045. const char *speed_s;
  2046. vers = hpriv->version;
  2047. cap = hpriv->cap;
  2048. cap2 = hpriv->cap2;
  2049. impl = hpriv->port_map;
  2050. speed = (cap >> 20) & 0xf;
  2051. if (speed == 1)
  2052. speed_s = "1.5";
  2053. else if (speed == 2)
  2054. speed_s = "3";
  2055. else if (speed == 3)
  2056. speed_s = "6";
  2057. else
  2058. speed_s = "?";
  2059. dev_info(host->dev,
  2060. "AHCI %02x%02x.%02x%02x "
  2061. "%u slots %u ports %s Gbps 0x%x impl %s mode\n"
  2062. ,
  2063. (vers >> 24) & 0xff,
  2064. (vers >> 16) & 0xff,
  2065. (vers >> 8) & 0xff,
  2066. vers & 0xff,
  2067. ((cap >> 8) & 0x1f) + 1,
  2068. (cap & 0x1f) + 1,
  2069. speed_s,
  2070. impl,
  2071. scc_s);
  2072. dev_info(host->dev,
  2073. "flags: "
  2074. "%s%s%s%s%s%s%s"
  2075. "%s%s%s%s%s%s%s"
  2076. "%s%s%s%s%s%s%s"
  2077. "%s%s\n"
  2078. ,
  2079. cap & HOST_CAP_64 ? "64bit " : "",
  2080. cap & HOST_CAP_NCQ ? "ncq " : "",
  2081. cap & HOST_CAP_SNTF ? "sntf " : "",
  2082. cap & HOST_CAP_MPS ? "ilck " : "",
  2083. cap & HOST_CAP_SSS ? "stag " : "",
  2084. cap & HOST_CAP_ALPM ? "pm " : "",
  2085. cap & HOST_CAP_LED ? "led " : "",
  2086. cap & HOST_CAP_CLO ? "clo " : "",
  2087. cap & HOST_CAP_ONLY ? "only " : "",
  2088. cap & HOST_CAP_PMP ? "pmp " : "",
  2089. cap & HOST_CAP_FBS ? "fbs " : "",
  2090. cap & HOST_CAP_PIO_MULTI ? "pio " : "",
  2091. cap & HOST_CAP_SSC ? "slum " : "",
  2092. cap & HOST_CAP_PART ? "part " : "",
  2093. cap & HOST_CAP_CCC ? "ccc " : "",
  2094. cap & HOST_CAP_EMS ? "ems " : "",
  2095. cap & HOST_CAP_SXS ? "sxs " : "",
  2096. cap2 & HOST_CAP2_DESO ? "deso " : "",
  2097. cap2 & HOST_CAP2_SADM ? "sadm " : "",
  2098. cap2 & HOST_CAP2_SDS ? "sds " : "",
  2099. cap2 & HOST_CAP2_APST ? "apst " : "",
  2100. cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
  2101. cap2 & HOST_CAP2_BOH ? "boh " : ""
  2102. );
  2103. }
  2104. EXPORT_SYMBOL_GPL(ahci_print_info);
  2105. void ahci_set_em_messages(struct ahci_host_priv *hpriv,
  2106. struct ata_port_info *pi)
  2107. {
  2108. u8 messages;
  2109. void __iomem *mmio = hpriv->mmio;
  2110. u32 em_loc = readl(mmio + HOST_EM_LOC);
  2111. u32 em_ctl = readl(mmio + HOST_EM_CTL);
  2112. if (!ahci_em_messages || !(hpriv->cap & HOST_CAP_EMS))
  2113. return;
  2114. messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
  2115. if (messages) {
  2116. /* store em_loc */
  2117. hpriv->em_loc = ((em_loc >> 16) * 4);
  2118. hpriv->em_buf_sz = ((em_loc & 0xff) * 4);
  2119. hpriv->em_msg_type = messages;
  2120. pi->flags |= ATA_FLAG_EM;
  2121. if (!(em_ctl & EM_CTL_ALHD))
  2122. pi->flags |= ATA_FLAG_SW_ACTIVITY;
  2123. }
  2124. }
  2125. EXPORT_SYMBOL_GPL(ahci_set_em_messages);
  2126. static int ahci_host_activate_multi_irqs(struct ata_host *host,
  2127. struct scsi_host_template *sht)
  2128. {
  2129. struct ahci_host_priv *hpriv = host->private_data;
  2130. int i, rc;
  2131. rc = ata_host_start(host);
  2132. if (rc)
  2133. return rc;
  2134. /*
  2135. * Requests IRQs according to AHCI-1.1 when multiple MSIs were
  2136. * allocated. That is one MSI per port, starting from @irq.
  2137. */
  2138. for (i = 0; i < host->n_ports; i++) {
  2139. struct ahci_port_priv *pp = host->ports[i]->private_data;
  2140. int irq = hpriv->get_irq_vector(host, i);
  2141. /* Do not receive interrupts sent by dummy ports */
  2142. if (!pp) {
  2143. disable_irq(irq);
  2144. continue;
  2145. }
  2146. rc = devm_request_irq(host->dev, irq, ahci_multi_irqs_intr_hard,
  2147. 0, pp->irq_desc, host->ports[i]);
  2148. if (rc)
  2149. return rc;
  2150. ata_port_desc(host->ports[i], "irq %d", irq);
  2151. }
  2152. return ata_host_register(host, sht);
  2153. }
  2154. /**
  2155. * ahci_host_activate - start AHCI host, request IRQs and register it
  2156. * @host: target ATA host
  2157. * @sht: scsi_host_template to use when registering the host
  2158. *
  2159. * LOCKING:
  2160. * Inherited from calling layer (may sleep).
  2161. *
  2162. * RETURNS:
  2163. * 0 on success, -errno otherwise.
  2164. */
  2165. int ahci_host_activate(struct ata_host *host, struct scsi_host_template *sht)
  2166. {
  2167. struct ahci_host_priv *hpriv = host->private_data;
  2168. int irq = hpriv->irq;
  2169. int rc;
  2170. if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
  2171. if (hpriv->irq_handler)
  2172. dev_warn(host->dev,
  2173. "both AHCI_HFLAG_MULTI_MSI flag set and custom irq handler implemented\n");
  2174. if (!hpriv->get_irq_vector) {
  2175. dev_err(host->dev,
  2176. "AHCI_HFLAG_MULTI_MSI requires ->get_irq_vector!\n");
  2177. return -EIO;
  2178. }
  2179. rc = ahci_host_activate_multi_irqs(host, sht);
  2180. } else {
  2181. rc = ata_host_activate(host, irq, hpriv->irq_handler,
  2182. IRQF_SHARED, sht);
  2183. }
  2184. return rc;
  2185. }
  2186. EXPORT_SYMBOL_GPL(ahci_host_activate);
  2187. MODULE_AUTHOR("Jeff Garzik");
  2188. MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
  2189. MODULE_LICENSE("GPL");