ntb_hw_amd.c 27 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156
  1. /*
  2. * This file is provided under a dual BSD/GPLv2 license. When using or
  3. * redistributing this file, you may do so under either license.
  4. *
  5. * GPL LICENSE SUMMARY
  6. *
  7. * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
  8. * Copyright (C) 2016 T-Platforms. All Rights Reserved.
  9. *
  10. * This program is free software; you can redistribute it and/or modify
  11. * it under the terms of version 2 of the GNU General Public License as
  12. * published by the Free Software Foundation.
  13. *
  14. * BSD LICENSE
  15. *
  16. * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
  17. * Copyright (C) 2016 T-Platforms. All Rights Reserved.
  18. *
  19. * Redistribution and use in source and binary forms, with or without
  20. * modification, are permitted provided that the following conditions
  21. * are met:
  22. *
  23. * * Redistributions of source code must retain the above copyright
  24. * notice, this list of conditions and the following disclaimer.
  25. * * Redistributions in binary form must reproduce the above copy
  26. * notice, this list of conditions and the following disclaimer in
  27. * the documentation and/or other materials provided with the
  28. * distribution.
  29. * * Neither the name of AMD Corporation nor the names of its
  30. * contributors may be used to endorse or promote products derived
  31. * from this software without specific prior written permission.
  32. *
  33. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  34. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  35. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  36. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  37. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  38. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  39. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  40. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  41. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  42. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  43. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44. *
  45. * AMD PCIe NTB Linux driver
  46. *
  47. * Contact Information:
  48. * Xiangliang Yu <Xiangliang.Yu@amd.com>
  49. */
  50. #include <linux/debugfs.h>
  51. #include <linux/delay.h>
  52. #include <linux/init.h>
  53. #include <linux/interrupt.h>
  54. #include <linux/module.h>
  55. #include <linux/acpi.h>
  56. #include <linux/pci.h>
  57. #include <linux/random.h>
  58. #include <linux/slab.h>
  59. #include <linux/ntb.h>
  60. #include "ntb_hw_amd.h"
  61. #define NTB_NAME "ntb_hw_amd"
  62. #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver"
  63. #define NTB_VER "1.0"
  64. MODULE_DESCRIPTION(NTB_DESC);
  65. MODULE_VERSION(NTB_VER);
  66. MODULE_LICENSE("Dual BSD/GPL");
  67. MODULE_AUTHOR("AMD Inc.");
  68. static const struct file_operations amd_ntb_debugfs_info;
  69. static struct dentry *debugfs_dir;
  70. static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
  71. {
  72. if (idx < 0 || idx > ndev->mw_count)
  73. return -EINVAL;
  74. return 1 << idx;
  75. }
  76. static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
  77. {
  78. if (pidx != NTB_DEF_PEER_IDX)
  79. return -EINVAL;
  80. return ntb_ndev(ntb)->mw_count;
  81. }
  82. static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
  83. resource_size_t *addr_align,
  84. resource_size_t *size_align,
  85. resource_size_t *size_max)
  86. {
  87. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  88. int bar;
  89. if (pidx != NTB_DEF_PEER_IDX)
  90. return -EINVAL;
  91. bar = ndev_mw_to_bar(ndev, idx);
  92. if (bar < 0)
  93. return bar;
  94. if (addr_align)
  95. *addr_align = SZ_4K;
  96. if (size_align)
  97. *size_align = 1;
  98. if (size_max)
  99. *size_max = pci_resource_len(ndev->ntb.pdev, bar);
  100. return 0;
  101. }
  102. static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
  103. dma_addr_t addr, resource_size_t size)
  104. {
  105. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  106. unsigned long xlat_reg, limit_reg = 0;
  107. resource_size_t mw_size;
  108. void __iomem *mmio, *peer_mmio;
  109. u64 base_addr, limit, reg_val;
  110. int bar;
  111. if (pidx != NTB_DEF_PEER_IDX)
  112. return -EINVAL;
  113. bar = ndev_mw_to_bar(ndev, idx);
  114. if (bar < 0)
  115. return bar;
  116. mw_size = pci_resource_len(ntb->pdev, bar);
  117. /* make sure the range fits in the usable mw size */
  118. if (size > mw_size)
  119. return -EINVAL;
  120. mmio = ndev->self_mmio;
  121. peer_mmio = ndev->peer_mmio;
  122. base_addr = pci_resource_start(ntb->pdev, bar);
  123. if (bar != 1) {
  124. xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2);
  125. limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2);
  126. /* Set the limit if supported */
  127. limit = size;
  128. /* set and verify setting the translation address */
  129. write64(addr, peer_mmio + xlat_reg);
  130. reg_val = read64(peer_mmio + xlat_reg);
  131. if (reg_val != addr) {
  132. write64(0, peer_mmio + xlat_reg);
  133. return -EIO;
  134. }
  135. /* set and verify setting the limit */
  136. write64(limit, mmio + limit_reg);
  137. reg_val = read64(mmio + limit_reg);
  138. if (reg_val != limit) {
  139. write64(base_addr, mmio + limit_reg);
  140. write64(0, peer_mmio + xlat_reg);
  141. return -EIO;
  142. }
  143. } else {
  144. xlat_reg = AMD_BAR1XLAT_OFFSET;
  145. limit_reg = AMD_BAR1LMT_OFFSET;
  146. /* Set the limit if supported */
  147. limit = size;
  148. /* set and verify setting the translation address */
  149. write64(addr, peer_mmio + xlat_reg);
  150. reg_val = read64(peer_mmio + xlat_reg);
  151. if (reg_val != addr) {
  152. write64(0, peer_mmio + xlat_reg);
  153. return -EIO;
  154. }
  155. /* set and verify setting the limit */
  156. writel(limit, mmio + limit_reg);
  157. reg_val = readl(mmio + limit_reg);
  158. if (reg_val != limit) {
  159. writel(base_addr, mmio + limit_reg);
  160. writel(0, peer_mmio + xlat_reg);
  161. return -EIO;
  162. }
  163. }
  164. return 0;
  165. }
  166. static int amd_link_is_up(struct amd_ntb_dev *ndev)
  167. {
  168. if (!ndev->peer_sta)
  169. return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
  170. if (ndev->peer_sta & AMD_LINK_UP_EVENT) {
  171. ndev->peer_sta = 0;
  172. return 1;
  173. }
  174. /* If peer_sta is reset or D0 event, the ISR has
  175. * started a timer to check link status of hardware.
  176. * So here just clear status bit. And if peer_sta is
  177. * D3 or PME_TO, D0/reset event will be happened when
  178. * system wakeup/poweron, so do nothing here.
  179. */
  180. if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
  181. ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
  182. else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT))
  183. ndev->peer_sta = 0;
  184. return 0;
  185. }
  186. static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
  187. enum ntb_speed *speed,
  188. enum ntb_width *width)
  189. {
  190. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  191. int ret = 0;
  192. if (amd_link_is_up(ndev)) {
  193. if (speed)
  194. *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
  195. if (width)
  196. *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
  197. dev_dbg(&ntb->pdev->dev, "link is up.\n");
  198. ret = 1;
  199. } else {
  200. if (speed)
  201. *speed = NTB_SPEED_NONE;
  202. if (width)
  203. *width = NTB_WIDTH_NONE;
  204. dev_dbg(&ntb->pdev->dev, "link is down.\n");
  205. }
  206. return ret;
  207. }
  208. static int amd_ntb_link_enable(struct ntb_dev *ntb,
  209. enum ntb_speed max_speed,
  210. enum ntb_width max_width)
  211. {
  212. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  213. void __iomem *mmio = ndev->self_mmio;
  214. u32 ntb_ctl;
  215. /* Enable event interrupt */
  216. ndev->int_mask &= ~AMD_EVENT_INTMASK;
  217. writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
  218. if (ndev->ntb.topo == NTB_TOPO_SEC)
  219. return -EINVAL;
  220. dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
  221. ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
  222. ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
  223. writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
  224. return 0;
  225. }
  226. static int amd_ntb_link_disable(struct ntb_dev *ntb)
  227. {
  228. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  229. void __iomem *mmio = ndev->self_mmio;
  230. u32 ntb_ctl;
  231. /* Disable event interrupt */
  232. ndev->int_mask |= AMD_EVENT_INTMASK;
  233. writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
  234. if (ndev->ntb.topo == NTB_TOPO_SEC)
  235. return -EINVAL;
  236. dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
  237. ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
  238. ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
  239. writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
  240. return 0;
  241. }
  242. static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
  243. {
  244. /* The same as for inbound MWs */
  245. return ntb_ndev(ntb)->mw_count;
  246. }
  247. static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
  248. phys_addr_t *base, resource_size_t *size)
  249. {
  250. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  251. int bar;
  252. bar = ndev_mw_to_bar(ndev, idx);
  253. if (bar < 0)
  254. return bar;
  255. if (base)
  256. *base = pci_resource_start(ndev->ntb.pdev, bar);
  257. if (size)
  258. *size = pci_resource_len(ndev->ntb.pdev, bar);
  259. return 0;
  260. }
  261. static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
  262. {
  263. return ntb_ndev(ntb)->db_valid_mask;
  264. }
  265. static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
  266. {
  267. return ntb_ndev(ntb)->db_count;
  268. }
  269. static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
  270. {
  271. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  272. if (db_vector < 0 || db_vector > ndev->db_count)
  273. return 0;
  274. return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector);
  275. }
  276. static u64 amd_ntb_db_read(struct ntb_dev *ntb)
  277. {
  278. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  279. void __iomem *mmio = ndev->self_mmio;
  280. return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
  281. }
  282. static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
  283. {
  284. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  285. void __iomem *mmio = ndev->self_mmio;
  286. writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
  287. return 0;
  288. }
  289. static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
  290. {
  291. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  292. void __iomem *mmio = ndev->self_mmio;
  293. unsigned long flags;
  294. if (db_bits & ~ndev->db_valid_mask)
  295. return -EINVAL;
  296. spin_lock_irqsave(&ndev->db_mask_lock, flags);
  297. ndev->db_mask |= db_bits;
  298. writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
  299. spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
  300. return 0;
  301. }
  302. static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
  303. {
  304. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  305. void __iomem *mmio = ndev->self_mmio;
  306. unsigned long flags;
  307. if (db_bits & ~ndev->db_valid_mask)
  308. return -EINVAL;
  309. spin_lock_irqsave(&ndev->db_mask_lock, flags);
  310. ndev->db_mask &= ~db_bits;
  311. writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
  312. spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
  313. return 0;
  314. }
  315. static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
  316. {
  317. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  318. void __iomem *mmio = ndev->self_mmio;
  319. writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
  320. return 0;
  321. }
  322. static int amd_ntb_spad_count(struct ntb_dev *ntb)
  323. {
  324. return ntb_ndev(ntb)->spad_count;
  325. }
  326. static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
  327. {
  328. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  329. void __iomem *mmio = ndev->self_mmio;
  330. u32 offset;
  331. if (idx < 0 || idx >= ndev->spad_count)
  332. return 0;
  333. offset = ndev->self_spad + (idx << 2);
  334. return readl(mmio + AMD_SPAD_OFFSET + offset);
  335. }
  336. static int amd_ntb_spad_write(struct ntb_dev *ntb,
  337. int idx, u32 val)
  338. {
  339. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  340. void __iomem *mmio = ndev->self_mmio;
  341. u32 offset;
  342. if (idx < 0 || idx >= ndev->spad_count)
  343. return -EINVAL;
  344. offset = ndev->self_spad + (idx << 2);
  345. writel(val, mmio + AMD_SPAD_OFFSET + offset);
  346. return 0;
  347. }
  348. static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
  349. {
  350. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  351. void __iomem *mmio = ndev->self_mmio;
  352. u32 offset;
  353. if (sidx < 0 || sidx >= ndev->spad_count)
  354. return -EINVAL;
  355. offset = ndev->peer_spad + (sidx << 2);
  356. return readl(mmio + AMD_SPAD_OFFSET + offset);
  357. }
  358. static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
  359. int sidx, u32 val)
  360. {
  361. struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  362. void __iomem *mmio = ndev->self_mmio;
  363. u32 offset;
  364. if (sidx < 0 || sidx >= ndev->spad_count)
  365. return -EINVAL;
  366. offset = ndev->peer_spad + (sidx << 2);
  367. writel(val, mmio + AMD_SPAD_OFFSET + offset);
  368. return 0;
  369. }
  370. static const struct ntb_dev_ops amd_ntb_ops = {
  371. .mw_count = amd_ntb_mw_count,
  372. .mw_get_align = amd_ntb_mw_get_align,
  373. .mw_set_trans = amd_ntb_mw_set_trans,
  374. .peer_mw_count = amd_ntb_peer_mw_count,
  375. .peer_mw_get_addr = amd_ntb_peer_mw_get_addr,
  376. .link_is_up = amd_ntb_link_is_up,
  377. .link_enable = amd_ntb_link_enable,
  378. .link_disable = amd_ntb_link_disable,
  379. .db_valid_mask = amd_ntb_db_valid_mask,
  380. .db_vector_count = amd_ntb_db_vector_count,
  381. .db_vector_mask = amd_ntb_db_vector_mask,
  382. .db_read = amd_ntb_db_read,
  383. .db_clear = amd_ntb_db_clear,
  384. .db_set_mask = amd_ntb_db_set_mask,
  385. .db_clear_mask = amd_ntb_db_clear_mask,
  386. .peer_db_set = amd_ntb_peer_db_set,
  387. .spad_count = amd_ntb_spad_count,
  388. .spad_read = amd_ntb_spad_read,
  389. .spad_write = amd_ntb_spad_write,
  390. .peer_spad_read = amd_ntb_peer_spad_read,
  391. .peer_spad_write = amd_ntb_peer_spad_write,
  392. };
  393. static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
  394. {
  395. void __iomem *mmio = ndev->self_mmio;
  396. int reg;
  397. reg = readl(mmio + AMD_SMUACK_OFFSET);
  398. reg |= bit;
  399. writel(reg, mmio + AMD_SMUACK_OFFSET);
  400. ndev->peer_sta |= bit;
  401. }
  402. static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
  403. {
  404. void __iomem *mmio = ndev->self_mmio;
  405. struct device *dev = &ndev->ntb.pdev->dev;
  406. u32 status;
  407. status = readl(mmio + AMD_INTSTAT_OFFSET);
  408. if (!(status & AMD_EVENT_INTMASK))
  409. return;
  410. dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec);
  411. status &= AMD_EVENT_INTMASK;
  412. switch (status) {
  413. case AMD_PEER_FLUSH_EVENT:
  414. dev_info(dev, "Flush is done.\n");
  415. break;
  416. case AMD_PEER_RESET_EVENT:
  417. amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
  418. /* link down first */
  419. ntb_link_event(&ndev->ntb);
  420. /* polling peer status */
  421. schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
  422. break;
  423. case AMD_PEER_D3_EVENT:
  424. case AMD_PEER_PMETO_EVENT:
  425. case AMD_LINK_UP_EVENT:
  426. case AMD_LINK_DOWN_EVENT:
  427. amd_ack_smu(ndev, status);
  428. /* link down */
  429. ntb_link_event(&ndev->ntb);
  430. break;
  431. case AMD_PEER_D0_EVENT:
  432. mmio = ndev->peer_mmio;
  433. status = readl(mmio + AMD_PMESTAT_OFFSET);
  434. /* check if this is WAKEUP event */
  435. if (status & 0x1)
  436. dev_info(dev, "Wakeup is done.\n");
  437. amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
  438. /* start a timer to poll link status */
  439. schedule_delayed_work(&ndev->hb_timer,
  440. AMD_LINK_HB_TIMEOUT);
  441. break;
  442. default:
  443. dev_info(dev, "event status = 0x%x.\n", status);
  444. break;
  445. }
  446. }
  447. static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
  448. {
  449. dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
  450. if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
  451. amd_handle_event(ndev, vec);
  452. if (vec < AMD_DB_CNT)
  453. ntb_db_event(&ndev->ntb, vec);
  454. return IRQ_HANDLED;
  455. }
  456. static irqreturn_t ndev_vec_isr(int irq, void *dev)
  457. {
  458. struct amd_ntb_vec *nvec = dev;
  459. return ndev_interrupt(nvec->ndev, nvec->num);
  460. }
  461. static irqreturn_t ndev_irq_isr(int irq, void *dev)
  462. {
  463. struct amd_ntb_dev *ndev = dev;
  464. return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
  465. }
  466. static int ndev_init_isr(struct amd_ntb_dev *ndev,
  467. int msix_min, int msix_max)
  468. {
  469. struct pci_dev *pdev;
  470. int rc, i, msix_count, node;
  471. pdev = ndev->ntb.pdev;
  472. node = dev_to_node(&pdev->dev);
  473. ndev->db_mask = ndev->db_valid_mask;
  474. /* Try to set up msix irq */
  475. ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
  476. GFP_KERNEL, node);
  477. if (!ndev->vec)
  478. goto err_msix_vec_alloc;
  479. ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
  480. GFP_KERNEL, node);
  481. if (!ndev->msix)
  482. goto err_msix_alloc;
  483. for (i = 0; i < msix_max; ++i)
  484. ndev->msix[i].entry = i;
  485. msix_count = pci_enable_msix_range(pdev, ndev->msix,
  486. msix_min, msix_max);
  487. if (msix_count < 0)
  488. goto err_msix_enable;
  489. /* NOTE: Disable MSIX if msix count is less than 16 because of
  490. * hardware limitation.
  491. */
  492. if (msix_count < msix_min) {
  493. pci_disable_msix(pdev);
  494. goto err_msix_enable;
  495. }
  496. for (i = 0; i < msix_count; ++i) {
  497. ndev->vec[i].ndev = ndev;
  498. ndev->vec[i].num = i;
  499. rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
  500. "ndev_vec_isr", &ndev->vec[i]);
  501. if (rc)
  502. goto err_msix_request;
  503. }
  504. dev_dbg(&pdev->dev, "Using msix interrupts\n");
  505. ndev->db_count = msix_min;
  506. ndev->msix_vec_count = msix_max;
  507. return 0;
  508. err_msix_request:
  509. while (i-- > 0)
  510. free_irq(ndev->msix[i].vector, &ndev->vec[i]);
  511. pci_disable_msix(pdev);
  512. err_msix_enable:
  513. kfree(ndev->msix);
  514. err_msix_alloc:
  515. kfree(ndev->vec);
  516. err_msix_vec_alloc:
  517. ndev->msix = NULL;
  518. ndev->vec = NULL;
  519. /* Try to set up msi irq */
  520. rc = pci_enable_msi(pdev);
  521. if (rc)
  522. goto err_msi_enable;
  523. rc = request_irq(pdev->irq, ndev_irq_isr, 0,
  524. "ndev_irq_isr", ndev);
  525. if (rc)
  526. goto err_msi_request;
  527. dev_dbg(&pdev->dev, "Using msi interrupts\n");
  528. ndev->db_count = 1;
  529. ndev->msix_vec_count = 1;
  530. return 0;
  531. err_msi_request:
  532. pci_disable_msi(pdev);
  533. err_msi_enable:
  534. /* Try to set up intx irq */
  535. pci_intx(pdev, 1);
  536. rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
  537. "ndev_irq_isr", ndev);
  538. if (rc)
  539. goto err_intx_request;
  540. dev_dbg(&pdev->dev, "Using intx interrupts\n");
  541. ndev->db_count = 1;
  542. ndev->msix_vec_count = 1;
  543. return 0;
  544. err_intx_request:
  545. return rc;
  546. }
  547. static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
  548. {
  549. struct pci_dev *pdev;
  550. void __iomem *mmio = ndev->self_mmio;
  551. int i;
  552. pdev = ndev->ntb.pdev;
  553. /* Mask all doorbell interrupts */
  554. ndev->db_mask = ndev->db_valid_mask;
  555. writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
  556. if (ndev->msix) {
  557. i = ndev->msix_vec_count;
  558. while (i--)
  559. free_irq(ndev->msix[i].vector, &ndev->vec[i]);
  560. pci_disable_msix(pdev);
  561. kfree(ndev->msix);
  562. kfree(ndev->vec);
  563. } else {
  564. free_irq(pdev->irq, ndev);
  565. if (pci_dev_msi_enabled(pdev))
  566. pci_disable_msi(pdev);
  567. else
  568. pci_intx(pdev, 0);
  569. }
  570. }
  571. static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
  572. size_t count, loff_t *offp)
  573. {
  574. struct amd_ntb_dev *ndev;
  575. void __iomem *mmio;
  576. char *buf;
  577. size_t buf_size;
  578. ssize_t ret, off;
  579. union { u64 v64; u32 v32; u16 v16; } u;
  580. ndev = filp->private_data;
  581. mmio = ndev->self_mmio;
  582. buf_size = min(count, 0x800ul);
  583. buf = kmalloc(buf_size, GFP_KERNEL);
  584. if (!buf)
  585. return -ENOMEM;
  586. off = 0;
  587. off += scnprintf(buf + off, buf_size - off,
  588. "NTB Device Information:\n");
  589. off += scnprintf(buf + off, buf_size - off,
  590. "Connection Topology -\t%s\n",
  591. ntb_topo_string(ndev->ntb.topo));
  592. off += scnprintf(buf + off, buf_size - off,
  593. "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
  594. if (!amd_link_is_up(ndev)) {
  595. off += scnprintf(buf + off, buf_size - off,
  596. "Link Status -\t\tDown\n");
  597. } else {
  598. off += scnprintf(buf + off, buf_size - off,
  599. "Link Status -\t\tUp\n");
  600. off += scnprintf(buf + off, buf_size - off,
  601. "Link Speed -\t\tPCI-E Gen %u\n",
  602. NTB_LNK_STA_SPEED(ndev->lnk_sta));
  603. off += scnprintf(buf + off, buf_size - off,
  604. "Link Width -\t\tx%u\n",
  605. NTB_LNK_STA_WIDTH(ndev->lnk_sta));
  606. }
  607. off += scnprintf(buf + off, buf_size - off,
  608. "Memory Window Count -\t%u\n", ndev->mw_count);
  609. off += scnprintf(buf + off, buf_size - off,
  610. "Scratchpad Count -\t%u\n", ndev->spad_count);
  611. off += scnprintf(buf + off, buf_size - off,
  612. "Doorbell Count -\t%u\n", ndev->db_count);
  613. off += scnprintf(buf + off, buf_size - off,
  614. "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
  615. off += scnprintf(buf + off, buf_size - off,
  616. "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
  617. u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
  618. off += scnprintf(buf + off, buf_size - off,
  619. "Doorbell Mask -\t\t\t%#06x\n", u.v32);
  620. u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
  621. off += scnprintf(buf + off, buf_size - off,
  622. "Doorbell Bell -\t\t\t%#06x\n", u.v32);
  623. off += scnprintf(buf + off, buf_size - off,
  624. "\nNTB Incoming XLAT:\n");
  625. u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
  626. off += scnprintf(buf + off, buf_size - off,
  627. "XLAT1 -\t\t%#018llx\n", u.v64);
  628. u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
  629. off += scnprintf(buf + off, buf_size - off,
  630. "XLAT23 -\t\t%#018llx\n", u.v64);
  631. u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
  632. off += scnprintf(buf + off, buf_size - off,
  633. "XLAT45 -\t\t%#018llx\n", u.v64);
  634. u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
  635. off += scnprintf(buf + off, buf_size - off,
  636. "LMT1 -\t\t\t%#06x\n", u.v32);
  637. u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
  638. off += scnprintf(buf + off, buf_size - off,
  639. "LMT23 -\t\t\t%#018llx\n", u.v64);
  640. u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
  641. off += scnprintf(buf + off, buf_size - off,
  642. "LMT45 -\t\t\t%#018llx\n", u.v64);
  643. ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
  644. kfree(buf);
  645. return ret;
  646. }
  647. static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
  648. {
  649. if (!debugfs_dir) {
  650. ndev->debugfs_dir = NULL;
  651. ndev->debugfs_info = NULL;
  652. } else {
  653. ndev->debugfs_dir =
  654. debugfs_create_dir(pci_name(ndev->ntb.pdev),
  655. debugfs_dir);
  656. if (!ndev->debugfs_dir)
  657. ndev->debugfs_info = NULL;
  658. else
  659. ndev->debugfs_info =
  660. debugfs_create_file("info", S_IRUSR,
  661. ndev->debugfs_dir, ndev,
  662. &amd_ntb_debugfs_info);
  663. }
  664. }
  665. static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
  666. {
  667. debugfs_remove_recursive(ndev->debugfs_dir);
  668. }
  669. static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
  670. struct pci_dev *pdev)
  671. {
  672. ndev->ntb.pdev = pdev;
  673. ndev->ntb.topo = NTB_TOPO_NONE;
  674. ndev->ntb.ops = &amd_ntb_ops;
  675. ndev->int_mask = AMD_EVENT_INTMASK;
  676. spin_lock_init(&ndev->db_mask_lock);
  677. }
  678. static int amd_poll_link(struct amd_ntb_dev *ndev)
  679. {
  680. void __iomem *mmio = ndev->peer_mmio;
  681. u32 reg, stat;
  682. int rc;
  683. reg = readl(mmio + AMD_SIDEINFO_OFFSET);
  684. reg &= NTB_LIN_STA_ACTIVE_BIT;
  685. dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
  686. if (reg == ndev->cntl_sta)
  687. return 0;
  688. ndev->cntl_sta = reg;
  689. rc = pci_read_config_dword(ndev->ntb.pdev,
  690. AMD_LINK_STATUS_OFFSET, &stat);
  691. if (rc)
  692. return 0;
  693. ndev->lnk_sta = stat;
  694. return 1;
  695. }
  696. static void amd_link_hb(struct work_struct *work)
  697. {
  698. struct amd_ntb_dev *ndev = hb_ndev(work);
  699. if (amd_poll_link(ndev))
  700. ntb_link_event(&ndev->ntb);
  701. if (!amd_link_is_up(ndev))
  702. schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
  703. }
  704. static int amd_init_isr(struct amd_ntb_dev *ndev)
  705. {
  706. return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
  707. }
  708. static void amd_init_side_info(struct amd_ntb_dev *ndev)
  709. {
  710. void __iomem *mmio = ndev->self_mmio;
  711. unsigned int reg;
  712. reg = readl(mmio + AMD_SIDEINFO_OFFSET);
  713. if (!(reg & AMD_SIDE_READY)) {
  714. reg |= AMD_SIDE_READY;
  715. writel(reg, mmio + AMD_SIDEINFO_OFFSET);
  716. }
  717. }
  718. static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
  719. {
  720. void __iomem *mmio = ndev->self_mmio;
  721. unsigned int reg;
  722. reg = readl(mmio + AMD_SIDEINFO_OFFSET);
  723. if (reg & AMD_SIDE_READY) {
  724. reg &= ~AMD_SIDE_READY;
  725. writel(reg, mmio + AMD_SIDEINFO_OFFSET);
  726. readl(mmio + AMD_SIDEINFO_OFFSET);
  727. }
  728. }
  729. static int amd_init_ntb(struct amd_ntb_dev *ndev)
  730. {
  731. void __iomem *mmio = ndev->self_mmio;
  732. ndev->mw_count = AMD_MW_CNT;
  733. ndev->spad_count = AMD_SPADS_CNT;
  734. ndev->db_count = AMD_DB_CNT;
  735. switch (ndev->ntb.topo) {
  736. case NTB_TOPO_PRI:
  737. case NTB_TOPO_SEC:
  738. ndev->spad_count >>= 1;
  739. if (ndev->ntb.topo == NTB_TOPO_PRI) {
  740. ndev->self_spad = 0;
  741. ndev->peer_spad = 0x20;
  742. } else {
  743. ndev->self_spad = 0x20;
  744. ndev->peer_spad = 0;
  745. }
  746. INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
  747. schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
  748. break;
  749. default:
  750. dev_err(&ndev->ntb.pdev->dev,
  751. "AMD NTB does not support B2B mode.\n");
  752. return -EINVAL;
  753. }
  754. ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
  755. /* Mask event interrupts */
  756. writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
  757. return 0;
  758. }
  759. static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
  760. {
  761. void __iomem *mmio = ndev->self_mmio;
  762. u32 info;
  763. info = readl(mmio + AMD_SIDEINFO_OFFSET);
  764. if (info & AMD_SIDE_MASK)
  765. return NTB_TOPO_SEC;
  766. else
  767. return NTB_TOPO_PRI;
  768. }
  769. static int amd_init_dev(struct amd_ntb_dev *ndev)
  770. {
  771. struct pci_dev *pdev;
  772. int rc = 0;
  773. pdev = ndev->ntb.pdev;
  774. ndev->ntb.topo = amd_get_topo(ndev);
  775. dev_dbg(&pdev->dev, "AMD NTB topo is %s\n",
  776. ntb_topo_string(ndev->ntb.topo));
  777. rc = amd_init_ntb(ndev);
  778. if (rc)
  779. return rc;
  780. rc = amd_init_isr(ndev);
  781. if (rc) {
  782. dev_err(&pdev->dev, "fail to init isr.\n");
  783. return rc;
  784. }
  785. ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
  786. return 0;
  787. }
  788. static void amd_deinit_dev(struct amd_ntb_dev *ndev)
  789. {
  790. cancel_delayed_work_sync(&ndev->hb_timer);
  791. ndev_deinit_isr(ndev);
  792. }
  793. static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
  794. struct pci_dev *pdev)
  795. {
  796. int rc;
  797. pci_set_drvdata(pdev, ndev);
  798. rc = pci_enable_device(pdev);
  799. if (rc)
  800. goto err_pci_enable;
  801. rc = pci_request_regions(pdev, NTB_NAME);
  802. if (rc)
  803. goto err_pci_regions;
  804. pci_set_master(pdev);
  805. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
  806. if (rc) {
  807. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
  808. if (rc)
  809. goto err_dma_mask;
  810. dev_warn(&pdev->dev, "Cannot DMA highmem\n");
  811. }
  812. rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
  813. if (rc) {
  814. rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
  815. if (rc)
  816. goto err_dma_mask;
  817. dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
  818. }
  819. rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
  820. dma_get_mask(&pdev->dev));
  821. if (rc)
  822. goto err_dma_mask;
  823. ndev->self_mmio = pci_iomap(pdev, 0, 0);
  824. if (!ndev->self_mmio) {
  825. rc = -EIO;
  826. goto err_dma_mask;
  827. }
  828. ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
  829. return 0;
  830. err_dma_mask:
  831. pci_clear_master(pdev);
  832. pci_release_regions(pdev);
  833. err_pci_regions:
  834. pci_disable_device(pdev);
  835. err_pci_enable:
  836. pci_set_drvdata(pdev, NULL);
  837. return rc;
  838. }
  839. static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
  840. {
  841. struct pci_dev *pdev = ndev->ntb.pdev;
  842. pci_iounmap(pdev, ndev->self_mmio);
  843. pci_clear_master(pdev);
  844. pci_release_regions(pdev);
  845. pci_disable_device(pdev);
  846. pci_set_drvdata(pdev, NULL);
  847. }
  848. static int amd_ntb_pci_probe(struct pci_dev *pdev,
  849. const struct pci_device_id *id)
  850. {
  851. struct amd_ntb_dev *ndev;
  852. int rc, node;
  853. node = dev_to_node(&pdev->dev);
  854. ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
  855. if (!ndev) {
  856. rc = -ENOMEM;
  857. goto err_ndev;
  858. }
  859. ndev_init_struct(ndev, pdev);
  860. rc = amd_ntb_init_pci(ndev, pdev);
  861. if (rc)
  862. goto err_init_pci;
  863. rc = amd_init_dev(ndev);
  864. if (rc)
  865. goto err_init_dev;
  866. /* write side info */
  867. amd_init_side_info(ndev);
  868. amd_poll_link(ndev);
  869. ndev_init_debugfs(ndev);
  870. rc = ntb_register_device(&ndev->ntb);
  871. if (rc)
  872. goto err_register;
  873. dev_info(&pdev->dev, "NTB device registered.\n");
  874. return 0;
  875. err_register:
  876. ndev_deinit_debugfs(ndev);
  877. amd_deinit_dev(ndev);
  878. err_init_dev:
  879. amd_ntb_deinit_pci(ndev);
  880. err_init_pci:
  881. kfree(ndev);
  882. err_ndev:
  883. return rc;
  884. }
  885. static void amd_ntb_pci_remove(struct pci_dev *pdev)
  886. {
  887. struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
  888. ntb_unregister_device(&ndev->ntb);
  889. ndev_deinit_debugfs(ndev);
  890. amd_deinit_side_info(ndev);
  891. amd_deinit_dev(ndev);
  892. amd_ntb_deinit_pci(ndev);
  893. kfree(ndev);
  894. }
  895. static const struct file_operations amd_ntb_debugfs_info = {
  896. .owner = THIS_MODULE,
  897. .open = simple_open,
  898. .read = ndev_debugfs_read,
  899. };
  900. static const struct pci_device_id amd_ntb_pci_tbl[] = {
  901. {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)},
  902. {0}
  903. };
  904. MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
  905. static struct pci_driver amd_ntb_pci_driver = {
  906. .name = KBUILD_MODNAME,
  907. .id_table = amd_ntb_pci_tbl,
  908. .probe = amd_ntb_pci_probe,
  909. .remove = amd_ntb_pci_remove,
  910. };
  911. static int __init amd_ntb_pci_driver_init(void)
  912. {
  913. pr_info("%s %s\n", NTB_DESC, NTB_VER);
  914. if (debugfs_initialized())
  915. debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
  916. return pci_register_driver(&amd_ntb_pci_driver);
  917. }
  918. module_init(amd_ntb_pci_driver_init);
  919. static void __exit amd_ntb_pci_driver_exit(void)
  920. {
  921. pci_unregister_driver(&amd_ntb_pci_driver);
  922. debugfs_remove_recursive(debugfs_dir);
  923. }
  924. module_exit(amd_ntb_pci_driver_exit);