ntb_hw_gen1.c 56 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070
  1. /*
  2. * This file is provided under a dual BSD/GPLv2 license. When using or
  3. * redistributing this file, you may do so under either license.
  4. *
  5. * GPL LICENSE SUMMARY
  6. *
  7. * Copyright(c) 2012 Intel Corporation. All rights reserved.
  8. * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  9. * Copyright (C) 2016 T-Platforms. All Rights Reserved.
  10. *
  11. * This program is free software; you can redistribute it and/or modify
  12. * it under the terms of version 2 of the GNU General Public License as
  13. * published by the Free Software Foundation.
  14. *
  15. * BSD LICENSE
  16. *
  17. * Copyright(c) 2012 Intel Corporation. All rights reserved.
  18. * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  19. * Copyright (C) 2016 T-Platforms. All Rights Reserved.
  20. *
  21. * Redistribution and use in source and binary forms, with or without
  22. * modification, are permitted provided that the following conditions
  23. * are met:
  24. *
  25. * * Redistributions of source code must retain the above copyright
  26. * notice, this list of conditions and the following disclaimer.
  27. * * Redistributions in binary form must reproduce the above copy
  28. * notice, this list of conditions and the following disclaimer in
  29. * the documentation and/or other materials provided with the
  30. * distribution.
  31. * * Neither the name of Intel Corporation nor the names of its
  32. * contributors may be used to endorse or promote products derived
  33. * from this software without specific prior written permission.
  34. *
  35. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46. *
  47. * Intel PCIe NTB Linux driver
  48. */
  49. #include <linux/debugfs.h>
  50. #include <linux/delay.h>
  51. #include <linux/init.h>
  52. #include <linux/interrupt.h>
  53. #include <linux/module.h>
  54. #include <linux/pci.h>
  55. #include <linux/random.h>
  56. #include <linux/slab.h>
  57. #include <linux/ntb.h>
  58. #include "ntb_hw_intel.h"
  59. #include "ntb_hw_gen1.h"
  60. #include "ntb_hw_gen3.h"
  61. #define NTB_NAME "ntb_hw_intel"
  62. #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
  63. #define NTB_VER "2.0"
  64. MODULE_DESCRIPTION(NTB_DESC);
  65. MODULE_VERSION(NTB_VER);
  66. MODULE_LICENSE("Dual BSD/GPL");
  67. MODULE_AUTHOR("Intel Corporation");
  68. #define bar0_off(base, bar) ((base) + ((bar) << 2))
  69. #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
  70. static const struct intel_ntb_reg xeon_reg;
  71. static const struct intel_ntb_alt_reg xeon_pri_reg;
  72. static const struct intel_ntb_alt_reg xeon_sec_reg;
  73. static const struct intel_ntb_alt_reg xeon_b2b_reg;
  74. static const struct intel_ntb_xlat_reg xeon_pri_xlat;
  75. static const struct intel_ntb_xlat_reg xeon_sec_xlat;
  76. static const struct ntb_dev_ops intel_ntb_ops;
  77. static const struct file_operations intel_ntb_debugfs_info;
  78. static struct dentry *debugfs_dir;
  79. static int b2b_mw_idx = -1;
  80. module_param(b2b_mw_idx, int, 0644);
  81. MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
  82. "value of zero or positive starts from first mw idx, and a "
  83. "negative value starts from last mw idx. Both sides MUST "
  84. "set the same value here!");
  85. static unsigned int b2b_mw_share;
  86. module_param(b2b_mw_share, uint, 0644);
  87. MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
  88. "ntb so that the peer ntb only occupies the first half of "
  89. "the mw, so the second half can still be used as a mw. Both "
  90. "sides MUST set the same value here!");
  91. module_param_named(xeon_b2b_usd_bar2_addr64,
  92. xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
  93. MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
  94. "XEON B2B USD BAR 2 64-bit address");
  95. module_param_named(xeon_b2b_usd_bar4_addr64,
  96. xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
  97. MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
  98. "XEON B2B USD BAR 4 64-bit address");
  99. module_param_named(xeon_b2b_usd_bar4_addr32,
  100. xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
  101. MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
  102. "XEON B2B USD split-BAR 4 32-bit address");
  103. module_param_named(xeon_b2b_usd_bar5_addr32,
  104. xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
  105. MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
  106. "XEON B2B USD split-BAR 5 32-bit address");
  107. module_param_named(xeon_b2b_dsd_bar2_addr64,
  108. xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
  109. MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
  110. "XEON B2B DSD BAR 2 64-bit address");
  111. module_param_named(xeon_b2b_dsd_bar4_addr64,
  112. xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
  113. MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
  114. "XEON B2B DSD BAR 4 64-bit address");
  115. module_param_named(xeon_b2b_dsd_bar4_addr32,
  116. xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
  117. MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
  118. "XEON B2B DSD split-BAR 4 32-bit address");
  119. module_param_named(xeon_b2b_dsd_bar5_addr32,
  120. xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
  121. MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
  122. "XEON B2B DSD split-BAR 5 32-bit address");
  123. static int xeon_init_isr(struct intel_ntb_dev *ndev);
  124. static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
  125. {
  126. ndev->unsafe_flags = 0;
  127. ndev->unsafe_flags_ignore = 0;
  128. /* Only B2B has a workaround to avoid SDOORBELL */
  129. if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
  130. if (!ntb_topo_is_b2b(ndev->ntb.topo))
  131. ndev->unsafe_flags |= NTB_UNSAFE_DB;
  132. /* No low level workaround to avoid SB01BASE */
  133. if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
  134. ndev->unsafe_flags |= NTB_UNSAFE_DB;
  135. ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
  136. }
  137. }
  138. static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
  139. unsigned long flag)
  140. {
  141. return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
  142. }
  143. static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
  144. unsigned long flag)
  145. {
  146. flag &= ndev->unsafe_flags;
  147. ndev->unsafe_flags_ignore |= flag;
  148. return !!flag;
  149. }
  150. int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
  151. {
  152. if (idx < 0 || idx >= ndev->mw_count)
  153. return -EINVAL;
  154. return ndev->reg->mw_bar[idx];
  155. }
  156. static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
  157. phys_addr_t *db_addr, resource_size_t *db_size,
  158. phys_addr_t reg_addr, unsigned long reg)
  159. {
  160. if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
  161. pr_warn_once("%s: NTB unsafe doorbell access", __func__);
  162. if (db_addr) {
  163. *db_addr = reg_addr + reg;
  164. dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
  165. }
  166. if (db_size) {
  167. *db_size = ndev->reg->db_size;
  168. dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
  169. }
  170. return 0;
  171. }
  172. u64 ndev_db_read(struct intel_ntb_dev *ndev,
  173. void __iomem *mmio)
  174. {
  175. if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
  176. pr_warn_once("%s: NTB unsafe doorbell access", __func__);
  177. return ndev->reg->db_ioread(mmio);
  178. }
  179. int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
  180. void __iomem *mmio)
  181. {
  182. if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
  183. pr_warn_once("%s: NTB unsafe doorbell access", __func__);
  184. if (db_bits & ~ndev->db_valid_mask)
  185. return -EINVAL;
  186. ndev->reg->db_iowrite(db_bits, mmio);
  187. return 0;
  188. }
  189. static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
  190. void __iomem *mmio)
  191. {
  192. unsigned long irqflags;
  193. if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
  194. pr_warn_once("%s: NTB unsafe doorbell access", __func__);
  195. if (db_bits & ~ndev->db_valid_mask)
  196. return -EINVAL;
  197. spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
  198. {
  199. ndev->db_mask |= db_bits;
  200. ndev->reg->db_iowrite(ndev->db_mask, mmio);
  201. }
  202. spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
  203. return 0;
  204. }
  205. static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
  206. void __iomem *mmio)
  207. {
  208. unsigned long irqflags;
  209. if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
  210. pr_warn_once("%s: NTB unsafe doorbell access", __func__);
  211. if (db_bits & ~ndev->db_valid_mask)
  212. return -EINVAL;
  213. spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
  214. {
  215. ndev->db_mask &= ~db_bits;
  216. ndev->reg->db_iowrite(ndev->db_mask, mmio);
  217. }
  218. spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
  219. return 0;
  220. }
  221. static inline u64 ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
  222. {
  223. u64 shift, mask;
  224. shift = ndev->db_vec_shift;
  225. mask = BIT_ULL(shift) - 1;
  226. return mask << (shift * db_vector);
  227. }
  228. static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
  229. phys_addr_t *spad_addr, phys_addr_t reg_addr,
  230. unsigned long reg)
  231. {
  232. if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
  233. pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
  234. if (idx < 0 || idx >= ndev->spad_count)
  235. return -EINVAL;
  236. if (spad_addr) {
  237. *spad_addr = reg_addr + reg + (idx << 2);
  238. dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
  239. *spad_addr);
  240. }
  241. return 0;
  242. }
  243. static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
  244. void __iomem *mmio)
  245. {
  246. if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
  247. pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
  248. if (idx < 0 || idx >= ndev->spad_count)
  249. return 0;
  250. return ioread32(mmio + (idx << 2));
  251. }
  252. static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
  253. void __iomem *mmio)
  254. {
  255. if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
  256. pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
  257. if (idx < 0 || idx >= ndev->spad_count)
  258. return -EINVAL;
  259. iowrite32(val, mmio + (idx << 2));
  260. return 0;
  261. }
  262. static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
  263. {
  264. u64 vec_mask;
  265. vec_mask = ndev_vec_mask(ndev, vec);
  266. if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
  267. vec_mask |= ndev->db_link_mask;
  268. dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
  269. ndev->last_ts = jiffies;
  270. if (vec_mask & ndev->db_link_mask) {
  271. if (ndev->reg->poll_link(ndev))
  272. ntb_link_event(&ndev->ntb);
  273. }
  274. if (vec_mask & ndev->db_valid_mask)
  275. ntb_db_event(&ndev->ntb, vec);
  276. return IRQ_HANDLED;
  277. }
  278. static irqreturn_t ndev_vec_isr(int irq, void *dev)
  279. {
  280. struct intel_ntb_vec *nvec = dev;
  281. dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d nvec->num: %d\n",
  282. irq, nvec->num);
  283. return ndev_interrupt(nvec->ndev, nvec->num);
  284. }
  285. static irqreturn_t ndev_irq_isr(int irq, void *dev)
  286. {
  287. struct intel_ntb_dev *ndev = dev;
  288. return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
  289. }
  290. int ndev_init_isr(struct intel_ntb_dev *ndev,
  291. int msix_min, int msix_max,
  292. int msix_shift, int total_shift)
  293. {
  294. struct pci_dev *pdev;
  295. int rc, i, msix_count, node;
  296. pdev = ndev->ntb.pdev;
  297. node = dev_to_node(&pdev->dev);
  298. /* Mask all doorbell interrupts */
  299. ndev->db_mask = ndev->db_valid_mask;
  300. ndev->reg->db_iowrite(ndev->db_mask,
  301. ndev->self_mmio +
  302. ndev->self_reg->db_mask);
  303. /* Try to set up msix irq */
  304. ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
  305. GFP_KERNEL, node);
  306. if (!ndev->vec)
  307. goto err_msix_vec_alloc;
  308. ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
  309. GFP_KERNEL, node);
  310. if (!ndev->msix)
  311. goto err_msix_alloc;
  312. for (i = 0; i < msix_max; ++i)
  313. ndev->msix[i].entry = i;
  314. msix_count = pci_enable_msix_range(pdev, ndev->msix,
  315. msix_min, msix_max);
  316. if (msix_count < 0)
  317. goto err_msix_enable;
  318. for (i = 0; i < msix_count; ++i) {
  319. ndev->vec[i].ndev = ndev;
  320. ndev->vec[i].num = i;
  321. rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
  322. "ndev_vec_isr", &ndev->vec[i]);
  323. if (rc)
  324. goto err_msix_request;
  325. }
  326. dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
  327. ndev->db_vec_count = msix_count;
  328. ndev->db_vec_shift = msix_shift;
  329. return 0;
  330. err_msix_request:
  331. while (i-- > 0)
  332. free_irq(ndev->msix[i].vector, &ndev->vec[i]);
  333. pci_disable_msix(pdev);
  334. err_msix_enable:
  335. kfree(ndev->msix);
  336. err_msix_alloc:
  337. kfree(ndev->vec);
  338. err_msix_vec_alloc:
  339. ndev->msix = NULL;
  340. ndev->vec = NULL;
  341. /* Try to set up msi irq */
  342. rc = pci_enable_msi(pdev);
  343. if (rc)
  344. goto err_msi_enable;
  345. rc = request_irq(pdev->irq, ndev_irq_isr, 0,
  346. "ndev_irq_isr", ndev);
  347. if (rc)
  348. goto err_msi_request;
  349. dev_dbg(&pdev->dev, "Using msi interrupts\n");
  350. ndev->db_vec_count = 1;
  351. ndev->db_vec_shift = total_shift;
  352. return 0;
  353. err_msi_request:
  354. pci_disable_msi(pdev);
  355. err_msi_enable:
  356. /* Try to set up intx irq */
  357. pci_intx(pdev, 1);
  358. rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
  359. "ndev_irq_isr", ndev);
  360. if (rc)
  361. goto err_intx_request;
  362. dev_dbg(&pdev->dev, "Using intx interrupts\n");
  363. ndev->db_vec_count = 1;
  364. ndev->db_vec_shift = total_shift;
  365. return 0;
  366. err_intx_request:
  367. return rc;
  368. }
  369. static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
  370. {
  371. struct pci_dev *pdev;
  372. int i;
  373. pdev = ndev->ntb.pdev;
  374. /* Mask all doorbell interrupts */
  375. ndev->db_mask = ndev->db_valid_mask;
  376. ndev->reg->db_iowrite(ndev->db_mask,
  377. ndev->self_mmio +
  378. ndev->self_reg->db_mask);
  379. if (ndev->msix) {
  380. i = ndev->db_vec_count;
  381. while (i--)
  382. free_irq(ndev->msix[i].vector, &ndev->vec[i]);
  383. pci_disable_msix(pdev);
  384. kfree(ndev->msix);
  385. kfree(ndev->vec);
  386. } else {
  387. free_irq(pdev->irq, ndev);
  388. if (pci_dev_msi_enabled(pdev))
  389. pci_disable_msi(pdev);
  390. }
  391. }
  392. static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
  393. size_t count, loff_t *offp)
  394. {
  395. struct intel_ntb_dev *ndev;
  396. struct pci_dev *pdev;
  397. void __iomem *mmio;
  398. char *buf;
  399. size_t buf_size;
  400. ssize_t ret, off;
  401. union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
  402. ndev = filp->private_data;
  403. pdev = ndev->ntb.pdev;
  404. mmio = ndev->self_mmio;
  405. buf_size = min(count, 0x800ul);
  406. buf = kmalloc(buf_size, GFP_KERNEL);
  407. if (!buf)
  408. return -ENOMEM;
  409. off = 0;
  410. off += scnprintf(buf + off, buf_size - off,
  411. "NTB Device Information:\n");
  412. off += scnprintf(buf + off, buf_size - off,
  413. "Connection Topology -\t%s\n",
  414. ntb_topo_string(ndev->ntb.topo));
  415. if (ndev->b2b_idx != UINT_MAX) {
  416. off += scnprintf(buf + off, buf_size - off,
  417. "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
  418. off += scnprintf(buf + off, buf_size - off,
  419. "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
  420. }
  421. off += scnprintf(buf + off, buf_size - off,
  422. "BAR4 Split -\t\t%s\n",
  423. ndev->bar4_split ? "yes" : "no");
  424. off += scnprintf(buf + off, buf_size - off,
  425. "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
  426. off += scnprintf(buf + off, buf_size - off,
  427. "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
  428. if (!ndev->reg->link_is_up(ndev)) {
  429. off += scnprintf(buf + off, buf_size - off,
  430. "Link Status -\t\tDown\n");
  431. } else {
  432. off += scnprintf(buf + off, buf_size - off,
  433. "Link Status -\t\tUp\n");
  434. off += scnprintf(buf + off, buf_size - off,
  435. "Link Speed -\t\tPCI-E Gen %u\n",
  436. NTB_LNK_STA_SPEED(ndev->lnk_sta));
  437. off += scnprintf(buf + off, buf_size - off,
  438. "Link Width -\t\tx%u\n",
  439. NTB_LNK_STA_WIDTH(ndev->lnk_sta));
  440. }
  441. off += scnprintf(buf + off, buf_size - off,
  442. "Memory Window Count -\t%u\n", ndev->mw_count);
  443. off += scnprintf(buf + off, buf_size - off,
  444. "Scratchpad Count -\t%u\n", ndev->spad_count);
  445. off += scnprintf(buf + off, buf_size - off,
  446. "Doorbell Count -\t%u\n", ndev->db_count);
  447. off += scnprintf(buf + off, buf_size - off,
  448. "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
  449. off += scnprintf(buf + off, buf_size - off,
  450. "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
  451. off += scnprintf(buf + off, buf_size - off,
  452. "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
  453. off += scnprintf(buf + off, buf_size - off,
  454. "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
  455. off += scnprintf(buf + off, buf_size - off,
  456. "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
  457. u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
  458. off += scnprintf(buf + off, buf_size - off,
  459. "Doorbell Mask -\t\t%#llx\n", u.v64);
  460. u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
  461. off += scnprintf(buf + off, buf_size - off,
  462. "Doorbell Bell -\t\t%#llx\n", u.v64);
  463. off += scnprintf(buf + off, buf_size - off,
  464. "\nNTB Window Size:\n");
  465. pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
  466. off += scnprintf(buf + off, buf_size - off,
  467. "PBAR23SZ %hhu\n", u.v8);
  468. if (!ndev->bar4_split) {
  469. pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
  470. off += scnprintf(buf + off, buf_size - off,
  471. "PBAR45SZ %hhu\n", u.v8);
  472. } else {
  473. pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
  474. off += scnprintf(buf + off, buf_size - off,
  475. "PBAR4SZ %hhu\n", u.v8);
  476. pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
  477. off += scnprintf(buf + off, buf_size - off,
  478. "PBAR5SZ %hhu\n", u.v8);
  479. }
  480. pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
  481. off += scnprintf(buf + off, buf_size - off,
  482. "SBAR23SZ %hhu\n", u.v8);
  483. if (!ndev->bar4_split) {
  484. pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
  485. off += scnprintf(buf + off, buf_size - off,
  486. "SBAR45SZ %hhu\n", u.v8);
  487. } else {
  488. pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
  489. off += scnprintf(buf + off, buf_size - off,
  490. "SBAR4SZ %hhu\n", u.v8);
  491. pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
  492. off += scnprintf(buf + off, buf_size - off,
  493. "SBAR5SZ %hhu\n", u.v8);
  494. }
  495. off += scnprintf(buf + off, buf_size - off,
  496. "\nNTB Incoming XLAT:\n");
  497. u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
  498. off += scnprintf(buf + off, buf_size - off,
  499. "XLAT23 -\t\t%#018llx\n", u.v64);
  500. if (ndev->bar4_split) {
  501. u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
  502. off += scnprintf(buf + off, buf_size - off,
  503. "XLAT4 -\t\t\t%#06x\n", u.v32);
  504. u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
  505. off += scnprintf(buf + off, buf_size - off,
  506. "XLAT5 -\t\t\t%#06x\n", u.v32);
  507. } else {
  508. u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
  509. off += scnprintf(buf + off, buf_size - off,
  510. "XLAT45 -\t\t%#018llx\n", u.v64);
  511. }
  512. u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
  513. off += scnprintf(buf + off, buf_size - off,
  514. "LMT23 -\t\t\t%#018llx\n", u.v64);
  515. if (ndev->bar4_split) {
  516. u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
  517. off += scnprintf(buf + off, buf_size - off,
  518. "LMT4 -\t\t\t%#06x\n", u.v32);
  519. u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
  520. off += scnprintf(buf + off, buf_size - off,
  521. "LMT5 -\t\t\t%#06x\n", u.v32);
  522. } else {
  523. u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
  524. off += scnprintf(buf + off, buf_size - off,
  525. "LMT45 -\t\t\t%#018llx\n", u.v64);
  526. }
  527. if (pdev_is_gen1(pdev)) {
  528. if (ntb_topo_is_b2b(ndev->ntb.topo)) {
  529. off += scnprintf(buf + off, buf_size - off,
  530. "\nNTB Outgoing B2B XLAT:\n");
  531. u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
  532. off += scnprintf(buf + off, buf_size - off,
  533. "B2B XLAT23 -\t\t%#018llx\n", u.v64);
  534. if (ndev->bar4_split) {
  535. u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
  536. off += scnprintf(buf + off, buf_size - off,
  537. "B2B XLAT4 -\t\t%#06x\n",
  538. u.v32);
  539. u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
  540. off += scnprintf(buf + off, buf_size - off,
  541. "B2B XLAT5 -\t\t%#06x\n",
  542. u.v32);
  543. } else {
  544. u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
  545. off += scnprintf(buf + off, buf_size - off,
  546. "B2B XLAT45 -\t\t%#018llx\n",
  547. u.v64);
  548. }
  549. u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
  550. off += scnprintf(buf + off, buf_size - off,
  551. "B2B LMT23 -\t\t%#018llx\n", u.v64);
  552. if (ndev->bar4_split) {
  553. u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
  554. off += scnprintf(buf + off, buf_size - off,
  555. "B2B LMT4 -\t\t%#06x\n",
  556. u.v32);
  557. u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
  558. off += scnprintf(buf + off, buf_size - off,
  559. "B2B LMT5 -\t\t%#06x\n",
  560. u.v32);
  561. } else {
  562. u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
  563. off += scnprintf(buf + off, buf_size - off,
  564. "B2B LMT45 -\t\t%#018llx\n",
  565. u.v64);
  566. }
  567. off += scnprintf(buf + off, buf_size - off,
  568. "\nNTB Secondary BAR:\n");
  569. u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
  570. off += scnprintf(buf + off, buf_size - off,
  571. "SBAR01 -\t\t%#018llx\n", u.v64);
  572. u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
  573. off += scnprintf(buf + off, buf_size - off,
  574. "SBAR23 -\t\t%#018llx\n", u.v64);
  575. if (ndev->bar4_split) {
  576. u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
  577. off += scnprintf(buf + off, buf_size - off,
  578. "SBAR4 -\t\t\t%#06x\n", u.v32);
  579. u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
  580. off += scnprintf(buf + off, buf_size - off,
  581. "SBAR5 -\t\t\t%#06x\n", u.v32);
  582. } else {
  583. u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
  584. off += scnprintf(buf + off, buf_size - off,
  585. "SBAR45 -\t\t%#018llx\n",
  586. u.v64);
  587. }
  588. }
  589. off += scnprintf(buf + off, buf_size - off,
  590. "\nXEON NTB Statistics:\n");
  591. u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
  592. off += scnprintf(buf + off, buf_size - off,
  593. "Upstream Memory Miss -\t%u\n", u.v16);
  594. off += scnprintf(buf + off, buf_size - off,
  595. "\nXEON NTB Hardware Errors:\n");
  596. if (!pci_read_config_word(pdev,
  597. XEON_DEVSTS_OFFSET, &u.v16))
  598. off += scnprintf(buf + off, buf_size - off,
  599. "DEVSTS -\t\t%#06x\n", u.v16);
  600. if (!pci_read_config_word(pdev,
  601. XEON_LINK_STATUS_OFFSET, &u.v16))
  602. off += scnprintf(buf + off, buf_size - off,
  603. "LNKSTS -\t\t%#06x\n", u.v16);
  604. if (!pci_read_config_dword(pdev,
  605. XEON_UNCERRSTS_OFFSET, &u.v32))
  606. off += scnprintf(buf + off, buf_size - off,
  607. "UNCERRSTS -\t\t%#06x\n", u.v32);
  608. if (!pci_read_config_dword(pdev,
  609. XEON_CORERRSTS_OFFSET, &u.v32))
  610. off += scnprintf(buf + off, buf_size - off,
  611. "CORERRSTS -\t\t%#06x\n", u.v32);
  612. }
  613. ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
  614. kfree(buf);
  615. return ret;
  616. }
  617. static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
  618. size_t count, loff_t *offp)
  619. {
  620. struct intel_ntb_dev *ndev = filp->private_data;
  621. if (pdev_is_gen1(ndev->ntb.pdev))
  622. return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
  623. else if (pdev_is_gen3(ndev->ntb.pdev))
  624. return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
  625. return -ENXIO;
  626. }
  627. static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
  628. {
  629. if (!debugfs_dir) {
  630. ndev->debugfs_dir = NULL;
  631. ndev->debugfs_info = NULL;
  632. } else {
  633. ndev->debugfs_dir =
  634. debugfs_create_dir(pci_name(ndev->ntb.pdev),
  635. debugfs_dir);
  636. if (!ndev->debugfs_dir)
  637. ndev->debugfs_info = NULL;
  638. else
  639. ndev->debugfs_info =
  640. debugfs_create_file("info", S_IRUSR,
  641. ndev->debugfs_dir, ndev,
  642. &intel_ntb_debugfs_info);
  643. }
  644. }
  645. static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
  646. {
  647. debugfs_remove_recursive(ndev->debugfs_dir);
  648. }
  649. int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
  650. {
  651. if (pidx != NTB_DEF_PEER_IDX)
  652. return -EINVAL;
  653. return ntb_ndev(ntb)->mw_count;
  654. }
  655. int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
  656. resource_size_t *addr_align,
  657. resource_size_t *size_align,
  658. resource_size_t *size_max)
  659. {
  660. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  661. resource_size_t bar_size, mw_size;
  662. int bar;
  663. if (pidx != NTB_DEF_PEER_IDX)
  664. return -EINVAL;
  665. if (idx >= ndev->b2b_idx && !ndev->b2b_off)
  666. idx += 1;
  667. bar = ndev_mw_to_bar(ndev, idx);
  668. if (bar < 0)
  669. return bar;
  670. bar_size = pci_resource_len(ndev->ntb.pdev, bar);
  671. if (idx == ndev->b2b_idx)
  672. mw_size = bar_size - ndev->b2b_off;
  673. else
  674. mw_size = bar_size;
  675. if (addr_align)
  676. *addr_align = pci_resource_len(ndev->ntb.pdev, bar);
  677. if (size_align)
  678. *size_align = 1;
  679. if (size_max)
  680. *size_max = mw_size;
  681. return 0;
  682. }
  683. static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
  684. dma_addr_t addr, resource_size_t size)
  685. {
  686. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  687. unsigned long base_reg, xlat_reg, limit_reg;
  688. resource_size_t bar_size, mw_size;
  689. void __iomem *mmio;
  690. u64 base, limit, reg_val;
  691. int bar;
  692. if (pidx != NTB_DEF_PEER_IDX)
  693. return -EINVAL;
  694. if (idx >= ndev->b2b_idx && !ndev->b2b_off)
  695. idx += 1;
  696. bar = ndev_mw_to_bar(ndev, idx);
  697. if (bar < 0)
  698. return bar;
  699. bar_size = pci_resource_len(ndev->ntb.pdev, bar);
  700. if (idx == ndev->b2b_idx)
  701. mw_size = bar_size - ndev->b2b_off;
  702. else
  703. mw_size = bar_size;
  704. /* hardware requires that addr is aligned to bar size */
  705. if (addr & (bar_size - 1))
  706. return -EINVAL;
  707. /* make sure the range fits in the usable mw size */
  708. if (size > mw_size)
  709. return -EINVAL;
  710. mmio = ndev->self_mmio;
  711. base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
  712. xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
  713. limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
  714. if (bar < 4 || !ndev->bar4_split) {
  715. base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
  716. /* Set the limit if supported, if size is not mw_size */
  717. if (limit_reg && size != mw_size)
  718. limit = base + size;
  719. else
  720. limit = 0;
  721. /* set and verify setting the translation address */
  722. iowrite64(addr, mmio + xlat_reg);
  723. reg_val = ioread64(mmio + xlat_reg);
  724. if (reg_val != addr) {
  725. iowrite64(0, mmio + xlat_reg);
  726. return -EIO;
  727. }
  728. /* set and verify setting the limit */
  729. iowrite64(limit, mmio + limit_reg);
  730. reg_val = ioread64(mmio + limit_reg);
  731. if (reg_val != limit) {
  732. iowrite64(base, mmio + limit_reg);
  733. iowrite64(0, mmio + xlat_reg);
  734. return -EIO;
  735. }
  736. } else {
  737. /* split bar addr range must all be 32 bit */
  738. if (addr & (~0ull << 32))
  739. return -EINVAL;
  740. if ((addr + size) & (~0ull << 32))
  741. return -EINVAL;
  742. base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
  743. /* Set the limit if supported, if size is not mw_size */
  744. if (limit_reg && size != mw_size)
  745. limit = base + size;
  746. else
  747. limit = 0;
  748. /* set and verify setting the translation address */
  749. iowrite32(addr, mmio + xlat_reg);
  750. reg_val = ioread32(mmio + xlat_reg);
  751. if (reg_val != addr) {
  752. iowrite32(0, mmio + xlat_reg);
  753. return -EIO;
  754. }
  755. /* set and verify setting the limit */
  756. iowrite32(limit, mmio + limit_reg);
  757. reg_val = ioread32(mmio + limit_reg);
  758. if (reg_val != limit) {
  759. iowrite32(base, mmio + limit_reg);
  760. iowrite32(0, mmio + xlat_reg);
  761. return -EIO;
  762. }
  763. }
  764. return 0;
  765. }
  766. u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed,
  767. enum ntb_width *width)
  768. {
  769. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  770. if (ndev->reg->link_is_up(ndev)) {
  771. if (speed)
  772. *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
  773. if (width)
  774. *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
  775. return 1;
  776. } else {
  777. /* TODO MAYBE: is it possible to observe the link speed and
  778. * width while link is training? */
  779. if (speed)
  780. *speed = NTB_SPEED_NONE;
  781. if (width)
  782. *width = NTB_WIDTH_NONE;
  783. return 0;
  784. }
  785. }
  786. static int intel_ntb_link_enable(struct ntb_dev *ntb,
  787. enum ntb_speed max_speed,
  788. enum ntb_width max_width)
  789. {
  790. struct intel_ntb_dev *ndev;
  791. u32 ntb_ctl;
  792. ndev = container_of(ntb, struct intel_ntb_dev, ntb);
  793. if (ndev->ntb.topo == NTB_TOPO_SEC)
  794. return -EINVAL;
  795. dev_dbg(&ntb->pdev->dev,
  796. "Enabling link with max_speed %d max_width %d\n",
  797. max_speed, max_width);
  798. if (max_speed != NTB_SPEED_AUTO)
  799. dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
  800. if (max_width != NTB_WIDTH_AUTO)
  801. dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
  802. ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
  803. ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
  804. ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
  805. ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
  806. if (ndev->bar4_split)
  807. ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
  808. iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
  809. return 0;
  810. }
  811. int intel_ntb_link_disable(struct ntb_dev *ntb)
  812. {
  813. struct intel_ntb_dev *ndev;
  814. u32 ntb_cntl;
  815. ndev = container_of(ntb, struct intel_ntb_dev, ntb);
  816. if (ndev->ntb.topo == NTB_TOPO_SEC)
  817. return -EINVAL;
  818. dev_dbg(&ntb->pdev->dev, "Disabling link\n");
  819. /* Bring NTB link down */
  820. ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
  821. ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
  822. ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
  823. if (ndev->bar4_split)
  824. ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
  825. ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
  826. iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
  827. return 0;
  828. }
  829. int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
  830. {
  831. /* Numbers of inbound and outbound memory windows match */
  832. return ntb_ndev(ntb)->mw_count;
  833. }
  834. int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
  835. phys_addr_t *base, resource_size_t *size)
  836. {
  837. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  838. int bar;
  839. if (idx >= ndev->b2b_idx && !ndev->b2b_off)
  840. idx += 1;
  841. bar = ndev_mw_to_bar(ndev, idx);
  842. if (bar < 0)
  843. return bar;
  844. if (base)
  845. *base = pci_resource_start(ndev->ntb.pdev, bar) +
  846. (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
  847. if (size)
  848. *size = pci_resource_len(ndev->ntb.pdev, bar) -
  849. (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
  850. return 0;
  851. }
  852. static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
  853. {
  854. return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
  855. }
  856. u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
  857. {
  858. return ntb_ndev(ntb)->db_valid_mask;
  859. }
  860. int intel_ntb_db_vector_count(struct ntb_dev *ntb)
  861. {
  862. struct intel_ntb_dev *ndev;
  863. ndev = container_of(ntb, struct intel_ntb_dev, ntb);
  864. return ndev->db_vec_count;
  865. }
  866. u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
  867. {
  868. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  869. if (db_vector < 0 || db_vector > ndev->db_vec_count)
  870. return 0;
  871. return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
  872. }
  873. static u64 intel_ntb_db_read(struct ntb_dev *ntb)
  874. {
  875. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  876. return ndev_db_read(ndev,
  877. ndev->self_mmio +
  878. ndev->self_reg->db_bell);
  879. }
  880. static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
  881. {
  882. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  883. return ndev_db_write(ndev, db_bits,
  884. ndev->self_mmio +
  885. ndev->self_reg->db_bell);
  886. }
  887. int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
  888. {
  889. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  890. return ndev_db_set_mask(ndev, db_bits,
  891. ndev->self_mmio +
  892. ndev->self_reg->db_mask);
  893. }
  894. int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
  895. {
  896. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  897. return ndev_db_clear_mask(ndev, db_bits,
  898. ndev->self_mmio +
  899. ndev->self_reg->db_mask);
  900. }
  901. int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
  902. resource_size_t *db_size)
  903. {
  904. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  905. return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
  906. ndev->peer_reg->db_bell);
  907. }
  908. static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
  909. {
  910. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  911. return ndev_db_write(ndev, db_bits,
  912. ndev->peer_mmio +
  913. ndev->peer_reg->db_bell);
  914. }
  915. int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
  916. {
  917. return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
  918. }
  919. int intel_ntb_spad_count(struct ntb_dev *ntb)
  920. {
  921. struct intel_ntb_dev *ndev;
  922. ndev = container_of(ntb, struct intel_ntb_dev, ntb);
  923. return ndev->spad_count;
  924. }
  925. u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
  926. {
  927. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  928. return ndev_spad_read(ndev, idx,
  929. ndev->self_mmio +
  930. ndev->self_reg->spad);
  931. }
  932. int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
  933. {
  934. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  935. return ndev_spad_write(ndev, idx, val,
  936. ndev->self_mmio +
  937. ndev->self_reg->spad);
  938. }
  939. int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
  940. phys_addr_t *spad_addr)
  941. {
  942. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  943. return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
  944. ndev->peer_reg->spad);
  945. }
  946. u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
  947. {
  948. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  949. return ndev_spad_read(ndev, sidx,
  950. ndev->peer_mmio +
  951. ndev->peer_reg->spad);
  952. }
  953. int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx,
  954. u32 val)
  955. {
  956. struct intel_ntb_dev *ndev = ntb_ndev(ntb);
  957. return ndev_spad_write(ndev, sidx, val,
  958. ndev->peer_mmio +
  959. ndev->peer_reg->spad);
  960. }
  961. static u64 xeon_db_ioread(void __iomem *mmio)
  962. {
  963. return (u64)ioread16(mmio);
  964. }
  965. static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
  966. {
  967. iowrite16((u16)bits, mmio);
  968. }
  969. static int xeon_poll_link(struct intel_ntb_dev *ndev)
  970. {
  971. u16 reg_val;
  972. int rc;
  973. ndev->reg->db_iowrite(ndev->db_link_mask,
  974. ndev->self_mmio +
  975. ndev->self_reg->db_bell);
  976. rc = pci_read_config_word(ndev->ntb.pdev,
  977. XEON_LINK_STATUS_OFFSET, &reg_val);
  978. if (rc)
  979. return 0;
  980. if (reg_val == ndev->lnk_sta)
  981. return 0;
  982. ndev->lnk_sta = reg_val;
  983. return 1;
  984. }
  985. int xeon_link_is_up(struct intel_ntb_dev *ndev)
  986. {
  987. if (ndev->ntb.topo == NTB_TOPO_SEC)
  988. return 1;
  989. return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
  990. }
  991. enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
  992. {
  993. switch (ppd & XEON_PPD_TOPO_MASK) {
  994. case XEON_PPD_TOPO_B2B_USD:
  995. return NTB_TOPO_B2B_USD;
  996. case XEON_PPD_TOPO_B2B_DSD:
  997. return NTB_TOPO_B2B_DSD;
  998. case XEON_PPD_TOPO_PRI_USD:
  999. case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
  1000. return NTB_TOPO_PRI;
  1001. case XEON_PPD_TOPO_SEC_USD:
  1002. case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
  1003. return NTB_TOPO_SEC;
  1004. }
  1005. return NTB_TOPO_NONE;
  1006. }
  1007. static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
  1008. {
  1009. if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
  1010. dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
  1011. return 1;
  1012. }
  1013. return 0;
  1014. }
  1015. static int xeon_init_isr(struct intel_ntb_dev *ndev)
  1016. {
  1017. return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
  1018. XEON_DB_MSIX_VECTOR_COUNT,
  1019. XEON_DB_MSIX_VECTOR_SHIFT,
  1020. XEON_DB_TOTAL_SHIFT);
  1021. }
  1022. static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
  1023. {
  1024. ndev_deinit_isr(ndev);
  1025. }
  1026. static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
  1027. const struct intel_b2b_addr *addr,
  1028. const struct intel_b2b_addr *peer_addr)
  1029. {
  1030. struct pci_dev *pdev;
  1031. void __iomem *mmio;
  1032. resource_size_t bar_size;
  1033. phys_addr_t bar_addr;
  1034. int b2b_bar;
  1035. u8 bar_sz;
  1036. pdev = ndev->ntb.pdev;
  1037. mmio = ndev->self_mmio;
  1038. if (ndev->b2b_idx == UINT_MAX) {
  1039. dev_dbg(&pdev->dev, "not using b2b mw\n");
  1040. b2b_bar = 0;
  1041. ndev->b2b_off = 0;
  1042. } else {
  1043. b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
  1044. if (b2b_bar < 0)
  1045. return -EIO;
  1046. dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
  1047. bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
  1048. dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
  1049. if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
  1050. dev_dbg(&pdev->dev, "b2b using first half of bar\n");
  1051. ndev->b2b_off = bar_size >> 1;
  1052. } else if (XEON_B2B_MIN_SIZE <= bar_size) {
  1053. dev_dbg(&pdev->dev, "b2b using whole bar\n");
  1054. ndev->b2b_off = 0;
  1055. --ndev->mw_count;
  1056. } else {
  1057. dev_dbg(&pdev->dev, "b2b bar size is too small\n");
  1058. return -EIO;
  1059. }
  1060. }
  1061. /* Reset the secondary bar sizes to match the primary bar sizes,
  1062. * except disable or halve the size of the b2b secondary bar.
  1063. *
  1064. * Note: code for each specific bar size register, because the register
  1065. * offsets are not in a consistent order (bar5sz comes after ppd, odd).
  1066. */
  1067. pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
  1068. dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
  1069. if (b2b_bar == 2) {
  1070. if (ndev->b2b_off)
  1071. bar_sz -= 1;
  1072. else
  1073. bar_sz = 0;
  1074. }
  1075. pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
  1076. pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
  1077. dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
  1078. if (!ndev->bar4_split) {
  1079. pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
  1080. dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
  1081. if (b2b_bar == 4) {
  1082. if (ndev->b2b_off)
  1083. bar_sz -= 1;
  1084. else
  1085. bar_sz = 0;
  1086. }
  1087. pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
  1088. pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
  1089. dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
  1090. } else {
  1091. pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
  1092. dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
  1093. if (b2b_bar == 4) {
  1094. if (ndev->b2b_off)
  1095. bar_sz -= 1;
  1096. else
  1097. bar_sz = 0;
  1098. }
  1099. pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
  1100. pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
  1101. dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
  1102. pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
  1103. dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
  1104. if (b2b_bar == 5) {
  1105. if (ndev->b2b_off)
  1106. bar_sz -= 1;
  1107. else
  1108. bar_sz = 0;
  1109. }
  1110. pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
  1111. pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
  1112. dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
  1113. }
  1114. /* SBAR01 hit by first part of the b2b bar */
  1115. if (b2b_bar == 0)
  1116. bar_addr = addr->bar0_addr;
  1117. else if (b2b_bar == 2)
  1118. bar_addr = addr->bar2_addr64;
  1119. else if (b2b_bar == 4 && !ndev->bar4_split)
  1120. bar_addr = addr->bar4_addr64;
  1121. else if (b2b_bar == 4)
  1122. bar_addr = addr->bar4_addr32;
  1123. else if (b2b_bar == 5)
  1124. bar_addr = addr->bar5_addr32;
  1125. else
  1126. return -EIO;
  1127. dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
  1128. iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
  1129. /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
  1130. * The b2b bar is either disabled above, or configured half-size, and
  1131. * it starts at the PBAR xlat + offset.
  1132. */
  1133. bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
  1134. iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
  1135. bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
  1136. dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
  1137. if (!ndev->bar4_split) {
  1138. bar_addr = addr->bar4_addr64 +
  1139. (b2b_bar == 4 ? ndev->b2b_off : 0);
  1140. iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
  1141. bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
  1142. dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
  1143. } else {
  1144. bar_addr = addr->bar4_addr32 +
  1145. (b2b_bar == 4 ? ndev->b2b_off : 0);
  1146. iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
  1147. bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
  1148. dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
  1149. bar_addr = addr->bar5_addr32 +
  1150. (b2b_bar == 5 ? ndev->b2b_off : 0);
  1151. iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
  1152. bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
  1153. dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
  1154. }
  1155. /* setup incoming bar limits == base addrs (zero length windows) */
  1156. bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
  1157. iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
  1158. bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
  1159. dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
  1160. if (!ndev->bar4_split) {
  1161. bar_addr = addr->bar4_addr64 +
  1162. (b2b_bar == 4 ? ndev->b2b_off : 0);
  1163. iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
  1164. bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
  1165. dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
  1166. } else {
  1167. bar_addr = addr->bar4_addr32 +
  1168. (b2b_bar == 4 ? ndev->b2b_off : 0);
  1169. iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
  1170. bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
  1171. dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
  1172. bar_addr = addr->bar5_addr32 +
  1173. (b2b_bar == 5 ? ndev->b2b_off : 0);
  1174. iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
  1175. bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
  1176. dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
  1177. }
  1178. /* zero incoming translation addrs */
  1179. iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
  1180. if (!ndev->bar4_split) {
  1181. iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
  1182. } else {
  1183. iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
  1184. iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
  1185. }
  1186. /* zero outgoing translation limits (whole bar size windows) */
  1187. iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
  1188. if (!ndev->bar4_split) {
  1189. iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
  1190. } else {
  1191. iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
  1192. iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
  1193. }
  1194. /* set outgoing translation offsets */
  1195. bar_addr = peer_addr->bar2_addr64;
  1196. iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
  1197. bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
  1198. dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
  1199. if (!ndev->bar4_split) {
  1200. bar_addr = peer_addr->bar4_addr64;
  1201. iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
  1202. bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
  1203. dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
  1204. } else {
  1205. bar_addr = peer_addr->bar4_addr32;
  1206. iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
  1207. bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
  1208. dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
  1209. bar_addr = peer_addr->bar5_addr32;
  1210. iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
  1211. bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
  1212. dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
  1213. }
  1214. /* set the translation offset for b2b registers */
  1215. if (b2b_bar == 0)
  1216. bar_addr = peer_addr->bar0_addr;
  1217. else if (b2b_bar == 2)
  1218. bar_addr = peer_addr->bar2_addr64;
  1219. else if (b2b_bar == 4 && !ndev->bar4_split)
  1220. bar_addr = peer_addr->bar4_addr64;
  1221. else if (b2b_bar == 4)
  1222. bar_addr = peer_addr->bar4_addr32;
  1223. else if (b2b_bar == 5)
  1224. bar_addr = peer_addr->bar5_addr32;
  1225. else
  1226. return -EIO;
  1227. /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
  1228. dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
  1229. iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
  1230. iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
  1231. if (b2b_bar) {
  1232. /* map peer ntb mmio config space registers */
  1233. ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
  1234. XEON_B2B_MIN_SIZE);
  1235. if (!ndev->peer_mmio)
  1236. return -EIO;
  1237. ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
  1238. }
  1239. return 0;
  1240. }
  1241. static int xeon_init_ntb(struct intel_ntb_dev *ndev)
  1242. {
  1243. struct device *dev = &ndev->ntb.pdev->dev;
  1244. int rc;
  1245. u32 ntb_ctl;
  1246. if (ndev->bar4_split)
  1247. ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
  1248. else
  1249. ndev->mw_count = XEON_MW_COUNT;
  1250. ndev->spad_count = XEON_SPAD_COUNT;
  1251. ndev->db_count = XEON_DB_COUNT;
  1252. ndev->db_link_mask = XEON_DB_LINK_BIT;
  1253. switch (ndev->ntb.topo) {
  1254. case NTB_TOPO_PRI:
  1255. if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
  1256. dev_err(dev, "NTB Primary config disabled\n");
  1257. return -EINVAL;
  1258. }
  1259. /* enable link to allow secondary side device to appear */
  1260. ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
  1261. ntb_ctl &= ~NTB_CTL_DISABLE;
  1262. iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
  1263. /* use half the spads for the peer */
  1264. ndev->spad_count >>= 1;
  1265. ndev->self_reg = &xeon_pri_reg;
  1266. ndev->peer_reg = &xeon_sec_reg;
  1267. ndev->xlat_reg = &xeon_sec_xlat;
  1268. break;
  1269. case NTB_TOPO_SEC:
  1270. if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
  1271. dev_err(dev, "NTB Secondary config disabled\n");
  1272. return -EINVAL;
  1273. }
  1274. /* use half the spads for the peer */
  1275. ndev->spad_count >>= 1;
  1276. ndev->self_reg = &xeon_sec_reg;
  1277. ndev->peer_reg = &xeon_pri_reg;
  1278. ndev->xlat_reg = &xeon_pri_xlat;
  1279. break;
  1280. case NTB_TOPO_B2B_USD:
  1281. case NTB_TOPO_B2B_DSD:
  1282. ndev->self_reg = &xeon_pri_reg;
  1283. ndev->peer_reg = &xeon_b2b_reg;
  1284. ndev->xlat_reg = &xeon_sec_xlat;
  1285. if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
  1286. ndev->peer_reg = &xeon_pri_reg;
  1287. if (b2b_mw_idx < 0)
  1288. ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
  1289. else
  1290. ndev->b2b_idx = b2b_mw_idx;
  1291. if (ndev->b2b_idx >= ndev->mw_count) {
  1292. dev_dbg(dev,
  1293. "b2b_mw_idx %d invalid for mw_count %u\n",
  1294. b2b_mw_idx, ndev->mw_count);
  1295. return -EINVAL;
  1296. }
  1297. dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
  1298. b2b_mw_idx, ndev->b2b_idx);
  1299. } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
  1300. dev_warn(dev, "Reduce doorbell count by 1\n");
  1301. ndev->db_count -= 1;
  1302. }
  1303. if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
  1304. rc = xeon_setup_b2b_mw(ndev,
  1305. &xeon_b2b_dsd_addr,
  1306. &xeon_b2b_usd_addr);
  1307. } else {
  1308. rc = xeon_setup_b2b_mw(ndev,
  1309. &xeon_b2b_usd_addr,
  1310. &xeon_b2b_dsd_addr);
  1311. }
  1312. if (rc)
  1313. return rc;
  1314. /* Enable Bus Master and Memory Space on the secondary side */
  1315. iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
  1316. ndev->self_mmio + XEON_SPCICMD_OFFSET);
  1317. break;
  1318. default:
  1319. return -EINVAL;
  1320. }
  1321. ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
  1322. ndev->reg->db_iowrite(ndev->db_valid_mask,
  1323. ndev->self_mmio +
  1324. ndev->self_reg->db_mask);
  1325. return 0;
  1326. }
  1327. static int xeon_init_dev(struct intel_ntb_dev *ndev)
  1328. {
  1329. struct pci_dev *pdev;
  1330. u8 ppd;
  1331. int rc, mem;
  1332. pdev = ndev->ntb.pdev;
  1333. switch (pdev->device) {
  1334. /* There is a Xeon hardware errata related to writes to SDOORBELL or
  1335. * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
  1336. * which may hang the system. To workaround this use the second memory
  1337. * window to access the interrupt and scratch pad registers on the
  1338. * remote system.
  1339. */
  1340. case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
  1341. case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
  1342. case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
  1343. case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
  1344. case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
  1345. case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
  1346. case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
  1347. case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
  1348. case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
  1349. case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
  1350. case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
  1351. case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
  1352. case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
  1353. case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
  1354. case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
  1355. ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
  1356. break;
  1357. }
  1358. switch (pdev->device) {
  1359. /* There is a hardware errata related to accessing any register in
  1360. * SB01BASE in the presence of bidirectional traffic crossing the NTB.
  1361. */
  1362. case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
  1363. case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
  1364. case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
  1365. case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
  1366. case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
  1367. case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
  1368. case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
  1369. case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
  1370. case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
  1371. ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
  1372. break;
  1373. }
  1374. switch (pdev->device) {
  1375. /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
  1376. * mirrored to the remote system. Shrink the number of bits by one,
  1377. * since bit 14 is the last bit.
  1378. */
  1379. case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
  1380. case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
  1381. case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
  1382. case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
  1383. case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
  1384. case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
  1385. case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
  1386. case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
  1387. case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
  1388. case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
  1389. case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
  1390. case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
  1391. case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
  1392. case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
  1393. case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
  1394. ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
  1395. break;
  1396. }
  1397. ndev->reg = &xeon_reg;
  1398. rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
  1399. if (rc)
  1400. return -EIO;
  1401. ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
  1402. dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
  1403. ntb_topo_string(ndev->ntb.topo));
  1404. if (ndev->ntb.topo == NTB_TOPO_NONE)
  1405. return -EINVAL;
  1406. if (ndev->ntb.topo != NTB_TOPO_SEC) {
  1407. ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
  1408. dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
  1409. ppd, ndev->bar4_split);
  1410. } else {
  1411. /* This is a way for transparent BAR to figure out if we are
  1412. * doing split BAR or not. There is no way for the hw on the
  1413. * transparent side to know and set the PPD.
  1414. */
  1415. mem = pci_select_bars(pdev, IORESOURCE_MEM);
  1416. ndev->bar4_split = hweight32(mem) ==
  1417. HSX_SPLIT_BAR_MW_COUNT + 1;
  1418. dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
  1419. mem, ndev->bar4_split);
  1420. }
  1421. rc = xeon_init_ntb(ndev);
  1422. if (rc)
  1423. return rc;
  1424. return xeon_init_isr(ndev);
  1425. }
  1426. static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
  1427. {
  1428. xeon_deinit_isr(ndev);
  1429. }
  1430. static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
  1431. {
  1432. int rc;
  1433. pci_set_drvdata(pdev, ndev);
  1434. rc = pci_enable_device(pdev);
  1435. if (rc)
  1436. goto err_pci_enable;
  1437. rc = pci_request_regions(pdev, NTB_NAME);
  1438. if (rc)
  1439. goto err_pci_regions;
  1440. pci_set_master(pdev);
  1441. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
  1442. if (rc) {
  1443. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
  1444. if (rc)
  1445. goto err_dma_mask;
  1446. dev_warn(&pdev->dev, "Cannot DMA highmem\n");
  1447. }
  1448. rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
  1449. if (rc) {
  1450. rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
  1451. if (rc)
  1452. goto err_dma_mask;
  1453. dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
  1454. }
  1455. rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
  1456. dma_get_mask(&pdev->dev));
  1457. if (rc)
  1458. goto err_dma_mask;
  1459. ndev->self_mmio = pci_iomap(pdev, 0, 0);
  1460. if (!ndev->self_mmio) {
  1461. rc = -EIO;
  1462. goto err_mmio;
  1463. }
  1464. ndev->peer_mmio = ndev->self_mmio;
  1465. ndev->peer_addr = pci_resource_start(pdev, 0);
  1466. return 0;
  1467. err_mmio:
  1468. err_dma_mask:
  1469. pci_clear_master(pdev);
  1470. pci_release_regions(pdev);
  1471. err_pci_regions:
  1472. pci_disable_device(pdev);
  1473. err_pci_enable:
  1474. pci_set_drvdata(pdev, NULL);
  1475. return rc;
  1476. }
  1477. static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
  1478. {
  1479. struct pci_dev *pdev = ndev->ntb.pdev;
  1480. if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
  1481. pci_iounmap(pdev, ndev->peer_mmio);
  1482. pci_iounmap(pdev, ndev->self_mmio);
  1483. pci_clear_master(pdev);
  1484. pci_release_regions(pdev);
  1485. pci_disable_device(pdev);
  1486. pci_set_drvdata(pdev, NULL);
  1487. }
  1488. static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
  1489. struct pci_dev *pdev)
  1490. {
  1491. ndev->ntb.pdev = pdev;
  1492. ndev->ntb.topo = NTB_TOPO_NONE;
  1493. ndev->ntb.ops = &intel_ntb_ops;
  1494. ndev->b2b_off = 0;
  1495. ndev->b2b_idx = UINT_MAX;
  1496. ndev->bar4_split = 0;
  1497. ndev->mw_count = 0;
  1498. ndev->spad_count = 0;
  1499. ndev->db_count = 0;
  1500. ndev->db_vec_count = 0;
  1501. ndev->db_vec_shift = 0;
  1502. ndev->ntb_ctl = 0;
  1503. ndev->lnk_sta = 0;
  1504. ndev->db_valid_mask = 0;
  1505. ndev->db_link_mask = 0;
  1506. ndev->db_mask = 0;
  1507. spin_lock_init(&ndev->db_mask_lock);
  1508. }
  1509. static int intel_ntb_pci_probe(struct pci_dev *pdev,
  1510. const struct pci_device_id *id)
  1511. {
  1512. struct intel_ntb_dev *ndev;
  1513. int rc, node;
  1514. node = dev_to_node(&pdev->dev);
  1515. if (pdev_is_gen1(pdev)) {
  1516. ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
  1517. if (!ndev) {
  1518. rc = -ENOMEM;
  1519. goto err_ndev;
  1520. }
  1521. ndev_init_struct(ndev, pdev);
  1522. rc = intel_ntb_init_pci(ndev, pdev);
  1523. if (rc)
  1524. goto err_init_pci;
  1525. rc = xeon_init_dev(ndev);
  1526. if (rc)
  1527. goto err_init_dev;
  1528. } else if (pdev_is_gen3(pdev)) {
  1529. ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
  1530. if (!ndev) {
  1531. rc = -ENOMEM;
  1532. goto err_ndev;
  1533. }
  1534. ndev_init_struct(ndev, pdev);
  1535. ndev->ntb.ops = &intel_ntb3_ops;
  1536. rc = intel_ntb_init_pci(ndev, pdev);
  1537. if (rc)
  1538. goto err_init_pci;
  1539. rc = gen3_init_dev(ndev);
  1540. if (rc)
  1541. goto err_init_dev;
  1542. } else {
  1543. rc = -EINVAL;
  1544. goto err_ndev;
  1545. }
  1546. ndev_reset_unsafe_flags(ndev);
  1547. ndev->reg->poll_link(ndev);
  1548. ndev_init_debugfs(ndev);
  1549. rc = ntb_register_device(&ndev->ntb);
  1550. if (rc)
  1551. goto err_register;
  1552. dev_info(&pdev->dev, "NTB device registered.\n");
  1553. return 0;
  1554. err_register:
  1555. ndev_deinit_debugfs(ndev);
  1556. if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
  1557. xeon_deinit_dev(ndev);
  1558. err_init_dev:
  1559. intel_ntb_deinit_pci(ndev);
  1560. err_init_pci:
  1561. kfree(ndev);
  1562. err_ndev:
  1563. return rc;
  1564. }
  1565. static void intel_ntb_pci_remove(struct pci_dev *pdev)
  1566. {
  1567. struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
  1568. ntb_unregister_device(&ndev->ntb);
  1569. ndev_deinit_debugfs(ndev);
  1570. if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
  1571. xeon_deinit_dev(ndev);
  1572. intel_ntb_deinit_pci(ndev);
  1573. kfree(ndev);
  1574. }
  1575. static const struct intel_ntb_reg xeon_reg = {
  1576. .poll_link = xeon_poll_link,
  1577. .link_is_up = xeon_link_is_up,
  1578. .db_ioread = xeon_db_ioread,
  1579. .db_iowrite = xeon_db_iowrite,
  1580. .db_size = sizeof(u32),
  1581. .ntb_ctl = XEON_NTBCNTL_OFFSET,
  1582. .mw_bar = {2, 4, 5},
  1583. };
  1584. static const struct intel_ntb_alt_reg xeon_pri_reg = {
  1585. .db_bell = XEON_PDOORBELL_OFFSET,
  1586. .db_mask = XEON_PDBMSK_OFFSET,
  1587. .spad = XEON_SPAD_OFFSET,
  1588. };
  1589. static const struct intel_ntb_alt_reg xeon_sec_reg = {
  1590. .db_bell = XEON_SDOORBELL_OFFSET,
  1591. .db_mask = XEON_SDBMSK_OFFSET,
  1592. /* second half of the scratchpads */
  1593. .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
  1594. };
  1595. static const struct intel_ntb_alt_reg xeon_b2b_reg = {
  1596. .db_bell = XEON_B2B_DOORBELL_OFFSET,
  1597. .spad = XEON_B2B_SPAD_OFFSET,
  1598. };
  1599. static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
  1600. /* Note: no primary .bar0_base visible to the secondary side.
  1601. *
  1602. * The secondary side cannot get the base address stored in primary
  1603. * bars. The base address is necessary to set the limit register to
  1604. * any value other than zero, or unlimited.
  1605. *
  1606. * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
  1607. * window by setting the limit equal to base, nor can it limit the size
  1608. * of the memory window by setting the limit to base + size.
  1609. */
  1610. .bar2_limit = XEON_PBAR23LMT_OFFSET,
  1611. .bar2_xlat = XEON_PBAR23XLAT_OFFSET,
  1612. };
  1613. static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
  1614. .bar0_base = XEON_SBAR0BASE_OFFSET,
  1615. .bar2_limit = XEON_SBAR23LMT_OFFSET,
  1616. .bar2_xlat = XEON_SBAR23XLAT_OFFSET,
  1617. };
  1618. struct intel_b2b_addr xeon_b2b_usd_addr = {
  1619. .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
  1620. .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
  1621. .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
  1622. .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
  1623. };
  1624. struct intel_b2b_addr xeon_b2b_dsd_addr = {
  1625. .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
  1626. .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
  1627. .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
  1628. .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
  1629. };
  1630. /* operations for primary side of local ntb */
  1631. static const struct ntb_dev_ops intel_ntb_ops = {
  1632. .mw_count = intel_ntb_mw_count,
  1633. .mw_get_align = intel_ntb_mw_get_align,
  1634. .mw_set_trans = intel_ntb_mw_set_trans,
  1635. .peer_mw_count = intel_ntb_peer_mw_count,
  1636. .peer_mw_get_addr = intel_ntb_peer_mw_get_addr,
  1637. .link_is_up = intel_ntb_link_is_up,
  1638. .link_enable = intel_ntb_link_enable,
  1639. .link_disable = intel_ntb_link_disable,
  1640. .db_is_unsafe = intel_ntb_db_is_unsafe,
  1641. .db_valid_mask = intel_ntb_db_valid_mask,
  1642. .db_vector_count = intel_ntb_db_vector_count,
  1643. .db_vector_mask = intel_ntb_db_vector_mask,
  1644. .db_read = intel_ntb_db_read,
  1645. .db_clear = intel_ntb_db_clear,
  1646. .db_set_mask = intel_ntb_db_set_mask,
  1647. .db_clear_mask = intel_ntb_db_clear_mask,
  1648. .peer_db_addr = intel_ntb_peer_db_addr,
  1649. .peer_db_set = intel_ntb_peer_db_set,
  1650. .spad_is_unsafe = intel_ntb_spad_is_unsafe,
  1651. .spad_count = intel_ntb_spad_count,
  1652. .spad_read = intel_ntb_spad_read,
  1653. .spad_write = intel_ntb_spad_write,
  1654. .peer_spad_addr = intel_ntb_peer_spad_addr,
  1655. .peer_spad_read = intel_ntb_peer_spad_read,
  1656. .peer_spad_write = intel_ntb_peer_spad_write,
  1657. };
  1658. static const struct file_operations intel_ntb_debugfs_info = {
  1659. .owner = THIS_MODULE,
  1660. .open = simple_open,
  1661. .read = ndev_debugfs_read,
  1662. };
  1663. static const struct pci_device_id intel_ntb_pci_tbl[] = {
  1664. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
  1665. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
  1666. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
  1667. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
  1668. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
  1669. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
  1670. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
  1671. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
  1672. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
  1673. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
  1674. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
  1675. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
  1676. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
  1677. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
  1678. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
  1679. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
  1680. {0}
  1681. };
  1682. MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
  1683. static struct pci_driver intel_ntb_pci_driver = {
  1684. .name = KBUILD_MODNAME,
  1685. .id_table = intel_ntb_pci_tbl,
  1686. .probe = intel_ntb_pci_probe,
  1687. .remove = intel_ntb_pci_remove,
  1688. };
  1689. static int __init intel_ntb_pci_driver_init(void)
  1690. {
  1691. pr_info("%s %s\n", NTB_DESC, NTB_VER);
  1692. if (debugfs_initialized())
  1693. debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
  1694. return pci_register_driver(&intel_ntb_pci_driver);
  1695. }
  1696. module_init(intel_ntb_pci_driver_init);
  1697. static void __exit intel_ntb_pci_driver_exit(void)
  1698. {
  1699. pci_unregister_driver(&intel_ntb_pci_driver);
  1700. debugfs_remove_recursive(debugfs_dir);
  1701. }
  1702. module_exit(intel_ntb_pci_driver_exit);