bus.c 30 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268
  1. /*
  2. * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of version 2 of the GNU General Public License as
  6. * published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful, but
  9. * WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. */
  13. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  14. #include <linux/libnvdimm.h>
  15. #include <linux/sched/mm.h>
  16. #include <linux/vmalloc.h>
  17. #include <linux/uaccess.h>
  18. #include <linux/module.h>
  19. #include <linux/blkdev.h>
  20. #include <linux/fcntl.h>
  21. #include <linux/async.h>
  22. #include <linux/genhd.h>
  23. #include <linux/ndctl.h>
  24. #include <linux/sched.h>
  25. #include <linux/slab.h>
  26. #include <linux/fs.h>
  27. #include <linux/io.h>
  28. #include <linux/mm.h>
  29. #include <linux/nd.h>
  30. #include "nd-core.h"
  31. #include "nd.h"
  32. #include "pfn.h"
  33. int nvdimm_major;
  34. static int nvdimm_bus_major;
  35. static struct class *nd_class;
  36. static DEFINE_IDA(nd_ida);
  37. static int to_nd_device_type(struct device *dev)
  38. {
  39. if (is_nvdimm(dev))
  40. return ND_DEVICE_DIMM;
  41. else if (is_memory(dev))
  42. return ND_DEVICE_REGION_PMEM;
  43. else if (is_nd_blk(dev))
  44. return ND_DEVICE_REGION_BLK;
  45. else if (is_nd_dax(dev))
  46. return ND_DEVICE_DAX_PMEM;
  47. else if (is_nd_region(dev->parent))
  48. return nd_region_to_nstype(to_nd_region(dev->parent));
  49. return 0;
  50. }
  51. static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
  52. {
  53. /*
  54. * Ensure that region devices always have their numa node set as
  55. * early as possible.
  56. */
  57. if (is_nd_region(dev))
  58. set_dev_node(dev, to_nd_region(dev)->numa_node);
  59. return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
  60. to_nd_device_type(dev));
  61. }
  62. static struct module *to_bus_provider(struct device *dev)
  63. {
  64. /* pin bus providers while regions are enabled */
  65. if (is_nd_region(dev)) {
  66. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  67. return nvdimm_bus->nd_desc->module;
  68. }
  69. return NULL;
  70. }
  71. static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
  72. {
  73. nvdimm_bus_lock(&nvdimm_bus->dev);
  74. nvdimm_bus->probe_active++;
  75. nvdimm_bus_unlock(&nvdimm_bus->dev);
  76. }
  77. static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
  78. {
  79. nvdimm_bus_lock(&nvdimm_bus->dev);
  80. if (--nvdimm_bus->probe_active == 0)
  81. wake_up(&nvdimm_bus->wait);
  82. nvdimm_bus_unlock(&nvdimm_bus->dev);
  83. }
  84. static int nvdimm_bus_probe(struct device *dev)
  85. {
  86. struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
  87. struct module *provider = to_bus_provider(dev);
  88. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  89. int rc;
  90. if (!try_module_get(provider))
  91. return -ENXIO;
  92. dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
  93. dev->driver->name, dev_name(dev));
  94. nvdimm_bus_probe_start(nvdimm_bus);
  95. rc = nd_drv->probe(dev);
  96. if (rc == 0)
  97. nd_region_probe_success(nvdimm_bus, dev);
  98. else
  99. nd_region_disable(nvdimm_bus, dev);
  100. nvdimm_bus_probe_end(nvdimm_bus);
  101. dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
  102. dev_name(dev), rc);
  103. if (rc != 0)
  104. module_put(provider);
  105. return rc;
  106. }
  107. static int nvdimm_bus_remove(struct device *dev)
  108. {
  109. struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
  110. struct module *provider = to_bus_provider(dev);
  111. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  112. int rc = 0;
  113. if (nd_drv->remove)
  114. rc = nd_drv->remove(dev);
  115. nd_region_disable(nvdimm_bus, dev);
  116. dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name,
  117. dev_name(dev), rc);
  118. module_put(provider);
  119. return rc;
  120. }
  121. static void nvdimm_bus_shutdown(struct device *dev)
  122. {
  123. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  124. struct nd_device_driver *nd_drv = NULL;
  125. if (dev->driver)
  126. nd_drv = to_nd_device_driver(dev->driver);
  127. if (nd_drv && nd_drv->shutdown) {
  128. nd_drv->shutdown(dev);
  129. dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
  130. dev->driver->name, dev_name(dev));
  131. }
  132. }
  133. void nd_device_notify(struct device *dev, enum nvdimm_event event)
  134. {
  135. device_lock(dev);
  136. if (dev->driver) {
  137. struct nd_device_driver *nd_drv;
  138. nd_drv = to_nd_device_driver(dev->driver);
  139. if (nd_drv->notify)
  140. nd_drv->notify(dev, event);
  141. }
  142. device_unlock(dev);
  143. }
  144. EXPORT_SYMBOL(nd_device_notify);
  145. void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
  146. {
  147. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
  148. if (!nvdimm_bus)
  149. return;
  150. /* caller is responsible for holding a reference on the device */
  151. nd_device_notify(&nd_region->dev, event);
  152. }
  153. EXPORT_SYMBOL_GPL(nvdimm_region_notify);
  154. struct clear_badblocks_context {
  155. resource_size_t phys, cleared;
  156. };
  157. static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
  158. {
  159. struct clear_badblocks_context *ctx = data;
  160. struct nd_region *nd_region;
  161. resource_size_t ndr_end;
  162. sector_t sector;
  163. /* make sure device is a region */
  164. if (!is_memory(dev))
  165. return 0;
  166. nd_region = to_nd_region(dev);
  167. ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
  168. /* make sure we are in the region */
  169. if (ctx->phys < nd_region->ndr_start
  170. || (ctx->phys + ctx->cleared) > ndr_end)
  171. return 0;
  172. sector = (ctx->phys - nd_region->ndr_start) / 512;
  173. badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
  174. if (nd_region->bb_state)
  175. sysfs_notify_dirent(nd_region->bb_state);
  176. return 0;
  177. }
  178. static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
  179. phys_addr_t phys, u64 cleared)
  180. {
  181. struct clear_badblocks_context ctx = {
  182. .phys = phys,
  183. .cleared = cleared,
  184. };
  185. device_for_each_child(&nvdimm_bus->dev, &ctx,
  186. nvdimm_clear_badblocks_region);
  187. }
  188. static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
  189. phys_addr_t phys, u64 cleared)
  190. {
  191. if (cleared > 0)
  192. badrange_forget(&nvdimm_bus->badrange, phys, cleared);
  193. if (cleared > 0 && cleared / 512)
  194. nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
  195. }
  196. long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
  197. unsigned int len)
  198. {
  199. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  200. struct nvdimm_bus_descriptor *nd_desc;
  201. struct nd_cmd_clear_error clear_err;
  202. struct nd_cmd_ars_cap ars_cap;
  203. u32 clear_err_unit, mask;
  204. unsigned int noio_flag;
  205. int cmd_rc, rc;
  206. if (!nvdimm_bus)
  207. return -ENXIO;
  208. nd_desc = nvdimm_bus->nd_desc;
  209. /*
  210. * if ndctl does not exist, it's PMEM_LEGACY and
  211. * we want to just pretend everything is handled.
  212. */
  213. if (!nd_desc->ndctl)
  214. return len;
  215. memset(&ars_cap, 0, sizeof(ars_cap));
  216. ars_cap.address = phys;
  217. ars_cap.length = len;
  218. noio_flag = memalloc_noio_save();
  219. rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
  220. sizeof(ars_cap), &cmd_rc);
  221. memalloc_noio_restore(noio_flag);
  222. if (rc < 0)
  223. return rc;
  224. if (cmd_rc < 0)
  225. return cmd_rc;
  226. clear_err_unit = ars_cap.clear_err_unit;
  227. if (!clear_err_unit || !is_power_of_2(clear_err_unit))
  228. return -ENXIO;
  229. mask = clear_err_unit - 1;
  230. if ((phys | len) & mask)
  231. return -ENXIO;
  232. memset(&clear_err, 0, sizeof(clear_err));
  233. clear_err.address = phys;
  234. clear_err.length = len;
  235. noio_flag = memalloc_noio_save();
  236. rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
  237. sizeof(clear_err), &cmd_rc);
  238. memalloc_noio_restore(noio_flag);
  239. if (rc < 0)
  240. return rc;
  241. if (cmd_rc < 0)
  242. return cmd_rc;
  243. nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
  244. return clear_err.cleared;
  245. }
  246. EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
  247. static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
  248. static struct bus_type nvdimm_bus_type = {
  249. .name = "nd",
  250. .uevent = nvdimm_bus_uevent,
  251. .match = nvdimm_bus_match,
  252. .probe = nvdimm_bus_probe,
  253. .remove = nvdimm_bus_remove,
  254. .shutdown = nvdimm_bus_shutdown,
  255. };
  256. static void nvdimm_bus_release(struct device *dev)
  257. {
  258. struct nvdimm_bus *nvdimm_bus;
  259. nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
  260. ida_simple_remove(&nd_ida, nvdimm_bus->id);
  261. kfree(nvdimm_bus);
  262. }
  263. static bool is_nvdimm_bus(struct device *dev)
  264. {
  265. return dev->release == nvdimm_bus_release;
  266. }
  267. struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
  268. {
  269. struct device *dev;
  270. for (dev = nd_dev; dev; dev = dev->parent)
  271. if (is_nvdimm_bus(dev))
  272. break;
  273. dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
  274. if (dev)
  275. return to_nvdimm_bus(dev);
  276. return NULL;
  277. }
  278. struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
  279. {
  280. struct nvdimm_bus *nvdimm_bus;
  281. nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
  282. WARN_ON(!is_nvdimm_bus(dev));
  283. return nvdimm_bus;
  284. }
  285. EXPORT_SYMBOL_GPL(to_nvdimm_bus);
  286. struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
  287. struct nvdimm_bus_descriptor *nd_desc)
  288. {
  289. struct nvdimm_bus *nvdimm_bus;
  290. int rc;
  291. nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
  292. if (!nvdimm_bus)
  293. return NULL;
  294. INIT_LIST_HEAD(&nvdimm_bus->list);
  295. INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
  296. init_waitqueue_head(&nvdimm_bus->wait);
  297. nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
  298. mutex_init(&nvdimm_bus->reconfig_mutex);
  299. badrange_init(&nvdimm_bus->badrange);
  300. if (nvdimm_bus->id < 0) {
  301. kfree(nvdimm_bus);
  302. return NULL;
  303. }
  304. nvdimm_bus->nd_desc = nd_desc;
  305. nvdimm_bus->dev.parent = parent;
  306. nvdimm_bus->dev.release = nvdimm_bus_release;
  307. nvdimm_bus->dev.groups = nd_desc->attr_groups;
  308. nvdimm_bus->dev.bus = &nvdimm_bus_type;
  309. nvdimm_bus->dev.of_node = nd_desc->of_node;
  310. dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
  311. rc = device_register(&nvdimm_bus->dev);
  312. if (rc) {
  313. dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
  314. goto err;
  315. }
  316. return nvdimm_bus;
  317. err:
  318. put_device(&nvdimm_bus->dev);
  319. return NULL;
  320. }
  321. EXPORT_SYMBOL_GPL(nvdimm_bus_register);
  322. void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
  323. {
  324. if (!nvdimm_bus)
  325. return;
  326. device_unregister(&nvdimm_bus->dev);
  327. }
  328. EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
  329. static int child_unregister(struct device *dev, void *data)
  330. {
  331. /*
  332. * the singular ndctl class device per bus needs to be
  333. * "device_destroy"ed, so skip it here
  334. *
  335. * i.e. remove classless children
  336. */
  337. if (dev->class)
  338. /* pass */;
  339. else
  340. nd_device_unregister(dev, ND_SYNC);
  341. return 0;
  342. }
  343. static void free_badrange_list(struct list_head *badrange_list)
  344. {
  345. struct badrange_entry *bre, *next;
  346. list_for_each_entry_safe(bre, next, badrange_list, list) {
  347. list_del(&bre->list);
  348. kfree(bre);
  349. }
  350. list_del_init(badrange_list);
  351. }
  352. static int nd_bus_remove(struct device *dev)
  353. {
  354. struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
  355. mutex_lock(&nvdimm_bus_list_mutex);
  356. list_del_init(&nvdimm_bus->list);
  357. mutex_unlock(&nvdimm_bus_list_mutex);
  358. wait_event(nvdimm_bus->wait,
  359. atomic_read(&nvdimm_bus->ioctl_active) == 0);
  360. nd_synchronize();
  361. device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
  362. spin_lock(&nvdimm_bus->badrange.lock);
  363. free_badrange_list(&nvdimm_bus->badrange.list);
  364. spin_unlock(&nvdimm_bus->badrange.lock);
  365. nvdimm_bus_destroy_ndctl(nvdimm_bus);
  366. return 0;
  367. }
  368. static int nd_bus_probe(struct device *dev)
  369. {
  370. struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
  371. int rc;
  372. rc = nvdimm_bus_create_ndctl(nvdimm_bus);
  373. if (rc)
  374. return rc;
  375. mutex_lock(&nvdimm_bus_list_mutex);
  376. list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
  377. mutex_unlock(&nvdimm_bus_list_mutex);
  378. /* enable bus provider attributes to look up their local context */
  379. dev_set_drvdata(dev, nvdimm_bus->nd_desc);
  380. return 0;
  381. }
  382. static struct nd_device_driver nd_bus_driver = {
  383. .probe = nd_bus_probe,
  384. .remove = nd_bus_remove,
  385. .drv = {
  386. .name = "nd_bus",
  387. .suppress_bind_attrs = true,
  388. .bus = &nvdimm_bus_type,
  389. .owner = THIS_MODULE,
  390. .mod_name = KBUILD_MODNAME,
  391. },
  392. };
  393. static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
  394. {
  395. struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
  396. if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
  397. return true;
  398. return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
  399. }
  400. static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
  401. void nd_synchronize(void)
  402. {
  403. async_synchronize_full_domain(&nd_async_domain);
  404. }
  405. EXPORT_SYMBOL_GPL(nd_synchronize);
  406. static void nd_async_device_register(void *d, async_cookie_t cookie)
  407. {
  408. struct device *dev = d;
  409. if (device_add(dev) != 0) {
  410. dev_err(dev, "%s: failed\n", __func__);
  411. put_device(dev);
  412. }
  413. put_device(dev);
  414. if (dev->parent)
  415. put_device(dev->parent);
  416. }
  417. static void nd_async_device_unregister(void *d, async_cookie_t cookie)
  418. {
  419. struct device *dev = d;
  420. /* flush bus operations before delete */
  421. nvdimm_bus_lock(dev);
  422. nvdimm_bus_unlock(dev);
  423. device_unregister(dev);
  424. put_device(dev);
  425. }
  426. void __nd_device_register(struct device *dev)
  427. {
  428. if (!dev)
  429. return;
  430. dev->bus = &nvdimm_bus_type;
  431. if (dev->parent)
  432. get_device(dev->parent);
  433. get_device(dev);
  434. async_schedule_domain(nd_async_device_register, dev,
  435. &nd_async_domain);
  436. }
  437. void nd_device_register(struct device *dev)
  438. {
  439. device_initialize(dev);
  440. __nd_device_register(dev);
  441. }
  442. EXPORT_SYMBOL(nd_device_register);
  443. void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
  444. {
  445. bool killed;
  446. switch (mode) {
  447. case ND_ASYNC:
  448. /*
  449. * In the async case this is being triggered with the
  450. * device lock held and the unregistration work needs to
  451. * be moved out of line iff this is thread has won the
  452. * race to schedule the deletion.
  453. */
  454. if (!kill_device(dev))
  455. return;
  456. get_device(dev);
  457. async_schedule_domain(nd_async_device_unregister, dev,
  458. &nd_async_domain);
  459. break;
  460. case ND_SYNC:
  461. /*
  462. * In the sync case the device is being unregistered due
  463. * to a state change of the parent. Claim the kill state
  464. * to synchronize against other unregistration requests,
  465. * or otherwise let the async path handle it if the
  466. * unregistration was already queued.
  467. */
  468. device_lock(dev);
  469. killed = kill_device(dev);
  470. device_unlock(dev);
  471. if (!killed)
  472. return;
  473. nd_synchronize();
  474. device_unregister(dev);
  475. break;
  476. }
  477. }
  478. EXPORT_SYMBOL(nd_device_unregister);
  479. /**
  480. * __nd_driver_register() - register a region or a namespace driver
  481. * @nd_drv: driver to register
  482. * @owner: automatically set by nd_driver_register() macro
  483. * @mod_name: automatically set by nd_driver_register() macro
  484. */
  485. int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
  486. const char *mod_name)
  487. {
  488. struct device_driver *drv = &nd_drv->drv;
  489. if (!nd_drv->type) {
  490. pr_debug("driver type bitmask not set (%pf)\n",
  491. __builtin_return_address(0));
  492. return -EINVAL;
  493. }
  494. if (!nd_drv->probe) {
  495. pr_debug("%s ->probe() must be specified\n", mod_name);
  496. return -EINVAL;
  497. }
  498. drv->bus = &nvdimm_bus_type;
  499. drv->owner = owner;
  500. drv->mod_name = mod_name;
  501. return driver_register(drv);
  502. }
  503. EXPORT_SYMBOL(__nd_driver_register);
  504. int nvdimm_revalidate_disk(struct gendisk *disk)
  505. {
  506. struct device *dev = disk_to_dev(disk)->parent;
  507. struct nd_region *nd_region = to_nd_region(dev->parent);
  508. int disk_ro = get_disk_ro(disk);
  509. /*
  510. * Upgrade to read-only if the region is read-only preserve as
  511. * read-only if the disk is already read-only.
  512. */
  513. if (disk_ro || nd_region->ro == disk_ro)
  514. return 0;
  515. dev_info(dev, "%s read-only, marking %s read-only\n",
  516. dev_name(&nd_region->dev), disk->disk_name);
  517. set_disk_ro(disk, 1);
  518. return 0;
  519. }
  520. EXPORT_SYMBOL(nvdimm_revalidate_disk);
  521. static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
  522. char *buf)
  523. {
  524. return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
  525. to_nd_device_type(dev));
  526. }
  527. static DEVICE_ATTR_RO(modalias);
  528. static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
  529. char *buf)
  530. {
  531. return sprintf(buf, "%s\n", dev->type->name);
  532. }
  533. static DEVICE_ATTR_RO(devtype);
  534. static struct attribute *nd_device_attributes[] = {
  535. &dev_attr_modalias.attr,
  536. &dev_attr_devtype.attr,
  537. NULL,
  538. };
  539. /*
  540. * nd_device_attribute_group - generic attributes for all devices on an nd bus
  541. */
  542. struct attribute_group nd_device_attribute_group = {
  543. .attrs = nd_device_attributes,
  544. };
  545. EXPORT_SYMBOL_GPL(nd_device_attribute_group);
  546. static ssize_t numa_node_show(struct device *dev,
  547. struct device_attribute *attr, char *buf)
  548. {
  549. return sprintf(buf, "%d\n", dev_to_node(dev));
  550. }
  551. static DEVICE_ATTR_RO(numa_node);
  552. static struct attribute *nd_numa_attributes[] = {
  553. &dev_attr_numa_node.attr,
  554. NULL,
  555. };
  556. static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
  557. int n)
  558. {
  559. if (!IS_ENABLED(CONFIG_NUMA))
  560. return 0;
  561. return a->mode;
  562. }
  563. /*
  564. * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
  565. */
  566. struct attribute_group nd_numa_attribute_group = {
  567. .attrs = nd_numa_attributes,
  568. .is_visible = nd_numa_attr_visible,
  569. };
  570. EXPORT_SYMBOL_GPL(nd_numa_attribute_group);
  571. int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
  572. {
  573. dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
  574. struct device *dev;
  575. dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus,
  576. "ndctl%d", nvdimm_bus->id);
  577. if (IS_ERR(dev))
  578. dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n",
  579. nvdimm_bus->id, PTR_ERR(dev));
  580. return PTR_ERR_OR_ZERO(dev);
  581. }
  582. void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
  583. {
  584. device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
  585. }
  586. static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
  587. [ND_CMD_IMPLEMENTED] = { },
  588. [ND_CMD_SMART] = {
  589. .out_num = 2,
  590. .out_sizes = { 4, 128, },
  591. },
  592. [ND_CMD_SMART_THRESHOLD] = {
  593. .out_num = 2,
  594. .out_sizes = { 4, 8, },
  595. },
  596. [ND_CMD_DIMM_FLAGS] = {
  597. .out_num = 2,
  598. .out_sizes = { 4, 4 },
  599. },
  600. [ND_CMD_GET_CONFIG_SIZE] = {
  601. .out_num = 3,
  602. .out_sizes = { 4, 4, 4, },
  603. },
  604. [ND_CMD_GET_CONFIG_DATA] = {
  605. .in_num = 2,
  606. .in_sizes = { 4, 4, },
  607. .out_num = 2,
  608. .out_sizes = { 4, UINT_MAX, },
  609. },
  610. [ND_CMD_SET_CONFIG_DATA] = {
  611. .in_num = 3,
  612. .in_sizes = { 4, 4, UINT_MAX, },
  613. .out_num = 1,
  614. .out_sizes = { 4, },
  615. },
  616. [ND_CMD_VENDOR] = {
  617. .in_num = 3,
  618. .in_sizes = { 4, 4, UINT_MAX, },
  619. .out_num = 3,
  620. .out_sizes = { 4, 4, UINT_MAX, },
  621. },
  622. [ND_CMD_CALL] = {
  623. .in_num = 2,
  624. .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
  625. .out_num = 1,
  626. .out_sizes = { UINT_MAX, },
  627. },
  628. };
  629. const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
  630. {
  631. if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
  632. return &__nd_cmd_dimm_descs[cmd];
  633. return NULL;
  634. }
  635. EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
  636. static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
  637. [ND_CMD_IMPLEMENTED] = { },
  638. [ND_CMD_ARS_CAP] = {
  639. .in_num = 2,
  640. .in_sizes = { 8, 8, },
  641. .out_num = 4,
  642. .out_sizes = { 4, 4, 4, 4, },
  643. },
  644. [ND_CMD_ARS_START] = {
  645. .in_num = 5,
  646. .in_sizes = { 8, 8, 2, 1, 5, },
  647. .out_num = 2,
  648. .out_sizes = { 4, 4, },
  649. },
  650. [ND_CMD_ARS_STATUS] = {
  651. .out_num = 3,
  652. .out_sizes = { 4, 4, UINT_MAX, },
  653. },
  654. [ND_CMD_CLEAR_ERROR] = {
  655. .in_num = 2,
  656. .in_sizes = { 8, 8, },
  657. .out_num = 3,
  658. .out_sizes = { 4, 4, 8, },
  659. },
  660. [ND_CMD_CALL] = {
  661. .in_num = 2,
  662. .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
  663. .out_num = 1,
  664. .out_sizes = { UINT_MAX, },
  665. },
  666. };
  667. const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
  668. {
  669. if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
  670. return &__nd_cmd_bus_descs[cmd];
  671. return NULL;
  672. }
  673. EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
  674. u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
  675. const struct nd_cmd_desc *desc, int idx, void *buf)
  676. {
  677. if (idx >= desc->in_num)
  678. return UINT_MAX;
  679. if (desc->in_sizes[idx] < UINT_MAX)
  680. return desc->in_sizes[idx];
  681. if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
  682. struct nd_cmd_set_config_hdr *hdr = buf;
  683. return hdr->in_length;
  684. } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
  685. struct nd_cmd_vendor_hdr *hdr = buf;
  686. return hdr->in_length;
  687. } else if (cmd == ND_CMD_CALL) {
  688. struct nd_cmd_pkg *pkg = buf;
  689. return pkg->nd_size_in;
  690. }
  691. return UINT_MAX;
  692. }
  693. EXPORT_SYMBOL_GPL(nd_cmd_in_size);
  694. u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
  695. const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
  696. const u32 *out_field, unsigned long remainder)
  697. {
  698. if (idx >= desc->out_num)
  699. return UINT_MAX;
  700. if (desc->out_sizes[idx] < UINT_MAX)
  701. return desc->out_sizes[idx];
  702. if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
  703. return in_field[1];
  704. else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
  705. return out_field[1];
  706. else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
  707. /*
  708. * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
  709. * "Size of Output Buffer in bytes, including this
  710. * field."
  711. */
  712. if (out_field[1] < 4)
  713. return 0;
  714. /*
  715. * ACPI 6.1 is ambiguous if 'status' is included in the
  716. * output size. If we encounter an output size that
  717. * overshoots the remainder by 4 bytes, assume it was
  718. * including 'status'.
  719. */
  720. if (out_field[1] - 4 == remainder)
  721. return remainder;
  722. return out_field[1] - 8;
  723. } else if (cmd == ND_CMD_CALL) {
  724. struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
  725. return pkg->nd_size_out;
  726. }
  727. return UINT_MAX;
  728. }
  729. EXPORT_SYMBOL_GPL(nd_cmd_out_size);
  730. void wait_nvdimm_bus_probe_idle(struct device *dev)
  731. {
  732. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  733. do {
  734. if (nvdimm_bus->probe_active == 0)
  735. break;
  736. nvdimm_bus_unlock(dev);
  737. device_unlock(dev);
  738. wait_event(nvdimm_bus->wait,
  739. nvdimm_bus->probe_active == 0);
  740. device_lock(dev);
  741. nvdimm_bus_lock(dev);
  742. } while (true);
  743. }
  744. static int nd_pmem_forget_poison_check(struct device *dev, void *data)
  745. {
  746. struct nd_cmd_clear_error *clear_err =
  747. (struct nd_cmd_clear_error *)data;
  748. struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
  749. struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
  750. struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
  751. struct nd_namespace_common *ndns = NULL;
  752. struct nd_namespace_io *nsio;
  753. resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
  754. if (nd_dax || !dev->driver)
  755. return 0;
  756. start = clear_err->address;
  757. end = clear_err->address + clear_err->cleared - 1;
  758. if (nd_btt || nd_pfn || nd_dax) {
  759. if (nd_btt)
  760. ndns = nd_btt->ndns;
  761. else if (nd_pfn)
  762. ndns = nd_pfn->ndns;
  763. else if (nd_dax)
  764. ndns = nd_dax->nd_pfn.ndns;
  765. if (!ndns)
  766. return 0;
  767. } else
  768. ndns = to_ndns(dev);
  769. nsio = to_nd_namespace_io(&ndns->dev);
  770. pstart = nsio->res.start + offset;
  771. pend = nsio->res.end - end_trunc;
  772. if ((pstart >= start) && (pend <= end))
  773. return -EBUSY;
  774. return 0;
  775. }
  776. static int nd_ns_forget_poison_check(struct device *dev, void *data)
  777. {
  778. return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
  779. }
  780. /* set_config requires an idle interleave set */
  781. static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
  782. struct nvdimm *nvdimm, unsigned int cmd, void *data)
  783. {
  784. struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
  785. /* ask the bus provider if it would like to block this request */
  786. if (nd_desc->clear_to_send) {
  787. int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd);
  788. if (rc)
  789. return rc;
  790. }
  791. /* require clear error to go through the pmem driver */
  792. if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
  793. return device_for_each_child(&nvdimm_bus->dev, data,
  794. nd_ns_forget_poison_check);
  795. if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
  796. return 0;
  797. /* prevent label manipulation while the kernel owns label updates */
  798. wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
  799. if (atomic_read(&nvdimm->busy))
  800. return -EBUSY;
  801. return 0;
  802. }
  803. static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
  804. int read_only, unsigned int ioctl_cmd, unsigned long arg)
  805. {
  806. struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
  807. const struct nd_cmd_desc *desc = NULL;
  808. unsigned int cmd = _IOC_NR(ioctl_cmd);
  809. struct device *dev = &nvdimm_bus->dev;
  810. void __user *p = (void __user *) arg;
  811. char *out_env = NULL, *in_env = NULL;
  812. const char *cmd_name, *dimm_name;
  813. u32 in_len = 0, out_len = 0;
  814. unsigned int func = cmd;
  815. unsigned long cmd_mask;
  816. struct nd_cmd_pkg pkg;
  817. int rc, i, cmd_rc;
  818. void *buf = NULL;
  819. u64 buf_len = 0;
  820. if (nvdimm) {
  821. desc = nd_cmd_dimm_desc(cmd);
  822. cmd_name = nvdimm_cmd_name(cmd);
  823. cmd_mask = nvdimm->cmd_mask;
  824. dimm_name = dev_name(&nvdimm->dev);
  825. } else {
  826. desc = nd_cmd_bus_desc(cmd);
  827. cmd_name = nvdimm_bus_cmd_name(cmd);
  828. cmd_mask = nd_desc->cmd_mask;
  829. dimm_name = "bus";
  830. }
  831. if (cmd == ND_CMD_CALL) {
  832. if (copy_from_user(&pkg, p, sizeof(pkg)))
  833. return -EFAULT;
  834. }
  835. if (!desc ||
  836. (desc->out_num + desc->in_num == 0) ||
  837. cmd > ND_CMD_CALL ||
  838. !test_bit(cmd, &cmd_mask))
  839. return -ENOTTY;
  840. /* fail write commands (when read-only) */
  841. if (read_only)
  842. switch (cmd) {
  843. case ND_CMD_VENDOR:
  844. case ND_CMD_SET_CONFIG_DATA:
  845. case ND_CMD_ARS_START:
  846. case ND_CMD_CLEAR_ERROR:
  847. case ND_CMD_CALL:
  848. dev_dbg(dev, "'%s' command while read-only.\n",
  849. nvdimm ? nvdimm_cmd_name(cmd)
  850. : nvdimm_bus_cmd_name(cmd));
  851. return -EPERM;
  852. default:
  853. break;
  854. }
  855. /* process an input envelope */
  856. in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
  857. if (!in_env)
  858. return -ENOMEM;
  859. for (i = 0; i < desc->in_num; i++) {
  860. u32 in_size, copy;
  861. in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
  862. if (in_size == UINT_MAX) {
  863. dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
  864. __func__, dimm_name, cmd_name, i);
  865. rc = -ENXIO;
  866. goto out;
  867. }
  868. if (in_len < ND_CMD_MAX_ENVELOPE)
  869. copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
  870. else
  871. copy = 0;
  872. if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
  873. rc = -EFAULT;
  874. goto out;
  875. }
  876. in_len += in_size;
  877. }
  878. if (cmd == ND_CMD_CALL) {
  879. func = pkg.nd_command;
  880. dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
  881. dimm_name, pkg.nd_command,
  882. in_len, out_len, buf_len);
  883. }
  884. /* process an output envelope */
  885. out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
  886. if (!out_env) {
  887. rc = -ENOMEM;
  888. goto out;
  889. }
  890. for (i = 0; i < desc->out_num; i++) {
  891. u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
  892. (u32 *) in_env, (u32 *) out_env, 0);
  893. u32 copy;
  894. if (out_size == UINT_MAX) {
  895. dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
  896. dimm_name, cmd_name, i);
  897. rc = -EFAULT;
  898. goto out;
  899. }
  900. if (out_len < ND_CMD_MAX_ENVELOPE)
  901. copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
  902. else
  903. copy = 0;
  904. if (copy && copy_from_user(&out_env[out_len],
  905. p + in_len + out_len, copy)) {
  906. rc = -EFAULT;
  907. goto out;
  908. }
  909. out_len += out_size;
  910. }
  911. buf_len = (u64) out_len + (u64) in_len;
  912. if (buf_len > ND_IOCTL_MAX_BUFLEN) {
  913. dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
  914. cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
  915. rc = -EINVAL;
  916. goto out;
  917. }
  918. buf = vmalloc(buf_len);
  919. if (!buf) {
  920. rc = -ENOMEM;
  921. goto out;
  922. }
  923. if (copy_from_user(buf, p, buf_len)) {
  924. rc = -EFAULT;
  925. goto out;
  926. }
  927. device_lock(dev);
  928. nvdimm_bus_lock(dev);
  929. rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
  930. if (rc)
  931. goto out_unlock;
  932. rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
  933. if (rc < 0)
  934. goto out_unlock;
  935. if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
  936. struct nd_cmd_clear_error *clear_err = buf;
  937. nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
  938. clear_err->cleared);
  939. }
  940. if (copy_to_user(p, buf, buf_len))
  941. rc = -EFAULT;
  942. out_unlock:
  943. nvdimm_bus_unlock(dev);
  944. device_unlock(dev);
  945. out:
  946. kfree(in_env);
  947. kfree(out_env);
  948. vfree(buf);
  949. return rc;
  950. }
  951. enum nd_ioctl_mode {
  952. BUS_IOCTL,
  953. DIMM_IOCTL,
  954. };
  955. static int match_dimm(struct device *dev, void *data)
  956. {
  957. long id = (long) data;
  958. if (is_nvdimm(dev)) {
  959. struct nvdimm *nvdimm = to_nvdimm(dev);
  960. return nvdimm->id == id;
  961. }
  962. return 0;
  963. }
  964. static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
  965. enum nd_ioctl_mode mode)
  966. {
  967. struct nvdimm_bus *nvdimm_bus, *found = NULL;
  968. long id = (long) file->private_data;
  969. struct nvdimm *nvdimm = NULL;
  970. int rc, ro;
  971. ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
  972. mutex_lock(&nvdimm_bus_list_mutex);
  973. list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
  974. if (mode == DIMM_IOCTL) {
  975. struct device *dev;
  976. dev = device_find_child(&nvdimm_bus->dev,
  977. file->private_data, match_dimm);
  978. if (!dev)
  979. continue;
  980. nvdimm = to_nvdimm(dev);
  981. found = nvdimm_bus;
  982. } else if (nvdimm_bus->id == id) {
  983. found = nvdimm_bus;
  984. }
  985. if (found) {
  986. atomic_inc(&nvdimm_bus->ioctl_active);
  987. break;
  988. }
  989. }
  990. mutex_unlock(&nvdimm_bus_list_mutex);
  991. if (!found)
  992. return -ENXIO;
  993. nvdimm_bus = found;
  994. rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
  995. if (nvdimm)
  996. put_device(&nvdimm->dev);
  997. if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
  998. wake_up(&nvdimm_bus->wait);
  999. return rc;
  1000. }
  1001. static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  1002. {
  1003. return nd_ioctl(file, cmd, arg, BUS_IOCTL);
  1004. }
  1005. static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  1006. {
  1007. return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
  1008. }
  1009. static int nd_open(struct inode *inode, struct file *file)
  1010. {
  1011. long minor = iminor(inode);
  1012. file->private_data = (void *) minor;
  1013. return 0;
  1014. }
  1015. static const struct file_operations nvdimm_bus_fops = {
  1016. .owner = THIS_MODULE,
  1017. .open = nd_open,
  1018. .unlocked_ioctl = bus_ioctl,
  1019. .compat_ioctl = bus_ioctl,
  1020. .llseek = noop_llseek,
  1021. };
  1022. static const struct file_operations nvdimm_fops = {
  1023. .owner = THIS_MODULE,
  1024. .open = nd_open,
  1025. .unlocked_ioctl = dimm_ioctl,
  1026. .compat_ioctl = dimm_ioctl,
  1027. .llseek = noop_llseek,
  1028. };
  1029. int __init nvdimm_bus_init(void)
  1030. {
  1031. int rc;
  1032. rc = bus_register(&nvdimm_bus_type);
  1033. if (rc)
  1034. return rc;
  1035. rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
  1036. if (rc < 0)
  1037. goto err_bus_chrdev;
  1038. nvdimm_bus_major = rc;
  1039. rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
  1040. if (rc < 0)
  1041. goto err_dimm_chrdev;
  1042. nvdimm_major = rc;
  1043. nd_class = class_create(THIS_MODULE, "nd");
  1044. if (IS_ERR(nd_class)) {
  1045. rc = PTR_ERR(nd_class);
  1046. goto err_class;
  1047. }
  1048. rc = driver_register(&nd_bus_driver.drv);
  1049. if (rc)
  1050. goto err_nd_bus;
  1051. return 0;
  1052. err_nd_bus:
  1053. class_destroy(nd_class);
  1054. err_class:
  1055. unregister_chrdev(nvdimm_major, "dimmctl");
  1056. err_dimm_chrdev:
  1057. unregister_chrdev(nvdimm_bus_major, "ndctl");
  1058. err_bus_chrdev:
  1059. bus_unregister(&nvdimm_bus_type);
  1060. return rc;
  1061. }
  1062. void nvdimm_bus_exit(void)
  1063. {
  1064. driver_unregister(&nd_bus_driver.drv);
  1065. class_destroy(nd_class);
  1066. unregister_chrdev(nvdimm_bus_major, "ndctl");
  1067. unregister_chrdev(nvdimm_major, "dimmctl");
  1068. bus_unregister(&nvdimm_bus_type);
  1069. ida_destroy(&nd_ida);
  1070. }