super.c 71 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * bcache setup/teardown code, and some metadata io - read a superblock and
  4. * figure out what to do with it.
  5. *
  6. * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
  7. * Copyright 2012 Google, Inc.
  8. */
  9. #include "bcache.h"
  10. #include "btree.h"
  11. #include "debug.h"
  12. #include "extents.h"
  13. #include "request.h"
  14. #include "writeback.h"
  15. #include "features.h"
  16. #include <linux/blkdev.h>
  17. #include <linux/pagemap.h>
  18. #include <linux/debugfs.h>
  19. #include <linux/idr.h>
  20. #include <linux/kthread.h>
  21. #include <linux/workqueue.h>
  22. #include <linux/module.h>
  23. #include <linux/random.h>
  24. #include <linux/reboot.h>
  25. #include <linux/sysfs.h>
  26. unsigned int bch_cutoff_writeback;
  27. unsigned int bch_cutoff_writeback_sync;
  28. static const char bcache_magic[] = {
  29. 0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca,
  30. 0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x81
  31. };
  32. static const char invalid_uuid[] = {
  33. 0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78,
  34. 0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99
  35. };
  36. static struct kobject *bcache_kobj;
  37. struct mutex bch_register_lock;
  38. bool bcache_is_reboot;
  39. LIST_HEAD(bch_cache_sets);
  40. static LIST_HEAD(uncached_devices);
  41. static int bcache_major;
  42. static DEFINE_IDA(bcache_device_idx);
  43. static wait_queue_head_t unregister_wait;
  44. struct workqueue_struct *bcache_wq;
  45. struct workqueue_struct *bch_flush_wq;
  46. struct workqueue_struct *bch_journal_wq;
  47. #define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE)
  48. /* limitation of partitions number on single bcache device */
  49. #define BCACHE_MINORS 128
  50. /* limitation of bcache devices number on single system */
  51. #define BCACHE_DEVICE_IDX_MAX ((1U << MINORBITS)/BCACHE_MINORS)
  52. /* Superblock */
  53. static unsigned int get_bucket_size(struct cache_sb *sb, struct cache_sb_disk *s)
  54. {
  55. unsigned int bucket_size = le16_to_cpu(s->bucket_size);
  56. if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) {
  57. if (bch_has_feature_large_bucket(sb)) {
  58. unsigned int max, order;
  59. max = sizeof(unsigned int) * BITS_PER_BYTE - 1;
  60. order = le16_to_cpu(s->bucket_size);
  61. /*
  62. * bcache tool will make sure the overflow won't
  63. * happen, an error message here is enough.
  64. */
  65. if (order > max)
  66. pr_err("Bucket size (1 << %u) overflows\n",
  67. order);
  68. bucket_size = 1 << order;
  69. } else if (bch_has_feature_obso_large_bucket(sb)) {
  70. bucket_size +=
  71. le16_to_cpu(s->obso_bucket_size_hi) << 16;
  72. }
  73. }
  74. return bucket_size;
  75. }
  76. static const char *read_super_common(struct cache_sb *sb, struct block_device *bdev,
  77. struct cache_sb_disk *s)
  78. {
  79. const char *err;
  80. unsigned int i;
  81. sb->first_bucket= le16_to_cpu(s->first_bucket);
  82. sb->nbuckets = le64_to_cpu(s->nbuckets);
  83. sb->bucket_size = get_bucket_size(sb, s);
  84. sb->nr_in_set = le16_to_cpu(s->nr_in_set);
  85. sb->nr_this_dev = le16_to_cpu(s->nr_this_dev);
  86. err = "Too many journal buckets";
  87. if (sb->keys > SB_JOURNAL_BUCKETS)
  88. goto err;
  89. err = "Too many buckets";
  90. if (sb->nbuckets > LONG_MAX)
  91. goto err;
  92. err = "Not enough buckets";
  93. if (sb->nbuckets < 1 << 7)
  94. goto err;
  95. err = "Bad block size (not power of 2)";
  96. if (!is_power_of_2(sb->block_size))
  97. goto err;
  98. err = "Bad block size (larger than page size)";
  99. if (sb->block_size > PAGE_SECTORS)
  100. goto err;
  101. err = "Bad bucket size (not power of 2)";
  102. if (!is_power_of_2(sb->bucket_size))
  103. goto err;
  104. err = "Bad bucket size (smaller than page size)";
  105. if (sb->bucket_size < PAGE_SECTORS)
  106. goto err;
  107. err = "Invalid superblock: device too small";
  108. if (get_capacity(bdev->bd_disk) <
  109. sb->bucket_size * sb->nbuckets)
  110. goto err;
  111. err = "Bad UUID";
  112. if (bch_is_zero(sb->set_uuid, 16))
  113. goto err;
  114. err = "Bad cache device number in set";
  115. if (!sb->nr_in_set ||
  116. sb->nr_in_set <= sb->nr_this_dev ||
  117. sb->nr_in_set > MAX_CACHES_PER_SET)
  118. goto err;
  119. err = "Journal buckets not sequential";
  120. for (i = 0; i < sb->keys; i++)
  121. if (sb->d[i] != sb->first_bucket + i)
  122. goto err;
  123. err = "Too many journal buckets";
  124. if (sb->first_bucket + sb->keys > sb->nbuckets)
  125. goto err;
  126. err = "Invalid superblock: first bucket comes before end of super";
  127. if (sb->first_bucket * sb->bucket_size < 16)
  128. goto err;
  129. err = NULL;
  130. err:
  131. return err;
  132. }
  133. static const char *read_super(struct cache_sb *sb, struct block_device *bdev,
  134. struct cache_sb_disk **res)
  135. {
  136. const char *err;
  137. struct cache_sb_disk *s;
  138. struct page *page;
  139. unsigned int i;
  140. page = read_cache_page_gfp(bdev->bd_mapping,
  141. SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL);
  142. if (IS_ERR(page))
  143. return "IO error";
  144. s = page_address(page) + offset_in_page(SB_OFFSET);
  145. sb->offset = le64_to_cpu(s->offset);
  146. sb->version = le64_to_cpu(s->version);
  147. memcpy(sb->magic, s->magic, 16);
  148. memcpy(sb->uuid, s->uuid, 16);
  149. memcpy(sb->set_uuid, s->set_uuid, 16);
  150. memcpy(sb->label, s->label, SB_LABEL_SIZE);
  151. sb->flags = le64_to_cpu(s->flags);
  152. sb->seq = le64_to_cpu(s->seq);
  153. sb->last_mount = le32_to_cpu(s->last_mount);
  154. sb->keys = le16_to_cpu(s->keys);
  155. for (i = 0; i < SB_JOURNAL_BUCKETS; i++)
  156. sb->d[i] = le64_to_cpu(s->d[i]);
  157. pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u\n",
  158. sb->version, sb->flags, sb->seq, sb->keys);
  159. err = "Not a bcache superblock (bad offset)";
  160. if (sb->offset != SB_SECTOR)
  161. goto err;
  162. err = "Not a bcache superblock (bad magic)";
  163. if (memcmp(sb->magic, bcache_magic, 16))
  164. goto err;
  165. err = "Bad checksum";
  166. if (s->csum != csum_set(s))
  167. goto err;
  168. err = "Bad UUID";
  169. if (bch_is_zero(sb->uuid, 16))
  170. goto err;
  171. sb->block_size = le16_to_cpu(s->block_size);
  172. err = "Superblock block size smaller than device block size";
  173. if (sb->block_size << 9 < bdev_logical_block_size(bdev))
  174. goto err;
  175. switch (sb->version) {
  176. case BCACHE_SB_VERSION_BDEV:
  177. sb->data_offset = BDEV_DATA_START_DEFAULT;
  178. break;
  179. case BCACHE_SB_VERSION_BDEV_WITH_OFFSET:
  180. case BCACHE_SB_VERSION_BDEV_WITH_FEATURES:
  181. sb->data_offset = le64_to_cpu(s->data_offset);
  182. err = "Bad data offset";
  183. if (sb->data_offset < BDEV_DATA_START_DEFAULT)
  184. goto err;
  185. break;
  186. case BCACHE_SB_VERSION_CDEV:
  187. case BCACHE_SB_VERSION_CDEV_WITH_UUID:
  188. err = read_super_common(sb, bdev, s);
  189. if (err)
  190. goto err;
  191. break;
  192. case BCACHE_SB_VERSION_CDEV_WITH_FEATURES:
  193. /*
  194. * Feature bits are needed in read_super_common(),
  195. * convert them firstly.
  196. */
  197. sb->feature_compat = le64_to_cpu(s->feature_compat);
  198. sb->feature_incompat = le64_to_cpu(s->feature_incompat);
  199. sb->feature_ro_compat = le64_to_cpu(s->feature_ro_compat);
  200. /* Check incompatible features */
  201. err = "Unsupported compatible feature found";
  202. if (bch_has_unknown_compat_features(sb))
  203. goto err;
  204. err = "Unsupported read-only compatible feature found";
  205. if (bch_has_unknown_ro_compat_features(sb))
  206. goto err;
  207. err = "Unsupported incompatible feature found";
  208. if (bch_has_unknown_incompat_features(sb))
  209. goto err;
  210. err = read_super_common(sb, bdev, s);
  211. if (err)
  212. goto err;
  213. break;
  214. default:
  215. err = "Unsupported superblock version";
  216. goto err;
  217. }
  218. sb->last_mount = (u32)ktime_get_real_seconds();
  219. *res = s;
  220. return NULL;
  221. err:
  222. put_page(page);
  223. return err;
  224. }
  225. static void write_bdev_super_endio(struct bio *bio)
  226. {
  227. struct cached_dev *dc = bio->bi_private;
  228. if (bio->bi_status)
  229. bch_count_backing_io_errors(dc, bio);
  230. closure_put(&dc->sb_write);
  231. }
  232. static void __write_super(struct cache_sb *sb, struct cache_sb_disk *out,
  233. struct bio *bio)
  234. {
  235. unsigned int i;
  236. bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_META;
  237. bio->bi_iter.bi_sector = SB_SECTOR;
  238. __bio_add_page(bio, virt_to_page(out), SB_SIZE,
  239. offset_in_page(out));
  240. out->offset = cpu_to_le64(sb->offset);
  241. memcpy(out->uuid, sb->uuid, 16);
  242. memcpy(out->set_uuid, sb->set_uuid, 16);
  243. memcpy(out->label, sb->label, SB_LABEL_SIZE);
  244. out->flags = cpu_to_le64(sb->flags);
  245. out->seq = cpu_to_le64(sb->seq);
  246. out->last_mount = cpu_to_le32(sb->last_mount);
  247. out->first_bucket = cpu_to_le16(sb->first_bucket);
  248. out->keys = cpu_to_le16(sb->keys);
  249. for (i = 0; i < sb->keys; i++)
  250. out->d[i] = cpu_to_le64(sb->d[i]);
  251. if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) {
  252. out->feature_compat = cpu_to_le64(sb->feature_compat);
  253. out->feature_incompat = cpu_to_le64(sb->feature_incompat);
  254. out->feature_ro_compat = cpu_to_le64(sb->feature_ro_compat);
  255. }
  256. out->version = cpu_to_le64(sb->version);
  257. out->csum = csum_set(out);
  258. pr_debug("ver %llu, flags %llu, seq %llu\n",
  259. sb->version, sb->flags, sb->seq);
  260. submit_bio(bio);
  261. }
  262. static CLOSURE_CALLBACK(bch_write_bdev_super_unlock)
  263. {
  264. closure_type(dc, struct cached_dev, sb_write);
  265. up(&dc->sb_write_mutex);
  266. }
  267. void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent)
  268. {
  269. struct closure *cl = &dc->sb_write;
  270. struct bio *bio = &dc->sb_bio;
  271. down(&dc->sb_write_mutex);
  272. closure_init(cl, parent);
  273. bio_init(bio, dc->bdev, dc->sb_bv, 1, 0);
  274. bio->bi_end_io = write_bdev_super_endio;
  275. bio->bi_private = dc;
  276. closure_get(cl);
  277. /* I/O request sent to backing device */
  278. __write_super(&dc->sb, dc->sb_disk, bio);
  279. closure_return_with_destructor(cl, bch_write_bdev_super_unlock);
  280. }
  281. static void write_super_endio(struct bio *bio)
  282. {
  283. struct cache *ca = bio->bi_private;
  284. /* is_read = 0 */
  285. bch_count_io_errors(ca, bio->bi_status, 0,
  286. "writing superblock");
  287. closure_put(&ca->set->sb_write);
  288. }
  289. static CLOSURE_CALLBACK(bcache_write_super_unlock)
  290. {
  291. closure_type(c, struct cache_set, sb_write);
  292. up(&c->sb_write_mutex);
  293. }
  294. void bcache_write_super(struct cache_set *c)
  295. {
  296. struct closure *cl = &c->sb_write;
  297. struct cache *ca = c->cache;
  298. struct bio *bio = &ca->sb_bio;
  299. unsigned int version = BCACHE_SB_VERSION_CDEV_WITH_UUID;
  300. down(&c->sb_write_mutex);
  301. closure_init(cl, &c->cl);
  302. ca->sb.seq++;
  303. if (ca->sb.version < version)
  304. ca->sb.version = version;
  305. bio_init(bio, ca->bdev, ca->sb_bv, 1, 0);
  306. bio->bi_end_io = write_super_endio;
  307. bio->bi_private = ca;
  308. closure_get(cl);
  309. __write_super(&ca->sb, ca->sb_disk, bio);
  310. closure_return_with_destructor(cl, bcache_write_super_unlock);
  311. }
  312. /* UUID io */
  313. static void uuid_endio(struct bio *bio)
  314. {
  315. struct closure *cl = bio->bi_private;
  316. struct cache_set *c = container_of(cl, struct cache_set, uuid_write);
  317. cache_set_err_on(bio->bi_status, c, "accessing uuids");
  318. bch_bbio_free(bio, c);
  319. closure_put(cl);
  320. }
  321. static CLOSURE_CALLBACK(uuid_io_unlock)
  322. {
  323. closure_type(c, struct cache_set, uuid_write);
  324. up(&c->uuid_write_mutex);
  325. }
  326. static void uuid_io(struct cache_set *c, blk_opf_t opf, struct bkey *k,
  327. struct closure *parent)
  328. {
  329. struct closure *cl = &c->uuid_write;
  330. struct uuid_entry *u;
  331. unsigned int i;
  332. char buf[80];
  333. BUG_ON(!parent);
  334. down(&c->uuid_write_mutex);
  335. closure_init(cl, parent);
  336. for (i = 0; i < KEY_PTRS(k); i++) {
  337. struct bio *bio = bch_bbio_alloc(c);
  338. bio->bi_opf = opf | REQ_SYNC | REQ_META;
  339. bio->bi_iter.bi_size = KEY_SIZE(k) << 9;
  340. bio->bi_end_io = uuid_endio;
  341. bio->bi_private = cl;
  342. bch_bio_map(bio, c->uuids);
  343. bch_submit_bbio(bio, c, k, i);
  344. if ((opf & REQ_OP_MASK) != REQ_OP_WRITE)
  345. break;
  346. }
  347. bch_extent_to_text(buf, sizeof(buf), k);
  348. pr_debug("%s UUIDs at %s\n", (opf & REQ_OP_MASK) == REQ_OP_WRITE ?
  349. "wrote" : "read", buf);
  350. for (u = c->uuids; u < c->uuids + c->nr_uuids; u++)
  351. if (!bch_is_zero(u->uuid, 16))
  352. pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u\n",
  353. u - c->uuids, u->uuid, u->label,
  354. u->first_reg, u->last_reg, u->invalidated);
  355. closure_return_with_destructor(cl, uuid_io_unlock);
  356. }
  357. static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl)
  358. {
  359. struct bkey *k = &j->uuid_bucket;
  360. if (__bch_btree_ptr_invalid(c, k))
  361. return "bad uuid pointer";
  362. bkey_copy(&c->uuid_bucket, k);
  363. uuid_io(c, REQ_OP_READ, k, cl);
  364. if (j->version < BCACHE_JSET_VERSION_UUIDv1) {
  365. struct uuid_entry_v0 *u0 = (void *) c->uuids;
  366. struct uuid_entry *u1 = (void *) c->uuids;
  367. int i;
  368. closure_sync(cl);
  369. /*
  370. * Since the new uuid entry is bigger than the old, we have to
  371. * convert starting at the highest memory address and work down
  372. * in order to do it in place
  373. */
  374. for (i = c->nr_uuids - 1;
  375. i >= 0;
  376. --i) {
  377. memcpy(u1[i].uuid, u0[i].uuid, 16);
  378. memcpy(u1[i].label, u0[i].label, 32);
  379. u1[i].first_reg = u0[i].first_reg;
  380. u1[i].last_reg = u0[i].last_reg;
  381. u1[i].invalidated = u0[i].invalidated;
  382. u1[i].flags = 0;
  383. u1[i].sectors = 0;
  384. }
  385. }
  386. return NULL;
  387. }
  388. static int __uuid_write(struct cache_set *c)
  389. {
  390. BKEY_PADDED(key) k;
  391. struct closure cl;
  392. struct cache *ca = c->cache;
  393. unsigned int size;
  394. closure_init_stack(&cl);
  395. lockdep_assert_held(&bch_register_lock);
  396. if (bch_bucket_alloc_set(c, RESERVE_BTREE, &k.key, true))
  397. return 1;
  398. size = meta_bucket_pages(&ca->sb) * PAGE_SECTORS;
  399. SET_KEY_SIZE(&k.key, size);
  400. uuid_io(c, REQ_OP_WRITE, &k.key, &cl);
  401. closure_sync(&cl);
  402. /* Only one bucket used for uuid write */
  403. atomic_long_add(ca->sb.bucket_size, &ca->meta_sectors_written);
  404. bkey_copy(&c->uuid_bucket, &k.key);
  405. bkey_put(c, &k.key);
  406. return 0;
  407. }
  408. int bch_uuid_write(struct cache_set *c)
  409. {
  410. int ret = __uuid_write(c);
  411. if (!ret)
  412. bch_journal_meta(c, NULL);
  413. return ret;
  414. }
  415. static struct uuid_entry *uuid_find(struct cache_set *c, const char *uuid)
  416. {
  417. struct uuid_entry *u;
  418. for (u = c->uuids;
  419. u < c->uuids + c->nr_uuids; u++)
  420. if (!memcmp(u->uuid, uuid, 16))
  421. return u;
  422. return NULL;
  423. }
  424. static struct uuid_entry *uuid_find_empty(struct cache_set *c)
  425. {
  426. static const char zero_uuid[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
  427. return uuid_find(c, zero_uuid);
  428. }
  429. /*
  430. * Bucket priorities/gens:
  431. *
  432. * For each bucket, we store on disk its
  433. * 8 bit gen
  434. * 16 bit priority
  435. *
  436. * See alloc.c for an explanation of the gen. The priority is used to implement
  437. * lru (and in the future other) cache replacement policies; for most purposes
  438. * it's just an opaque integer.
  439. *
  440. * The gens and the priorities don't have a whole lot to do with each other, and
  441. * it's actually the gens that must be written out at specific times - it's no
  442. * big deal if the priorities don't get written, if we lose them we just reuse
  443. * buckets in suboptimal order.
  444. *
  445. * On disk they're stored in a packed array, and in as many buckets are required
  446. * to fit them all. The buckets we use to store them form a list; the journal
  447. * header points to the first bucket, the first bucket points to the second
  448. * bucket, et cetera.
  449. *
  450. * This code is used by the allocation code; periodically (whenever it runs out
  451. * of buckets to allocate from) the allocation code will invalidate some
  452. * buckets, but it can't use those buckets until their new gens are safely on
  453. * disk.
  454. */
  455. static void prio_endio(struct bio *bio)
  456. {
  457. struct cache *ca = bio->bi_private;
  458. cache_set_err_on(bio->bi_status, ca->set, "accessing priorities");
  459. bch_bbio_free(bio, ca->set);
  460. closure_put(&ca->prio);
  461. }
  462. static void prio_io(struct cache *ca, uint64_t bucket, blk_opf_t opf)
  463. {
  464. struct closure *cl = &ca->prio;
  465. struct bio *bio = bch_bbio_alloc(ca->set);
  466. closure_init_stack(cl);
  467. bio->bi_iter.bi_sector = bucket * ca->sb.bucket_size;
  468. bio_set_dev(bio, ca->bdev);
  469. bio->bi_iter.bi_size = meta_bucket_bytes(&ca->sb);
  470. bio->bi_end_io = prio_endio;
  471. bio->bi_private = ca;
  472. bio->bi_opf = opf | REQ_SYNC | REQ_META;
  473. bch_bio_map(bio, ca->disk_buckets);
  474. closure_bio_submit(ca->set, bio, &ca->prio);
  475. closure_sync(cl);
  476. }
  477. int bch_prio_write(struct cache *ca, bool wait)
  478. {
  479. int i;
  480. struct bucket *b;
  481. struct closure cl;
  482. pr_debug("free_prio=%zu, free_none=%zu, free_inc=%zu\n",
  483. fifo_used(&ca->free[RESERVE_PRIO]),
  484. fifo_used(&ca->free[RESERVE_NONE]),
  485. fifo_used(&ca->free_inc));
  486. /*
  487. * Pre-check if there are enough free buckets. In the non-blocking
  488. * scenario it's better to fail early rather than starting to allocate
  489. * buckets and do a cleanup later in case of failure.
  490. */
  491. if (!wait) {
  492. size_t avail = fifo_used(&ca->free[RESERVE_PRIO]) +
  493. fifo_used(&ca->free[RESERVE_NONE]);
  494. if (prio_buckets(ca) > avail)
  495. return -ENOMEM;
  496. }
  497. closure_init_stack(&cl);
  498. lockdep_assert_held(&ca->set->bucket_lock);
  499. ca->disk_buckets->seq++;
  500. atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),
  501. &ca->meta_sectors_written);
  502. for (i = prio_buckets(ca) - 1; i >= 0; --i) {
  503. long bucket;
  504. struct prio_set *p = ca->disk_buckets;
  505. struct bucket_disk *d = p->data;
  506. struct bucket_disk *end = d + prios_per_bucket(ca);
  507. for (b = ca->buckets + i * prios_per_bucket(ca);
  508. b < ca->buckets + ca->sb.nbuckets && d < end;
  509. b++, d++) {
  510. d->prio = cpu_to_le16(b->prio);
  511. d->gen = b->gen;
  512. }
  513. p->next_bucket = ca->prio_buckets[i + 1];
  514. p->magic = pset_magic(&ca->sb);
  515. p->csum = bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8);
  516. bucket = bch_bucket_alloc(ca, RESERVE_PRIO, wait);
  517. BUG_ON(bucket == -1);
  518. mutex_unlock(&ca->set->bucket_lock);
  519. prio_io(ca, bucket, REQ_OP_WRITE);
  520. mutex_lock(&ca->set->bucket_lock);
  521. ca->prio_buckets[i] = bucket;
  522. atomic_dec_bug(&ca->buckets[bucket].pin);
  523. }
  524. mutex_unlock(&ca->set->bucket_lock);
  525. bch_journal_meta(ca->set, &cl);
  526. closure_sync(&cl);
  527. mutex_lock(&ca->set->bucket_lock);
  528. /*
  529. * Don't want the old priorities to get garbage collected until after we
  530. * finish writing the new ones, and they're journalled
  531. */
  532. for (i = 0; i < prio_buckets(ca); i++) {
  533. if (ca->prio_last_buckets[i])
  534. __bch_bucket_free(ca,
  535. &ca->buckets[ca->prio_last_buckets[i]]);
  536. ca->prio_last_buckets[i] = ca->prio_buckets[i];
  537. }
  538. return 0;
  539. }
  540. static int prio_read(struct cache *ca, uint64_t bucket)
  541. {
  542. struct prio_set *p = ca->disk_buckets;
  543. struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;
  544. struct bucket *b;
  545. unsigned int bucket_nr = 0;
  546. int ret = -EIO;
  547. for (b = ca->buckets;
  548. b < ca->buckets + ca->sb.nbuckets;
  549. b++, d++) {
  550. if (d == end) {
  551. ca->prio_buckets[bucket_nr] = bucket;
  552. ca->prio_last_buckets[bucket_nr] = bucket;
  553. bucket_nr++;
  554. prio_io(ca, bucket, REQ_OP_READ);
  555. if (p->csum !=
  556. bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8)) {
  557. pr_warn("bad csum reading priorities\n");
  558. goto out;
  559. }
  560. if (p->magic != pset_magic(&ca->sb)) {
  561. pr_warn("bad magic reading priorities\n");
  562. goto out;
  563. }
  564. bucket = p->next_bucket;
  565. d = p->data;
  566. }
  567. b->prio = le16_to_cpu(d->prio);
  568. b->gen = b->last_gc = d->gen;
  569. }
  570. ret = 0;
  571. out:
  572. return ret;
  573. }
  574. /* Bcache device */
  575. static int open_dev(struct gendisk *disk, blk_mode_t mode)
  576. {
  577. struct bcache_device *d = disk->private_data;
  578. if (test_bit(BCACHE_DEV_CLOSING, &d->flags))
  579. return -ENXIO;
  580. closure_get(&d->cl);
  581. return 0;
  582. }
  583. static void release_dev(struct gendisk *b)
  584. {
  585. struct bcache_device *d = b->private_data;
  586. closure_put(&d->cl);
  587. }
  588. static int ioctl_dev(struct block_device *b, blk_mode_t mode,
  589. unsigned int cmd, unsigned long arg)
  590. {
  591. struct bcache_device *d = b->bd_disk->private_data;
  592. return d->ioctl(d, mode, cmd, arg);
  593. }
  594. static const struct block_device_operations bcache_cached_ops = {
  595. .submit_bio = cached_dev_submit_bio,
  596. .open = open_dev,
  597. .release = release_dev,
  598. .ioctl = ioctl_dev,
  599. .owner = THIS_MODULE,
  600. };
  601. static const struct block_device_operations bcache_flash_ops = {
  602. .submit_bio = flash_dev_submit_bio,
  603. .open = open_dev,
  604. .release = release_dev,
  605. .ioctl = ioctl_dev,
  606. .owner = THIS_MODULE,
  607. };
  608. void bcache_device_stop(struct bcache_device *d)
  609. {
  610. if (!test_and_set_bit(BCACHE_DEV_CLOSING, &d->flags))
  611. /*
  612. * closure_fn set to
  613. * - cached device: cached_dev_flush()
  614. * - flash dev: flash_dev_flush()
  615. */
  616. closure_queue(&d->cl);
  617. }
  618. static void bcache_device_unlink(struct bcache_device *d)
  619. {
  620. lockdep_assert_held(&bch_register_lock);
  621. if (d->c && !test_and_set_bit(BCACHE_DEV_UNLINK_DONE, &d->flags)) {
  622. struct cache *ca = d->c->cache;
  623. sysfs_remove_link(&d->c->kobj, d->name);
  624. sysfs_remove_link(&d->kobj, "cache");
  625. bd_unlink_disk_holder(ca->bdev, d->disk);
  626. }
  627. }
  628. static void bcache_device_link(struct bcache_device *d, struct cache_set *c,
  629. const char *name)
  630. {
  631. struct cache *ca = c->cache;
  632. int ret;
  633. bd_link_disk_holder(ca->bdev, d->disk);
  634. snprintf(d->name, BCACHEDEVNAME_SIZE,
  635. "%s%u", name, d->id);
  636. ret = sysfs_create_link(&d->kobj, &c->kobj, "cache");
  637. if (ret < 0)
  638. pr_err("Couldn't create device -> cache set symlink\n");
  639. ret = sysfs_create_link(&c->kobj, &d->kobj, d->name);
  640. if (ret < 0)
  641. pr_err("Couldn't create cache set -> device symlink\n");
  642. clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags);
  643. }
  644. static void bcache_device_detach(struct bcache_device *d)
  645. {
  646. lockdep_assert_held(&bch_register_lock);
  647. atomic_dec(&d->c->attached_dev_nr);
  648. if (test_bit(BCACHE_DEV_DETACHING, &d->flags)) {
  649. struct uuid_entry *u = d->c->uuids + d->id;
  650. SET_UUID_FLASH_ONLY(u, 0);
  651. memcpy(u->uuid, invalid_uuid, 16);
  652. u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds());
  653. bch_uuid_write(d->c);
  654. }
  655. bcache_device_unlink(d);
  656. d->c->devices[d->id] = NULL;
  657. closure_put(&d->c->caching);
  658. d->c = NULL;
  659. }
  660. static void bcache_device_attach(struct bcache_device *d, struct cache_set *c,
  661. unsigned int id)
  662. {
  663. d->id = id;
  664. d->c = c;
  665. c->devices[id] = d;
  666. if (id >= c->devices_max_used)
  667. c->devices_max_used = id + 1;
  668. closure_get(&c->caching);
  669. }
  670. static inline int first_minor_to_idx(int first_minor)
  671. {
  672. return (first_minor/BCACHE_MINORS);
  673. }
  674. static inline int idx_to_first_minor(int idx)
  675. {
  676. return (idx * BCACHE_MINORS);
  677. }
  678. static void bcache_device_free(struct bcache_device *d)
  679. {
  680. struct gendisk *disk = d->disk;
  681. lockdep_assert_held(&bch_register_lock);
  682. if (disk)
  683. pr_info("%s stopped\n", disk->disk_name);
  684. else
  685. pr_err("bcache device (NULL gendisk) stopped\n");
  686. if (d->c)
  687. bcache_device_detach(d);
  688. if (disk) {
  689. ida_free(&bcache_device_idx,
  690. first_minor_to_idx(disk->first_minor));
  691. put_disk(disk);
  692. }
  693. bioset_exit(&d->bio_split);
  694. kvfree(d->full_dirty_stripes);
  695. kvfree(d->stripe_sectors_dirty);
  696. closure_debug_destroy(&d->cl);
  697. }
  698. static int bcache_device_init(struct bcache_device *d, unsigned int block_size,
  699. sector_t sectors, struct block_device *cached_bdev,
  700. const struct block_device_operations *ops)
  701. {
  702. const size_t max_stripes = min_t(size_t, INT_MAX,
  703. SIZE_MAX / sizeof(atomic_t));
  704. struct queue_limits lim = {
  705. .max_hw_sectors = UINT_MAX,
  706. .max_sectors = UINT_MAX,
  707. .max_segment_size = UINT_MAX,
  708. .max_segments = BIO_MAX_VECS,
  709. .max_hw_discard_sectors = UINT_MAX,
  710. .io_min = block_size,
  711. .logical_block_size = block_size,
  712. .physical_block_size = block_size,
  713. .features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA,
  714. };
  715. uint64_t n;
  716. int idx;
  717. if (cached_bdev) {
  718. d->stripe_size = bdev_io_opt(cached_bdev) >> SECTOR_SHIFT;
  719. lim.io_opt = umax(block_size, bdev_io_opt(cached_bdev));
  720. }
  721. if (!d->stripe_size)
  722. d->stripe_size = 1 << 31;
  723. else if (d->stripe_size < BCH_MIN_STRIPE_SZ)
  724. d->stripe_size = roundup(BCH_MIN_STRIPE_SZ, d->stripe_size);
  725. n = DIV_ROUND_UP_ULL(sectors, d->stripe_size);
  726. if (!n || n > max_stripes) {
  727. pr_err("nr_stripes too large or invalid: %llu (start sector beyond end of disk?)\n",
  728. n);
  729. return -ENOMEM;
  730. }
  731. d->nr_stripes = n;
  732. n = d->nr_stripes * sizeof(atomic_t);
  733. d->stripe_sectors_dirty = kvzalloc(n, GFP_KERNEL);
  734. if (!d->stripe_sectors_dirty)
  735. return -ENOMEM;
  736. n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long);
  737. d->full_dirty_stripes = kvzalloc(n, GFP_KERNEL);
  738. if (!d->full_dirty_stripes)
  739. goto out_free_stripe_sectors_dirty;
  740. idx = ida_alloc_max(&bcache_device_idx, BCACHE_DEVICE_IDX_MAX - 1,
  741. GFP_KERNEL);
  742. if (idx < 0)
  743. goto out_free_full_dirty_stripes;
  744. if (bioset_init(&d->bio_split, 4, offsetof(struct bbio, bio),
  745. BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER))
  746. goto out_ida_remove;
  747. if (lim.logical_block_size > PAGE_SIZE && cached_bdev) {
  748. /*
  749. * This should only happen with BCACHE_SB_VERSION_BDEV.
  750. * Block/page size is checked for BCACHE_SB_VERSION_CDEV.
  751. */
  752. pr_info("bcache%i: sb/logical block size (%u) greater than page size (%lu) falling back to device logical block size (%u)\n",
  753. idx, lim.logical_block_size,
  754. PAGE_SIZE, bdev_logical_block_size(cached_bdev));
  755. /* This also adjusts physical block size/min io size if needed */
  756. lim.logical_block_size = bdev_logical_block_size(cached_bdev);
  757. }
  758. d->disk = blk_alloc_disk(&lim, NUMA_NO_NODE);
  759. if (IS_ERR(d->disk))
  760. goto out_bioset_exit;
  761. set_capacity(d->disk, sectors);
  762. snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", idx);
  763. d->disk->major = bcache_major;
  764. d->disk->first_minor = idx_to_first_minor(idx);
  765. d->disk->minors = BCACHE_MINORS;
  766. d->disk->fops = ops;
  767. d->disk->private_data = d;
  768. return 0;
  769. out_bioset_exit:
  770. bioset_exit(&d->bio_split);
  771. out_ida_remove:
  772. ida_free(&bcache_device_idx, idx);
  773. out_free_full_dirty_stripes:
  774. kvfree(d->full_dirty_stripes);
  775. out_free_stripe_sectors_dirty:
  776. kvfree(d->stripe_sectors_dirty);
  777. return -ENOMEM;
  778. }
  779. /* Cached device */
  780. static void calc_cached_dev_sectors(struct cache_set *c)
  781. {
  782. uint64_t sectors = 0;
  783. struct cached_dev *dc;
  784. list_for_each_entry(dc, &c->cached_devs, list)
  785. sectors += bdev_nr_sectors(dc->bdev);
  786. c->cached_dev_sectors = sectors;
  787. }
  788. #define BACKING_DEV_OFFLINE_TIMEOUT 5
  789. static int cached_dev_status_update(void *arg)
  790. {
  791. struct cached_dev *dc = arg;
  792. struct request_queue *q;
  793. /*
  794. * If this delayed worker is stopping outside, directly quit here.
  795. * dc->io_disable might be set via sysfs interface, so check it
  796. * here too.
  797. */
  798. while (!kthread_should_stop() && !dc->io_disable) {
  799. q = bdev_get_queue(dc->bdev);
  800. if (blk_queue_dying(q))
  801. dc->offline_seconds++;
  802. else
  803. dc->offline_seconds = 0;
  804. if (dc->offline_seconds >= BACKING_DEV_OFFLINE_TIMEOUT) {
  805. pr_err("%pg: device offline for %d seconds\n",
  806. dc->bdev,
  807. BACKING_DEV_OFFLINE_TIMEOUT);
  808. pr_err("%s: disable I/O request due to backing device offline\n",
  809. dc->disk.name);
  810. dc->io_disable = true;
  811. /* let others know earlier that io_disable is true */
  812. smp_mb();
  813. bcache_device_stop(&dc->disk);
  814. break;
  815. }
  816. schedule_timeout_interruptible(HZ);
  817. }
  818. wait_for_kthread_stop();
  819. return 0;
  820. }
  821. int bch_cached_dev_run(struct cached_dev *dc)
  822. {
  823. int ret = 0;
  824. struct bcache_device *d = &dc->disk;
  825. char *buf = kmemdup_nul(dc->sb.label, SB_LABEL_SIZE, GFP_KERNEL);
  826. char *env[] = {
  827. "DRIVER=bcache",
  828. kasprintf(GFP_KERNEL, "CACHED_UUID=%pU", dc->sb.uuid),
  829. kasprintf(GFP_KERNEL, "CACHED_LABEL=%s", buf ? : ""),
  830. NULL,
  831. };
  832. if (dc->io_disable) {
  833. pr_err("I/O disabled on cached dev %pg\n", dc->bdev);
  834. ret = -EIO;
  835. goto out;
  836. }
  837. if (atomic_xchg(&dc->running, 1)) {
  838. pr_info("cached dev %pg is running already\n", dc->bdev);
  839. ret = -EBUSY;
  840. goto out;
  841. }
  842. if (!d->c &&
  843. BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) {
  844. struct closure cl;
  845. closure_init_stack(&cl);
  846. SET_BDEV_STATE(&dc->sb, BDEV_STATE_STALE);
  847. bch_write_bdev_super(dc, &cl);
  848. closure_sync(&cl);
  849. }
  850. ret = add_disk(d->disk);
  851. if (ret)
  852. goto out;
  853. bd_link_disk_holder(dc->bdev, dc->disk.disk);
  854. /*
  855. * won't show up in the uevent file, use udevadm monitor -e instead
  856. * only class / kset properties are persistent
  857. */
  858. kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env);
  859. if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
  860. sysfs_create_link(&disk_to_dev(d->disk)->kobj,
  861. &d->kobj, "bcache")) {
  862. pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n");
  863. ret = -ENOMEM;
  864. goto out;
  865. }
  866. dc->status_update_thread = kthread_run(cached_dev_status_update,
  867. dc, "bcache_status_update");
  868. if (IS_ERR(dc->status_update_thread)) {
  869. pr_warn("failed to create bcache_status_update kthread, continue to run without monitoring backing device status\n");
  870. }
  871. out:
  872. kfree(env[1]);
  873. kfree(env[2]);
  874. kfree(buf);
  875. return ret;
  876. }
  877. /*
  878. * If BCACHE_DEV_RATE_DW_RUNNING is set, it means routine of the delayed
  879. * work dc->writeback_rate_update is running. Wait until the routine
  880. * quits (BCACHE_DEV_RATE_DW_RUNNING is clear), then continue to
  881. * cancel it. If BCACHE_DEV_RATE_DW_RUNNING is not clear after time_out
  882. * seconds, give up waiting here and continue to cancel it too.
  883. */
  884. static void cancel_writeback_rate_update_dwork(struct cached_dev *dc)
  885. {
  886. int time_out = WRITEBACK_RATE_UPDATE_SECS_MAX * HZ;
  887. do {
  888. if (!test_bit(BCACHE_DEV_RATE_DW_RUNNING,
  889. &dc->disk.flags))
  890. break;
  891. time_out--;
  892. schedule_timeout_interruptible(1);
  893. } while (time_out > 0);
  894. if (time_out == 0)
  895. pr_warn("give up waiting for dc->writeback_write_update to quit\n");
  896. cancel_delayed_work_sync(&dc->writeback_rate_update);
  897. }
  898. static void cached_dev_detach_finish(struct work_struct *w)
  899. {
  900. struct cached_dev *dc = container_of(w, struct cached_dev, detach);
  901. struct cache_set *c = dc->disk.c;
  902. BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags));
  903. BUG_ON(refcount_read(&dc->count));
  904. if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags))
  905. cancel_writeback_rate_update_dwork(dc);
  906. if (!IS_ERR_OR_NULL(dc->writeback_thread)) {
  907. kthread_stop(dc->writeback_thread);
  908. dc->writeback_thread = NULL;
  909. }
  910. mutex_lock(&bch_register_lock);
  911. bcache_device_detach(&dc->disk);
  912. list_move(&dc->list, &uncached_devices);
  913. calc_cached_dev_sectors(c);
  914. clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags);
  915. clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags);
  916. mutex_unlock(&bch_register_lock);
  917. pr_info("Caching disabled for %pg\n", dc->bdev);
  918. /* Drop ref we took in cached_dev_detach() */
  919. closure_put(&dc->disk.cl);
  920. }
  921. void bch_cached_dev_detach(struct cached_dev *dc)
  922. {
  923. lockdep_assert_held(&bch_register_lock);
  924. if (test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))
  925. return;
  926. if (test_and_set_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
  927. return;
  928. /*
  929. * Block the device from being closed and freed until we're finished
  930. * detaching
  931. */
  932. closure_get(&dc->disk.cl);
  933. bch_writeback_queue(dc);
  934. cached_dev_put(dc);
  935. }
  936. int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c,
  937. uint8_t *set_uuid)
  938. {
  939. uint32_t rtime = cpu_to_le32((u32)ktime_get_real_seconds());
  940. struct uuid_entry *u;
  941. struct cached_dev *exist_dc, *t;
  942. int ret = 0;
  943. if ((set_uuid && memcmp(set_uuid, c->set_uuid, 16)) ||
  944. (!set_uuid && memcmp(dc->sb.set_uuid, c->set_uuid, 16)))
  945. return -ENOENT;
  946. if (dc->disk.c) {
  947. pr_err("Can't attach %pg: already attached\n", dc->bdev);
  948. return -EINVAL;
  949. }
  950. if (test_bit(CACHE_SET_STOPPING, &c->flags)) {
  951. pr_err("Can't attach %pg: shutting down\n", dc->bdev);
  952. return -EINVAL;
  953. }
  954. if (dc->sb.block_size < c->cache->sb.block_size) {
  955. /* Will die */
  956. pr_err("Couldn't attach %pg: block size less than set's block size\n",
  957. dc->bdev);
  958. return -EINVAL;
  959. }
  960. /* Check whether already attached */
  961. list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) {
  962. if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) {
  963. pr_err("Tried to attach %pg but duplicate UUID already attached\n",
  964. dc->bdev);
  965. return -EINVAL;
  966. }
  967. }
  968. u = uuid_find(c, dc->sb.uuid);
  969. if (u &&
  970. (BDEV_STATE(&dc->sb) == BDEV_STATE_STALE ||
  971. BDEV_STATE(&dc->sb) == BDEV_STATE_NONE)) {
  972. memcpy(u->uuid, invalid_uuid, 16);
  973. u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds());
  974. u = NULL;
  975. }
  976. if (!u) {
  977. if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
  978. pr_err("Couldn't find uuid for %pg in set\n", dc->bdev);
  979. return -ENOENT;
  980. }
  981. u = uuid_find_empty(c);
  982. if (!u) {
  983. pr_err("Not caching %pg, no room for UUID\n", dc->bdev);
  984. return -EINVAL;
  985. }
  986. }
  987. /*
  988. * Deadlocks since we're called via sysfs...
  989. * sysfs_remove_file(&dc->kobj, &sysfs_attach);
  990. */
  991. if (bch_is_zero(u->uuid, 16)) {
  992. struct closure cl;
  993. closure_init_stack(&cl);
  994. memcpy(u->uuid, dc->sb.uuid, 16);
  995. memcpy(u->label, dc->sb.label, SB_LABEL_SIZE);
  996. u->first_reg = u->last_reg = rtime;
  997. bch_uuid_write(c);
  998. memcpy(dc->sb.set_uuid, c->set_uuid, 16);
  999. SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
  1000. bch_write_bdev_super(dc, &cl);
  1001. closure_sync(&cl);
  1002. } else {
  1003. u->last_reg = rtime;
  1004. bch_uuid_write(c);
  1005. }
  1006. bcache_device_attach(&dc->disk, c, u - c->uuids);
  1007. list_move(&dc->list, &c->cached_devs);
  1008. calc_cached_dev_sectors(c);
  1009. /*
  1010. * dc->c must be set before dc->count != 0 - paired with the mb in
  1011. * cached_dev_get()
  1012. */
  1013. smp_wmb();
  1014. refcount_set(&dc->count, 1);
  1015. /* Block writeback thread, but spawn it */
  1016. down_write(&dc->writeback_lock);
  1017. if (bch_cached_dev_writeback_start(dc)) {
  1018. up_write(&dc->writeback_lock);
  1019. pr_err("Couldn't start writeback facilities for %s\n",
  1020. dc->disk.disk->disk_name);
  1021. return -ENOMEM;
  1022. }
  1023. if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
  1024. atomic_set(&dc->has_dirty, 1);
  1025. bch_writeback_queue(dc);
  1026. }
  1027. bch_sectors_dirty_init(&dc->disk);
  1028. ret = bch_cached_dev_run(dc);
  1029. if (ret && (ret != -EBUSY)) {
  1030. up_write(&dc->writeback_lock);
  1031. /*
  1032. * bch_register_lock is held, bcache_device_stop() is not
  1033. * able to be directly called. The kthread and kworker
  1034. * created previously in bch_cached_dev_writeback_start()
  1035. * have to be stopped manually here.
  1036. */
  1037. kthread_stop(dc->writeback_thread);
  1038. cancel_writeback_rate_update_dwork(dc);
  1039. pr_err("Couldn't run cached device %pg\n", dc->bdev);
  1040. return ret;
  1041. }
  1042. bcache_device_link(&dc->disk, c, "bdev");
  1043. atomic_inc(&c->attached_dev_nr);
  1044. if (bch_has_feature_obso_large_bucket(&(c->cache->sb))) {
  1045. pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n");
  1046. pr_err("Please update to the latest bcache-tools to create the cache device\n");
  1047. set_disk_ro(dc->disk.disk, 1);
  1048. }
  1049. /* Allow the writeback thread to proceed */
  1050. up_write(&dc->writeback_lock);
  1051. pr_info("Caching %pg as %s on set %pU\n",
  1052. dc->bdev,
  1053. dc->disk.disk->disk_name,
  1054. dc->disk.c->set_uuid);
  1055. return 0;
  1056. }
  1057. /* when dc->disk.kobj released */
  1058. void bch_cached_dev_release(struct kobject *kobj)
  1059. {
  1060. struct cached_dev *dc = container_of(kobj, struct cached_dev,
  1061. disk.kobj);
  1062. kfree(dc);
  1063. module_put(THIS_MODULE);
  1064. }
  1065. static CLOSURE_CALLBACK(cached_dev_free)
  1066. {
  1067. closure_type(dc, struct cached_dev, disk.cl);
  1068. if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags))
  1069. cancel_writeback_rate_update_dwork(dc);
  1070. if (!IS_ERR_OR_NULL(dc->writeback_thread))
  1071. kthread_stop(dc->writeback_thread);
  1072. if (!IS_ERR_OR_NULL(dc->status_update_thread))
  1073. kthread_stop(dc->status_update_thread);
  1074. mutex_lock(&bch_register_lock);
  1075. if (atomic_read(&dc->running)) {
  1076. bd_unlink_disk_holder(dc->bdev, dc->disk.disk);
  1077. del_gendisk(dc->disk.disk);
  1078. }
  1079. bcache_device_free(&dc->disk);
  1080. list_del(&dc->list);
  1081. mutex_unlock(&bch_register_lock);
  1082. if (dc->sb_disk)
  1083. put_page(virt_to_page(dc->sb_disk));
  1084. if (dc->bdev_file)
  1085. fput(dc->bdev_file);
  1086. wake_up(&unregister_wait);
  1087. kobject_put(&dc->disk.kobj);
  1088. }
  1089. static CLOSURE_CALLBACK(cached_dev_flush)
  1090. {
  1091. closure_type(dc, struct cached_dev, disk.cl);
  1092. struct bcache_device *d = &dc->disk;
  1093. mutex_lock(&bch_register_lock);
  1094. bcache_device_unlink(d);
  1095. mutex_unlock(&bch_register_lock);
  1096. bch_cache_accounting_destroy(&dc->accounting);
  1097. kobject_del(&d->kobj);
  1098. continue_at(cl, cached_dev_free, system_wq);
  1099. }
  1100. static int cached_dev_init(struct cached_dev *dc, unsigned int block_size)
  1101. {
  1102. int ret;
  1103. struct io *io;
  1104. struct request_queue *q = bdev_get_queue(dc->bdev);
  1105. __module_get(THIS_MODULE);
  1106. INIT_LIST_HEAD(&dc->list);
  1107. closure_init(&dc->disk.cl, NULL);
  1108. set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq);
  1109. kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype);
  1110. INIT_WORK(&dc->detach, cached_dev_detach_finish);
  1111. sema_init(&dc->sb_write_mutex, 1);
  1112. INIT_LIST_HEAD(&dc->io_lru);
  1113. spin_lock_init(&dc->io_lock);
  1114. bch_cache_accounting_init(&dc->accounting, &dc->disk.cl);
  1115. dc->sequential_cutoff = 4 << 20;
  1116. for (io = dc->io; io < dc->io + RECENT_IO; io++) {
  1117. list_add(&io->lru, &dc->io_lru);
  1118. hlist_add_head(&io->hash, dc->io_hash + RECENT_IO);
  1119. }
  1120. if (bdev_io_opt(dc->bdev))
  1121. dc->partial_stripes_expensive = !!(q->limits.features &
  1122. BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE);
  1123. ret = bcache_device_init(&dc->disk, block_size,
  1124. bdev_nr_sectors(dc->bdev) - dc->sb.data_offset,
  1125. dc->bdev, &bcache_cached_ops);
  1126. if (ret)
  1127. return ret;
  1128. atomic_set(&dc->io_errors, 0);
  1129. dc->io_disable = false;
  1130. dc->error_limit = DEFAULT_CACHED_DEV_ERROR_LIMIT;
  1131. /* default to auto */
  1132. dc->stop_when_cache_set_failed = BCH_CACHED_DEV_STOP_AUTO;
  1133. bch_cached_dev_request_init(dc);
  1134. bch_cached_dev_writeback_init(dc);
  1135. return 0;
  1136. }
  1137. /* Cached device - bcache superblock */
  1138. static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
  1139. struct file *bdev_file,
  1140. struct cached_dev *dc)
  1141. {
  1142. const char *err = "cannot allocate memory";
  1143. struct cache_set *c;
  1144. int ret = -ENOMEM;
  1145. memcpy(&dc->sb, sb, sizeof(struct cache_sb));
  1146. dc->bdev_file = bdev_file;
  1147. dc->bdev = file_bdev(bdev_file);
  1148. dc->sb_disk = sb_disk;
  1149. if (cached_dev_init(dc, sb->block_size << 9))
  1150. goto err;
  1151. err = "error creating kobject";
  1152. if (kobject_add(&dc->disk.kobj, bdev_kobj(dc->bdev), "bcache"))
  1153. goto err;
  1154. if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
  1155. goto err;
  1156. pr_info("registered backing device %pg\n", dc->bdev);
  1157. list_add(&dc->list, &uncached_devices);
  1158. /* attach to a matched cache set if it exists */
  1159. list_for_each_entry(c, &bch_cache_sets, list)
  1160. bch_cached_dev_attach(dc, c, NULL);
  1161. if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
  1162. BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) {
  1163. err = "failed to run cached device";
  1164. ret = bch_cached_dev_run(dc);
  1165. if (ret)
  1166. goto err;
  1167. }
  1168. return 0;
  1169. err:
  1170. pr_notice("error %pg: %s\n", dc->bdev, err);
  1171. bcache_device_stop(&dc->disk);
  1172. return ret;
  1173. }
  1174. /* Flash only volumes */
  1175. /* When d->kobj released */
  1176. void bch_flash_dev_release(struct kobject *kobj)
  1177. {
  1178. struct bcache_device *d = container_of(kobj, struct bcache_device,
  1179. kobj);
  1180. kfree(d);
  1181. }
  1182. static CLOSURE_CALLBACK(flash_dev_free)
  1183. {
  1184. closure_type(d, struct bcache_device, cl);
  1185. mutex_lock(&bch_register_lock);
  1186. atomic_long_sub(bcache_dev_sectors_dirty(d),
  1187. &d->c->flash_dev_dirty_sectors);
  1188. del_gendisk(d->disk);
  1189. bcache_device_free(d);
  1190. mutex_unlock(&bch_register_lock);
  1191. kobject_put(&d->kobj);
  1192. }
  1193. static CLOSURE_CALLBACK(flash_dev_flush)
  1194. {
  1195. closure_type(d, struct bcache_device, cl);
  1196. mutex_lock(&bch_register_lock);
  1197. bcache_device_unlink(d);
  1198. mutex_unlock(&bch_register_lock);
  1199. kobject_del(&d->kobj);
  1200. continue_at(cl, flash_dev_free, system_wq);
  1201. }
  1202. static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
  1203. {
  1204. int err = -ENOMEM;
  1205. struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
  1206. GFP_KERNEL);
  1207. if (!d)
  1208. goto err_ret;
  1209. closure_init(&d->cl, NULL);
  1210. set_closure_fn(&d->cl, flash_dev_flush, system_wq);
  1211. kobject_init(&d->kobj, &bch_flash_dev_ktype);
  1212. if (bcache_device_init(d, block_bytes(c->cache), u->sectors,
  1213. NULL, &bcache_flash_ops))
  1214. goto err;
  1215. bcache_device_attach(d, c, u - c->uuids);
  1216. bch_sectors_dirty_init(d);
  1217. bch_flash_dev_request_init(d);
  1218. err = add_disk(d->disk);
  1219. if (err)
  1220. goto err;
  1221. err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache");
  1222. if (err)
  1223. goto err;
  1224. bcache_device_link(d, c, "volume");
  1225. if (bch_has_feature_obso_large_bucket(&c->cache->sb)) {
  1226. pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n");
  1227. pr_err("Please update to the latest bcache-tools to create the cache device\n");
  1228. set_disk_ro(d->disk, 1);
  1229. }
  1230. return 0;
  1231. err:
  1232. kobject_put(&d->kobj);
  1233. err_ret:
  1234. return err;
  1235. }
  1236. static int flash_devs_run(struct cache_set *c)
  1237. {
  1238. int ret = 0;
  1239. struct uuid_entry *u;
  1240. for (u = c->uuids;
  1241. u < c->uuids + c->nr_uuids && !ret;
  1242. u++)
  1243. if (UUID_FLASH_ONLY(u))
  1244. ret = flash_dev_run(c, u);
  1245. return ret;
  1246. }
  1247. int bch_flash_dev_create(struct cache_set *c, uint64_t size)
  1248. {
  1249. struct uuid_entry *u;
  1250. if (test_bit(CACHE_SET_STOPPING, &c->flags))
  1251. return -EINTR;
  1252. if (!test_bit(CACHE_SET_RUNNING, &c->flags))
  1253. return -EPERM;
  1254. u = uuid_find_empty(c);
  1255. if (!u) {
  1256. pr_err("Can't create volume, no room for UUID\n");
  1257. return -EINVAL;
  1258. }
  1259. get_random_bytes(u->uuid, 16);
  1260. memset(u->label, 0, 32);
  1261. u->first_reg = u->last_reg = cpu_to_le32((u32)ktime_get_real_seconds());
  1262. SET_UUID_FLASH_ONLY(u, 1);
  1263. u->sectors = size >> 9;
  1264. bch_uuid_write(c);
  1265. return flash_dev_run(c, u);
  1266. }
  1267. bool bch_cached_dev_error(struct cached_dev *dc)
  1268. {
  1269. if (!dc || test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))
  1270. return false;
  1271. dc->io_disable = true;
  1272. /* make others know io_disable is true earlier */
  1273. smp_mb();
  1274. pr_err("stop %s: too many IO errors on backing device %pg\n",
  1275. dc->disk.disk->disk_name, dc->bdev);
  1276. bcache_device_stop(&dc->disk);
  1277. return true;
  1278. }
  1279. /* Cache set */
  1280. __printf(2, 3)
  1281. bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...)
  1282. {
  1283. struct va_format vaf;
  1284. va_list args;
  1285. if (c->on_error != ON_ERROR_PANIC &&
  1286. test_bit(CACHE_SET_STOPPING, &c->flags))
  1287. return false;
  1288. if (test_and_set_bit(CACHE_SET_IO_DISABLE, &c->flags))
  1289. pr_info("CACHE_SET_IO_DISABLE already set\n");
  1290. /*
  1291. * XXX: we can be called from atomic context
  1292. * acquire_console_sem();
  1293. */
  1294. va_start(args, fmt);
  1295. vaf.fmt = fmt;
  1296. vaf.va = &args;
  1297. pr_err("error on %pU: %pV, disabling caching\n",
  1298. c->set_uuid, &vaf);
  1299. va_end(args);
  1300. if (c->on_error == ON_ERROR_PANIC)
  1301. panic("panic forced after error\n");
  1302. bch_cache_set_unregister(c);
  1303. return true;
  1304. }
  1305. /* When c->kobj released */
  1306. void bch_cache_set_release(struct kobject *kobj)
  1307. {
  1308. struct cache_set *c = container_of(kobj, struct cache_set, kobj);
  1309. kfree(c);
  1310. module_put(THIS_MODULE);
  1311. }
  1312. static CLOSURE_CALLBACK(cache_set_free)
  1313. {
  1314. closure_type(c, struct cache_set, cl);
  1315. struct cache *ca;
  1316. debugfs_remove(c->debug);
  1317. bch_open_buckets_free(c);
  1318. bch_btree_cache_free(c);
  1319. bch_journal_free(c);
  1320. mutex_lock(&bch_register_lock);
  1321. bch_bset_sort_state_free(&c->sort);
  1322. free_pages((unsigned long) c->uuids, ilog2(meta_bucket_pages(&c->cache->sb)));
  1323. ca = c->cache;
  1324. if (ca) {
  1325. ca->set = NULL;
  1326. c->cache = NULL;
  1327. kobject_put(&ca->kobj);
  1328. }
  1329. if (c->moving_gc_wq)
  1330. destroy_workqueue(c->moving_gc_wq);
  1331. bioset_exit(&c->bio_split);
  1332. mempool_exit(&c->fill_iter);
  1333. mempool_exit(&c->bio_meta);
  1334. mempool_exit(&c->search);
  1335. kfree(c->devices);
  1336. list_del(&c->list);
  1337. mutex_unlock(&bch_register_lock);
  1338. pr_info("Cache set %pU unregistered\n", c->set_uuid);
  1339. wake_up(&unregister_wait);
  1340. closure_debug_destroy(&c->cl);
  1341. kobject_put(&c->kobj);
  1342. }
  1343. static CLOSURE_CALLBACK(cache_set_flush)
  1344. {
  1345. closure_type(c, struct cache_set, caching);
  1346. struct cache *ca = c->cache;
  1347. struct btree *b;
  1348. bch_cache_accounting_destroy(&c->accounting);
  1349. kobject_put(&c->internal);
  1350. kobject_del(&c->kobj);
  1351. if (!IS_ERR_OR_NULL(c->gc_thread))
  1352. kthread_stop(c->gc_thread);
  1353. if (!IS_ERR_OR_NULL(c->root))
  1354. list_add(&c->root->list, &c->btree_cache);
  1355. /*
  1356. * Avoid flushing cached nodes if cache set is retiring
  1357. * due to too many I/O errors detected.
  1358. */
  1359. if (!test_bit(CACHE_SET_IO_DISABLE, &c->flags))
  1360. list_for_each_entry(b, &c->btree_cache, list) {
  1361. mutex_lock(&b->write_lock);
  1362. if (btree_node_dirty(b))
  1363. __bch_btree_node_write(b, NULL);
  1364. mutex_unlock(&b->write_lock);
  1365. }
  1366. /*
  1367. * If the register_cache_set() call to bch_cache_set_alloc() failed,
  1368. * ca has not been assigned a value and return error.
  1369. * So we need check ca is not NULL during bch_cache_set_unregister().
  1370. */
  1371. if (ca && ca->alloc_thread)
  1372. kthread_stop(ca->alloc_thread);
  1373. if (c->journal.cur) {
  1374. cancel_delayed_work_sync(&c->journal.work);
  1375. /* flush last journal entry if needed */
  1376. c->journal.work.work.func(&c->journal.work.work);
  1377. }
  1378. closure_return(cl);
  1379. }
  1380. /*
  1381. * This function is only called when CACHE_SET_IO_DISABLE is set, which means
  1382. * cache set is unregistering due to too many I/O errors. In this condition,
  1383. * the bcache device might be stopped, it depends on stop_when_cache_set_failed
  1384. * value and whether the broken cache has dirty data:
  1385. *
  1386. * dc->stop_when_cache_set_failed dc->has_dirty stop bcache device
  1387. * BCH_CACHED_STOP_AUTO 0 NO
  1388. * BCH_CACHED_STOP_AUTO 1 YES
  1389. * BCH_CACHED_DEV_STOP_ALWAYS 0 YES
  1390. * BCH_CACHED_DEV_STOP_ALWAYS 1 YES
  1391. *
  1392. * The expected behavior is, if stop_when_cache_set_failed is configured to
  1393. * "auto" via sysfs interface, the bcache device will not be stopped if the
  1394. * backing device is clean on the broken cache device.
  1395. */
  1396. static void conditional_stop_bcache_device(struct cache_set *c,
  1397. struct bcache_device *d,
  1398. struct cached_dev *dc)
  1399. {
  1400. if (dc->stop_when_cache_set_failed == BCH_CACHED_DEV_STOP_ALWAYS) {
  1401. pr_warn("stop_when_cache_set_failed of %s is \"always\", stop it for failed cache set %pU.\n",
  1402. d->disk->disk_name, c->set_uuid);
  1403. bcache_device_stop(d);
  1404. } else if (atomic_read(&dc->has_dirty)) {
  1405. /*
  1406. * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO
  1407. * and dc->has_dirty == 1
  1408. */
  1409. pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is dirty, stop it to avoid potential data corruption.\n",
  1410. d->disk->disk_name);
  1411. /*
  1412. * There might be a small time gap that cache set is
  1413. * released but bcache device is not. Inside this time
  1414. * gap, regular I/O requests will directly go into
  1415. * backing device as no cache set attached to. This
  1416. * behavior may also introduce potential inconsistence
  1417. * data in writeback mode while cache is dirty.
  1418. * Therefore before calling bcache_device_stop() due
  1419. * to a broken cache device, dc->io_disable should be
  1420. * explicitly set to true.
  1421. */
  1422. dc->io_disable = true;
  1423. /* make others know io_disable is true earlier */
  1424. smp_mb();
  1425. bcache_device_stop(d);
  1426. } else {
  1427. /*
  1428. * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO
  1429. * and dc->has_dirty == 0
  1430. */
  1431. pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is clean, keep it alive.\n",
  1432. d->disk->disk_name);
  1433. }
  1434. }
  1435. static CLOSURE_CALLBACK(__cache_set_unregister)
  1436. {
  1437. closure_type(c, struct cache_set, caching);
  1438. struct cached_dev *dc;
  1439. struct bcache_device *d;
  1440. size_t i;
  1441. mutex_lock(&bch_register_lock);
  1442. for (i = 0; i < c->devices_max_used; i++) {
  1443. d = c->devices[i];
  1444. if (!d)
  1445. continue;
  1446. if (!UUID_FLASH_ONLY(&c->uuids[i]) &&
  1447. test_bit(CACHE_SET_UNREGISTERING, &c->flags)) {
  1448. dc = container_of(d, struct cached_dev, disk);
  1449. bch_cached_dev_detach(dc);
  1450. if (test_bit(CACHE_SET_IO_DISABLE, &c->flags))
  1451. conditional_stop_bcache_device(c, d, dc);
  1452. } else {
  1453. bcache_device_stop(d);
  1454. }
  1455. }
  1456. mutex_unlock(&bch_register_lock);
  1457. continue_at(cl, cache_set_flush, system_wq);
  1458. }
  1459. void bch_cache_set_stop(struct cache_set *c)
  1460. {
  1461. if (!test_and_set_bit(CACHE_SET_STOPPING, &c->flags))
  1462. /* closure_fn set to __cache_set_unregister() */
  1463. closure_queue(&c->caching);
  1464. }
  1465. void bch_cache_set_unregister(struct cache_set *c)
  1466. {
  1467. set_bit(CACHE_SET_UNREGISTERING, &c->flags);
  1468. bch_cache_set_stop(c);
  1469. }
  1470. #define alloc_meta_bucket_pages(gfp, sb) \
  1471. ((void *) __get_free_pages(__GFP_ZERO|__GFP_COMP|gfp, ilog2(meta_bucket_pages(sb))))
  1472. struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
  1473. {
  1474. int iter_size;
  1475. struct cache *ca = container_of(sb, struct cache, sb);
  1476. struct cache_set *c = kzalloc(sizeof(struct cache_set), GFP_KERNEL);
  1477. if (!c)
  1478. return NULL;
  1479. __module_get(THIS_MODULE);
  1480. closure_init(&c->cl, NULL);
  1481. set_closure_fn(&c->cl, cache_set_free, system_wq);
  1482. closure_init(&c->caching, &c->cl);
  1483. set_closure_fn(&c->caching, __cache_set_unregister, system_wq);
  1484. /* Maybe create continue_at_noreturn() and use it here? */
  1485. closure_set_stopped(&c->cl);
  1486. closure_put(&c->cl);
  1487. kobject_init(&c->kobj, &bch_cache_set_ktype);
  1488. kobject_init(&c->internal, &bch_cache_set_internal_ktype);
  1489. bch_cache_accounting_init(&c->accounting, &c->cl);
  1490. memcpy(c->set_uuid, sb->set_uuid, 16);
  1491. c->cache = ca;
  1492. c->cache->set = c;
  1493. c->bucket_bits = ilog2(sb->bucket_size);
  1494. c->block_bits = ilog2(sb->block_size);
  1495. c->nr_uuids = meta_bucket_bytes(sb) / sizeof(struct uuid_entry);
  1496. c->devices_max_used = 0;
  1497. atomic_set(&c->attached_dev_nr, 0);
  1498. c->btree_pages = meta_bucket_pages(sb);
  1499. if (c->btree_pages > BTREE_MAX_PAGES)
  1500. c->btree_pages = max_t(int, c->btree_pages / 4,
  1501. BTREE_MAX_PAGES);
  1502. sema_init(&c->sb_write_mutex, 1);
  1503. mutex_init(&c->bucket_lock);
  1504. init_waitqueue_head(&c->btree_cache_wait);
  1505. spin_lock_init(&c->btree_cannibalize_lock);
  1506. init_waitqueue_head(&c->bucket_wait);
  1507. init_waitqueue_head(&c->gc_wait);
  1508. sema_init(&c->uuid_write_mutex, 1);
  1509. spin_lock_init(&c->btree_gc_time.lock);
  1510. spin_lock_init(&c->btree_split_time.lock);
  1511. spin_lock_init(&c->btree_read_time.lock);
  1512. bch_moving_init_cache_set(c);
  1513. INIT_LIST_HEAD(&c->list);
  1514. INIT_LIST_HEAD(&c->cached_devs);
  1515. INIT_LIST_HEAD(&c->btree_cache);
  1516. INIT_LIST_HEAD(&c->btree_cache_freeable);
  1517. INIT_LIST_HEAD(&c->btree_cache_freed);
  1518. INIT_LIST_HEAD(&c->data_buckets);
  1519. iter_size = sizeof(struct btree_iter) +
  1520. ((meta_bucket_pages(sb) * PAGE_SECTORS) / sb->block_size) *
  1521. sizeof(struct btree_iter_set);
  1522. c->devices = kcalloc(c->nr_uuids, sizeof(void *), GFP_KERNEL);
  1523. if (!c->devices)
  1524. goto err;
  1525. if (mempool_init_slab_pool(&c->search, 32, bch_search_cache))
  1526. goto err;
  1527. if (mempool_init_kmalloc_pool(&c->bio_meta, 2,
  1528. sizeof(struct bbio) +
  1529. sizeof(struct bio_vec) * meta_bucket_pages(sb)))
  1530. goto err;
  1531. if (mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size))
  1532. goto err;
  1533. if (bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio),
  1534. BIOSET_NEED_RESCUER))
  1535. goto err;
  1536. c->uuids = alloc_meta_bucket_pages(GFP_KERNEL, sb);
  1537. if (!c->uuids)
  1538. goto err;
  1539. c->moving_gc_wq = alloc_workqueue("bcache_gc", WQ_MEM_RECLAIM, 0);
  1540. if (!c->moving_gc_wq)
  1541. goto err;
  1542. if (bch_journal_alloc(c))
  1543. goto err;
  1544. if (bch_btree_cache_alloc(c))
  1545. goto err;
  1546. if (bch_open_buckets_alloc(c))
  1547. goto err;
  1548. if (bch_bset_sort_state_init(&c->sort, ilog2(c->btree_pages)))
  1549. goto err;
  1550. c->congested_read_threshold_us = 2000;
  1551. c->congested_write_threshold_us = 20000;
  1552. c->error_limit = DEFAULT_IO_ERROR_LIMIT;
  1553. c->idle_max_writeback_rate_enabled = 1;
  1554. WARN_ON(test_and_clear_bit(CACHE_SET_IO_DISABLE, &c->flags));
  1555. return c;
  1556. err:
  1557. bch_cache_set_unregister(c);
  1558. return NULL;
  1559. }
  1560. static int run_cache_set(struct cache_set *c)
  1561. {
  1562. const char *err = "cannot allocate memory";
  1563. struct cached_dev *dc, *t;
  1564. struct cache *ca = c->cache;
  1565. struct closure cl;
  1566. LIST_HEAD(journal);
  1567. struct journal_replay *l;
  1568. closure_init_stack(&cl);
  1569. c->nbuckets = ca->sb.nbuckets;
  1570. set_gc_sectors(c);
  1571. if (CACHE_SYNC(&c->cache->sb)) {
  1572. struct bkey *k;
  1573. struct jset *j;
  1574. err = "cannot allocate memory for journal";
  1575. if (bch_journal_read(c, &journal))
  1576. goto err;
  1577. pr_debug("btree_journal_read() done\n");
  1578. err = "no journal entries found";
  1579. if (list_empty(&journal))
  1580. goto err;
  1581. j = &list_entry(journal.prev, struct journal_replay, list)->j;
  1582. err = "IO error reading priorities";
  1583. if (prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]))
  1584. goto err;
  1585. /*
  1586. * If prio_read() fails it'll call cache_set_error and we'll
  1587. * tear everything down right away, but if we perhaps checked
  1588. * sooner we could avoid journal replay.
  1589. */
  1590. k = &j->btree_root;
  1591. err = "bad btree root";
  1592. if (__bch_btree_ptr_invalid(c, k))
  1593. goto err;
  1594. err = "error reading btree root";
  1595. c->root = bch_btree_node_get(c, NULL, k,
  1596. j->btree_level,
  1597. true, NULL);
  1598. if (IS_ERR(c->root))
  1599. goto err;
  1600. list_del_init(&c->root->list);
  1601. rw_unlock(true, c->root);
  1602. err = uuid_read(c, j, &cl);
  1603. if (err)
  1604. goto err;
  1605. err = "error in recovery";
  1606. if (bch_btree_check(c))
  1607. goto err;
  1608. bch_journal_mark(c, &journal);
  1609. bch_initial_gc_finish(c);
  1610. pr_debug("btree_check() done\n");
  1611. /*
  1612. * bcache_journal_next() can't happen sooner, or
  1613. * btree_gc_finish() will give spurious errors about last_gc >
  1614. * gc_gen - this is a hack but oh well.
  1615. */
  1616. bch_journal_next(&c->journal);
  1617. err = "error starting allocator thread";
  1618. if (bch_cache_allocator_start(ca))
  1619. goto err;
  1620. /*
  1621. * First place it's safe to allocate: btree_check() and
  1622. * btree_gc_finish() have to run before we have buckets to
  1623. * allocate, and bch_bucket_alloc_set() might cause a journal
  1624. * entry to be written so bcache_journal_next() has to be called
  1625. * first.
  1626. *
  1627. * If the uuids were in the old format we have to rewrite them
  1628. * before the next journal entry is written:
  1629. */
  1630. if (j->version < BCACHE_JSET_VERSION_UUID)
  1631. __uuid_write(c);
  1632. err = "bcache: replay journal failed";
  1633. if (bch_journal_replay(c, &journal))
  1634. goto err;
  1635. } else {
  1636. unsigned int j;
  1637. pr_notice("invalidating existing data\n");
  1638. ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,
  1639. 2, SB_JOURNAL_BUCKETS);
  1640. for (j = 0; j < ca->sb.keys; j++)
  1641. ca->sb.d[j] = ca->sb.first_bucket + j;
  1642. bch_initial_gc_finish(c);
  1643. err = "error starting allocator thread";
  1644. if (bch_cache_allocator_start(ca))
  1645. goto err;
  1646. mutex_lock(&c->bucket_lock);
  1647. bch_prio_write(ca, true);
  1648. mutex_unlock(&c->bucket_lock);
  1649. err = "cannot allocate new UUID bucket";
  1650. if (__uuid_write(c))
  1651. goto err;
  1652. err = "cannot allocate new btree root";
  1653. c->root = __bch_btree_node_alloc(c, NULL, 0, true, NULL);
  1654. if (IS_ERR(c->root))
  1655. goto err;
  1656. mutex_lock(&c->root->write_lock);
  1657. bkey_copy_key(&c->root->key, &MAX_KEY);
  1658. bch_btree_node_write(c->root, &cl);
  1659. mutex_unlock(&c->root->write_lock);
  1660. bch_btree_set_root(c->root);
  1661. rw_unlock(true, c->root);
  1662. /*
  1663. * We don't want to write the first journal entry until
  1664. * everything is set up - fortunately journal entries won't be
  1665. * written until the SET_CACHE_SYNC() here:
  1666. */
  1667. SET_CACHE_SYNC(&c->cache->sb, true);
  1668. bch_journal_next(&c->journal);
  1669. bch_journal_meta(c, &cl);
  1670. }
  1671. err = "error starting gc thread";
  1672. if (bch_gc_thread_start(c))
  1673. goto err;
  1674. closure_sync(&cl);
  1675. c->cache->sb.last_mount = (u32)ktime_get_real_seconds();
  1676. bcache_write_super(c);
  1677. if (bch_has_feature_obso_large_bucket(&c->cache->sb))
  1678. pr_err("Detect obsoleted large bucket layout, all attached bcache device will be read-only\n");
  1679. list_for_each_entry_safe(dc, t, &uncached_devices, list)
  1680. bch_cached_dev_attach(dc, c, NULL);
  1681. flash_devs_run(c);
  1682. bch_journal_space_reserve(&c->journal);
  1683. set_bit(CACHE_SET_RUNNING, &c->flags);
  1684. return 0;
  1685. err:
  1686. while (!list_empty(&journal)) {
  1687. l = list_first_entry(&journal, struct journal_replay, list);
  1688. list_del(&l->list);
  1689. kfree(l);
  1690. }
  1691. closure_sync(&cl);
  1692. bch_cache_set_error(c, "%s", err);
  1693. return -EIO;
  1694. }
  1695. static const char *register_cache_set(struct cache *ca)
  1696. {
  1697. char buf[12];
  1698. const char *err = "cannot allocate memory";
  1699. struct cache_set *c;
  1700. list_for_each_entry(c, &bch_cache_sets, list)
  1701. if (!memcmp(c->set_uuid, ca->sb.set_uuid, 16)) {
  1702. if (c->cache)
  1703. return "duplicate cache set member";
  1704. goto found;
  1705. }
  1706. c = bch_cache_set_alloc(&ca->sb);
  1707. if (!c)
  1708. return err;
  1709. err = "error creating kobject";
  1710. if (kobject_add(&c->kobj, bcache_kobj, "%pU", c->set_uuid) ||
  1711. kobject_add(&c->internal, &c->kobj, "internal"))
  1712. goto err;
  1713. if (bch_cache_accounting_add_kobjs(&c->accounting, &c->kobj))
  1714. goto err;
  1715. bch_debug_init_cache_set(c);
  1716. list_add(&c->list, &bch_cache_sets);
  1717. found:
  1718. sprintf(buf, "cache%i", ca->sb.nr_this_dev);
  1719. if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||
  1720. sysfs_create_link(&c->kobj, &ca->kobj, buf))
  1721. goto err;
  1722. kobject_get(&ca->kobj);
  1723. ca->set = c;
  1724. ca->set->cache = ca;
  1725. err = "failed to run cache set";
  1726. if (run_cache_set(c) < 0)
  1727. goto err;
  1728. return NULL;
  1729. err:
  1730. bch_cache_set_unregister(c);
  1731. return err;
  1732. }
  1733. /* Cache device */
  1734. /* When ca->kobj released */
  1735. void bch_cache_release(struct kobject *kobj)
  1736. {
  1737. struct cache *ca = container_of(kobj, struct cache, kobj);
  1738. unsigned int i;
  1739. if (ca->set) {
  1740. BUG_ON(ca->set->cache != ca);
  1741. ca->set->cache = NULL;
  1742. }
  1743. free_pages((unsigned long) ca->disk_buckets, ilog2(meta_bucket_pages(&ca->sb)));
  1744. kfree(ca->prio_buckets);
  1745. vfree(ca->buckets);
  1746. free_heap(&ca->heap);
  1747. free_fifo(&ca->free_inc);
  1748. for (i = 0; i < RESERVE_NR; i++)
  1749. free_fifo(&ca->free[i]);
  1750. if (ca->sb_disk)
  1751. put_page(virt_to_page(ca->sb_disk));
  1752. if (ca->bdev_file)
  1753. fput(ca->bdev_file);
  1754. kfree(ca);
  1755. module_put(THIS_MODULE);
  1756. }
  1757. static int cache_alloc(struct cache *ca)
  1758. {
  1759. size_t free;
  1760. size_t btree_buckets;
  1761. struct bucket *b;
  1762. int ret = -ENOMEM;
  1763. const char *err = NULL;
  1764. __module_get(THIS_MODULE);
  1765. kobject_init(&ca->kobj, &bch_cache_ktype);
  1766. bio_init(&ca->journal.bio, NULL, ca->journal.bio.bi_inline_vecs, 8, 0);
  1767. /*
  1768. * when ca->sb.njournal_buckets is not zero, journal exists,
  1769. * and in bch_journal_replay(), tree node may split,
  1770. * so bucket of RESERVE_BTREE type is needed,
  1771. * the worst situation is all journal buckets are valid journal,
  1772. * and all the keys need to replay,
  1773. * so the number of RESERVE_BTREE type buckets should be as much
  1774. * as journal buckets
  1775. */
  1776. btree_buckets = ca->sb.njournal_buckets ?: 8;
  1777. free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;
  1778. if (!free) {
  1779. ret = -EPERM;
  1780. err = "ca->sb.nbuckets is too small";
  1781. goto err_free;
  1782. }
  1783. if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets,
  1784. GFP_KERNEL)) {
  1785. err = "ca->free[RESERVE_BTREE] alloc failed";
  1786. goto err_btree_alloc;
  1787. }
  1788. if (!init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca),
  1789. GFP_KERNEL)) {
  1790. err = "ca->free[RESERVE_PRIO] alloc failed";
  1791. goto err_prio_alloc;
  1792. }
  1793. if (!init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL)) {
  1794. err = "ca->free[RESERVE_MOVINGGC] alloc failed";
  1795. goto err_movinggc_alloc;
  1796. }
  1797. if (!init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL)) {
  1798. err = "ca->free[RESERVE_NONE] alloc failed";
  1799. goto err_none_alloc;
  1800. }
  1801. if (!init_fifo(&ca->free_inc, free << 2, GFP_KERNEL)) {
  1802. err = "ca->free_inc alloc failed";
  1803. goto err_free_inc_alloc;
  1804. }
  1805. if (!init_heap(&ca->heap, free << 3, GFP_KERNEL)) {
  1806. err = "ca->heap alloc failed";
  1807. goto err_heap_alloc;
  1808. }
  1809. ca->buckets = vzalloc(array_size(sizeof(struct bucket),
  1810. ca->sb.nbuckets));
  1811. if (!ca->buckets) {
  1812. err = "ca->buckets alloc failed";
  1813. goto err_buckets_alloc;
  1814. }
  1815. ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t),
  1816. prio_buckets(ca), 2),
  1817. GFP_KERNEL);
  1818. if (!ca->prio_buckets) {
  1819. err = "ca->prio_buckets alloc failed";
  1820. goto err_prio_buckets_alloc;
  1821. }
  1822. ca->disk_buckets = alloc_meta_bucket_pages(GFP_KERNEL, &ca->sb);
  1823. if (!ca->disk_buckets) {
  1824. err = "ca->disk_buckets alloc failed";
  1825. goto err_disk_buckets_alloc;
  1826. }
  1827. ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);
  1828. for_each_bucket(b, ca)
  1829. atomic_set(&b->pin, 0);
  1830. return 0;
  1831. err_disk_buckets_alloc:
  1832. kfree(ca->prio_buckets);
  1833. err_prio_buckets_alloc:
  1834. vfree(ca->buckets);
  1835. err_buckets_alloc:
  1836. free_heap(&ca->heap);
  1837. err_heap_alloc:
  1838. free_fifo(&ca->free_inc);
  1839. err_free_inc_alloc:
  1840. free_fifo(&ca->free[RESERVE_NONE]);
  1841. err_none_alloc:
  1842. free_fifo(&ca->free[RESERVE_MOVINGGC]);
  1843. err_movinggc_alloc:
  1844. free_fifo(&ca->free[RESERVE_PRIO]);
  1845. err_prio_alloc:
  1846. free_fifo(&ca->free[RESERVE_BTREE]);
  1847. err_btree_alloc:
  1848. err_free:
  1849. module_put(THIS_MODULE);
  1850. if (err)
  1851. pr_notice("error %pg: %s\n", ca->bdev, err);
  1852. return ret;
  1853. }
  1854. static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
  1855. struct file *bdev_file,
  1856. struct cache *ca)
  1857. {
  1858. const char *err = NULL; /* must be set for any error case */
  1859. int ret = 0;
  1860. memcpy(&ca->sb, sb, sizeof(struct cache_sb));
  1861. ca->bdev_file = bdev_file;
  1862. ca->bdev = file_bdev(bdev_file);
  1863. ca->sb_disk = sb_disk;
  1864. if (bdev_max_discard_sectors(file_bdev(bdev_file)))
  1865. ca->discard = CACHE_DISCARD(&ca->sb);
  1866. ret = cache_alloc(ca);
  1867. if (ret != 0) {
  1868. if (ret == -ENOMEM)
  1869. err = "cache_alloc(): -ENOMEM";
  1870. else if (ret == -EPERM)
  1871. err = "cache_alloc(): cache device is too small";
  1872. else
  1873. err = "cache_alloc(): unknown error";
  1874. pr_notice("error %pg: %s\n", file_bdev(bdev_file), err);
  1875. /*
  1876. * If we failed here, it means ca->kobj is not initialized yet,
  1877. * kobject_put() won't be called and there is no chance to
  1878. * call fput() to bdev in bch_cache_release(). So
  1879. * we explicitly call fput() on the block device here.
  1880. */
  1881. fput(bdev_file);
  1882. return ret;
  1883. }
  1884. if (kobject_add(&ca->kobj, bdev_kobj(file_bdev(bdev_file)), "bcache")) {
  1885. pr_notice("error %pg: error calling kobject_add\n",
  1886. file_bdev(bdev_file));
  1887. ret = -ENOMEM;
  1888. goto out;
  1889. }
  1890. mutex_lock(&bch_register_lock);
  1891. err = register_cache_set(ca);
  1892. mutex_unlock(&bch_register_lock);
  1893. if (err) {
  1894. ret = -ENODEV;
  1895. goto out;
  1896. }
  1897. pr_info("registered cache device %pg\n", file_bdev(ca->bdev_file));
  1898. out:
  1899. kobject_put(&ca->kobj);
  1900. return ret;
  1901. }
  1902. /* Global interfaces/init */
  1903. static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
  1904. const char *buffer, size_t size);
  1905. static ssize_t bch_pending_bdevs_cleanup(struct kobject *k,
  1906. struct kobj_attribute *attr,
  1907. const char *buffer, size_t size);
  1908. kobj_attribute_write(register, register_bcache);
  1909. kobj_attribute_write(register_quiet, register_bcache);
  1910. kobj_attribute_write(pendings_cleanup, bch_pending_bdevs_cleanup);
  1911. static bool bch_is_open_backing(dev_t dev)
  1912. {
  1913. struct cache_set *c, *tc;
  1914. struct cached_dev *dc, *t;
  1915. list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
  1916. list_for_each_entry_safe(dc, t, &c->cached_devs, list)
  1917. if (dc->bdev->bd_dev == dev)
  1918. return true;
  1919. list_for_each_entry_safe(dc, t, &uncached_devices, list)
  1920. if (dc->bdev->bd_dev == dev)
  1921. return true;
  1922. return false;
  1923. }
  1924. static bool bch_is_open_cache(dev_t dev)
  1925. {
  1926. struct cache_set *c, *tc;
  1927. list_for_each_entry_safe(c, tc, &bch_cache_sets, list) {
  1928. struct cache *ca = c->cache;
  1929. if (ca->bdev->bd_dev == dev)
  1930. return true;
  1931. }
  1932. return false;
  1933. }
  1934. static bool bch_is_open(dev_t dev)
  1935. {
  1936. return bch_is_open_cache(dev) || bch_is_open_backing(dev);
  1937. }
  1938. struct async_reg_args {
  1939. struct delayed_work reg_work;
  1940. char *path;
  1941. struct cache_sb *sb;
  1942. struct cache_sb_disk *sb_disk;
  1943. struct file *bdev_file;
  1944. void *holder;
  1945. };
  1946. static void register_bdev_worker(struct work_struct *work)
  1947. {
  1948. int fail = false;
  1949. struct async_reg_args *args =
  1950. container_of(work, struct async_reg_args, reg_work.work);
  1951. mutex_lock(&bch_register_lock);
  1952. if (register_bdev(args->sb, args->sb_disk, args->bdev_file,
  1953. args->holder) < 0)
  1954. fail = true;
  1955. mutex_unlock(&bch_register_lock);
  1956. if (fail)
  1957. pr_info("error %s: fail to register backing device\n",
  1958. args->path);
  1959. kfree(args->sb);
  1960. kfree(args->path);
  1961. kfree(args);
  1962. module_put(THIS_MODULE);
  1963. }
  1964. static void register_cache_worker(struct work_struct *work)
  1965. {
  1966. int fail = false;
  1967. struct async_reg_args *args =
  1968. container_of(work, struct async_reg_args, reg_work.work);
  1969. /* blkdev_put() will be called in bch_cache_release() */
  1970. if (register_cache(args->sb, args->sb_disk, args->bdev_file,
  1971. args->holder))
  1972. fail = true;
  1973. if (fail)
  1974. pr_info("error %s: fail to register cache device\n",
  1975. args->path);
  1976. kfree(args->sb);
  1977. kfree(args->path);
  1978. kfree(args);
  1979. module_put(THIS_MODULE);
  1980. }
  1981. static void register_device_async(struct async_reg_args *args)
  1982. {
  1983. if (SB_IS_BDEV(args->sb))
  1984. INIT_DELAYED_WORK(&args->reg_work, register_bdev_worker);
  1985. else
  1986. INIT_DELAYED_WORK(&args->reg_work, register_cache_worker);
  1987. /* 10 jiffies is enough for a delay */
  1988. queue_delayed_work(system_wq, &args->reg_work, 10);
  1989. }
  1990. static void *alloc_holder_object(struct cache_sb *sb)
  1991. {
  1992. if (SB_IS_BDEV(sb))
  1993. return kzalloc(sizeof(struct cached_dev), GFP_KERNEL);
  1994. return kzalloc(sizeof(struct cache), GFP_KERNEL);
  1995. }
  1996. static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
  1997. const char *buffer, size_t size)
  1998. {
  1999. const char *err;
  2000. char *path = NULL;
  2001. struct cache_sb *sb;
  2002. struct cache_sb_disk *sb_disk;
  2003. struct file *bdev_file, *bdev_file2;
  2004. void *holder = NULL;
  2005. ssize_t ret;
  2006. bool async_registration = false;
  2007. bool quiet = false;
  2008. #ifdef CONFIG_BCACHE_ASYNC_REGISTRATION
  2009. async_registration = true;
  2010. #endif
  2011. ret = -EBUSY;
  2012. err = "failed to reference bcache module";
  2013. if (!try_module_get(THIS_MODULE))
  2014. goto out;
  2015. /* For latest state of bcache_is_reboot */
  2016. smp_mb();
  2017. err = "bcache is in reboot";
  2018. if (bcache_is_reboot)
  2019. goto out_module_put;
  2020. ret = -ENOMEM;
  2021. err = "cannot allocate memory";
  2022. path = kstrndup(buffer, size, GFP_KERNEL);
  2023. if (!path)
  2024. goto out_module_put;
  2025. sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL);
  2026. if (!sb)
  2027. goto out_free_path;
  2028. ret = -EINVAL;
  2029. err = "failed to open device";
  2030. bdev_file = bdev_file_open_by_path(strim(path), BLK_OPEN_READ, NULL, NULL);
  2031. if (IS_ERR(bdev_file))
  2032. goto out_free_sb;
  2033. err = read_super(sb, file_bdev(bdev_file), &sb_disk);
  2034. if (err)
  2035. goto out_blkdev_put;
  2036. holder = alloc_holder_object(sb);
  2037. if (!holder) {
  2038. ret = -ENOMEM;
  2039. err = "cannot allocate memory";
  2040. goto out_put_sb_page;
  2041. }
  2042. /* Now reopen in exclusive mode with proper holder */
  2043. bdev_file2 = bdev_file_open_by_dev(file_bdev(bdev_file)->bd_dev,
  2044. BLK_OPEN_READ | BLK_OPEN_WRITE, holder, NULL);
  2045. fput(bdev_file);
  2046. bdev_file = bdev_file2;
  2047. if (IS_ERR(bdev_file)) {
  2048. ret = PTR_ERR(bdev_file);
  2049. bdev_file = NULL;
  2050. if (ret == -EBUSY) {
  2051. dev_t dev;
  2052. mutex_lock(&bch_register_lock);
  2053. if (lookup_bdev(strim(path), &dev) == 0 &&
  2054. bch_is_open(dev))
  2055. err = "device already registered";
  2056. else
  2057. err = "device busy";
  2058. mutex_unlock(&bch_register_lock);
  2059. if (attr == &ksysfs_register_quiet) {
  2060. quiet = true;
  2061. ret = size;
  2062. }
  2063. }
  2064. goto out_free_holder;
  2065. }
  2066. err = "failed to register device";
  2067. if (async_registration) {
  2068. /* register in asynchronous way */
  2069. struct async_reg_args *args =
  2070. kzalloc(sizeof(struct async_reg_args), GFP_KERNEL);
  2071. if (!args) {
  2072. ret = -ENOMEM;
  2073. err = "cannot allocate memory";
  2074. goto out_free_holder;
  2075. }
  2076. args->path = path;
  2077. args->sb = sb;
  2078. args->sb_disk = sb_disk;
  2079. args->bdev_file = bdev_file;
  2080. args->holder = holder;
  2081. register_device_async(args);
  2082. /* No wait and returns to user space */
  2083. goto async_done;
  2084. }
  2085. if (SB_IS_BDEV(sb)) {
  2086. mutex_lock(&bch_register_lock);
  2087. ret = register_bdev(sb, sb_disk, bdev_file, holder);
  2088. mutex_unlock(&bch_register_lock);
  2089. /* blkdev_put() will be called in cached_dev_free() */
  2090. if (ret < 0)
  2091. goto out_free_sb;
  2092. } else {
  2093. /* blkdev_put() will be called in bch_cache_release() */
  2094. ret = register_cache(sb, sb_disk, bdev_file, holder);
  2095. if (ret)
  2096. goto out_free_sb;
  2097. }
  2098. kfree(sb);
  2099. kfree(path);
  2100. module_put(THIS_MODULE);
  2101. async_done:
  2102. return size;
  2103. out_free_holder:
  2104. kfree(holder);
  2105. out_put_sb_page:
  2106. put_page(virt_to_page(sb_disk));
  2107. out_blkdev_put:
  2108. if (bdev_file)
  2109. fput(bdev_file);
  2110. out_free_sb:
  2111. kfree(sb);
  2112. out_free_path:
  2113. kfree(path);
  2114. path = NULL;
  2115. out_module_put:
  2116. module_put(THIS_MODULE);
  2117. out:
  2118. if (!quiet)
  2119. pr_info("error %s: %s\n", path?path:"", err);
  2120. return ret;
  2121. }
  2122. struct pdev {
  2123. struct list_head list;
  2124. struct cached_dev *dc;
  2125. };
  2126. static ssize_t bch_pending_bdevs_cleanup(struct kobject *k,
  2127. struct kobj_attribute *attr,
  2128. const char *buffer,
  2129. size_t size)
  2130. {
  2131. LIST_HEAD(pending_devs);
  2132. ssize_t ret = size;
  2133. struct cached_dev *dc, *tdc;
  2134. struct pdev *pdev, *tpdev;
  2135. struct cache_set *c, *tc;
  2136. mutex_lock(&bch_register_lock);
  2137. list_for_each_entry_safe(dc, tdc, &uncached_devices, list) {
  2138. pdev = kmalloc(sizeof(struct pdev), GFP_KERNEL);
  2139. if (!pdev)
  2140. break;
  2141. pdev->dc = dc;
  2142. list_add(&pdev->list, &pending_devs);
  2143. }
  2144. list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) {
  2145. char *pdev_set_uuid = pdev->dc->sb.set_uuid;
  2146. list_for_each_entry_safe(c, tc, &bch_cache_sets, list) {
  2147. char *set_uuid = c->set_uuid;
  2148. if (!memcmp(pdev_set_uuid, set_uuid, 16)) {
  2149. list_del(&pdev->list);
  2150. kfree(pdev);
  2151. break;
  2152. }
  2153. }
  2154. }
  2155. mutex_unlock(&bch_register_lock);
  2156. list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) {
  2157. pr_info("delete pdev %p\n", pdev);
  2158. list_del(&pdev->list);
  2159. bcache_device_stop(&pdev->dc->disk);
  2160. kfree(pdev);
  2161. }
  2162. return ret;
  2163. }
  2164. static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x)
  2165. {
  2166. if (bcache_is_reboot)
  2167. return NOTIFY_DONE;
  2168. if (code == SYS_DOWN ||
  2169. code == SYS_HALT ||
  2170. code == SYS_POWER_OFF) {
  2171. DEFINE_WAIT(wait);
  2172. unsigned long start = jiffies;
  2173. bool stopped = false;
  2174. struct cache_set *c, *tc;
  2175. struct cached_dev *dc, *tdc;
  2176. mutex_lock(&bch_register_lock);
  2177. if (bcache_is_reboot)
  2178. goto out;
  2179. /* New registration is rejected since now */
  2180. bcache_is_reboot = true;
  2181. /*
  2182. * Make registering caller (if there is) on other CPU
  2183. * core know bcache_is_reboot set to true earlier
  2184. */
  2185. smp_mb();
  2186. if (list_empty(&bch_cache_sets) &&
  2187. list_empty(&uncached_devices))
  2188. goto out;
  2189. mutex_unlock(&bch_register_lock);
  2190. pr_info("Stopping all devices:\n");
  2191. /*
  2192. * The reason bch_register_lock is not held to call
  2193. * bch_cache_set_stop() and bcache_device_stop() is to
  2194. * avoid potential deadlock during reboot, because cache
  2195. * set or bcache device stopping process will acquire
  2196. * bch_register_lock too.
  2197. *
  2198. * We are safe here because bcache_is_reboot sets to
  2199. * true already, register_bcache() will reject new
  2200. * registration now. bcache_is_reboot also makes sure
  2201. * bcache_reboot() won't be re-entered on by other thread,
  2202. * so there is no race in following list iteration by
  2203. * list_for_each_entry_safe().
  2204. */
  2205. list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
  2206. bch_cache_set_stop(c);
  2207. list_for_each_entry_safe(dc, tdc, &uncached_devices, list)
  2208. bcache_device_stop(&dc->disk);
  2209. /*
  2210. * Give an early chance for other kthreads and
  2211. * kworkers to stop themselves
  2212. */
  2213. schedule();
  2214. /* What's a condition variable? */
  2215. while (1) {
  2216. long timeout = start + 10 * HZ - jiffies;
  2217. mutex_lock(&bch_register_lock);
  2218. stopped = list_empty(&bch_cache_sets) &&
  2219. list_empty(&uncached_devices);
  2220. if (timeout < 0 || stopped)
  2221. break;
  2222. prepare_to_wait(&unregister_wait, &wait,
  2223. TASK_UNINTERRUPTIBLE);
  2224. mutex_unlock(&bch_register_lock);
  2225. schedule_timeout(timeout);
  2226. }
  2227. finish_wait(&unregister_wait, &wait);
  2228. if (stopped)
  2229. pr_info("All devices stopped\n");
  2230. else
  2231. pr_notice("Timeout waiting for devices to be closed\n");
  2232. out:
  2233. mutex_unlock(&bch_register_lock);
  2234. }
  2235. return NOTIFY_DONE;
  2236. }
  2237. static struct notifier_block reboot = {
  2238. .notifier_call = bcache_reboot,
  2239. .priority = INT_MAX, /* before any real devices */
  2240. };
  2241. static void bcache_exit(void)
  2242. {
  2243. bch_debug_exit();
  2244. bch_request_exit();
  2245. if (bcache_kobj)
  2246. kobject_put(bcache_kobj);
  2247. if (bcache_wq)
  2248. destroy_workqueue(bcache_wq);
  2249. if (bch_journal_wq)
  2250. destroy_workqueue(bch_journal_wq);
  2251. if (bch_flush_wq)
  2252. destroy_workqueue(bch_flush_wq);
  2253. bch_btree_exit();
  2254. if (bcache_major)
  2255. unregister_blkdev(bcache_major, "bcache");
  2256. unregister_reboot_notifier(&reboot);
  2257. mutex_destroy(&bch_register_lock);
  2258. }
  2259. /* Check and fixup module parameters */
  2260. static void check_module_parameters(void)
  2261. {
  2262. if (bch_cutoff_writeback_sync == 0)
  2263. bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC;
  2264. else if (bch_cutoff_writeback_sync > CUTOFF_WRITEBACK_SYNC_MAX) {
  2265. pr_warn("set bch_cutoff_writeback_sync (%u) to max value %u\n",
  2266. bch_cutoff_writeback_sync, CUTOFF_WRITEBACK_SYNC_MAX);
  2267. bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC_MAX;
  2268. }
  2269. if (bch_cutoff_writeback == 0)
  2270. bch_cutoff_writeback = CUTOFF_WRITEBACK;
  2271. else if (bch_cutoff_writeback > CUTOFF_WRITEBACK_MAX) {
  2272. pr_warn("set bch_cutoff_writeback (%u) to max value %u\n",
  2273. bch_cutoff_writeback, CUTOFF_WRITEBACK_MAX);
  2274. bch_cutoff_writeback = CUTOFF_WRITEBACK_MAX;
  2275. }
  2276. if (bch_cutoff_writeback > bch_cutoff_writeback_sync) {
  2277. pr_warn("set bch_cutoff_writeback (%u) to %u\n",
  2278. bch_cutoff_writeback, bch_cutoff_writeback_sync);
  2279. bch_cutoff_writeback = bch_cutoff_writeback_sync;
  2280. }
  2281. }
  2282. static int __init bcache_init(void)
  2283. {
  2284. static const struct attribute *files[] = {
  2285. &ksysfs_register.attr,
  2286. &ksysfs_register_quiet.attr,
  2287. &ksysfs_pendings_cleanup.attr,
  2288. NULL
  2289. };
  2290. check_module_parameters();
  2291. mutex_init(&bch_register_lock);
  2292. init_waitqueue_head(&unregister_wait);
  2293. register_reboot_notifier(&reboot);
  2294. bcache_major = register_blkdev(0, "bcache");
  2295. if (bcache_major < 0) {
  2296. unregister_reboot_notifier(&reboot);
  2297. mutex_destroy(&bch_register_lock);
  2298. return bcache_major;
  2299. }
  2300. if (bch_btree_init())
  2301. goto err;
  2302. bcache_wq = alloc_workqueue("bcache", WQ_MEM_RECLAIM, 0);
  2303. if (!bcache_wq)
  2304. goto err;
  2305. /*
  2306. * Let's not make this `WQ_MEM_RECLAIM` for the following reasons:
  2307. *
  2308. * 1. It used `system_wq` before which also does no memory reclaim.
  2309. * 2. With `WQ_MEM_RECLAIM` desktop stalls, increased boot times, and
  2310. * reduced throughput can be observed.
  2311. *
  2312. * We still want to user our own queue to not congest the `system_wq`.
  2313. */
  2314. bch_flush_wq = alloc_workqueue("bch_flush", 0, 0);
  2315. if (!bch_flush_wq)
  2316. goto err;
  2317. bch_journal_wq = alloc_workqueue("bch_journal", WQ_MEM_RECLAIM, 0);
  2318. if (!bch_journal_wq)
  2319. goto err;
  2320. bcache_kobj = kobject_create_and_add("bcache", fs_kobj);
  2321. if (!bcache_kobj)
  2322. goto err;
  2323. if (bch_request_init() ||
  2324. sysfs_create_files(bcache_kobj, files))
  2325. goto err;
  2326. bch_debug_init();
  2327. bcache_is_reboot = false;
  2328. return 0;
  2329. err:
  2330. bcache_exit();
  2331. return -ENOMEM;
  2332. }
  2333. /*
  2334. * Module hooks
  2335. */
  2336. module_exit(bcache_exit);
  2337. module_init(bcache_init);
  2338. module_param(bch_cutoff_writeback, uint, 0);
  2339. MODULE_PARM_DESC(bch_cutoff_writeback, "threshold to cutoff writeback");
  2340. module_param(bch_cutoff_writeback_sync, uint, 0);
  2341. MODULE_PARM_DESC(bch_cutoff_writeback_sync, "hard threshold to cutoff writeback");
  2342. MODULE_DESCRIPTION("Bcache: a Linux block layer cache");
  2343. MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
  2344. MODULE_LICENSE("GPL");