mon_client.c 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/ceph/ceph_debug.h>
  3. #include <linux/module.h>
  4. #include <linux/types.h>
  5. #include <linux/slab.h>
  6. #include <linux/random.h>
  7. #include <linux/sched.h>
  8. #include <linux/ceph/ceph_features.h>
  9. #include <linux/ceph/mon_client.h>
  10. #include <linux/ceph/libceph.h>
  11. #include <linux/ceph/debugfs.h>
  12. #include <linux/ceph/decode.h>
  13. #include <linux/ceph/auth.h>
  14. /*
  15. * Interact with Ceph monitor cluster. Handle requests for new map
  16. * versions, and periodically resend as needed. Also implement
  17. * statfs() and umount().
  18. *
  19. * A small cluster of Ceph "monitors" are responsible for managing critical
  20. * cluster configuration and state information. An odd number (e.g., 3, 5)
  21. * of cmon daemons use a modified version of the Paxos part-time parliament
  22. * algorithm to manage the MDS map (mds cluster membership), OSD map, and
  23. * list of clients who have mounted the file system.
  24. *
  25. * We maintain an open, active session with a monitor at all times in order to
  26. * receive timely MDSMap updates. We periodically send a keepalive byte on the
  27. * TCP socket to ensure we detect a failure. If the connection does break, we
  28. * randomly hunt for a new monitor. Once the connection is reestablished, we
  29. * resend any outstanding requests.
  30. */
  31. static const struct ceph_connection_operations mon_con_ops;
  32. static int __validate_auth(struct ceph_mon_client *monc);
  33. static int decode_mon_info(void **p, void *end, bool msgr2,
  34. struct ceph_entity_addr *addr)
  35. {
  36. void *mon_info_end;
  37. u32 struct_len;
  38. u8 struct_v;
  39. int ret;
  40. ret = ceph_start_decoding(p, end, 1, "mon_info_t", &struct_v,
  41. &struct_len);
  42. if (ret)
  43. return ret;
  44. mon_info_end = *p + struct_len;
  45. ceph_decode_skip_string(p, end, e_inval); /* skip mon name */
  46. ret = ceph_decode_entity_addrvec(p, end, msgr2, addr);
  47. if (ret)
  48. return ret;
  49. *p = mon_info_end;
  50. return 0;
  51. e_inval:
  52. return -EINVAL;
  53. }
  54. /*
  55. * Decode a monmap blob (e.g., during mount).
  56. *
  57. * Assume MonMap v3 (i.e. encoding with MONNAMES and MONENC).
  58. */
  59. static struct ceph_monmap *ceph_monmap_decode(void **p, void *end, bool msgr2)
  60. {
  61. struct ceph_monmap *monmap = NULL;
  62. struct ceph_fsid fsid;
  63. u32 struct_len;
  64. int blob_len;
  65. int num_mon;
  66. u8 struct_v;
  67. u32 epoch;
  68. int ret;
  69. int i;
  70. ceph_decode_32_safe(p, end, blob_len, e_inval);
  71. ceph_decode_need(p, end, blob_len, e_inval);
  72. ret = ceph_start_decoding(p, end, 6, "monmap", &struct_v, &struct_len);
  73. if (ret)
  74. goto fail;
  75. dout("%s struct_v %d\n", __func__, struct_v);
  76. ceph_decode_copy_safe(p, end, &fsid, sizeof(fsid), e_inval);
  77. ceph_decode_32_safe(p, end, epoch, e_inval);
  78. if (struct_v >= 6) {
  79. u32 feat_struct_len;
  80. u8 feat_struct_v;
  81. *p += sizeof(struct ceph_timespec); /* skip last_changed */
  82. *p += sizeof(struct ceph_timespec); /* skip created */
  83. ret = ceph_start_decoding(p, end, 1, "mon_feature_t",
  84. &feat_struct_v, &feat_struct_len);
  85. if (ret)
  86. goto fail;
  87. *p += feat_struct_len; /* skip persistent_features */
  88. ret = ceph_start_decoding(p, end, 1, "mon_feature_t",
  89. &feat_struct_v, &feat_struct_len);
  90. if (ret)
  91. goto fail;
  92. *p += feat_struct_len; /* skip optional_features */
  93. }
  94. ceph_decode_32_safe(p, end, num_mon, e_inval);
  95. dout("%s fsid %pU epoch %u num_mon %d\n", __func__, &fsid, epoch,
  96. num_mon);
  97. if (num_mon > CEPH_MAX_MON)
  98. goto e_inval;
  99. monmap = kmalloc(struct_size(monmap, mon_inst, num_mon), GFP_NOIO);
  100. if (!monmap) {
  101. ret = -ENOMEM;
  102. goto fail;
  103. }
  104. monmap->fsid = fsid;
  105. monmap->epoch = epoch;
  106. monmap->num_mon = num_mon;
  107. /* legacy_mon_addr map or mon_info map */
  108. for (i = 0; i < num_mon; i++) {
  109. struct ceph_entity_inst *inst = &monmap->mon_inst[i];
  110. ceph_decode_skip_string(p, end, e_inval); /* skip mon name */
  111. inst->name.type = CEPH_ENTITY_TYPE_MON;
  112. inst->name.num = cpu_to_le64(i);
  113. if (struct_v >= 6)
  114. ret = decode_mon_info(p, end, msgr2, &inst->addr);
  115. else
  116. ret = ceph_decode_entity_addr(p, end, &inst->addr);
  117. if (ret)
  118. goto fail;
  119. dout("%s mon%d addr %s\n", __func__, i,
  120. ceph_pr_addr(&inst->addr));
  121. }
  122. return monmap;
  123. e_inval:
  124. ret = -EINVAL;
  125. fail:
  126. kfree(monmap);
  127. return ERR_PTR(ret);
  128. }
  129. /*
  130. * return true if *addr is included in the monmap.
  131. */
  132. int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr)
  133. {
  134. int i;
  135. for (i = 0; i < m->num_mon; i++) {
  136. if (ceph_addr_equal_no_type(addr, &m->mon_inst[i].addr))
  137. return 1;
  138. }
  139. return 0;
  140. }
  141. /*
  142. * Send an auth request.
  143. */
  144. static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len)
  145. {
  146. monc->pending_auth = 1;
  147. monc->m_auth->front.iov_len = len;
  148. monc->m_auth->hdr.front_len = cpu_to_le32(len);
  149. ceph_msg_revoke(monc->m_auth);
  150. ceph_msg_get(monc->m_auth); /* keep our ref */
  151. ceph_con_send(&monc->con, monc->m_auth);
  152. }
  153. /*
  154. * Close monitor session, if any.
  155. */
  156. static void __close_session(struct ceph_mon_client *monc)
  157. {
  158. dout("__close_session closing mon%d\n", monc->cur_mon);
  159. ceph_msg_revoke(monc->m_auth);
  160. ceph_msg_revoke_incoming(monc->m_auth_reply);
  161. ceph_msg_revoke(monc->m_subscribe);
  162. ceph_msg_revoke_incoming(monc->m_subscribe_ack);
  163. ceph_con_close(&monc->con);
  164. monc->pending_auth = 0;
  165. ceph_auth_reset(monc->auth);
  166. }
  167. /*
  168. * Pick a new monitor at random and set cur_mon. If we are repicking
  169. * (i.e. cur_mon is already set), be sure to pick a different one.
  170. */
  171. static void pick_new_mon(struct ceph_mon_client *monc)
  172. {
  173. int old_mon = monc->cur_mon;
  174. BUG_ON(monc->monmap->num_mon < 1);
  175. if (monc->monmap->num_mon == 1) {
  176. monc->cur_mon = 0;
  177. } else {
  178. int max = monc->monmap->num_mon;
  179. int o = -1;
  180. int n;
  181. if (monc->cur_mon >= 0) {
  182. if (monc->cur_mon < monc->monmap->num_mon)
  183. o = monc->cur_mon;
  184. if (o >= 0)
  185. max--;
  186. }
  187. n = get_random_u32_below(max);
  188. if (o >= 0 && n >= o)
  189. n++;
  190. monc->cur_mon = n;
  191. }
  192. dout("%s mon%d -> mon%d out of %d mons\n", __func__, old_mon,
  193. monc->cur_mon, monc->monmap->num_mon);
  194. }
  195. /*
  196. * Open a session with a new monitor.
  197. */
  198. static void __open_session(struct ceph_mon_client *monc)
  199. {
  200. int ret;
  201. pick_new_mon(monc);
  202. monc->hunting = true;
  203. if (monc->had_a_connection) {
  204. monc->hunt_mult *= CEPH_MONC_HUNT_BACKOFF;
  205. if (monc->hunt_mult > CEPH_MONC_HUNT_MAX_MULT)
  206. monc->hunt_mult = CEPH_MONC_HUNT_MAX_MULT;
  207. }
  208. monc->sub_renew_after = jiffies; /* i.e., expired */
  209. monc->sub_renew_sent = 0;
  210. dout("%s opening mon%d\n", __func__, monc->cur_mon);
  211. ceph_con_open(&monc->con, CEPH_ENTITY_TYPE_MON, monc->cur_mon,
  212. &monc->monmap->mon_inst[monc->cur_mon].addr);
  213. /*
  214. * Queue a keepalive to ensure that in case of an early fault
  215. * the messenger doesn't put us into STANDBY state and instead
  216. * retries. This also ensures that our timestamp is valid by
  217. * the time we finish hunting and delayed_work() checks it.
  218. */
  219. ceph_con_keepalive(&monc->con);
  220. if (ceph_msgr2(monc->client)) {
  221. monc->pending_auth = 1;
  222. return;
  223. }
  224. /* initiate authentication handshake */
  225. ret = ceph_auth_build_hello(monc->auth,
  226. monc->m_auth->front.iov_base,
  227. monc->m_auth->front_alloc_len);
  228. BUG_ON(ret <= 0);
  229. __send_prepared_auth_request(monc, ret);
  230. }
  231. static void reopen_session(struct ceph_mon_client *monc)
  232. {
  233. if (!monc->hunting)
  234. pr_info("mon%d %s session lost, hunting for new mon\n",
  235. monc->cur_mon, ceph_pr_addr(&monc->con.peer_addr));
  236. __close_session(monc);
  237. __open_session(monc);
  238. }
  239. void ceph_monc_reopen_session(struct ceph_mon_client *monc)
  240. {
  241. mutex_lock(&monc->mutex);
  242. reopen_session(monc);
  243. mutex_unlock(&monc->mutex);
  244. }
  245. static void un_backoff(struct ceph_mon_client *monc)
  246. {
  247. monc->hunt_mult /= 2; /* reduce by 50% */
  248. if (monc->hunt_mult < 1)
  249. monc->hunt_mult = 1;
  250. dout("%s hunt_mult now %d\n", __func__, monc->hunt_mult);
  251. }
  252. /*
  253. * Reschedule delayed work timer.
  254. */
  255. static void __schedule_delayed(struct ceph_mon_client *monc)
  256. {
  257. unsigned long delay;
  258. if (monc->hunting)
  259. delay = CEPH_MONC_HUNT_INTERVAL * monc->hunt_mult;
  260. else
  261. delay = CEPH_MONC_PING_INTERVAL;
  262. dout("__schedule_delayed after %lu\n", delay);
  263. mod_delayed_work(system_wq, &monc->delayed_work,
  264. round_jiffies_relative(delay));
  265. }
  266. const char *ceph_sub_str[] = {
  267. [CEPH_SUB_MONMAP] = "monmap",
  268. [CEPH_SUB_OSDMAP] = "osdmap",
  269. [CEPH_SUB_FSMAP] = "fsmap.user",
  270. [CEPH_SUB_MDSMAP] = "mdsmap",
  271. };
  272. /*
  273. * Send subscribe request for one or more maps, according to
  274. * monc->subs.
  275. */
  276. static void __send_subscribe(struct ceph_mon_client *monc)
  277. {
  278. struct ceph_msg *msg = monc->m_subscribe;
  279. void *p = msg->front.iov_base;
  280. void *const end = p + msg->front_alloc_len;
  281. int num = 0;
  282. int i;
  283. dout("%s sent %lu\n", __func__, monc->sub_renew_sent);
  284. BUG_ON(monc->cur_mon < 0);
  285. if (!monc->sub_renew_sent)
  286. monc->sub_renew_sent = jiffies | 1; /* never 0 */
  287. msg->hdr.version = cpu_to_le16(2);
  288. for (i = 0; i < ARRAY_SIZE(monc->subs); i++) {
  289. if (monc->subs[i].want)
  290. num++;
  291. }
  292. BUG_ON(num < 1); /* monmap sub is always there */
  293. ceph_encode_32(&p, num);
  294. for (i = 0; i < ARRAY_SIZE(monc->subs); i++) {
  295. char buf[32];
  296. int len;
  297. if (!monc->subs[i].want)
  298. continue;
  299. len = sprintf(buf, "%s", ceph_sub_str[i]);
  300. if (i == CEPH_SUB_MDSMAP &&
  301. monc->fs_cluster_id != CEPH_FS_CLUSTER_ID_NONE)
  302. len += sprintf(buf + len, ".%d", monc->fs_cluster_id);
  303. dout("%s %s start %llu flags 0x%x\n", __func__, buf,
  304. le64_to_cpu(monc->subs[i].item.start),
  305. monc->subs[i].item.flags);
  306. ceph_encode_string(&p, end, buf, len);
  307. memcpy(p, &monc->subs[i].item, sizeof(monc->subs[i].item));
  308. p += sizeof(monc->subs[i].item);
  309. }
  310. BUG_ON(p > end);
  311. msg->front.iov_len = p - msg->front.iov_base;
  312. msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
  313. ceph_msg_revoke(msg);
  314. ceph_con_send(&monc->con, ceph_msg_get(msg));
  315. }
  316. static void handle_subscribe_ack(struct ceph_mon_client *monc,
  317. struct ceph_msg *msg)
  318. {
  319. unsigned int seconds;
  320. struct ceph_mon_subscribe_ack *h = msg->front.iov_base;
  321. if (msg->front.iov_len < sizeof(*h))
  322. goto bad;
  323. seconds = le32_to_cpu(h->duration);
  324. mutex_lock(&monc->mutex);
  325. if (monc->sub_renew_sent) {
  326. /*
  327. * This is only needed for legacy (infernalis or older)
  328. * MONs -- see delayed_work().
  329. */
  330. monc->sub_renew_after = monc->sub_renew_sent +
  331. (seconds >> 1) * HZ - 1;
  332. dout("%s sent %lu duration %d renew after %lu\n", __func__,
  333. monc->sub_renew_sent, seconds, monc->sub_renew_after);
  334. monc->sub_renew_sent = 0;
  335. } else {
  336. dout("%s sent %lu renew after %lu, ignoring\n", __func__,
  337. monc->sub_renew_sent, monc->sub_renew_after);
  338. }
  339. mutex_unlock(&monc->mutex);
  340. return;
  341. bad:
  342. pr_err("got corrupt subscribe-ack msg\n");
  343. ceph_msg_dump(msg);
  344. }
  345. /*
  346. * Register interest in a map
  347. *
  348. * @sub: one of CEPH_SUB_*
  349. * @epoch: X for "every map since X", or 0 for "just the latest"
  350. */
  351. static bool __ceph_monc_want_map(struct ceph_mon_client *monc, int sub,
  352. u32 epoch, bool continuous)
  353. {
  354. __le64 start = cpu_to_le64(epoch);
  355. u8 flags = !continuous ? CEPH_SUBSCRIBE_ONETIME : 0;
  356. dout("%s %s epoch %u continuous %d\n", __func__, ceph_sub_str[sub],
  357. epoch, continuous);
  358. if (monc->subs[sub].want &&
  359. monc->subs[sub].item.start == start &&
  360. monc->subs[sub].item.flags == flags)
  361. return false;
  362. monc->subs[sub].item.start = start;
  363. monc->subs[sub].item.flags = flags;
  364. monc->subs[sub].want = true;
  365. return true;
  366. }
  367. bool ceph_monc_want_map(struct ceph_mon_client *monc, int sub, u32 epoch,
  368. bool continuous)
  369. {
  370. bool need_request;
  371. mutex_lock(&monc->mutex);
  372. need_request = __ceph_monc_want_map(monc, sub, epoch, continuous);
  373. mutex_unlock(&monc->mutex);
  374. return need_request;
  375. }
  376. EXPORT_SYMBOL(ceph_monc_want_map);
  377. /*
  378. * Keep track of which maps we have
  379. *
  380. * @sub: one of CEPH_SUB_*
  381. */
  382. static void __ceph_monc_got_map(struct ceph_mon_client *monc, int sub,
  383. u32 epoch)
  384. {
  385. dout("%s %s epoch %u\n", __func__, ceph_sub_str[sub], epoch);
  386. if (monc->subs[sub].want) {
  387. if (monc->subs[sub].item.flags & CEPH_SUBSCRIBE_ONETIME)
  388. monc->subs[sub].want = false;
  389. else
  390. monc->subs[sub].item.start = cpu_to_le64(epoch + 1);
  391. }
  392. monc->subs[sub].have = epoch;
  393. }
  394. void ceph_monc_got_map(struct ceph_mon_client *monc, int sub, u32 epoch)
  395. {
  396. mutex_lock(&monc->mutex);
  397. __ceph_monc_got_map(monc, sub, epoch);
  398. mutex_unlock(&monc->mutex);
  399. }
  400. EXPORT_SYMBOL(ceph_monc_got_map);
  401. void ceph_monc_renew_subs(struct ceph_mon_client *monc)
  402. {
  403. mutex_lock(&monc->mutex);
  404. __send_subscribe(monc);
  405. mutex_unlock(&monc->mutex);
  406. }
  407. EXPORT_SYMBOL(ceph_monc_renew_subs);
  408. /*
  409. * Wait for an osdmap with a given epoch.
  410. *
  411. * @epoch: epoch to wait for
  412. * @timeout: in jiffies, 0 means "wait forever"
  413. */
  414. int ceph_monc_wait_osdmap(struct ceph_mon_client *monc, u32 epoch,
  415. unsigned long timeout)
  416. {
  417. unsigned long started = jiffies;
  418. long ret;
  419. mutex_lock(&monc->mutex);
  420. while (monc->subs[CEPH_SUB_OSDMAP].have < epoch) {
  421. mutex_unlock(&monc->mutex);
  422. if (timeout && time_after_eq(jiffies, started + timeout))
  423. return -ETIMEDOUT;
  424. ret = wait_event_interruptible_timeout(monc->client->auth_wq,
  425. monc->subs[CEPH_SUB_OSDMAP].have >= epoch,
  426. ceph_timeout_jiffies(timeout));
  427. if (ret < 0)
  428. return ret;
  429. mutex_lock(&monc->mutex);
  430. }
  431. mutex_unlock(&monc->mutex);
  432. return 0;
  433. }
  434. EXPORT_SYMBOL(ceph_monc_wait_osdmap);
  435. /*
  436. * Open a session with a random monitor. Request monmap and osdmap,
  437. * which are waited upon in __ceph_open_session().
  438. */
  439. int ceph_monc_open_session(struct ceph_mon_client *monc)
  440. {
  441. mutex_lock(&monc->mutex);
  442. __ceph_monc_want_map(monc, CEPH_SUB_MONMAP, 0, true);
  443. __ceph_monc_want_map(monc, CEPH_SUB_OSDMAP, 0, false);
  444. __open_session(monc);
  445. __schedule_delayed(monc);
  446. mutex_unlock(&monc->mutex);
  447. return 0;
  448. }
  449. EXPORT_SYMBOL(ceph_monc_open_session);
  450. static void ceph_monc_handle_map(struct ceph_mon_client *monc,
  451. struct ceph_msg *msg)
  452. {
  453. struct ceph_client *client = monc->client;
  454. struct ceph_monmap *monmap;
  455. void *p, *end;
  456. mutex_lock(&monc->mutex);
  457. dout("handle_monmap\n");
  458. p = msg->front.iov_base;
  459. end = p + msg->front.iov_len;
  460. monmap = ceph_monmap_decode(&p, end, ceph_msgr2(client));
  461. if (IS_ERR(monmap)) {
  462. pr_err("problem decoding monmap, %d\n",
  463. (int)PTR_ERR(monmap));
  464. ceph_msg_dump(msg);
  465. goto out;
  466. }
  467. if (ceph_check_fsid(client, &monmap->fsid) < 0) {
  468. kfree(monmap);
  469. goto out;
  470. }
  471. kfree(monc->monmap);
  472. monc->monmap = monmap;
  473. __ceph_monc_got_map(monc, CEPH_SUB_MONMAP, monc->monmap->epoch);
  474. client->have_fsid = true;
  475. out:
  476. mutex_unlock(&monc->mutex);
  477. wake_up_all(&client->auth_wq);
  478. }
  479. /*
  480. * generic requests (currently statfs, mon_get_version)
  481. */
  482. DEFINE_RB_FUNCS(generic_request, struct ceph_mon_generic_request, tid, node)
  483. static void release_generic_request(struct kref *kref)
  484. {
  485. struct ceph_mon_generic_request *req =
  486. container_of(kref, struct ceph_mon_generic_request, kref);
  487. dout("%s greq %p request %p reply %p\n", __func__, req, req->request,
  488. req->reply);
  489. WARN_ON(!RB_EMPTY_NODE(&req->node));
  490. if (req->reply)
  491. ceph_msg_put(req->reply);
  492. if (req->request)
  493. ceph_msg_put(req->request);
  494. kfree(req);
  495. }
  496. static void put_generic_request(struct ceph_mon_generic_request *req)
  497. {
  498. if (req)
  499. kref_put(&req->kref, release_generic_request);
  500. }
  501. static void get_generic_request(struct ceph_mon_generic_request *req)
  502. {
  503. kref_get(&req->kref);
  504. }
  505. static struct ceph_mon_generic_request *
  506. alloc_generic_request(struct ceph_mon_client *monc, gfp_t gfp)
  507. {
  508. struct ceph_mon_generic_request *req;
  509. req = kzalloc(sizeof(*req), gfp);
  510. if (!req)
  511. return NULL;
  512. req->monc = monc;
  513. kref_init(&req->kref);
  514. RB_CLEAR_NODE(&req->node);
  515. init_completion(&req->completion);
  516. dout("%s greq %p\n", __func__, req);
  517. return req;
  518. }
  519. static void register_generic_request(struct ceph_mon_generic_request *req)
  520. {
  521. struct ceph_mon_client *monc = req->monc;
  522. WARN_ON(req->tid);
  523. get_generic_request(req);
  524. req->tid = ++monc->last_tid;
  525. insert_generic_request(&monc->generic_request_tree, req);
  526. }
  527. static void send_generic_request(struct ceph_mon_client *monc,
  528. struct ceph_mon_generic_request *req)
  529. {
  530. WARN_ON(!req->tid);
  531. dout("%s greq %p tid %llu\n", __func__, req, req->tid);
  532. req->request->hdr.tid = cpu_to_le64(req->tid);
  533. ceph_con_send(&monc->con, ceph_msg_get(req->request));
  534. }
  535. static void __finish_generic_request(struct ceph_mon_generic_request *req)
  536. {
  537. struct ceph_mon_client *monc = req->monc;
  538. dout("%s greq %p tid %llu\n", __func__, req, req->tid);
  539. erase_generic_request(&monc->generic_request_tree, req);
  540. ceph_msg_revoke(req->request);
  541. ceph_msg_revoke_incoming(req->reply);
  542. }
  543. static void finish_generic_request(struct ceph_mon_generic_request *req)
  544. {
  545. __finish_generic_request(req);
  546. put_generic_request(req);
  547. }
  548. static void complete_generic_request(struct ceph_mon_generic_request *req)
  549. {
  550. if (req->complete_cb)
  551. req->complete_cb(req);
  552. else
  553. complete_all(&req->completion);
  554. put_generic_request(req);
  555. }
  556. static void cancel_generic_request(struct ceph_mon_generic_request *req)
  557. {
  558. struct ceph_mon_client *monc = req->monc;
  559. struct ceph_mon_generic_request *lookup_req;
  560. dout("%s greq %p tid %llu\n", __func__, req, req->tid);
  561. mutex_lock(&monc->mutex);
  562. lookup_req = lookup_generic_request(&monc->generic_request_tree,
  563. req->tid);
  564. if (lookup_req) {
  565. WARN_ON(lookup_req != req);
  566. finish_generic_request(req);
  567. }
  568. mutex_unlock(&monc->mutex);
  569. }
  570. static int wait_generic_request(struct ceph_mon_generic_request *req)
  571. {
  572. int ret;
  573. dout("%s greq %p tid %llu\n", __func__, req, req->tid);
  574. ret = wait_for_completion_interruptible(&req->completion);
  575. if (ret)
  576. cancel_generic_request(req);
  577. else
  578. ret = req->result; /* completed */
  579. return ret;
  580. }
  581. static struct ceph_msg *get_generic_reply(struct ceph_connection *con,
  582. struct ceph_msg_header *hdr,
  583. int *skip)
  584. {
  585. struct ceph_mon_client *monc = con->private;
  586. struct ceph_mon_generic_request *req;
  587. u64 tid = le64_to_cpu(hdr->tid);
  588. struct ceph_msg *m;
  589. mutex_lock(&monc->mutex);
  590. req = lookup_generic_request(&monc->generic_request_tree, tid);
  591. if (!req) {
  592. dout("get_generic_reply %lld dne\n", tid);
  593. *skip = 1;
  594. m = NULL;
  595. } else {
  596. dout("get_generic_reply %lld got %p\n", tid, req->reply);
  597. *skip = 0;
  598. m = ceph_msg_get(req->reply);
  599. /*
  600. * we don't need to track the connection reading into
  601. * this reply because we only have one open connection
  602. * at a time, ever.
  603. */
  604. }
  605. mutex_unlock(&monc->mutex);
  606. return m;
  607. }
  608. /*
  609. * statfs
  610. */
  611. static void handle_statfs_reply(struct ceph_mon_client *monc,
  612. struct ceph_msg *msg)
  613. {
  614. struct ceph_mon_generic_request *req;
  615. struct ceph_mon_statfs_reply *reply = msg->front.iov_base;
  616. u64 tid = le64_to_cpu(msg->hdr.tid);
  617. dout("%s msg %p tid %llu\n", __func__, msg, tid);
  618. if (msg->front.iov_len != sizeof(*reply))
  619. goto bad;
  620. mutex_lock(&monc->mutex);
  621. req = lookup_generic_request(&monc->generic_request_tree, tid);
  622. if (!req) {
  623. mutex_unlock(&monc->mutex);
  624. return;
  625. }
  626. req->result = 0;
  627. *req->u.st = reply->st; /* struct */
  628. __finish_generic_request(req);
  629. mutex_unlock(&monc->mutex);
  630. complete_generic_request(req);
  631. return;
  632. bad:
  633. pr_err("corrupt statfs reply, tid %llu\n", tid);
  634. ceph_msg_dump(msg);
  635. }
  636. /*
  637. * Do a synchronous statfs().
  638. */
  639. int ceph_monc_do_statfs(struct ceph_mon_client *monc, u64 data_pool,
  640. struct ceph_statfs *buf)
  641. {
  642. struct ceph_mon_generic_request *req;
  643. struct ceph_mon_statfs *h;
  644. int ret = -ENOMEM;
  645. req = alloc_generic_request(monc, GFP_NOFS);
  646. if (!req)
  647. goto out;
  648. req->request = ceph_msg_new(CEPH_MSG_STATFS, sizeof(*h), GFP_NOFS,
  649. true);
  650. if (!req->request)
  651. goto out;
  652. req->reply = ceph_msg_new(CEPH_MSG_STATFS_REPLY, 64, GFP_NOFS, true);
  653. if (!req->reply)
  654. goto out;
  655. req->u.st = buf;
  656. req->request->hdr.version = cpu_to_le16(2);
  657. mutex_lock(&monc->mutex);
  658. register_generic_request(req);
  659. /* fill out request */
  660. h = req->request->front.iov_base;
  661. h->monhdr.have_version = 0;
  662. h->monhdr.session_mon = cpu_to_le16(-1);
  663. h->monhdr.session_mon_tid = 0;
  664. h->fsid = monc->monmap->fsid;
  665. h->contains_data_pool = (data_pool != CEPH_NOPOOL);
  666. h->data_pool = cpu_to_le64(data_pool);
  667. send_generic_request(monc, req);
  668. mutex_unlock(&monc->mutex);
  669. ret = wait_generic_request(req);
  670. out:
  671. put_generic_request(req);
  672. return ret;
  673. }
  674. EXPORT_SYMBOL(ceph_monc_do_statfs);
  675. static void handle_get_version_reply(struct ceph_mon_client *monc,
  676. struct ceph_msg *msg)
  677. {
  678. struct ceph_mon_generic_request *req;
  679. u64 tid = le64_to_cpu(msg->hdr.tid);
  680. void *p = msg->front.iov_base;
  681. void *end = p + msg->front_alloc_len;
  682. u64 handle;
  683. dout("%s msg %p tid %llu\n", __func__, msg, tid);
  684. ceph_decode_need(&p, end, 2*sizeof(u64), bad);
  685. handle = ceph_decode_64(&p);
  686. if (tid != 0 && tid != handle)
  687. goto bad;
  688. mutex_lock(&monc->mutex);
  689. req = lookup_generic_request(&monc->generic_request_tree, handle);
  690. if (!req) {
  691. mutex_unlock(&monc->mutex);
  692. return;
  693. }
  694. req->result = 0;
  695. req->u.newest = ceph_decode_64(&p);
  696. __finish_generic_request(req);
  697. mutex_unlock(&monc->mutex);
  698. complete_generic_request(req);
  699. return;
  700. bad:
  701. pr_err("corrupt mon_get_version reply, tid %llu\n", tid);
  702. ceph_msg_dump(msg);
  703. }
  704. static struct ceph_mon_generic_request *
  705. __ceph_monc_get_version(struct ceph_mon_client *monc, const char *what,
  706. ceph_monc_callback_t cb, u64 private_data)
  707. {
  708. struct ceph_mon_generic_request *req;
  709. req = alloc_generic_request(monc, GFP_NOIO);
  710. if (!req)
  711. goto err_put_req;
  712. req->request = ceph_msg_new(CEPH_MSG_MON_GET_VERSION,
  713. sizeof(u64) + sizeof(u32) + strlen(what),
  714. GFP_NOIO, true);
  715. if (!req->request)
  716. goto err_put_req;
  717. req->reply = ceph_msg_new(CEPH_MSG_MON_GET_VERSION_REPLY, 32, GFP_NOIO,
  718. true);
  719. if (!req->reply)
  720. goto err_put_req;
  721. req->complete_cb = cb;
  722. req->private_data = private_data;
  723. mutex_lock(&monc->mutex);
  724. register_generic_request(req);
  725. {
  726. void *p = req->request->front.iov_base;
  727. void *const end = p + req->request->front_alloc_len;
  728. ceph_encode_64(&p, req->tid); /* handle */
  729. ceph_encode_string(&p, end, what, strlen(what));
  730. WARN_ON(p != end);
  731. }
  732. send_generic_request(monc, req);
  733. mutex_unlock(&monc->mutex);
  734. return req;
  735. err_put_req:
  736. put_generic_request(req);
  737. return ERR_PTR(-ENOMEM);
  738. }
  739. /*
  740. * Send MMonGetVersion and wait for the reply.
  741. *
  742. * @what: one of "mdsmap", "osdmap" or "monmap"
  743. */
  744. int ceph_monc_get_version(struct ceph_mon_client *monc, const char *what,
  745. u64 *newest)
  746. {
  747. struct ceph_mon_generic_request *req;
  748. int ret;
  749. req = __ceph_monc_get_version(monc, what, NULL, 0);
  750. if (IS_ERR(req))
  751. return PTR_ERR(req);
  752. ret = wait_generic_request(req);
  753. if (!ret)
  754. *newest = req->u.newest;
  755. put_generic_request(req);
  756. return ret;
  757. }
  758. EXPORT_SYMBOL(ceph_monc_get_version);
  759. /*
  760. * Send MMonGetVersion,
  761. *
  762. * @what: one of "mdsmap", "osdmap" or "monmap"
  763. */
  764. int ceph_monc_get_version_async(struct ceph_mon_client *monc, const char *what,
  765. ceph_monc_callback_t cb, u64 private_data)
  766. {
  767. struct ceph_mon_generic_request *req;
  768. req = __ceph_monc_get_version(monc, what, cb, private_data);
  769. if (IS_ERR(req))
  770. return PTR_ERR(req);
  771. put_generic_request(req);
  772. return 0;
  773. }
  774. EXPORT_SYMBOL(ceph_monc_get_version_async);
  775. static void handle_command_ack(struct ceph_mon_client *monc,
  776. struct ceph_msg *msg)
  777. {
  778. struct ceph_mon_generic_request *req;
  779. void *p = msg->front.iov_base;
  780. void *const end = p + msg->front_alloc_len;
  781. u64 tid = le64_to_cpu(msg->hdr.tid);
  782. dout("%s msg %p tid %llu\n", __func__, msg, tid);
  783. ceph_decode_need(&p, end, sizeof(struct ceph_mon_request_header) +
  784. sizeof(u32), bad);
  785. p += sizeof(struct ceph_mon_request_header);
  786. mutex_lock(&monc->mutex);
  787. req = lookup_generic_request(&monc->generic_request_tree, tid);
  788. if (!req) {
  789. mutex_unlock(&monc->mutex);
  790. return;
  791. }
  792. req->result = ceph_decode_32(&p);
  793. __finish_generic_request(req);
  794. mutex_unlock(&monc->mutex);
  795. complete_generic_request(req);
  796. return;
  797. bad:
  798. pr_err("corrupt mon_command ack, tid %llu\n", tid);
  799. ceph_msg_dump(msg);
  800. }
  801. static __printf(2, 0)
  802. int do_mon_command_vargs(struct ceph_mon_client *monc, const char *fmt,
  803. va_list ap)
  804. {
  805. struct ceph_mon_generic_request *req;
  806. struct ceph_mon_command *h;
  807. int ret = -ENOMEM;
  808. int len;
  809. req = alloc_generic_request(monc, GFP_NOIO);
  810. if (!req)
  811. goto out;
  812. req->request = ceph_msg_new(CEPH_MSG_MON_COMMAND, 256, GFP_NOIO, true);
  813. if (!req->request)
  814. goto out;
  815. req->reply = ceph_msg_new(CEPH_MSG_MON_COMMAND_ACK, 512, GFP_NOIO,
  816. true);
  817. if (!req->reply)
  818. goto out;
  819. mutex_lock(&monc->mutex);
  820. register_generic_request(req);
  821. h = req->request->front.iov_base;
  822. h->monhdr.have_version = 0;
  823. h->monhdr.session_mon = cpu_to_le16(-1);
  824. h->monhdr.session_mon_tid = 0;
  825. h->fsid = monc->monmap->fsid;
  826. h->num_strs = cpu_to_le32(1);
  827. len = vsprintf(h->str, fmt, ap);
  828. h->str_len = cpu_to_le32(len);
  829. send_generic_request(monc, req);
  830. mutex_unlock(&monc->mutex);
  831. ret = wait_generic_request(req);
  832. out:
  833. put_generic_request(req);
  834. return ret;
  835. }
  836. static __printf(2, 3)
  837. int do_mon_command(struct ceph_mon_client *monc, const char *fmt, ...)
  838. {
  839. va_list ap;
  840. int ret;
  841. va_start(ap, fmt);
  842. ret = do_mon_command_vargs(monc, fmt, ap);
  843. va_end(ap);
  844. return ret;
  845. }
  846. int ceph_monc_blocklist_add(struct ceph_mon_client *monc,
  847. struct ceph_entity_addr *client_addr)
  848. {
  849. int ret;
  850. ret = do_mon_command(monc,
  851. "{ \"prefix\": \"osd blocklist\", \
  852. \"blocklistop\": \"add\", \
  853. \"addr\": \"%pISpc/%u\" }",
  854. &client_addr->in_addr,
  855. le32_to_cpu(client_addr->nonce));
  856. if (ret == -EINVAL) {
  857. /*
  858. * The monitor returns EINVAL on an unrecognized command.
  859. * Try the legacy command -- it is exactly the same except
  860. * for the name.
  861. */
  862. ret = do_mon_command(monc,
  863. "{ \"prefix\": \"osd blacklist\", \
  864. \"blacklistop\": \"add\", \
  865. \"addr\": \"%pISpc/%u\" }",
  866. &client_addr->in_addr,
  867. le32_to_cpu(client_addr->nonce));
  868. }
  869. if (ret)
  870. return ret;
  871. /*
  872. * Make sure we have the osdmap that includes the blocklist
  873. * entry. This is needed to ensure that the OSDs pick up the
  874. * new blocklist before processing any future requests from
  875. * this client.
  876. */
  877. return ceph_wait_for_latest_osdmap(monc->client, 0);
  878. }
  879. EXPORT_SYMBOL(ceph_monc_blocklist_add);
  880. /*
  881. * Resend pending generic requests.
  882. */
  883. static void __resend_generic_request(struct ceph_mon_client *monc)
  884. {
  885. struct ceph_mon_generic_request *req;
  886. struct rb_node *p;
  887. for (p = rb_first(&monc->generic_request_tree); p; p = rb_next(p)) {
  888. req = rb_entry(p, struct ceph_mon_generic_request, node);
  889. ceph_msg_revoke(req->request);
  890. ceph_msg_revoke_incoming(req->reply);
  891. ceph_con_send(&monc->con, ceph_msg_get(req->request));
  892. }
  893. }
  894. /*
  895. * Delayed work. If we haven't mounted yet, retry. Otherwise,
  896. * renew/retry subscription as needed (in case it is timing out, or we
  897. * got an ENOMEM). And keep the monitor connection alive.
  898. */
  899. static void delayed_work(struct work_struct *work)
  900. {
  901. struct ceph_mon_client *monc =
  902. container_of(work, struct ceph_mon_client, delayed_work.work);
  903. mutex_lock(&monc->mutex);
  904. dout("%s mon%d\n", __func__, monc->cur_mon);
  905. if (monc->cur_mon < 0) {
  906. goto out;
  907. }
  908. if (monc->hunting) {
  909. dout("%s continuing hunt\n", __func__);
  910. reopen_session(monc);
  911. } else {
  912. int is_auth = ceph_auth_is_authenticated(monc->auth);
  913. dout("%s is_authed %d\n", __func__, is_auth);
  914. if (ceph_con_keepalive_expired(&monc->con,
  915. CEPH_MONC_PING_TIMEOUT)) {
  916. dout("monc keepalive timeout\n");
  917. is_auth = 0;
  918. reopen_session(monc);
  919. }
  920. if (!monc->hunting) {
  921. ceph_con_keepalive(&monc->con);
  922. __validate_auth(monc);
  923. un_backoff(monc);
  924. }
  925. if (is_auth &&
  926. !(monc->con.peer_features & CEPH_FEATURE_MON_STATEFUL_SUB)) {
  927. unsigned long now = jiffies;
  928. dout("%s renew subs? now %lu renew after %lu\n",
  929. __func__, now, monc->sub_renew_after);
  930. if (time_after_eq(now, monc->sub_renew_after))
  931. __send_subscribe(monc);
  932. }
  933. }
  934. __schedule_delayed(monc);
  935. out:
  936. mutex_unlock(&monc->mutex);
  937. }
  938. /*
  939. * On startup, we build a temporary monmap populated with the IPs
  940. * provided by mount(2).
  941. */
  942. static int build_initial_monmap(struct ceph_mon_client *monc)
  943. {
  944. __le32 my_type = ceph_msgr2(monc->client) ?
  945. CEPH_ENTITY_ADDR_TYPE_MSGR2 : CEPH_ENTITY_ADDR_TYPE_LEGACY;
  946. struct ceph_options *opt = monc->client->options;
  947. int num_mon = opt->num_mon;
  948. int i;
  949. /* build initial monmap */
  950. monc->monmap = kzalloc(struct_size(monc->monmap, mon_inst, num_mon),
  951. GFP_KERNEL);
  952. if (!monc->monmap)
  953. return -ENOMEM;
  954. monc->monmap->num_mon = num_mon;
  955. for (i = 0; i < num_mon; i++) {
  956. struct ceph_entity_inst *inst = &monc->monmap->mon_inst[i];
  957. memcpy(&inst->addr.in_addr, &opt->mon_addr[i].in_addr,
  958. sizeof(inst->addr.in_addr));
  959. inst->addr.type = my_type;
  960. inst->addr.nonce = 0;
  961. inst->name.type = CEPH_ENTITY_TYPE_MON;
  962. inst->name.num = cpu_to_le64(i);
  963. }
  964. return 0;
  965. }
  966. int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl)
  967. {
  968. int err;
  969. dout("init\n");
  970. memset(monc, 0, sizeof(*monc));
  971. monc->client = cl;
  972. mutex_init(&monc->mutex);
  973. err = build_initial_monmap(monc);
  974. if (err)
  975. goto out;
  976. /* connection */
  977. /* authentication */
  978. monc->auth = ceph_auth_init(cl->options->name, cl->options->key,
  979. cl->options->con_modes);
  980. if (IS_ERR(monc->auth)) {
  981. err = PTR_ERR(monc->auth);
  982. goto out_monmap;
  983. }
  984. monc->auth->want_keys =
  985. CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |
  986. CEPH_ENTITY_TYPE_OSD | CEPH_ENTITY_TYPE_MDS;
  987. /* msgs */
  988. err = -ENOMEM;
  989. monc->m_subscribe_ack = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE_ACK,
  990. sizeof(struct ceph_mon_subscribe_ack),
  991. GFP_KERNEL, true);
  992. if (!monc->m_subscribe_ack)
  993. goto out_auth;
  994. monc->m_subscribe = ceph_msg_new(CEPH_MSG_MON_SUBSCRIBE, 128,
  995. GFP_KERNEL, true);
  996. if (!monc->m_subscribe)
  997. goto out_subscribe_ack;
  998. monc->m_auth_reply = ceph_msg_new(CEPH_MSG_AUTH_REPLY, 4096,
  999. GFP_KERNEL, true);
  1000. if (!monc->m_auth_reply)
  1001. goto out_subscribe;
  1002. monc->m_auth = ceph_msg_new(CEPH_MSG_AUTH, 4096, GFP_KERNEL, true);
  1003. monc->pending_auth = 0;
  1004. if (!monc->m_auth)
  1005. goto out_auth_reply;
  1006. ceph_con_init(&monc->con, monc, &mon_con_ops,
  1007. &monc->client->msgr);
  1008. monc->cur_mon = -1;
  1009. monc->had_a_connection = false;
  1010. monc->hunt_mult = 1;
  1011. INIT_DELAYED_WORK(&monc->delayed_work, delayed_work);
  1012. monc->generic_request_tree = RB_ROOT;
  1013. monc->last_tid = 0;
  1014. monc->fs_cluster_id = CEPH_FS_CLUSTER_ID_NONE;
  1015. return 0;
  1016. out_auth_reply:
  1017. ceph_msg_put(monc->m_auth_reply);
  1018. out_subscribe:
  1019. ceph_msg_put(monc->m_subscribe);
  1020. out_subscribe_ack:
  1021. ceph_msg_put(monc->m_subscribe_ack);
  1022. out_auth:
  1023. ceph_auth_destroy(monc->auth);
  1024. out_monmap:
  1025. kfree(monc->monmap);
  1026. out:
  1027. return err;
  1028. }
  1029. EXPORT_SYMBOL(ceph_monc_init);
  1030. void ceph_monc_stop(struct ceph_mon_client *monc)
  1031. {
  1032. dout("stop\n");
  1033. mutex_lock(&monc->mutex);
  1034. __close_session(monc);
  1035. monc->hunting = false;
  1036. monc->cur_mon = -1;
  1037. mutex_unlock(&monc->mutex);
  1038. cancel_delayed_work_sync(&monc->delayed_work);
  1039. /*
  1040. * flush msgr queue before we destroy ourselves to ensure that:
  1041. * - any work that references our embedded con is finished.
  1042. * - any osd_client or other work that may reference an authorizer
  1043. * finishes before we shut down the auth subsystem.
  1044. */
  1045. ceph_msgr_flush();
  1046. ceph_auth_destroy(monc->auth);
  1047. WARN_ON(!RB_EMPTY_ROOT(&monc->generic_request_tree));
  1048. ceph_msg_put(monc->m_auth);
  1049. ceph_msg_put(monc->m_auth_reply);
  1050. ceph_msg_put(monc->m_subscribe);
  1051. ceph_msg_put(monc->m_subscribe_ack);
  1052. kfree(monc->monmap);
  1053. }
  1054. EXPORT_SYMBOL(ceph_monc_stop);
  1055. static void finish_hunting(struct ceph_mon_client *monc)
  1056. {
  1057. if (monc->hunting) {
  1058. dout("%s found mon%d\n", __func__, monc->cur_mon);
  1059. monc->hunting = false;
  1060. monc->had_a_connection = true;
  1061. un_backoff(monc);
  1062. __schedule_delayed(monc);
  1063. }
  1064. }
  1065. static void finish_auth(struct ceph_mon_client *monc, int auth_err,
  1066. bool was_authed)
  1067. {
  1068. dout("%s auth_err %d was_authed %d\n", __func__, auth_err, was_authed);
  1069. WARN_ON(auth_err > 0);
  1070. monc->pending_auth = 0;
  1071. if (auth_err) {
  1072. monc->client->auth_err = auth_err;
  1073. wake_up_all(&monc->client->auth_wq);
  1074. return;
  1075. }
  1076. if (!was_authed && ceph_auth_is_authenticated(monc->auth)) {
  1077. dout("%s authenticated, starting session global_id %llu\n",
  1078. __func__, monc->auth->global_id);
  1079. monc->client->msgr.inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
  1080. monc->client->msgr.inst.name.num =
  1081. cpu_to_le64(monc->auth->global_id);
  1082. __send_subscribe(monc);
  1083. __resend_generic_request(monc);
  1084. pr_info("mon%d %s session established\n", monc->cur_mon,
  1085. ceph_pr_addr(&monc->con.peer_addr));
  1086. }
  1087. }
  1088. static void handle_auth_reply(struct ceph_mon_client *monc,
  1089. struct ceph_msg *msg)
  1090. {
  1091. bool was_authed;
  1092. int ret;
  1093. mutex_lock(&monc->mutex);
  1094. was_authed = ceph_auth_is_authenticated(monc->auth);
  1095. ret = ceph_handle_auth_reply(monc->auth, msg->front.iov_base,
  1096. msg->front.iov_len,
  1097. monc->m_auth->front.iov_base,
  1098. monc->m_auth->front_alloc_len);
  1099. if (ret > 0) {
  1100. __send_prepared_auth_request(monc, ret);
  1101. } else {
  1102. finish_auth(monc, ret, was_authed);
  1103. finish_hunting(monc);
  1104. }
  1105. mutex_unlock(&monc->mutex);
  1106. }
  1107. static int __validate_auth(struct ceph_mon_client *monc)
  1108. {
  1109. int ret;
  1110. if (monc->pending_auth)
  1111. return 0;
  1112. ret = ceph_build_auth(monc->auth, monc->m_auth->front.iov_base,
  1113. monc->m_auth->front_alloc_len);
  1114. if (ret <= 0)
  1115. return ret; /* either an error, or no need to authenticate */
  1116. __send_prepared_auth_request(monc, ret);
  1117. return 0;
  1118. }
  1119. int ceph_monc_validate_auth(struct ceph_mon_client *monc)
  1120. {
  1121. int ret;
  1122. mutex_lock(&monc->mutex);
  1123. ret = __validate_auth(monc);
  1124. mutex_unlock(&monc->mutex);
  1125. return ret;
  1126. }
  1127. EXPORT_SYMBOL(ceph_monc_validate_auth);
  1128. static int mon_get_auth_request(struct ceph_connection *con,
  1129. void *buf, int *buf_len,
  1130. void **authorizer, int *authorizer_len)
  1131. {
  1132. struct ceph_mon_client *monc = con->private;
  1133. int ret;
  1134. mutex_lock(&monc->mutex);
  1135. ret = ceph_auth_get_request(monc->auth, buf, *buf_len);
  1136. mutex_unlock(&monc->mutex);
  1137. if (ret < 0)
  1138. return ret;
  1139. *buf_len = ret;
  1140. *authorizer = NULL;
  1141. *authorizer_len = 0;
  1142. return 0;
  1143. }
  1144. static int mon_handle_auth_reply_more(struct ceph_connection *con,
  1145. void *reply, int reply_len,
  1146. void *buf, int *buf_len,
  1147. void **authorizer, int *authorizer_len)
  1148. {
  1149. struct ceph_mon_client *monc = con->private;
  1150. int ret;
  1151. mutex_lock(&monc->mutex);
  1152. ret = ceph_auth_handle_reply_more(monc->auth, reply, reply_len,
  1153. buf, *buf_len);
  1154. mutex_unlock(&monc->mutex);
  1155. if (ret < 0)
  1156. return ret;
  1157. *buf_len = ret;
  1158. *authorizer = NULL;
  1159. *authorizer_len = 0;
  1160. return 0;
  1161. }
  1162. static int mon_handle_auth_done(struct ceph_connection *con,
  1163. u64 global_id, void *reply, int reply_len,
  1164. u8 *session_key, int *session_key_len,
  1165. u8 *con_secret, int *con_secret_len)
  1166. {
  1167. struct ceph_mon_client *monc = con->private;
  1168. bool was_authed;
  1169. int ret;
  1170. mutex_lock(&monc->mutex);
  1171. WARN_ON(!monc->hunting);
  1172. was_authed = ceph_auth_is_authenticated(monc->auth);
  1173. ret = ceph_auth_handle_reply_done(monc->auth, global_id,
  1174. reply, reply_len,
  1175. session_key, session_key_len,
  1176. con_secret, con_secret_len);
  1177. finish_auth(monc, ret, was_authed);
  1178. if (!ret)
  1179. finish_hunting(monc);
  1180. mutex_unlock(&monc->mutex);
  1181. return 0;
  1182. }
  1183. static int mon_handle_auth_bad_method(struct ceph_connection *con,
  1184. int used_proto, int result,
  1185. const int *allowed_protos, int proto_cnt,
  1186. const int *allowed_modes, int mode_cnt)
  1187. {
  1188. struct ceph_mon_client *monc = con->private;
  1189. bool was_authed;
  1190. mutex_lock(&monc->mutex);
  1191. WARN_ON(!monc->hunting);
  1192. was_authed = ceph_auth_is_authenticated(monc->auth);
  1193. ceph_auth_handle_bad_method(monc->auth, used_proto, result,
  1194. allowed_protos, proto_cnt,
  1195. allowed_modes, mode_cnt);
  1196. finish_auth(monc, -EACCES, was_authed);
  1197. mutex_unlock(&monc->mutex);
  1198. return 0;
  1199. }
  1200. /*
  1201. * handle incoming message
  1202. */
  1203. static void mon_dispatch(struct ceph_connection *con, struct ceph_msg *msg)
  1204. {
  1205. struct ceph_mon_client *monc = con->private;
  1206. int type = le16_to_cpu(msg->hdr.type);
  1207. switch (type) {
  1208. case CEPH_MSG_AUTH_REPLY:
  1209. handle_auth_reply(monc, msg);
  1210. break;
  1211. case CEPH_MSG_MON_SUBSCRIBE_ACK:
  1212. handle_subscribe_ack(monc, msg);
  1213. break;
  1214. case CEPH_MSG_STATFS_REPLY:
  1215. handle_statfs_reply(monc, msg);
  1216. break;
  1217. case CEPH_MSG_MON_GET_VERSION_REPLY:
  1218. handle_get_version_reply(monc, msg);
  1219. break;
  1220. case CEPH_MSG_MON_COMMAND_ACK:
  1221. handle_command_ack(monc, msg);
  1222. break;
  1223. case CEPH_MSG_MON_MAP:
  1224. ceph_monc_handle_map(monc, msg);
  1225. break;
  1226. case CEPH_MSG_OSD_MAP:
  1227. ceph_osdc_handle_map(&monc->client->osdc, msg);
  1228. break;
  1229. default:
  1230. /* can the chained handler handle it? */
  1231. if (monc->client->extra_mon_dispatch &&
  1232. monc->client->extra_mon_dispatch(monc->client, msg) == 0)
  1233. break;
  1234. pr_err("received unknown message type %d %s\n", type,
  1235. ceph_msg_type_name(type));
  1236. }
  1237. ceph_msg_put(msg);
  1238. }
  1239. /*
  1240. * Allocate memory for incoming message
  1241. */
  1242. static struct ceph_msg *mon_alloc_msg(struct ceph_connection *con,
  1243. struct ceph_msg_header *hdr,
  1244. int *skip)
  1245. {
  1246. struct ceph_mon_client *monc = con->private;
  1247. int type = le16_to_cpu(hdr->type);
  1248. int front_len = le32_to_cpu(hdr->front_len);
  1249. struct ceph_msg *m = NULL;
  1250. *skip = 0;
  1251. switch (type) {
  1252. case CEPH_MSG_MON_SUBSCRIBE_ACK:
  1253. m = ceph_msg_get(monc->m_subscribe_ack);
  1254. break;
  1255. case CEPH_MSG_STATFS_REPLY:
  1256. case CEPH_MSG_MON_COMMAND_ACK:
  1257. return get_generic_reply(con, hdr, skip);
  1258. case CEPH_MSG_AUTH_REPLY:
  1259. m = ceph_msg_get(monc->m_auth_reply);
  1260. break;
  1261. case CEPH_MSG_MON_GET_VERSION_REPLY:
  1262. if (le64_to_cpu(hdr->tid) != 0)
  1263. return get_generic_reply(con, hdr, skip);
  1264. /*
  1265. * Older OSDs don't set reply tid even if the original
  1266. * request had a non-zero tid. Work around this weirdness
  1267. * by allocating a new message.
  1268. */
  1269. fallthrough;
  1270. case CEPH_MSG_MON_MAP:
  1271. case CEPH_MSG_MDS_MAP:
  1272. case CEPH_MSG_OSD_MAP:
  1273. case CEPH_MSG_FS_MAP_USER:
  1274. m = ceph_msg_new(type, front_len, GFP_NOFS, false);
  1275. if (!m)
  1276. return NULL; /* ENOMEM--return skip == 0 */
  1277. break;
  1278. }
  1279. if (!m) {
  1280. pr_info("alloc_msg unknown type %d\n", type);
  1281. *skip = 1;
  1282. } else if (front_len > m->front_alloc_len) {
  1283. pr_warn("mon_alloc_msg front %d > prealloc %d (%u#%llu)\n",
  1284. front_len, m->front_alloc_len,
  1285. (unsigned int)con->peer_name.type,
  1286. le64_to_cpu(con->peer_name.num));
  1287. ceph_msg_put(m);
  1288. m = ceph_msg_new(type, front_len, GFP_NOFS, false);
  1289. }
  1290. return m;
  1291. }
  1292. /*
  1293. * If the monitor connection resets, pick a new monitor and resubmit
  1294. * any pending requests.
  1295. */
  1296. static void mon_fault(struct ceph_connection *con)
  1297. {
  1298. struct ceph_mon_client *monc = con->private;
  1299. mutex_lock(&monc->mutex);
  1300. dout("%s mon%d\n", __func__, monc->cur_mon);
  1301. if (monc->cur_mon >= 0) {
  1302. if (!monc->hunting) {
  1303. dout("%s hunting for new mon\n", __func__);
  1304. reopen_session(monc);
  1305. __schedule_delayed(monc);
  1306. } else {
  1307. dout("%s already hunting\n", __func__);
  1308. }
  1309. }
  1310. mutex_unlock(&monc->mutex);
  1311. }
  1312. /*
  1313. * We can ignore refcounting on the connection struct, as all references
  1314. * will come from the messenger workqueue, which is drained prior to
  1315. * mon_client destruction.
  1316. */
  1317. static struct ceph_connection *mon_get_con(struct ceph_connection *con)
  1318. {
  1319. return con;
  1320. }
  1321. static void mon_put_con(struct ceph_connection *con)
  1322. {
  1323. }
  1324. static const struct ceph_connection_operations mon_con_ops = {
  1325. .get = mon_get_con,
  1326. .put = mon_put_con,
  1327. .alloc_msg = mon_alloc_msg,
  1328. .dispatch = mon_dispatch,
  1329. .fault = mon_fault,
  1330. .get_auth_request = mon_get_auth_request,
  1331. .handle_auth_reply_more = mon_handle_auth_reply_more,
  1332. .handle_auth_done = mon_handle_auth_done,
  1333. .handle_auth_bad_method = mon_handle_auth_bad_method,
  1334. };