test_xdp_noinline.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833
  1. // SPDX-License-Identifier: GPL-2.0
  2. // Copyright (c) 2017 Facebook
  3. #include <stddef.h>
  4. #include <stdbool.h>
  5. #include <string.h>
  6. #include <linux/pkt_cls.h>
  7. #include <linux/bpf.h>
  8. #include <linux/in.h>
  9. #include <linux/if_ether.h>
  10. #include <linux/ip.h>
  11. #include <linux/ipv6.h>
  12. #include <linux/icmp.h>
  13. #include <linux/icmpv6.h>
  14. #include <linux/tcp.h>
  15. #include <linux/udp.h>
  16. #include "bpf_helpers.h"
  17. #define bpf_printk(fmt, ...) \
  18. ({ \
  19. char ____fmt[] = fmt; \
  20. bpf_trace_printk(____fmt, sizeof(____fmt), \
  21. ##__VA_ARGS__); \
  22. })
  23. static __u32 rol32(__u32 word, unsigned int shift)
  24. {
  25. return (word << shift) | (word >> ((-shift) & 31));
  26. }
  27. /* copy paste of jhash from kernel sources to make sure llvm
  28. * can compile it into valid sequence of bpf instructions
  29. */
  30. #define __jhash_mix(a, b, c) \
  31. { \
  32. a -= c; a ^= rol32(c, 4); c += b; \
  33. b -= a; b ^= rol32(a, 6); a += c; \
  34. c -= b; c ^= rol32(b, 8); b += a; \
  35. a -= c; a ^= rol32(c, 16); c += b; \
  36. b -= a; b ^= rol32(a, 19); a += c; \
  37. c -= b; c ^= rol32(b, 4); b += a; \
  38. }
  39. #define __jhash_final(a, b, c) \
  40. { \
  41. c ^= b; c -= rol32(b, 14); \
  42. a ^= c; a -= rol32(c, 11); \
  43. b ^= a; b -= rol32(a, 25); \
  44. c ^= b; c -= rol32(b, 16); \
  45. a ^= c; a -= rol32(c, 4); \
  46. b ^= a; b -= rol32(a, 14); \
  47. c ^= b; c -= rol32(b, 24); \
  48. }
  49. #define JHASH_INITVAL 0xdeadbeef
  50. typedef unsigned int u32;
  51. static __attribute__ ((noinline))
  52. u32 jhash(const void *key, u32 length, u32 initval)
  53. {
  54. u32 a, b, c;
  55. const unsigned char *k = key;
  56. a = b = c = JHASH_INITVAL + length + initval;
  57. while (length > 12) {
  58. a += *(u32 *)(k);
  59. b += *(u32 *)(k + 4);
  60. c += *(u32 *)(k + 8);
  61. __jhash_mix(a, b, c);
  62. length -= 12;
  63. k += 12;
  64. }
  65. switch (length) {
  66. case 12: c += (u32)k[11]<<24;
  67. case 11: c += (u32)k[10]<<16;
  68. case 10: c += (u32)k[9]<<8;
  69. case 9: c += k[8];
  70. case 8: b += (u32)k[7]<<24;
  71. case 7: b += (u32)k[6]<<16;
  72. case 6: b += (u32)k[5]<<8;
  73. case 5: b += k[4];
  74. case 4: a += (u32)k[3]<<24;
  75. case 3: a += (u32)k[2]<<16;
  76. case 2: a += (u32)k[1]<<8;
  77. case 1: a += k[0];
  78. __jhash_final(a, b, c);
  79. case 0: /* Nothing left to add */
  80. break;
  81. }
  82. return c;
  83. }
  84. static __attribute__ ((noinline))
  85. u32 __jhash_nwords(u32 a, u32 b, u32 c, u32 initval)
  86. {
  87. a += initval;
  88. b += initval;
  89. c += initval;
  90. __jhash_final(a, b, c);
  91. return c;
  92. }
  93. static __attribute__ ((noinline))
  94. u32 jhash_2words(u32 a, u32 b, u32 initval)
  95. {
  96. return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2));
  97. }
  98. struct flow_key {
  99. union {
  100. __be32 src;
  101. __be32 srcv6[4];
  102. };
  103. union {
  104. __be32 dst;
  105. __be32 dstv6[4];
  106. };
  107. union {
  108. __u32 ports;
  109. __u16 port16[2];
  110. };
  111. __u8 proto;
  112. };
  113. struct packet_description {
  114. struct flow_key flow;
  115. __u8 flags;
  116. };
  117. struct ctl_value {
  118. union {
  119. __u64 value;
  120. __u32 ifindex;
  121. __u8 mac[6];
  122. };
  123. };
  124. struct vip_definition {
  125. union {
  126. __be32 vip;
  127. __be32 vipv6[4];
  128. };
  129. __u16 port;
  130. __u16 family;
  131. __u8 proto;
  132. };
  133. struct vip_meta {
  134. __u32 flags;
  135. __u32 vip_num;
  136. };
  137. struct real_pos_lru {
  138. __u32 pos;
  139. __u64 atime;
  140. };
  141. struct real_definition {
  142. union {
  143. __be32 dst;
  144. __be32 dstv6[4];
  145. };
  146. __u8 flags;
  147. };
  148. struct lb_stats {
  149. __u64 v2;
  150. __u64 v1;
  151. };
  152. struct bpf_map_def __attribute__ ((section("maps"), used)) vip_map = {
  153. .type = BPF_MAP_TYPE_HASH,
  154. .key_size = sizeof(struct vip_definition),
  155. .value_size = sizeof(struct vip_meta),
  156. .max_entries = 512,
  157. .map_flags = 0,
  158. };
  159. struct bpf_map_def __attribute__ ((section("maps"), used)) lru_cache = {
  160. .type = BPF_MAP_TYPE_LRU_HASH,
  161. .key_size = sizeof(struct flow_key),
  162. .value_size = sizeof(struct real_pos_lru),
  163. .max_entries = 300,
  164. .map_flags = 1U << 1,
  165. };
  166. struct bpf_map_def __attribute__ ((section("maps"), used)) ch_rings = {
  167. .type = BPF_MAP_TYPE_ARRAY,
  168. .key_size = sizeof(__u32),
  169. .value_size = sizeof(__u32),
  170. .max_entries = 12 * 655,
  171. .map_flags = 0,
  172. };
  173. struct bpf_map_def __attribute__ ((section("maps"), used)) reals = {
  174. .type = BPF_MAP_TYPE_ARRAY,
  175. .key_size = sizeof(__u32),
  176. .value_size = sizeof(struct real_definition),
  177. .max_entries = 40,
  178. .map_flags = 0,
  179. };
  180. struct bpf_map_def __attribute__ ((section("maps"), used)) stats = {
  181. .type = BPF_MAP_TYPE_PERCPU_ARRAY,
  182. .key_size = sizeof(__u32),
  183. .value_size = sizeof(struct lb_stats),
  184. .max_entries = 515,
  185. .map_flags = 0,
  186. };
  187. struct bpf_map_def __attribute__ ((section("maps"), used)) ctl_array = {
  188. .type = BPF_MAP_TYPE_ARRAY,
  189. .key_size = sizeof(__u32),
  190. .value_size = sizeof(struct ctl_value),
  191. .max_entries = 16,
  192. .map_flags = 0,
  193. };
  194. struct eth_hdr {
  195. unsigned char eth_dest[6];
  196. unsigned char eth_source[6];
  197. unsigned short eth_proto;
  198. };
  199. static inline __u64 calc_offset(bool is_ipv6, bool is_icmp)
  200. {
  201. __u64 off = sizeof(struct eth_hdr);
  202. if (is_ipv6) {
  203. off += sizeof(struct ipv6hdr);
  204. if (is_icmp)
  205. off += sizeof(struct icmp6hdr) + sizeof(struct ipv6hdr);
  206. } else {
  207. off += sizeof(struct iphdr);
  208. if (is_icmp)
  209. off += sizeof(struct icmphdr) + sizeof(struct iphdr);
  210. }
  211. return off;
  212. }
  213. static __attribute__ ((noinline))
  214. bool parse_udp(void *data, void *data_end,
  215. bool is_ipv6, struct packet_description *pckt)
  216. {
  217. bool is_icmp = !((pckt->flags & (1 << 0)) == 0);
  218. __u64 off = calc_offset(is_ipv6, is_icmp);
  219. struct udphdr *udp;
  220. udp = data + off;
  221. if (udp + 1 > data_end)
  222. return 0;
  223. if (!is_icmp) {
  224. pckt->flow.port16[0] = udp->source;
  225. pckt->flow.port16[1] = udp->dest;
  226. } else {
  227. pckt->flow.port16[0] = udp->dest;
  228. pckt->flow.port16[1] = udp->source;
  229. }
  230. return 1;
  231. }
  232. static __attribute__ ((noinline))
  233. bool parse_tcp(void *data, void *data_end,
  234. bool is_ipv6, struct packet_description *pckt)
  235. {
  236. bool is_icmp = !((pckt->flags & (1 << 0)) == 0);
  237. __u64 off = calc_offset(is_ipv6, is_icmp);
  238. struct tcphdr *tcp;
  239. tcp = data + off;
  240. if (tcp + 1 > data_end)
  241. return 0;
  242. if (tcp->syn)
  243. pckt->flags |= (1 << 1);
  244. if (!is_icmp) {
  245. pckt->flow.port16[0] = tcp->source;
  246. pckt->flow.port16[1] = tcp->dest;
  247. } else {
  248. pckt->flow.port16[0] = tcp->dest;
  249. pckt->flow.port16[1] = tcp->source;
  250. }
  251. return 1;
  252. }
  253. static __attribute__ ((noinline))
  254. bool encap_v6(struct xdp_md *xdp, struct ctl_value *cval,
  255. struct packet_description *pckt,
  256. struct real_definition *dst, __u32 pkt_bytes)
  257. {
  258. struct eth_hdr *new_eth;
  259. struct eth_hdr *old_eth;
  260. struct ipv6hdr *ip6h;
  261. __u32 ip_suffix;
  262. void *data_end;
  263. void *data;
  264. if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr)))
  265. return 0;
  266. data = (void *)(long)xdp->data;
  267. data_end = (void *)(long)xdp->data_end;
  268. new_eth = data;
  269. ip6h = data + sizeof(struct eth_hdr);
  270. old_eth = data + sizeof(struct ipv6hdr);
  271. if (new_eth + 1 > data_end ||
  272. old_eth + 1 > data_end || ip6h + 1 > data_end)
  273. return 0;
  274. memcpy(new_eth->eth_dest, cval->mac, 6);
  275. memcpy(new_eth->eth_source, old_eth->eth_dest, 6);
  276. new_eth->eth_proto = 56710;
  277. ip6h->version = 6;
  278. ip6h->priority = 0;
  279. memset(ip6h->flow_lbl, 0, sizeof(ip6h->flow_lbl));
  280. ip6h->nexthdr = IPPROTO_IPV6;
  281. ip_suffix = pckt->flow.srcv6[3] ^ pckt->flow.port16[0];
  282. ip6h->payload_len =
  283. __builtin_bswap16(pkt_bytes + sizeof(struct ipv6hdr));
  284. ip6h->hop_limit = 4;
  285. ip6h->saddr.in6_u.u6_addr32[0] = 1;
  286. ip6h->saddr.in6_u.u6_addr32[1] = 2;
  287. ip6h->saddr.in6_u.u6_addr32[2] = 3;
  288. ip6h->saddr.in6_u.u6_addr32[3] = ip_suffix;
  289. memcpy(ip6h->daddr.in6_u.u6_addr32, dst->dstv6, 16);
  290. return 1;
  291. }
  292. static __attribute__ ((noinline))
  293. bool encap_v4(struct xdp_md *xdp, struct ctl_value *cval,
  294. struct packet_description *pckt,
  295. struct real_definition *dst, __u32 pkt_bytes)
  296. {
  297. __u32 ip_suffix = __builtin_bswap16(pckt->flow.port16[0]);
  298. struct eth_hdr *new_eth;
  299. struct eth_hdr *old_eth;
  300. __u16 *next_iph_u16;
  301. struct iphdr *iph;
  302. __u32 csum = 0;
  303. void *data_end;
  304. void *data;
  305. ip_suffix <<= 15;
  306. ip_suffix ^= pckt->flow.src;
  307. if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr)))
  308. return 0;
  309. data = (void *)(long)xdp->data;
  310. data_end = (void *)(long)xdp->data_end;
  311. new_eth = data;
  312. iph = data + sizeof(struct eth_hdr);
  313. old_eth = data + sizeof(struct iphdr);
  314. if (new_eth + 1 > data_end ||
  315. old_eth + 1 > data_end || iph + 1 > data_end)
  316. return 0;
  317. memcpy(new_eth->eth_dest, cval->mac, 6);
  318. memcpy(new_eth->eth_source, old_eth->eth_dest, 6);
  319. new_eth->eth_proto = 8;
  320. iph->version = 4;
  321. iph->ihl = 5;
  322. iph->frag_off = 0;
  323. iph->protocol = IPPROTO_IPIP;
  324. iph->check = 0;
  325. iph->tos = 1;
  326. iph->tot_len = __builtin_bswap16(pkt_bytes + sizeof(struct iphdr));
  327. /* don't update iph->daddr, since it will overwrite old eth_proto
  328. * and multiple iterations of bpf_prog_run() will fail
  329. */
  330. iph->saddr = ((0xFFFF0000 & ip_suffix) | 4268) ^ dst->dst;
  331. iph->ttl = 4;
  332. next_iph_u16 = (__u16 *) iph;
  333. #pragma clang loop unroll(full)
  334. for (int i = 0; i < sizeof(struct iphdr) >> 1; i++)
  335. csum += *next_iph_u16++;
  336. iph->check = ~((csum & 0xffff) + (csum >> 16));
  337. if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr)))
  338. return 0;
  339. return 1;
  340. }
  341. static __attribute__ ((noinline))
  342. bool decap_v6(struct xdp_md *xdp, void **data, void **data_end, bool inner_v4)
  343. {
  344. struct eth_hdr *new_eth;
  345. struct eth_hdr *old_eth;
  346. old_eth = *data;
  347. new_eth = *data + sizeof(struct ipv6hdr);
  348. memcpy(new_eth->eth_source, old_eth->eth_source, 6);
  349. memcpy(new_eth->eth_dest, old_eth->eth_dest, 6);
  350. if (inner_v4)
  351. new_eth->eth_proto = 8;
  352. else
  353. new_eth->eth_proto = 56710;
  354. if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct ipv6hdr)))
  355. return 0;
  356. *data = (void *)(long)xdp->data;
  357. *data_end = (void *)(long)xdp->data_end;
  358. return 1;
  359. }
  360. static __attribute__ ((noinline))
  361. bool decap_v4(struct xdp_md *xdp, void **data, void **data_end)
  362. {
  363. struct eth_hdr *new_eth;
  364. struct eth_hdr *old_eth;
  365. old_eth = *data;
  366. new_eth = *data + sizeof(struct iphdr);
  367. memcpy(new_eth->eth_source, old_eth->eth_source, 6);
  368. memcpy(new_eth->eth_dest, old_eth->eth_dest, 6);
  369. new_eth->eth_proto = 8;
  370. if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr)))
  371. return 0;
  372. *data = (void *)(long)xdp->data;
  373. *data_end = (void *)(long)xdp->data_end;
  374. return 1;
  375. }
  376. static __attribute__ ((noinline))
  377. int swap_mac_and_send(void *data, void *data_end)
  378. {
  379. unsigned char tmp_mac[6];
  380. struct eth_hdr *eth;
  381. eth = data;
  382. memcpy(tmp_mac, eth->eth_source, 6);
  383. memcpy(eth->eth_source, eth->eth_dest, 6);
  384. memcpy(eth->eth_dest, tmp_mac, 6);
  385. return XDP_TX;
  386. }
  387. static __attribute__ ((noinline))
  388. int send_icmp_reply(void *data, void *data_end)
  389. {
  390. struct icmphdr *icmp_hdr;
  391. __u16 *next_iph_u16;
  392. __u32 tmp_addr = 0;
  393. struct iphdr *iph;
  394. __u32 csum1 = 0;
  395. __u32 csum = 0;
  396. __u64 off = 0;
  397. if (data + sizeof(struct eth_hdr)
  398. + sizeof(struct iphdr) + sizeof(struct icmphdr) > data_end)
  399. return XDP_DROP;
  400. off += sizeof(struct eth_hdr);
  401. iph = data + off;
  402. off += sizeof(struct iphdr);
  403. icmp_hdr = data + off;
  404. icmp_hdr->type = 0;
  405. icmp_hdr->checksum += 0x0007;
  406. iph->ttl = 4;
  407. tmp_addr = iph->daddr;
  408. iph->daddr = iph->saddr;
  409. iph->saddr = tmp_addr;
  410. iph->check = 0;
  411. next_iph_u16 = (__u16 *) iph;
  412. #pragma clang loop unroll(full)
  413. for (int i = 0; i < sizeof(struct iphdr) >> 1; i++)
  414. csum += *next_iph_u16++;
  415. iph->check = ~((csum & 0xffff) + (csum >> 16));
  416. return swap_mac_and_send(data, data_end);
  417. }
  418. static __attribute__ ((noinline))
  419. int send_icmp6_reply(void *data, void *data_end)
  420. {
  421. struct icmp6hdr *icmp_hdr;
  422. struct ipv6hdr *ip6h;
  423. __be32 tmp_addr[4];
  424. __u64 off = 0;
  425. if (data + sizeof(struct eth_hdr)
  426. + sizeof(struct ipv6hdr) + sizeof(struct icmp6hdr) > data_end)
  427. return XDP_DROP;
  428. off += sizeof(struct eth_hdr);
  429. ip6h = data + off;
  430. off += sizeof(struct ipv6hdr);
  431. icmp_hdr = data + off;
  432. icmp_hdr->icmp6_type = 129;
  433. icmp_hdr->icmp6_cksum -= 0x0001;
  434. ip6h->hop_limit = 4;
  435. memcpy(tmp_addr, ip6h->saddr.in6_u.u6_addr32, 16);
  436. memcpy(ip6h->saddr.in6_u.u6_addr32, ip6h->daddr.in6_u.u6_addr32, 16);
  437. memcpy(ip6h->daddr.in6_u.u6_addr32, tmp_addr, 16);
  438. return swap_mac_and_send(data, data_end);
  439. }
  440. static __attribute__ ((noinline))
  441. int parse_icmpv6(void *data, void *data_end, __u64 off,
  442. struct packet_description *pckt)
  443. {
  444. struct icmp6hdr *icmp_hdr;
  445. struct ipv6hdr *ip6h;
  446. icmp_hdr = data + off;
  447. if (icmp_hdr + 1 > data_end)
  448. return XDP_DROP;
  449. if (icmp_hdr->icmp6_type == 128)
  450. return send_icmp6_reply(data, data_end);
  451. if (icmp_hdr->icmp6_type != 3)
  452. return XDP_PASS;
  453. off += sizeof(struct icmp6hdr);
  454. ip6h = data + off;
  455. if (ip6h + 1 > data_end)
  456. return XDP_DROP;
  457. pckt->flow.proto = ip6h->nexthdr;
  458. pckt->flags |= (1 << 0);
  459. memcpy(pckt->flow.srcv6, ip6h->daddr.in6_u.u6_addr32, 16);
  460. memcpy(pckt->flow.dstv6, ip6h->saddr.in6_u.u6_addr32, 16);
  461. return -1;
  462. }
  463. static __attribute__ ((noinline))
  464. int parse_icmp(void *data, void *data_end, __u64 off,
  465. struct packet_description *pckt)
  466. {
  467. struct icmphdr *icmp_hdr;
  468. struct iphdr *iph;
  469. icmp_hdr = data + off;
  470. if (icmp_hdr + 1 > data_end)
  471. return XDP_DROP;
  472. if (icmp_hdr->type == 8)
  473. return send_icmp_reply(data, data_end);
  474. if ((icmp_hdr->type != 3) || (icmp_hdr->code != 4))
  475. return XDP_PASS;
  476. off += sizeof(struct icmphdr);
  477. iph = data + off;
  478. if (iph + 1 > data_end)
  479. return XDP_DROP;
  480. if (iph->ihl != 5)
  481. return XDP_DROP;
  482. pckt->flow.proto = iph->protocol;
  483. pckt->flags |= (1 << 0);
  484. pckt->flow.src = iph->daddr;
  485. pckt->flow.dst = iph->saddr;
  486. return -1;
  487. }
  488. static __attribute__ ((noinline))
  489. __u32 get_packet_hash(struct packet_description *pckt,
  490. bool hash_16bytes)
  491. {
  492. if (hash_16bytes)
  493. return jhash_2words(jhash(pckt->flow.srcv6, 16, 12),
  494. pckt->flow.ports, 24);
  495. else
  496. return jhash_2words(pckt->flow.src, pckt->flow.ports,
  497. 24);
  498. }
  499. __attribute__ ((noinline))
  500. static bool get_packet_dst(struct real_definition **real,
  501. struct packet_description *pckt,
  502. struct vip_meta *vip_info,
  503. bool is_ipv6, void *lru_map)
  504. {
  505. struct real_pos_lru new_dst_lru = { };
  506. bool hash_16bytes = is_ipv6;
  507. __u32 *real_pos, hash, key;
  508. __u64 cur_time;
  509. if (vip_info->flags & (1 << 2))
  510. hash_16bytes = 1;
  511. if (vip_info->flags & (1 << 3)) {
  512. pckt->flow.port16[0] = pckt->flow.port16[1];
  513. memset(pckt->flow.srcv6, 0, 16);
  514. }
  515. hash = get_packet_hash(pckt, hash_16bytes);
  516. if (hash != 0x358459b7 /* jhash of ipv4 packet */ &&
  517. hash != 0x2f4bc6bb /* jhash of ipv6 packet */)
  518. return 0;
  519. key = 2 * vip_info->vip_num + hash % 2;
  520. real_pos = bpf_map_lookup_elem(&ch_rings, &key);
  521. if (!real_pos)
  522. return 0;
  523. key = *real_pos;
  524. *real = bpf_map_lookup_elem(&reals, &key);
  525. if (!(*real))
  526. return 0;
  527. if (!(vip_info->flags & (1 << 1))) {
  528. __u32 conn_rate_key = 512 + 2;
  529. struct lb_stats *conn_rate_stats =
  530. bpf_map_lookup_elem(&stats, &conn_rate_key);
  531. if (!conn_rate_stats)
  532. return 1;
  533. cur_time = bpf_ktime_get_ns();
  534. if ((cur_time - conn_rate_stats->v2) >> 32 > 0xffFFFF) {
  535. conn_rate_stats->v1 = 1;
  536. conn_rate_stats->v2 = cur_time;
  537. } else {
  538. conn_rate_stats->v1 += 1;
  539. if (conn_rate_stats->v1 >= 1)
  540. return 1;
  541. }
  542. if (pckt->flow.proto == IPPROTO_UDP)
  543. new_dst_lru.atime = cur_time;
  544. new_dst_lru.pos = key;
  545. bpf_map_update_elem(lru_map, &pckt->flow, &new_dst_lru, 0);
  546. }
  547. return 1;
  548. }
  549. __attribute__ ((noinline))
  550. static void connection_table_lookup(struct real_definition **real,
  551. struct packet_description *pckt,
  552. void *lru_map)
  553. {
  554. struct real_pos_lru *dst_lru;
  555. __u64 cur_time;
  556. __u32 key;
  557. dst_lru = bpf_map_lookup_elem(lru_map, &pckt->flow);
  558. if (!dst_lru)
  559. return;
  560. if (pckt->flow.proto == IPPROTO_UDP) {
  561. cur_time = bpf_ktime_get_ns();
  562. if (cur_time - dst_lru->atime > 300000)
  563. return;
  564. dst_lru->atime = cur_time;
  565. }
  566. key = dst_lru->pos;
  567. *real = bpf_map_lookup_elem(&reals, &key);
  568. }
  569. /* don't believe your eyes!
  570. * below function has 6 arguments whereas bpf and llvm allow maximum of 5
  571. * but since it's _static_ llvm can optimize one argument away
  572. */
  573. __attribute__ ((noinline))
  574. static int process_l3_headers_v6(struct packet_description *pckt,
  575. __u8 *protocol, __u64 off,
  576. __u16 *pkt_bytes, void *data,
  577. void *data_end)
  578. {
  579. struct ipv6hdr *ip6h;
  580. __u64 iph_len;
  581. int action;
  582. ip6h = data + off;
  583. if (ip6h + 1 > data_end)
  584. return XDP_DROP;
  585. iph_len = sizeof(struct ipv6hdr);
  586. *protocol = ip6h->nexthdr;
  587. pckt->flow.proto = *protocol;
  588. *pkt_bytes = __builtin_bswap16(ip6h->payload_len);
  589. off += iph_len;
  590. if (*protocol == 45) {
  591. return XDP_DROP;
  592. } else if (*protocol == 59) {
  593. action = parse_icmpv6(data, data_end, off, pckt);
  594. if (action >= 0)
  595. return action;
  596. } else {
  597. memcpy(pckt->flow.srcv6, ip6h->saddr.in6_u.u6_addr32, 16);
  598. memcpy(pckt->flow.dstv6, ip6h->daddr.in6_u.u6_addr32, 16);
  599. }
  600. return -1;
  601. }
  602. __attribute__ ((noinline))
  603. static int process_l3_headers_v4(struct packet_description *pckt,
  604. __u8 *protocol, __u64 off,
  605. __u16 *pkt_bytes, void *data,
  606. void *data_end)
  607. {
  608. struct iphdr *iph;
  609. __u64 iph_len;
  610. int action;
  611. iph = data + off;
  612. if (iph + 1 > data_end)
  613. return XDP_DROP;
  614. if (iph->ihl != 5)
  615. return XDP_DROP;
  616. *protocol = iph->protocol;
  617. pckt->flow.proto = *protocol;
  618. *pkt_bytes = __builtin_bswap16(iph->tot_len);
  619. off += 20;
  620. if (iph->frag_off & 65343)
  621. return XDP_DROP;
  622. if (*protocol == IPPROTO_ICMP) {
  623. action = parse_icmp(data, data_end, off, pckt);
  624. if (action >= 0)
  625. return action;
  626. } else {
  627. pckt->flow.src = iph->saddr;
  628. pckt->flow.dst = iph->daddr;
  629. }
  630. return -1;
  631. }
  632. __attribute__ ((noinline))
  633. static int process_packet(void *data, __u64 off, void *data_end,
  634. bool is_ipv6, struct xdp_md *xdp)
  635. {
  636. struct real_definition *dst = NULL;
  637. struct packet_description pckt = { };
  638. struct vip_definition vip = { };
  639. struct lb_stats *data_stats;
  640. struct eth_hdr *eth = data;
  641. void *lru_map = &lru_cache;
  642. struct vip_meta *vip_info;
  643. __u32 lru_stats_key = 513;
  644. __u32 mac_addr_pos = 0;
  645. __u32 stats_key = 512;
  646. struct ctl_value *cval;
  647. __u16 pkt_bytes;
  648. __u64 iph_len;
  649. __u8 protocol;
  650. __u32 vip_num;
  651. int action;
  652. if (is_ipv6)
  653. action = process_l3_headers_v6(&pckt, &protocol, off,
  654. &pkt_bytes, data, data_end);
  655. else
  656. action = process_l3_headers_v4(&pckt, &protocol, off,
  657. &pkt_bytes, data, data_end);
  658. if (action >= 0)
  659. return action;
  660. protocol = pckt.flow.proto;
  661. if (protocol == IPPROTO_TCP) {
  662. if (!parse_tcp(data, data_end, is_ipv6, &pckt))
  663. return XDP_DROP;
  664. } else if (protocol == IPPROTO_UDP) {
  665. if (!parse_udp(data, data_end, is_ipv6, &pckt))
  666. return XDP_DROP;
  667. } else {
  668. return XDP_TX;
  669. }
  670. if (is_ipv6)
  671. memcpy(vip.vipv6, pckt.flow.dstv6, 16);
  672. else
  673. vip.vip = pckt.flow.dst;
  674. vip.port = pckt.flow.port16[1];
  675. vip.proto = pckt.flow.proto;
  676. vip_info = bpf_map_lookup_elem(&vip_map, &vip);
  677. if (!vip_info) {
  678. vip.port = 0;
  679. vip_info = bpf_map_lookup_elem(&vip_map, &vip);
  680. if (!vip_info)
  681. return XDP_PASS;
  682. if (!(vip_info->flags & (1 << 4)))
  683. pckt.flow.port16[1] = 0;
  684. }
  685. if (data_end - data > 1400)
  686. return XDP_DROP;
  687. data_stats = bpf_map_lookup_elem(&stats, &stats_key);
  688. if (!data_stats)
  689. return XDP_DROP;
  690. data_stats->v1 += 1;
  691. if (!dst) {
  692. if (vip_info->flags & (1 << 0))
  693. pckt.flow.port16[0] = 0;
  694. if (!(pckt.flags & (1 << 1)) && !(vip_info->flags & (1 << 1)))
  695. connection_table_lookup(&dst, &pckt, lru_map);
  696. if (dst)
  697. goto out;
  698. if (pckt.flow.proto == IPPROTO_TCP) {
  699. struct lb_stats *lru_stats =
  700. bpf_map_lookup_elem(&stats, &lru_stats_key);
  701. if (!lru_stats)
  702. return XDP_DROP;
  703. if (pckt.flags & (1 << 1))
  704. lru_stats->v1 += 1;
  705. else
  706. lru_stats->v2 += 1;
  707. }
  708. if (!get_packet_dst(&dst, &pckt, vip_info, is_ipv6, lru_map))
  709. return XDP_DROP;
  710. data_stats->v2 += 1;
  711. }
  712. out:
  713. cval = bpf_map_lookup_elem(&ctl_array, &mac_addr_pos);
  714. if (!cval)
  715. return XDP_DROP;
  716. if (dst->flags & (1 << 0)) {
  717. if (!encap_v6(xdp, cval, &pckt, dst, pkt_bytes))
  718. return XDP_DROP;
  719. } else {
  720. if (!encap_v4(xdp, cval, &pckt, dst, pkt_bytes))
  721. return XDP_DROP;
  722. }
  723. vip_num = vip_info->vip_num;
  724. data_stats = bpf_map_lookup_elem(&stats, &vip_num);
  725. if (!data_stats)
  726. return XDP_DROP;
  727. data_stats->v1 += 1;
  728. data_stats->v2 += pkt_bytes;
  729. data = (void *)(long)xdp->data;
  730. data_end = (void *)(long)xdp->data_end;
  731. if (data + 4 > data_end)
  732. return XDP_DROP;
  733. *(u32 *)data = dst->dst;
  734. return XDP_DROP;
  735. }
  736. __attribute__ ((section("xdp-test"), used))
  737. int balancer_ingress(struct xdp_md *ctx)
  738. {
  739. void *data = (void *)(long)ctx->data;
  740. void *data_end = (void *)(long)ctx->data_end;
  741. struct eth_hdr *eth = data;
  742. __u32 eth_proto;
  743. __u32 nh_off;
  744. nh_off = sizeof(struct eth_hdr);
  745. if (data + nh_off > data_end)
  746. return XDP_DROP;
  747. eth_proto = eth->eth_proto;
  748. if (eth_proto == 8)
  749. return process_packet(data, nh_off, data_end, 0, ctx);
  750. else if (eth_proto == 56710)
  751. return process_packet(data, nh_off, data_end, 1, ctx);
  752. else
  753. return XDP_DROP;
  754. }
  755. char _license[] __attribute__ ((section("license"), used)) = "GPL";
  756. int _version __attribute__ ((section("version"), used)) = 1;