cache.c 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332
  1. /*
  2. * ARC Cache Management
  3. *
  4. * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com)
  5. * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
  6. *
  7. * This program is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License version 2 as
  9. * published by the Free Software Foundation.
  10. */
  11. #include <linux/module.h>
  12. #include <linux/mm.h>
  13. #include <linux/sched.h>
  14. #include <linux/cache.h>
  15. #include <linux/mmu_context.h>
  16. #include <linux/syscalls.h>
  17. #include <linux/uaccess.h>
  18. #include <linux/pagemap.h>
  19. #include <asm/cacheflush.h>
  20. #include <asm/cachectl.h>
  21. #include <asm/setup.h>
  22. #ifdef CONFIG_ISA_ARCV2
  23. #define USE_RGN_FLSH 1
  24. #endif
  25. static int l2_line_sz;
  26. static int ioc_exists;
  27. int slc_enable = 1, ioc_enable = 1;
  28. unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */
  29. unsigned long perip_end = 0xFFFFFFFF; /* legacy value */
  30. void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr,
  31. unsigned long sz, const int op, const int full_page);
  32. void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz);
  33. void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz);
  34. void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz);
  35. char *arc_cache_mumbojumbo(int c, char *buf, int len)
  36. {
  37. int n = 0;
  38. struct cpuinfo_arc_cache *p;
  39. #define PR_CACHE(p, cfg, str) \
  40. if (!(p)->line_len) \
  41. n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \
  42. else \
  43. n += scnprintf(buf + n, len - n, \
  44. str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", \
  45. (p)->sz_k, (p)->assoc, (p)->line_len, \
  46. (p)->vipt ? "VIPT" : "PIPT", \
  47. (p)->alias ? " aliasing" : "", \
  48. IS_USED_CFG(cfg));
  49. PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache");
  50. PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache");
  51. p = &cpuinfo_arc700[c].slc;
  52. if (p->line_len)
  53. n += scnprintf(buf + n, len - n,
  54. "SLC\t\t: %uK, %uB Line%s\n",
  55. p->sz_k, p->line_len, IS_USED_RUN(slc_enable));
  56. n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n",
  57. perip_base,
  58. IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency (per-device) "));
  59. return buf;
  60. }
  61. /*
  62. * Read the Cache Build Confuration Registers, Decode them and save into
  63. * the cpuinfo structure for later use.
  64. * No Validation done here, simply read/convert the BCRs
  65. */
  66. static void read_decode_cache_bcr_arcv2(int cpu)
  67. {
  68. struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc;
  69. struct bcr_generic sbcr;
  70. struct bcr_slc_cfg {
  71. #ifdef CONFIG_CPU_BIG_ENDIAN
  72. unsigned int pad:24, way:2, lsz:2, sz:4;
  73. #else
  74. unsigned int sz:4, lsz:2, way:2, pad:24;
  75. #endif
  76. } slc_cfg;
  77. struct bcr_clust_cfg {
  78. #ifdef CONFIG_CPU_BIG_ENDIAN
  79. unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8;
  80. #else
  81. unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7;
  82. #endif
  83. } cbcr;
  84. struct bcr_volatile {
  85. #ifdef CONFIG_CPU_BIG_ENDIAN
  86. unsigned int start:4, limit:4, pad:22, order:1, disable:1;
  87. #else
  88. unsigned int disable:1, order:1, pad:22, limit:4, start:4;
  89. #endif
  90. } vol;
  91. READ_BCR(ARC_REG_SLC_BCR, sbcr);
  92. if (sbcr.ver) {
  93. READ_BCR(ARC_REG_SLC_CFG, slc_cfg);
  94. p_slc->sz_k = 128 << slc_cfg.sz;
  95. l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64;
  96. }
  97. READ_BCR(ARC_REG_CLUSTER_BCR, cbcr);
  98. if (cbcr.c)
  99. ioc_exists = 1;
  100. else
  101. ioc_enable = 0;
  102. /* HS 2.0 didn't have AUX_VOL */
  103. if (cpuinfo_arc700[cpu].core.family > 0x51) {
  104. READ_BCR(AUX_VOL, vol);
  105. perip_base = vol.start << 28;
  106. /* HS 3.0 has limit and strict-ordering fields */
  107. if (cpuinfo_arc700[cpu].core.family > 0x52)
  108. perip_end = (vol.limit << 28) - 1;
  109. }
  110. }
  111. void read_decode_cache_bcr(void)
  112. {
  113. struct cpuinfo_arc_cache *p_ic, *p_dc;
  114. unsigned int cpu = smp_processor_id();
  115. struct bcr_cache {
  116. #ifdef CONFIG_CPU_BIG_ENDIAN
  117. unsigned int pad:12, line_len:4, sz:4, config:4, ver:8;
  118. #else
  119. unsigned int ver:8, config:4, sz:4, line_len:4, pad:12;
  120. #endif
  121. } ibcr, dbcr;
  122. p_ic = &cpuinfo_arc700[cpu].icache;
  123. READ_BCR(ARC_REG_IC_BCR, ibcr);
  124. if (!ibcr.ver)
  125. goto dc_chk;
  126. if (ibcr.ver <= 3) {
  127. BUG_ON(ibcr.config != 3);
  128. p_ic->assoc = 2; /* Fixed to 2w set assoc */
  129. } else if (ibcr.ver >= 4) {
  130. p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */
  131. }
  132. p_ic->line_len = 8 << ibcr.line_len;
  133. p_ic->sz_k = 1 << (ibcr.sz - 1);
  134. p_ic->vipt = 1;
  135. p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1;
  136. dc_chk:
  137. p_dc = &cpuinfo_arc700[cpu].dcache;
  138. READ_BCR(ARC_REG_DC_BCR, dbcr);
  139. if (!dbcr.ver)
  140. goto slc_chk;
  141. if (dbcr.ver <= 3) {
  142. BUG_ON(dbcr.config != 2);
  143. p_dc->assoc = 4; /* Fixed to 4w set assoc */
  144. p_dc->vipt = 1;
  145. p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1;
  146. } else if (dbcr.ver >= 4) {
  147. p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */
  148. p_dc->vipt = 0;
  149. p_dc->alias = 0; /* PIPT so can't VIPT alias */
  150. }
  151. p_dc->line_len = 16 << dbcr.line_len;
  152. p_dc->sz_k = 1 << (dbcr.sz - 1);
  153. slc_chk:
  154. if (is_isa_arcv2())
  155. read_decode_cache_bcr_arcv2(cpu);
  156. }
  157. /*
  158. * Line Operation on {I,D}-Cache
  159. */
  160. #define OP_INV 0x1
  161. #define OP_FLUSH 0x2
  162. #define OP_FLUSH_N_INV 0x3
  163. #define OP_INV_IC 0x4
  164. /*
  165. * I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3)
  166. *
  167. * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag.
  168. * The orig Cache Management Module "CDU" only required paddr to invalidate a
  169. * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry.
  170. * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching
  171. * the exact same line.
  172. *
  173. * However for larger Caches (way-size > page-size) - i.e. in Aliasing config,
  174. * paddr alone could not be used to correctly index the cache.
  175. *
  176. * ------------------
  177. * MMU v1/v2 (Fixed Page Size 8k)
  178. * ------------------
  179. * The solution was to provide CDU with these additonal vaddr bits. These
  180. * would be bits [x:13], x would depend on cache-geometry, 13 comes from
  181. * standard page size of 8k.
  182. * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits
  183. * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the
  184. * orig 5 bits of paddr were anyways ignored by CDU line ops, as they
  185. * represent the offset within cache-line. The adv of using this "clumsy"
  186. * interface for additional info was no new reg was needed in CDU programming
  187. * model.
  188. *
  189. * 17:13 represented the max num of bits passable, actual bits needed were
  190. * fewer, based on the num-of-aliases possible.
  191. * -for 2 alias possibility, only bit 13 needed (32K cache)
  192. * -for 4 alias possibility, bits 14:13 needed (64K cache)
  193. *
  194. * ------------------
  195. * MMU v3
  196. * ------------------
  197. * This ver of MMU supports variable page sizes (1k-16k): although Linux will
  198. * only support 8k (default), 16k and 4k.
  199. * However from hardware perspective, smaller page sizes aggravate aliasing
  200. * meaning more vaddr bits needed to disambiguate the cache-line-op ;
  201. * the existing scheme of piggybacking won't work for certain configurations.
  202. * Two new registers IC_PTAG and DC_PTAG inttoduced.
  203. * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs
  204. */
  205. static inline
  206. void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr,
  207. unsigned long sz, const int op, const int full_page)
  208. {
  209. unsigned int aux_cmd;
  210. int num_lines;
  211. if (op == OP_INV_IC) {
  212. aux_cmd = ARC_REG_IC_IVIL;
  213. } else {
  214. /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
  215. aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL;
  216. }
  217. /* Ensure we properly floor/ceil the non-line aligned/sized requests
  218. * and have @paddr - aligned to cache line and integral @num_lines.
  219. * This however can be avoided for page sized since:
  220. * -@paddr will be cache-line aligned already (being page aligned)
  221. * -@sz will be integral multiple of line size (being page sized).
  222. */
  223. if (!full_page) {
  224. sz += paddr & ~CACHE_LINE_MASK;
  225. paddr &= CACHE_LINE_MASK;
  226. vaddr &= CACHE_LINE_MASK;
  227. }
  228. num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES);
  229. /* MMUv2 and before: paddr contains stuffed vaddrs bits */
  230. paddr |= (vaddr >> PAGE_SHIFT) & 0x1F;
  231. while (num_lines-- > 0) {
  232. write_aux_reg(aux_cmd, paddr);
  233. paddr += L1_CACHE_BYTES;
  234. }
  235. }
  236. /*
  237. * For ARC700 MMUv3 I-cache and D-cache flushes
  238. * - ARC700 programming model requires paddr and vaddr be passed in seperate
  239. * AUX registers (*_IV*L and *_PTAG respectively) irrespective of whether the
  240. * caches actually alias or not.
  241. * - For HS38, only the aliasing I-cache configuration uses the PTAG reg
  242. * (non aliasing I-cache version doesn't; while D-cache can't possibly alias)
  243. */
  244. static inline
  245. void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr,
  246. unsigned long sz, const int op, const int full_page)
  247. {
  248. unsigned int aux_cmd, aux_tag;
  249. int num_lines;
  250. if (op == OP_INV_IC) {
  251. aux_cmd = ARC_REG_IC_IVIL;
  252. aux_tag = ARC_REG_IC_PTAG;
  253. } else {
  254. aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL;
  255. aux_tag = ARC_REG_DC_PTAG;
  256. }
  257. /* Ensure we properly floor/ceil the non-line aligned/sized requests
  258. * and have @paddr - aligned to cache line and integral @num_lines.
  259. * This however can be avoided for page sized since:
  260. * -@paddr will be cache-line aligned already (being page aligned)
  261. * -@sz will be integral multiple of line size (being page sized).
  262. */
  263. if (!full_page) {
  264. sz += paddr & ~CACHE_LINE_MASK;
  265. paddr &= CACHE_LINE_MASK;
  266. vaddr &= CACHE_LINE_MASK;
  267. }
  268. num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES);
  269. /*
  270. * MMUv3, cache ops require paddr in PTAG reg
  271. * if V-P const for loop, PTAG can be written once outside loop
  272. */
  273. if (full_page)
  274. write_aux_reg(aux_tag, paddr);
  275. /*
  276. * This is technically for MMU v4, using the MMU v3 programming model
  277. * Special work for HS38 aliasing I-cache configuration with PAE40
  278. * - upper 8 bits of paddr need to be written into PTAG_HI
  279. * - (and needs to be written before the lower 32 bits)
  280. * Note that PTAG_HI is hoisted outside the line loop
  281. */
  282. if (is_pae40_enabled() && op == OP_INV_IC)
  283. write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);
  284. while (num_lines-- > 0) {
  285. if (!full_page) {
  286. write_aux_reg(aux_tag, paddr);
  287. paddr += L1_CACHE_BYTES;
  288. }
  289. write_aux_reg(aux_cmd, vaddr);
  290. vaddr += L1_CACHE_BYTES;
  291. }
  292. }
  293. #ifndef USE_RGN_FLSH
  294. /*
  295. * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT
  296. * Here's how cache ops are implemented
  297. *
  298. * - D-cache: only paddr needed (in DC_IVDL/DC_FLDL)
  299. * - I-cache Non Aliasing: Despite VIPT, only paddr needed (in IC_IVIL)
  300. * - I-cache Aliasing: Both vaddr and paddr needed (in IC_IVIL, IC_PTAG
  301. * respectively, similar to MMU v3 programming model, hence
  302. * __cache_line_loop_v3() is used)
  303. *
  304. * If PAE40 is enabled, independent of aliasing considerations, the higher bits
  305. * needs to be written into PTAG_HI
  306. */
  307. static inline
  308. void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr,
  309. unsigned long sz, const int op, const int full_page)
  310. {
  311. unsigned int aux_cmd;
  312. int num_lines;
  313. if (op == OP_INV_IC) {
  314. aux_cmd = ARC_REG_IC_IVIL;
  315. } else {
  316. /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
  317. aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL;
  318. }
  319. /* Ensure we properly floor/ceil the non-line aligned/sized requests
  320. * and have @paddr - aligned to cache line and integral @num_lines.
  321. * This however can be avoided for page sized since:
  322. * -@paddr will be cache-line aligned already (being page aligned)
  323. * -@sz will be integral multiple of line size (being page sized).
  324. */
  325. if (!full_page) {
  326. sz += paddr & ~CACHE_LINE_MASK;
  327. paddr &= CACHE_LINE_MASK;
  328. }
  329. num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES);
  330. /*
  331. * For HS38 PAE40 configuration
  332. * - upper 8 bits of paddr need to be written into PTAG_HI
  333. * - (and needs to be written before the lower 32 bits)
  334. */
  335. if (is_pae40_enabled()) {
  336. if (op == OP_INV_IC)
  337. /*
  338. * Non aliasing I-cache in HS38,
  339. * aliasing I-cache handled in __cache_line_loop_v3()
  340. */
  341. write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);
  342. else
  343. write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32);
  344. }
  345. while (num_lines-- > 0) {
  346. write_aux_reg(aux_cmd, paddr);
  347. paddr += L1_CACHE_BYTES;
  348. }
  349. }
  350. #else
  351. /*
  352. * optimized flush operation which takes a region as opposed to iterating per line
  353. */
  354. static inline
  355. void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr,
  356. unsigned long sz, const int op, const int full_page)
  357. {
  358. unsigned int s, e;
  359. /* Only for Non aliasing I-cache in HS38 */
  360. if (op == OP_INV_IC) {
  361. s = ARC_REG_IC_IVIR;
  362. e = ARC_REG_IC_ENDR;
  363. } else {
  364. s = ARC_REG_DC_STARTR;
  365. e = ARC_REG_DC_ENDR;
  366. }
  367. if (!full_page) {
  368. /* for any leading gap between @paddr and start of cache line */
  369. sz += paddr & ~CACHE_LINE_MASK;
  370. paddr &= CACHE_LINE_MASK;
  371. /*
  372. * account for any trailing gap to end of cache line
  373. * this is equivalent to DIV_ROUND_UP() in line ops above
  374. */
  375. sz += L1_CACHE_BYTES - 1;
  376. }
  377. if (is_pae40_enabled()) {
  378. /* TBD: check if crossing 4TB boundary */
  379. if (op == OP_INV_IC)
  380. write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);
  381. else
  382. write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32);
  383. }
  384. /* ENDR needs to be set ahead of START */
  385. write_aux_reg(e, paddr + sz); /* ENDR is exclusive */
  386. write_aux_reg(s, paddr);
  387. /* caller waits on DC_CTRL.FS */
  388. }
  389. #endif
  390. #if (CONFIG_ARC_MMU_VER < 3)
  391. #define __cache_line_loop __cache_line_loop_v2
  392. #elif (CONFIG_ARC_MMU_VER == 3)
  393. #define __cache_line_loop __cache_line_loop_v3
  394. #elif (CONFIG_ARC_MMU_VER > 3)
  395. #define __cache_line_loop __cache_line_loop_v4
  396. #endif
  397. #ifdef CONFIG_ARC_HAS_DCACHE
  398. /***************************************************************
  399. * Machine specific helpers for Entire D-Cache or Per Line ops
  400. */
  401. #ifndef USE_RGN_FLSH
  402. /*
  403. * this version avoids extra read/write of DC_CTRL for flush or invalid ops
  404. * in the non region flush regime (such as for ARCompact)
  405. */
  406. static inline void __before_dc_op(const int op)
  407. {
  408. if (op == OP_FLUSH_N_INV) {
  409. /* Dcache provides 2 cmd: FLUSH or INV
  410. * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE
  411. * flush-n-inv is achieved by INV cmd but with IM=1
  412. * So toggle INV sub-mode depending on op request and default
  413. */
  414. const unsigned int ctl = ARC_REG_DC_CTRL;
  415. write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH);
  416. }
  417. }
  418. #else
  419. static inline void __before_dc_op(const int op)
  420. {
  421. const unsigned int ctl = ARC_REG_DC_CTRL;
  422. unsigned int val = read_aux_reg(ctl);
  423. if (op == OP_FLUSH_N_INV) {
  424. val |= DC_CTRL_INV_MODE_FLUSH;
  425. }
  426. if (op != OP_INV_IC) {
  427. /*
  428. * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1
  429. * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above
  430. */
  431. val &= ~DC_CTRL_RGN_OP_MSK;
  432. if (op & OP_INV)
  433. val |= DC_CTRL_RGN_OP_INV;
  434. }
  435. write_aux_reg(ctl, val);
  436. }
  437. #endif
  438. static inline void __after_dc_op(const int op)
  439. {
  440. if (op & OP_FLUSH) {
  441. const unsigned int ctl = ARC_REG_DC_CTRL;
  442. unsigned int reg;
  443. /* flush / flush-n-inv both wait */
  444. while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS)
  445. ;
  446. /* Switch back to default Invalidate mode */
  447. if (op == OP_FLUSH_N_INV)
  448. write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH);
  449. }
  450. }
  451. /*
  452. * Operation on Entire D-Cache
  453. * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV}
  454. * Note that constant propagation ensures all the checks are gone
  455. * in generated code
  456. */
  457. static inline void __dc_entire_op(const int op)
  458. {
  459. int aux;
  460. __before_dc_op(op);
  461. if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */
  462. aux = ARC_REG_DC_IVDC;
  463. else
  464. aux = ARC_REG_DC_FLSH;
  465. write_aux_reg(aux, 0x1);
  466. __after_dc_op(op);
  467. }
  468. static inline void __dc_disable(void)
  469. {
  470. const int r = ARC_REG_DC_CTRL;
  471. __dc_entire_op(OP_FLUSH_N_INV);
  472. write_aux_reg(r, read_aux_reg(r) | DC_CTRL_DIS);
  473. }
  474. static void __dc_enable(void)
  475. {
  476. const int r = ARC_REG_DC_CTRL;
  477. write_aux_reg(r, read_aux_reg(r) & ~DC_CTRL_DIS);
  478. }
  479. /* For kernel mappings cache operation: index is same as paddr */
  480. #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op)
  481. /*
  482. * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback)
  483. */
  484. static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr,
  485. unsigned long sz, const int op)
  486. {
  487. const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE;
  488. unsigned long flags;
  489. local_irq_save(flags);
  490. __before_dc_op(op);
  491. __cache_line_loop(paddr, vaddr, sz, op, full_page);
  492. __after_dc_op(op);
  493. local_irq_restore(flags);
  494. }
  495. #else
  496. #define __dc_entire_op(op)
  497. #define __dc_disable()
  498. #define __dc_enable()
  499. #define __dc_line_op(paddr, vaddr, sz, op)
  500. #define __dc_line_op_k(paddr, sz, op)
  501. #endif /* CONFIG_ARC_HAS_DCACHE */
  502. #ifdef CONFIG_ARC_HAS_ICACHE
  503. static inline void __ic_entire_inv(void)
  504. {
  505. write_aux_reg(ARC_REG_IC_IVIC, 1);
  506. read_aux_reg(ARC_REG_IC_CTRL); /* blocks */
  507. }
  508. static inline void
  509. __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr,
  510. unsigned long sz)
  511. {
  512. const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE;
  513. unsigned long flags;
  514. local_irq_save(flags);
  515. (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page);
  516. local_irq_restore(flags);
  517. }
  518. #ifndef CONFIG_SMP
  519. #define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s)
  520. #else
  521. struct ic_inv_args {
  522. phys_addr_t paddr, vaddr;
  523. int sz;
  524. };
  525. static void __ic_line_inv_vaddr_helper(void *info)
  526. {
  527. struct ic_inv_args *ic_inv = info;
  528. __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz);
  529. }
  530. static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr,
  531. unsigned long sz)
  532. {
  533. struct ic_inv_args ic_inv = {
  534. .paddr = paddr,
  535. .vaddr = vaddr,
  536. .sz = sz
  537. };
  538. on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1);
  539. }
  540. #endif /* CONFIG_SMP */
  541. #else /* !CONFIG_ARC_HAS_ICACHE */
  542. #define __ic_entire_inv()
  543. #define __ic_line_inv_vaddr(pstart, vstart, sz)
  544. #endif /* CONFIG_ARC_HAS_ICACHE */
  545. noinline void slc_op_rgn(phys_addr_t paddr, unsigned long sz, const int op)
  546. {
  547. #ifdef CONFIG_ISA_ARCV2
  548. /*
  549. * SLC is shared between all cores and concurrent aux operations from
  550. * multiple cores need to be serialized using a spinlock
  551. * A concurrent operation can be silently ignored and/or the old/new
  552. * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop
  553. * below)
  554. */
  555. static DEFINE_SPINLOCK(lock);
  556. unsigned long flags;
  557. unsigned int ctrl;
  558. phys_addr_t end;
  559. spin_lock_irqsave(&lock, flags);
  560. /*
  561. * The Region Flush operation is specified by CTRL.RGN_OP[11..9]
  562. * - b'000 (default) is Flush,
  563. * - b'001 is Invalidate if CTRL.IM == 0
  564. * - b'001 is Flush-n-Invalidate if CTRL.IM == 1
  565. */
  566. ctrl = read_aux_reg(ARC_REG_SLC_CTRL);
  567. /* Don't rely on default value of IM bit */
  568. if (!(op & OP_FLUSH)) /* i.e. OP_INV */
  569. ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */
  570. else
  571. ctrl |= SLC_CTRL_IM;
  572. if (op & OP_INV)
  573. ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */
  574. else
  575. ctrl &= ~SLC_CTRL_RGN_OP_INV;
  576. write_aux_reg(ARC_REG_SLC_CTRL, ctrl);
  577. /*
  578. * Lower bits are ignored, no need to clip
  579. * END needs to be setup before START (latter triggers the operation)
  580. * END can't be same as START, so add (l2_line_sz - 1) to sz
  581. */
  582. end = paddr + sz + l2_line_sz - 1;
  583. if (is_pae40_enabled())
  584. write_aux_reg(ARC_REG_SLC_RGN_END1, upper_32_bits(end));
  585. write_aux_reg(ARC_REG_SLC_RGN_END, lower_32_bits(end));
  586. if (is_pae40_enabled())
  587. write_aux_reg(ARC_REG_SLC_RGN_START1, upper_32_bits(paddr));
  588. write_aux_reg(ARC_REG_SLC_RGN_START, lower_32_bits(paddr));
  589. /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
  590. read_aux_reg(ARC_REG_SLC_CTRL);
  591. while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY);
  592. spin_unlock_irqrestore(&lock, flags);
  593. #endif
  594. }
  595. noinline void slc_op_line(phys_addr_t paddr, unsigned long sz, const int op)
  596. {
  597. #ifdef CONFIG_ISA_ARCV2
  598. /*
  599. * SLC is shared between all cores and concurrent aux operations from
  600. * multiple cores need to be serialized using a spinlock
  601. * A concurrent operation can be silently ignored and/or the old/new
  602. * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop
  603. * below)
  604. */
  605. static DEFINE_SPINLOCK(lock);
  606. const unsigned long SLC_LINE_MASK = ~(l2_line_sz - 1);
  607. unsigned int ctrl, cmd;
  608. unsigned long flags;
  609. int num_lines;
  610. spin_lock_irqsave(&lock, flags);
  611. ctrl = read_aux_reg(ARC_REG_SLC_CTRL);
  612. /* Don't rely on default value of IM bit */
  613. if (!(op & OP_FLUSH)) /* i.e. OP_INV */
  614. ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */
  615. else
  616. ctrl |= SLC_CTRL_IM;
  617. write_aux_reg(ARC_REG_SLC_CTRL, ctrl);
  618. cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL;
  619. sz += paddr & ~SLC_LINE_MASK;
  620. paddr &= SLC_LINE_MASK;
  621. num_lines = DIV_ROUND_UP(sz, l2_line_sz);
  622. while (num_lines-- > 0) {
  623. write_aux_reg(cmd, paddr);
  624. paddr += l2_line_sz;
  625. }
  626. /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
  627. read_aux_reg(ARC_REG_SLC_CTRL);
  628. while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY);
  629. spin_unlock_irqrestore(&lock, flags);
  630. #endif
  631. }
  632. #define slc_op(paddr, sz, op) slc_op_rgn(paddr, sz, op)
  633. noinline static void slc_entire_op(const int op)
  634. {
  635. unsigned int ctrl, r = ARC_REG_SLC_CTRL;
  636. ctrl = read_aux_reg(r);
  637. if (!(op & OP_FLUSH)) /* i.e. OP_INV */
  638. ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */
  639. else
  640. ctrl |= SLC_CTRL_IM;
  641. write_aux_reg(r, ctrl);
  642. if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */
  643. write_aux_reg(ARC_REG_SLC_INVALIDATE, 0x1);
  644. else
  645. write_aux_reg(ARC_REG_SLC_FLUSH, 0x1);
  646. /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
  647. read_aux_reg(r);
  648. /* Important to wait for flush to complete */
  649. while (read_aux_reg(r) & SLC_CTRL_BUSY);
  650. }
  651. static inline void arc_slc_disable(void)
  652. {
  653. const int r = ARC_REG_SLC_CTRL;
  654. slc_entire_op(OP_FLUSH_N_INV);
  655. write_aux_reg(r, read_aux_reg(r) | SLC_CTRL_DIS);
  656. }
  657. static inline void arc_slc_enable(void)
  658. {
  659. const int r = ARC_REG_SLC_CTRL;
  660. write_aux_reg(r, read_aux_reg(r) & ~SLC_CTRL_DIS);
  661. }
  662. /***********************************************************
  663. * Exported APIs
  664. */
  665. /*
  666. * Handle cache congruency of kernel and userspace mappings of page when kernel
  667. * writes-to/reads-from
  668. *
  669. * The idea is to defer flushing of kernel mapping after a WRITE, possible if:
  670. * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent
  671. * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache)
  672. * -In SMP, if hardware caches are coherent
  673. *
  674. * There's a corollary case, where kernel READs from a userspace mapped page.
  675. * If the U-mapping is not congruent to to K-mapping, former needs flushing.
  676. */
  677. void flush_dcache_page(struct page *page)
  678. {
  679. struct address_space *mapping;
  680. if (!cache_is_vipt_aliasing()) {
  681. clear_bit(PG_dc_clean, &page->flags);
  682. return;
  683. }
  684. /* don't handle anon pages here */
  685. mapping = page_mapping_file(page);
  686. if (!mapping)
  687. return;
  688. /*
  689. * pagecache page, file not yet mapped to userspace
  690. * Make a note that K-mapping is dirty
  691. */
  692. if (!mapping_mapped(mapping)) {
  693. clear_bit(PG_dc_clean, &page->flags);
  694. } else if (page_mapcount(page)) {
  695. /* kernel reading from page with U-mapping */
  696. phys_addr_t paddr = (unsigned long)page_address(page);
  697. unsigned long vaddr = page->index << PAGE_SHIFT;
  698. if (addr_not_cache_congruent(paddr, vaddr))
  699. __flush_dcache_page(paddr, vaddr);
  700. }
  701. }
  702. EXPORT_SYMBOL(flush_dcache_page);
  703. /*
  704. * DMA ops for systems with L1 cache only
  705. * Make memory coherent with L1 cache by flushing/invalidating L1 lines
  706. */
  707. static void __dma_cache_wback_inv_l1(phys_addr_t start, unsigned long sz)
  708. {
  709. __dc_line_op_k(start, sz, OP_FLUSH_N_INV);
  710. }
  711. static void __dma_cache_inv_l1(phys_addr_t start, unsigned long sz)
  712. {
  713. __dc_line_op_k(start, sz, OP_INV);
  714. }
  715. static void __dma_cache_wback_l1(phys_addr_t start, unsigned long sz)
  716. {
  717. __dc_line_op_k(start, sz, OP_FLUSH);
  718. }
  719. /*
  720. * DMA ops for systems with both L1 and L2 caches, but without IOC
  721. * Both L1 and L2 lines need to be explicitly flushed/invalidated
  722. */
  723. static void __dma_cache_wback_inv_slc(phys_addr_t start, unsigned long sz)
  724. {
  725. __dc_line_op_k(start, sz, OP_FLUSH_N_INV);
  726. slc_op(start, sz, OP_FLUSH_N_INV);
  727. }
  728. static void __dma_cache_inv_slc(phys_addr_t start, unsigned long sz)
  729. {
  730. __dc_line_op_k(start, sz, OP_INV);
  731. slc_op(start, sz, OP_INV);
  732. }
  733. static void __dma_cache_wback_slc(phys_addr_t start, unsigned long sz)
  734. {
  735. __dc_line_op_k(start, sz, OP_FLUSH);
  736. slc_op(start, sz, OP_FLUSH);
  737. }
  738. /*
  739. * Exported DMA API
  740. */
  741. void dma_cache_wback_inv(phys_addr_t start, unsigned long sz)
  742. {
  743. __dma_cache_wback_inv(start, sz);
  744. }
  745. EXPORT_SYMBOL(dma_cache_wback_inv);
  746. void dma_cache_inv(phys_addr_t start, unsigned long sz)
  747. {
  748. __dma_cache_inv(start, sz);
  749. }
  750. EXPORT_SYMBOL(dma_cache_inv);
  751. void dma_cache_wback(phys_addr_t start, unsigned long sz)
  752. {
  753. __dma_cache_wback(start, sz);
  754. }
  755. EXPORT_SYMBOL(dma_cache_wback);
  756. /*
  757. * This is API for making I/D Caches consistent when modifying
  758. * kernel code (loadable modules, kprobes, kgdb...)
  759. * This is called on insmod, with kernel virtual address for CODE of
  760. * the module. ARC cache maintenance ops require PHY address thus we
  761. * need to convert vmalloc addr to PHY addr
  762. */
  763. void flush_icache_range(unsigned long kstart, unsigned long kend)
  764. {
  765. unsigned int tot_sz;
  766. WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__);
  767. /* Shortcut for bigger flush ranges.
  768. * Here we don't care if this was kernel virtual or phy addr
  769. */
  770. tot_sz = kend - kstart;
  771. if (tot_sz > PAGE_SIZE) {
  772. flush_cache_all();
  773. return;
  774. }
  775. /* Case: Kernel Phy addr (0x8000_0000 onwards) */
  776. if (likely(kstart > PAGE_OFFSET)) {
  777. /*
  778. * The 2nd arg despite being paddr will be used to index icache
  779. * This is OK since no alternate virtual mappings will exist
  780. * given the callers for this case: kprobe/kgdb in built-in
  781. * kernel code only.
  782. */
  783. __sync_icache_dcache(kstart, kstart, kend - kstart);
  784. return;
  785. }
  786. /*
  787. * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff)
  788. * (1) ARC Cache Maintenance ops only take Phy addr, hence special
  789. * handling of kernel vaddr.
  790. *
  791. * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already),
  792. * it still needs to handle a 2 page scenario, where the range
  793. * straddles across 2 virtual pages and hence need for loop
  794. */
  795. while (tot_sz > 0) {
  796. unsigned int off, sz;
  797. unsigned long phy, pfn;
  798. off = kstart % PAGE_SIZE;
  799. pfn = vmalloc_to_pfn((void *)kstart);
  800. phy = (pfn << PAGE_SHIFT) + off;
  801. sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off);
  802. __sync_icache_dcache(phy, kstart, sz);
  803. kstart += sz;
  804. tot_sz -= sz;
  805. }
  806. }
  807. EXPORT_SYMBOL(flush_icache_range);
  808. /*
  809. * General purpose helper to make I and D cache lines consistent.
  810. * @paddr is phy addr of region
  811. * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc)
  812. * However in one instance, when called by kprobe (for a breakpt in
  813. * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will
  814. * use a paddr to index the cache (despite VIPT). This is fine since since a
  815. * builtin kernel page will not have any virtual mappings.
  816. * kprobe on loadable module will be kernel vaddr.
  817. */
  818. void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len)
  819. {
  820. __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV);
  821. __ic_line_inv_vaddr(paddr, vaddr, len);
  822. }
  823. /* wrapper to compile time eliminate alignment checks in flush loop */
  824. void __inv_icache_page(phys_addr_t paddr, unsigned long vaddr)
  825. {
  826. __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE);
  827. }
  828. /*
  829. * wrapper to clearout kernel or userspace mappings of a page
  830. * For kernel mappings @vaddr == @paddr
  831. */
  832. void __flush_dcache_page(phys_addr_t paddr, unsigned long vaddr)
  833. {
  834. __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV);
  835. }
  836. noinline void flush_cache_all(void)
  837. {
  838. unsigned long flags;
  839. local_irq_save(flags);
  840. __ic_entire_inv();
  841. __dc_entire_op(OP_FLUSH_N_INV);
  842. local_irq_restore(flags);
  843. }
  844. #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING
  845. void flush_cache_mm(struct mm_struct *mm)
  846. {
  847. flush_cache_all();
  848. }
  849. void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr,
  850. unsigned long pfn)
  851. {
  852. phys_addr_t paddr = pfn << PAGE_SHIFT;
  853. u_vaddr &= PAGE_MASK;
  854. __flush_dcache_page(paddr, u_vaddr);
  855. if (vma->vm_flags & VM_EXEC)
  856. __inv_icache_page(paddr, u_vaddr);
  857. }
  858. void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
  859. unsigned long end)
  860. {
  861. flush_cache_all();
  862. }
  863. void flush_anon_page(struct vm_area_struct *vma, struct page *page,
  864. unsigned long u_vaddr)
  865. {
  866. /* TBD: do we really need to clear the kernel mapping */
  867. __flush_dcache_page((phys_addr_t)page_address(page), u_vaddr);
  868. __flush_dcache_page((phys_addr_t)page_address(page),
  869. (phys_addr_t)page_address(page));
  870. }
  871. #endif
  872. void copy_user_highpage(struct page *to, struct page *from,
  873. unsigned long u_vaddr, struct vm_area_struct *vma)
  874. {
  875. void *kfrom = kmap_atomic(from);
  876. void *kto = kmap_atomic(to);
  877. int clean_src_k_mappings = 0;
  878. /*
  879. * If SRC page was already mapped in userspace AND it's U-mapping is
  880. * not congruent with K-mapping, sync former to physical page so that
  881. * K-mapping in memcpy below, sees the right data
  882. *
  883. * Note that while @u_vaddr refers to DST page's userspace vaddr, it is
  884. * equally valid for SRC page as well
  885. *
  886. * For !VIPT cache, all of this gets compiled out as
  887. * addr_not_cache_congruent() is 0
  888. */
  889. if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) {
  890. __flush_dcache_page((unsigned long)kfrom, u_vaddr);
  891. clean_src_k_mappings = 1;
  892. }
  893. copy_page(kto, kfrom);
  894. /*
  895. * Mark DST page K-mapping as dirty for a later finalization by
  896. * update_mmu_cache(). Although the finalization could have been done
  897. * here as well (given that both vaddr/paddr are available).
  898. * But update_mmu_cache() already has code to do that for other
  899. * non copied user pages (e.g. read faults which wire in pagecache page
  900. * directly).
  901. */
  902. clear_bit(PG_dc_clean, &to->flags);
  903. /*
  904. * if SRC was already usermapped and non-congruent to kernel mapping
  905. * sync the kernel mapping back to physical page
  906. */
  907. if (clean_src_k_mappings) {
  908. __flush_dcache_page((unsigned long)kfrom, (unsigned long)kfrom);
  909. set_bit(PG_dc_clean, &from->flags);
  910. } else {
  911. clear_bit(PG_dc_clean, &from->flags);
  912. }
  913. kunmap_atomic(kto);
  914. kunmap_atomic(kfrom);
  915. }
  916. void clear_user_page(void *to, unsigned long u_vaddr, struct page *page)
  917. {
  918. clear_page(to);
  919. clear_bit(PG_dc_clean, &page->flags);
  920. }
  921. /**********************************************************************
  922. * Explicit Cache flush request from user space via syscall
  923. * Needed for JITs which generate code on the fly
  924. */
  925. SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags)
  926. {
  927. /* TBD: optimize this */
  928. flush_cache_all();
  929. return 0;
  930. }
  931. /*
  932. * IO-Coherency (IOC) setup rules:
  933. *
  934. * 1. Needs to be at system level, so only once by Master core
  935. * Non-Masters need not be accessing caches at that time
  936. * - They are either HALT_ON_RESET and kick started much later or
  937. * - if run on reset, need to ensure that arc_platform_smp_wait_to_boot()
  938. * doesn't perturb caches or coherency unit
  939. *
  940. * 2. caches (L1 and SLC) need to be purged (flush+inv) before setting up IOC,
  941. * otherwise any straggler data might behave strangely post IOC enabling
  942. *
  943. * 3. All Caches need to be disabled when setting up IOC to elide any in-flight
  944. * Coherency transactions
  945. */
  946. noinline void __init arc_ioc_setup(void)
  947. {
  948. unsigned int ioc_base, mem_sz;
  949. /*
  950. * If IOC was already enabled (due to bootloader) it technically needs to
  951. * be reconfigured with aperture base,size corresponding to Linux memory map
  952. * which will certainly be different than uboot's. But disabling and
  953. * reenabling IOC when DMA might be potentially active is tricky business.
  954. * To avoid random memory issues later, just panic here and ask user to
  955. * upgrade bootloader to one which doesn't enable IOC
  956. */
  957. if (read_aux_reg(ARC_REG_IO_COH_ENABLE) & ARC_IO_COH_ENABLE_BIT)
  958. panic("IOC already enabled, please upgrade bootloader!\n");
  959. if (!ioc_enable)
  960. return;
  961. /*
  962. * As for today we don't support both IOC and ZONE_HIGHMEM enabled
  963. * simultaneously. This happens because as of today IOC aperture covers
  964. * only ZONE_NORMAL (low mem) and any dma transactions outside this
  965. * region won't be HW coherent.
  966. * If we want to use both IOC and ZONE_HIGHMEM we can use
  967. * bounce_buffer to handle dma transactions to HIGHMEM.
  968. * Also it is possible to modify dma_direct cache ops or increase IOC
  969. * aperture size if we are planning to use HIGHMEM without PAE.
  970. */
  971. if (IS_ENABLED(CONFIG_HIGHMEM))
  972. panic("IOC and HIGHMEM can't be used simultaneously");
  973. /* Flush + invalidate + disable L1 dcache */
  974. __dc_disable();
  975. /* Flush + invalidate SLC */
  976. if (read_aux_reg(ARC_REG_SLC_BCR))
  977. slc_entire_op(OP_FLUSH_N_INV);
  978. /*
  979. * currently IOC Aperture covers entire DDR
  980. * TBD: fix for PGU + 1GB of low mem
  981. * TBD: fix for PAE
  982. */
  983. mem_sz = arc_get_mem_sz();
  984. if (!is_power_of_2(mem_sz) || mem_sz < 4096)
  985. panic("IOC Aperture size must be power of 2 larger than 4KB");
  986. /*
  987. * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB,
  988. * so setting 0x11 implies 512MB, 0x12 implies 1GB...
  989. */
  990. write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, order_base_2(mem_sz >> 10) - 2);
  991. /* for now assume kernel base is start of IOC aperture */
  992. ioc_base = CONFIG_LINUX_RAM_BASE;
  993. if (ioc_base % mem_sz != 0)
  994. panic("IOC Aperture start must be aligned to the size of the aperture");
  995. write_aux_reg(ARC_REG_IO_COH_AP0_BASE, ioc_base >> 12);
  996. write_aux_reg(ARC_REG_IO_COH_PARTIAL, ARC_IO_COH_PARTIAL_BIT);
  997. write_aux_reg(ARC_REG_IO_COH_ENABLE, ARC_IO_COH_ENABLE_BIT);
  998. /* Re-enable L1 dcache */
  999. __dc_enable();
  1000. }
  1001. /*
  1002. * Cache related boot time checks/setups only needed on master CPU:
  1003. * - Geometry checks (kernel build and hardware agree: e.g. L1_CACHE_BYTES)
  1004. * Assume SMP only, so all cores will have same cache config. A check on
  1005. * one core suffices for all
  1006. * - IOC setup / dma callbacks only need to be done once
  1007. */
  1008. void __init arc_cache_init_master(void)
  1009. {
  1010. unsigned int __maybe_unused cpu = smp_processor_id();
  1011. if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) {
  1012. struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache;
  1013. if (!ic->line_len)
  1014. panic("cache support enabled but non-existent cache\n");
  1015. if (ic->line_len != L1_CACHE_BYTES)
  1016. panic("ICache line [%d] != kernel Config [%d]",
  1017. ic->line_len, L1_CACHE_BYTES);
  1018. /*
  1019. * In MMU v4 (HS38x) the aliasing icache config uses IVIL/PTAG
  1020. * pair to provide vaddr/paddr respectively, just as in MMU v3
  1021. */
  1022. if (is_isa_arcv2() && ic->alias)
  1023. _cache_line_loop_ic_fn = __cache_line_loop_v3;
  1024. else
  1025. _cache_line_loop_ic_fn = __cache_line_loop;
  1026. }
  1027. if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) {
  1028. struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache;
  1029. if (!dc->line_len)
  1030. panic("cache support enabled but non-existent cache\n");
  1031. if (dc->line_len != L1_CACHE_BYTES)
  1032. panic("DCache line [%d] != kernel Config [%d]",
  1033. dc->line_len, L1_CACHE_BYTES);
  1034. /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */
  1035. if (is_isa_arcompact()) {
  1036. int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING);
  1037. int num_colors = dc->sz_k/dc->assoc/TO_KB(PAGE_SIZE);
  1038. if (dc->alias) {
  1039. if (!handled)
  1040. panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n");
  1041. if (CACHE_COLORS_NUM != num_colors)
  1042. panic("CACHE_COLORS_NUM not optimized for config\n");
  1043. } else if (!dc->alias && handled) {
  1044. panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n");
  1045. }
  1046. }
  1047. }
  1048. /*
  1049. * Check that SMP_CACHE_BYTES (and hence ARCH_DMA_MINALIGN) is larger
  1050. * or equal to any cache line length.
  1051. */
  1052. BUILD_BUG_ON_MSG(L1_CACHE_BYTES > SMP_CACHE_BYTES,
  1053. "SMP_CACHE_BYTES must be >= any cache line length");
  1054. if (is_isa_arcv2() && (l2_line_sz > SMP_CACHE_BYTES))
  1055. panic("L2 Cache line [%d] > kernel Config [%d]\n",
  1056. l2_line_sz, SMP_CACHE_BYTES);
  1057. /* Note that SLC disable not formally supported till HS 3.0 */
  1058. if (is_isa_arcv2() && l2_line_sz && !slc_enable)
  1059. arc_slc_disable();
  1060. if (is_isa_arcv2() && ioc_exists)
  1061. arc_ioc_setup();
  1062. if (is_isa_arcv2() && l2_line_sz && slc_enable) {
  1063. __dma_cache_wback_inv = __dma_cache_wback_inv_slc;
  1064. __dma_cache_inv = __dma_cache_inv_slc;
  1065. __dma_cache_wback = __dma_cache_wback_slc;
  1066. } else {
  1067. __dma_cache_wback_inv = __dma_cache_wback_inv_l1;
  1068. __dma_cache_inv = __dma_cache_inv_l1;
  1069. __dma_cache_wback = __dma_cache_wback_l1;
  1070. }
  1071. /*
  1072. * In case of IOC (say IOC+SLC case), pointers above could still be set
  1073. * but end up not being relevant as the first function in chain is not
  1074. * called at all for @dma_direct_ops
  1075. * arch_sync_dma_for_cpu() -> dma_cache_*() -> __dma_cache_*()
  1076. */
  1077. }
  1078. void __ref arc_cache_init(void)
  1079. {
  1080. unsigned int __maybe_unused cpu = smp_processor_id();
  1081. char str[256];
  1082. pr_info("%s", arc_cache_mumbojumbo(0, str, sizeof(str)));
  1083. if (!cpu)
  1084. arc_cache_init_master();
  1085. /*
  1086. * In PAE regime, TLB and cache maintenance ops take wider addresses
  1087. * And even if PAE is not enabled in kernel, the upper 32-bits still need
  1088. * to be zeroed to keep the ops sane.
  1089. * As an optimization for more common !PAE enabled case, zero them out
  1090. * once at init, rather than checking/setting to 0 for every runtime op
  1091. */
  1092. if (is_isa_arcv2() && pae40_exist_but_not_enab()) {
  1093. if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE))
  1094. write_aux_reg(ARC_REG_IC_PTAG_HI, 0);
  1095. if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE))
  1096. write_aux_reg(ARC_REG_DC_PTAG_HI, 0);
  1097. if (l2_line_sz) {
  1098. write_aux_reg(ARC_REG_SLC_RGN_END1, 0);
  1099. write_aux_reg(ARC_REG_SLC_RGN_START1, 0);
  1100. }
  1101. }
  1102. }