ccp-ops.c 62 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518
  1. /*
  2. * AMD Cryptographic Coprocessor (CCP) driver
  3. *
  4. * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  5. *
  6. * Author: Tom Lendacky <thomas.lendacky@amd.com>
  7. * Author: Gary R Hook <gary.hook@amd.com>
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License version 2 as
  11. * published by the Free Software Foundation.
  12. */
  13. #include <linux/module.h>
  14. #include <linux/kernel.h>
  15. #include <linux/pci.h>
  16. #include <linux/interrupt.h>
  17. #include <crypto/scatterwalk.h>
  18. #include <crypto/des.h>
  19. #include <linux/ccp.h>
  20. #include "ccp-dev.h"
  21. /* SHA initial context values */
  22. static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  23. cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  24. cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  25. cpu_to_be32(SHA1_H4),
  26. };
  27. static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  28. cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  29. cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  30. cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  31. cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  32. };
  33. static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  34. cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  35. cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  36. cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  37. cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  38. };
  39. static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  40. cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  41. cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  42. cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  43. cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  44. };
  45. static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  46. cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  47. cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  48. cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  49. cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  50. };
  51. #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  52. ccp_gen_jobid(ccp) : 0)
  53. static u32 ccp_gen_jobid(struct ccp_device *ccp)
  54. {
  55. return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  56. }
  57. static void ccp_sg_free(struct ccp_sg_workarea *wa)
  58. {
  59. if (wa->dma_count)
  60. dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
  61. wa->dma_count = 0;
  62. }
  63. static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  64. struct scatterlist *sg, u64 len,
  65. enum dma_data_direction dma_dir)
  66. {
  67. memset(wa, 0, sizeof(*wa));
  68. wa->sg = sg;
  69. if (!sg)
  70. return 0;
  71. wa->nents = sg_nents_for_len(sg, len);
  72. if (wa->nents < 0)
  73. return wa->nents;
  74. wa->bytes_left = len;
  75. wa->sg_used = 0;
  76. if (len == 0)
  77. return 0;
  78. if (dma_dir == DMA_NONE)
  79. return 0;
  80. wa->dma_sg = sg;
  81. wa->dma_sg_head = sg;
  82. wa->dma_dev = dev;
  83. wa->dma_dir = dma_dir;
  84. wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
  85. if (!wa->dma_count)
  86. return -ENOMEM;
  87. return 0;
  88. }
  89. static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
  90. {
  91. unsigned int nbytes = min_t(u64, len, wa->bytes_left);
  92. unsigned int sg_combined_len = 0;
  93. if (!wa->sg)
  94. return;
  95. wa->sg_used += nbytes;
  96. wa->bytes_left -= nbytes;
  97. if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
  98. /* Advance to the next DMA scatterlist entry */
  99. wa->dma_sg = sg_next(wa->dma_sg);
  100. /* In the case that the DMA mapped scatterlist has entries
  101. * that have been merged, the non-DMA mapped scatterlist
  102. * must be advanced multiple times for each merged entry.
  103. * This ensures that the current non-DMA mapped entry
  104. * corresponds to the current DMA mapped entry.
  105. */
  106. do {
  107. sg_combined_len += wa->sg->length;
  108. wa->sg = sg_next(wa->sg);
  109. } while (wa->sg_used > sg_combined_len);
  110. wa->sg_used = 0;
  111. }
  112. }
  113. static void ccp_dm_free(struct ccp_dm_workarea *wa)
  114. {
  115. if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
  116. if (wa->address)
  117. dma_pool_free(wa->dma_pool, wa->address,
  118. wa->dma.address);
  119. } else {
  120. if (wa->dma.address)
  121. dma_unmap_single(wa->dev, wa->dma.address, wa->length,
  122. wa->dma.dir);
  123. kfree(wa->address);
  124. }
  125. wa->address = NULL;
  126. wa->dma.address = 0;
  127. }
  128. static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
  129. struct ccp_cmd_queue *cmd_q,
  130. unsigned int len,
  131. enum dma_data_direction dir)
  132. {
  133. memset(wa, 0, sizeof(*wa));
  134. if (!len)
  135. return 0;
  136. wa->dev = cmd_q->ccp->dev;
  137. wa->length = len;
  138. if (len <= CCP_DMAPOOL_MAX_SIZE) {
  139. wa->dma_pool = cmd_q->dma_pool;
  140. wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
  141. &wa->dma.address);
  142. if (!wa->address)
  143. return -ENOMEM;
  144. wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
  145. memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
  146. } else {
  147. wa->address = kzalloc(len, GFP_KERNEL);
  148. if (!wa->address)
  149. return -ENOMEM;
  150. wa->dma.address = dma_map_single(wa->dev, wa->address, len,
  151. dir);
  152. if (dma_mapping_error(wa->dev, wa->dma.address))
  153. return -ENOMEM;
  154. wa->dma.length = len;
  155. }
  156. wa->dma.dir = dir;
  157. return 0;
  158. }
  159. static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
  160. struct scatterlist *sg, unsigned int sg_offset,
  161. unsigned int len)
  162. {
  163. WARN_ON(!wa->address);
  164. if (len > (wa->length - wa_offset))
  165. return -EINVAL;
  166. scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
  167. 0);
  168. return 0;
  169. }
  170. static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
  171. struct scatterlist *sg, unsigned int sg_offset,
  172. unsigned int len)
  173. {
  174. WARN_ON(!wa->address);
  175. scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
  176. 1);
  177. }
  178. static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
  179. unsigned int wa_offset,
  180. struct scatterlist *sg,
  181. unsigned int sg_offset,
  182. unsigned int len)
  183. {
  184. u8 *p, *q;
  185. int rc;
  186. rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
  187. if (rc)
  188. return rc;
  189. p = wa->address + wa_offset;
  190. q = p + len - 1;
  191. while (p < q) {
  192. *p = *p ^ *q;
  193. *q = *p ^ *q;
  194. *p = *p ^ *q;
  195. p++;
  196. q--;
  197. }
  198. return 0;
  199. }
  200. static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
  201. unsigned int wa_offset,
  202. struct scatterlist *sg,
  203. unsigned int sg_offset,
  204. unsigned int len)
  205. {
  206. u8 *p, *q;
  207. p = wa->address + wa_offset;
  208. q = p + len - 1;
  209. while (p < q) {
  210. *p = *p ^ *q;
  211. *q = *p ^ *q;
  212. *p = *p ^ *q;
  213. p++;
  214. q--;
  215. }
  216. ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
  217. }
  218. static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
  219. {
  220. ccp_dm_free(&data->dm_wa);
  221. ccp_sg_free(&data->sg_wa);
  222. }
  223. static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
  224. struct scatterlist *sg, u64 sg_len,
  225. unsigned int dm_len,
  226. enum dma_data_direction dir)
  227. {
  228. int ret;
  229. memset(data, 0, sizeof(*data));
  230. ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
  231. dir);
  232. if (ret)
  233. goto e_err;
  234. ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
  235. if (ret)
  236. goto e_err;
  237. return 0;
  238. e_err:
  239. ccp_free_data(data, cmd_q);
  240. return ret;
  241. }
  242. static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
  243. {
  244. struct ccp_sg_workarea *sg_wa = &data->sg_wa;
  245. struct ccp_dm_workarea *dm_wa = &data->dm_wa;
  246. unsigned int buf_count, nbytes;
  247. /* Clear the buffer if setting it */
  248. if (!from)
  249. memset(dm_wa->address, 0, dm_wa->length);
  250. if (!sg_wa->sg)
  251. return 0;
  252. /* Perform the copy operation
  253. * nbytes will always be <= UINT_MAX because dm_wa->length is
  254. * an unsigned int
  255. */
  256. nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
  257. scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
  258. nbytes, from);
  259. /* Update the structures and generate the count */
  260. buf_count = 0;
  261. while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
  262. nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
  263. dm_wa->length - buf_count);
  264. nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
  265. buf_count += nbytes;
  266. ccp_update_sg_workarea(sg_wa, nbytes);
  267. }
  268. return buf_count;
  269. }
  270. static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
  271. {
  272. return ccp_queue_buf(data, 0);
  273. }
  274. static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
  275. {
  276. return ccp_queue_buf(data, 1);
  277. }
  278. static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
  279. struct ccp_op *op, unsigned int block_size,
  280. bool blocksize_op)
  281. {
  282. unsigned int sg_src_len, sg_dst_len, op_len;
  283. /* The CCP can only DMA from/to one address each per operation. This
  284. * requires that we find the smallest DMA area between the source
  285. * and destination. The resulting len values will always be <= UINT_MAX
  286. * because the dma length is an unsigned int.
  287. */
  288. sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
  289. sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
  290. if (dst) {
  291. sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
  292. sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
  293. op_len = min(sg_src_len, sg_dst_len);
  294. } else {
  295. op_len = sg_src_len;
  296. }
  297. /* The data operation length will be at least block_size in length
  298. * or the smaller of available sg room remaining for the source or
  299. * the destination
  300. */
  301. op_len = max(op_len, block_size);
  302. /* Unless we have to buffer data, there's no reason to wait */
  303. op->soc = 0;
  304. if (sg_src_len < block_size) {
  305. /* Not enough data in the sg element, so it
  306. * needs to be buffered into a blocksize chunk
  307. */
  308. int cp_len = ccp_fill_queue_buf(src);
  309. op->soc = 1;
  310. op->src.u.dma.address = src->dm_wa.dma.address;
  311. op->src.u.dma.offset = 0;
  312. op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
  313. } else {
  314. /* Enough data in the sg element, but we need to
  315. * adjust for any previously copied data
  316. */
  317. op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
  318. op->src.u.dma.offset = src->sg_wa.sg_used;
  319. op->src.u.dma.length = op_len & ~(block_size - 1);
  320. ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
  321. }
  322. if (dst) {
  323. if (sg_dst_len < block_size) {
  324. /* Not enough room in the sg element or we're on the
  325. * last piece of data (when using padding), so the
  326. * output needs to be buffered into a blocksize chunk
  327. */
  328. op->soc = 1;
  329. op->dst.u.dma.address = dst->dm_wa.dma.address;
  330. op->dst.u.dma.offset = 0;
  331. op->dst.u.dma.length = op->src.u.dma.length;
  332. } else {
  333. /* Enough room in the sg element, but we need to
  334. * adjust for any previously used area
  335. */
  336. op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
  337. op->dst.u.dma.offset = dst->sg_wa.sg_used;
  338. op->dst.u.dma.length = op->src.u.dma.length;
  339. }
  340. }
  341. }
  342. static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
  343. struct ccp_op *op)
  344. {
  345. op->init = 0;
  346. if (dst) {
  347. if (op->dst.u.dma.address == dst->dm_wa.dma.address)
  348. ccp_empty_queue_buf(dst);
  349. else
  350. ccp_update_sg_workarea(&dst->sg_wa,
  351. op->dst.u.dma.length);
  352. }
  353. }
  354. static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
  355. struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
  356. u32 byte_swap, bool from)
  357. {
  358. struct ccp_op op;
  359. memset(&op, 0, sizeof(op));
  360. op.cmd_q = cmd_q;
  361. op.jobid = jobid;
  362. op.eom = 1;
  363. if (from) {
  364. op.soc = 1;
  365. op.src.type = CCP_MEMTYPE_SB;
  366. op.src.u.sb = sb;
  367. op.dst.type = CCP_MEMTYPE_SYSTEM;
  368. op.dst.u.dma.address = wa->dma.address;
  369. op.dst.u.dma.length = wa->length;
  370. } else {
  371. op.src.type = CCP_MEMTYPE_SYSTEM;
  372. op.src.u.dma.address = wa->dma.address;
  373. op.src.u.dma.length = wa->length;
  374. op.dst.type = CCP_MEMTYPE_SB;
  375. op.dst.u.sb = sb;
  376. }
  377. op.u.passthru.byte_swap = byte_swap;
  378. return cmd_q->ccp->vdata->perform->passthru(&op);
  379. }
  380. static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
  381. struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
  382. u32 byte_swap)
  383. {
  384. return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
  385. }
  386. static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
  387. struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
  388. u32 byte_swap)
  389. {
  390. return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
  391. }
  392. static noinline_for_stack int
  393. ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  394. {
  395. struct ccp_aes_engine *aes = &cmd->u.aes;
  396. struct ccp_dm_workarea key, ctx;
  397. struct ccp_data src;
  398. struct ccp_op op;
  399. unsigned int dm_offset;
  400. int ret;
  401. if (!((aes->key_len == AES_KEYSIZE_128) ||
  402. (aes->key_len == AES_KEYSIZE_192) ||
  403. (aes->key_len == AES_KEYSIZE_256)))
  404. return -EINVAL;
  405. if (aes->src_len & (AES_BLOCK_SIZE - 1))
  406. return -EINVAL;
  407. if (aes->iv_len != AES_BLOCK_SIZE)
  408. return -EINVAL;
  409. if (!aes->key || !aes->iv || !aes->src)
  410. return -EINVAL;
  411. if (aes->cmac_final) {
  412. if (aes->cmac_key_len != AES_BLOCK_SIZE)
  413. return -EINVAL;
  414. if (!aes->cmac_key)
  415. return -EINVAL;
  416. }
  417. BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
  418. BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
  419. ret = -EIO;
  420. memset(&op, 0, sizeof(op));
  421. op.cmd_q = cmd_q;
  422. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  423. op.sb_key = cmd_q->sb_key;
  424. op.sb_ctx = cmd_q->sb_ctx;
  425. op.init = 1;
  426. op.u.aes.type = aes->type;
  427. op.u.aes.mode = aes->mode;
  428. op.u.aes.action = aes->action;
  429. /* All supported key sizes fit in a single (32-byte) SB entry
  430. * and must be in little endian format. Use the 256-bit byte
  431. * swap passthru option to convert from big endian to little
  432. * endian.
  433. */
  434. ret = ccp_init_dm_workarea(&key, cmd_q,
  435. CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
  436. DMA_TO_DEVICE);
  437. if (ret)
  438. return ret;
  439. dm_offset = CCP_SB_BYTES - aes->key_len;
  440. ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
  441. if (ret)
  442. goto e_key;
  443. ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
  444. CCP_PASSTHRU_BYTESWAP_256BIT);
  445. if (ret) {
  446. cmd->engine_error = cmd_q->cmd_error;
  447. goto e_key;
  448. }
  449. /* The AES context fits in a single (32-byte) SB entry and
  450. * must be in little endian format. Use the 256-bit byte swap
  451. * passthru option to convert from big endian to little endian.
  452. */
  453. ret = ccp_init_dm_workarea(&ctx, cmd_q,
  454. CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
  455. DMA_BIDIRECTIONAL);
  456. if (ret)
  457. goto e_key;
  458. dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
  459. ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
  460. if (ret)
  461. goto e_ctx;
  462. ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  463. CCP_PASSTHRU_BYTESWAP_256BIT);
  464. if (ret) {
  465. cmd->engine_error = cmd_q->cmd_error;
  466. goto e_ctx;
  467. }
  468. /* Send data to the CCP AES engine */
  469. ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
  470. AES_BLOCK_SIZE, DMA_TO_DEVICE);
  471. if (ret)
  472. goto e_ctx;
  473. while (src.sg_wa.bytes_left) {
  474. ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
  475. if (aes->cmac_final && !src.sg_wa.bytes_left) {
  476. op.eom = 1;
  477. /* Push the K1/K2 key to the CCP now */
  478. ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
  479. op.sb_ctx,
  480. CCP_PASSTHRU_BYTESWAP_256BIT);
  481. if (ret) {
  482. cmd->engine_error = cmd_q->cmd_error;
  483. goto e_src;
  484. }
  485. ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
  486. aes->cmac_key_len);
  487. if (ret)
  488. goto e_src;
  489. ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  490. CCP_PASSTHRU_BYTESWAP_256BIT);
  491. if (ret) {
  492. cmd->engine_error = cmd_q->cmd_error;
  493. goto e_src;
  494. }
  495. }
  496. ret = cmd_q->ccp->vdata->perform->aes(&op);
  497. if (ret) {
  498. cmd->engine_error = cmd_q->cmd_error;
  499. goto e_src;
  500. }
  501. ccp_process_data(&src, NULL, &op);
  502. }
  503. /* Retrieve the AES context - convert from LE to BE using
  504. * 32-byte (256-bit) byteswapping
  505. */
  506. ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  507. CCP_PASSTHRU_BYTESWAP_256BIT);
  508. if (ret) {
  509. cmd->engine_error = cmd_q->cmd_error;
  510. goto e_src;
  511. }
  512. /* ...but we only need AES_BLOCK_SIZE bytes */
  513. dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
  514. ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
  515. e_src:
  516. ccp_free_data(&src, cmd_q);
  517. e_ctx:
  518. ccp_dm_free(&ctx);
  519. e_key:
  520. ccp_dm_free(&key);
  521. return ret;
  522. }
  523. static noinline_for_stack int
  524. ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  525. {
  526. struct ccp_aes_engine *aes = &cmd->u.aes;
  527. struct ccp_dm_workarea key, ctx, final_wa, tag;
  528. struct ccp_data src, dst;
  529. struct ccp_data aad;
  530. struct ccp_op op;
  531. unsigned long long *final;
  532. unsigned int dm_offset;
  533. unsigned int authsize;
  534. unsigned int jobid;
  535. unsigned int ilen;
  536. bool in_place = true; /* Default value */
  537. int ret;
  538. struct scatterlist *p_inp, sg_inp[2];
  539. struct scatterlist *p_tag, sg_tag[2];
  540. struct scatterlist *p_outp, sg_outp[2];
  541. struct scatterlist *p_aad;
  542. if (!aes->iv)
  543. return -EINVAL;
  544. if (!((aes->key_len == AES_KEYSIZE_128) ||
  545. (aes->key_len == AES_KEYSIZE_192) ||
  546. (aes->key_len == AES_KEYSIZE_256)))
  547. return -EINVAL;
  548. if (!aes->key) /* Gotta have a key SGL */
  549. return -EINVAL;
  550. /* Zero defaults to 16 bytes, the maximum size */
  551. authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
  552. switch (authsize) {
  553. case 16:
  554. case 15:
  555. case 14:
  556. case 13:
  557. case 12:
  558. case 8:
  559. case 4:
  560. break;
  561. default:
  562. return -EINVAL;
  563. }
  564. /* First, decompose the source buffer into AAD & PT,
  565. * and the destination buffer into AAD, CT & tag, or
  566. * the input into CT & tag.
  567. * It is expected that the input and output SGs will
  568. * be valid, even if the AAD and input lengths are 0.
  569. */
  570. p_aad = aes->src;
  571. p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
  572. p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
  573. if (aes->action == CCP_AES_ACTION_ENCRYPT) {
  574. ilen = aes->src_len;
  575. p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
  576. } else {
  577. /* Input length for decryption includes tag */
  578. ilen = aes->src_len - authsize;
  579. p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
  580. }
  581. jobid = CCP_NEW_JOBID(cmd_q->ccp);
  582. memset(&op, 0, sizeof(op));
  583. op.cmd_q = cmd_q;
  584. op.jobid = jobid;
  585. op.sb_key = cmd_q->sb_key; /* Pre-allocated */
  586. op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
  587. op.init = 1;
  588. op.u.aes.type = aes->type;
  589. /* Copy the key to the LSB */
  590. ret = ccp_init_dm_workarea(&key, cmd_q,
  591. CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
  592. DMA_TO_DEVICE);
  593. if (ret)
  594. return ret;
  595. dm_offset = CCP_SB_BYTES - aes->key_len;
  596. ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
  597. if (ret)
  598. goto e_key;
  599. ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
  600. CCP_PASSTHRU_BYTESWAP_256BIT);
  601. if (ret) {
  602. cmd->engine_error = cmd_q->cmd_error;
  603. goto e_key;
  604. }
  605. /* Copy the context (IV) to the LSB.
  606. * There is an assumption here that the IV is 96 bits in length, plus
  607. * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
  608. */
  609. ret = ccp_init_dm_workarea(&ctx, cmd_q,
  610. CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
  611. DMA_BIDIRECTIONAL);
  612. if (ret)
  613. goto e_key;
  614. dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
  615. ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
  616. if (ret)
  617. goto e_ctx;
  618. ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  619. CCP_PASSTHRU_BYTESWAP_256BIT);
  620. if (ret) {
  621. cmd->engine_error = cmd_q->cmd_error;
  622. goto e_ctx;
  623. }
  624. op.init = 1;
  625. if (aes->aad_len > 0) {
  626. /* Step 1: Run a GHASH over the Additional Authenticated Data */
  627. ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
  628. AES_BLOCK_SIZE,
  629. DMA_TO_DEVICE);
  630. if (ret)
  631. goto e_ctx;
  632. op.u.aes.mode = CCP_AES_MODE_GHASH;
  633. op.u.aes.action = CCP_AES_GHASHAAD;
  634. while (aad.sg_wa.bytes_left) {
  635. ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
  636. ret = cmd_q->ccp->vdata->perform->aes(&op);
  637. if (ret) {
  638. cmd->engine_error = cmd_q->cmd_error;
  639. goto e_aad;
  640. }
  641. ccp_process_data(&aad, NULL, &op);
  642. op.init = 0;
  643. }
  644. }
  645. op.u.aes.mode = CCP_AES_MODE_GCTR;
  646. op.u.aes.action = aes->action;
  647. if (ilen > 0) {
  648. /* Step 2: Run a GCTR over the plaintext */
  649. in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
  650. ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
  651. AES_BLOCK_SIZE,
  652. in_place ? DMA_BIDIRECTIONAL
  653. : DMA_TO_DEVICE);
  654. if (ret)
  655. goto e_ctx;
  656. if (in_place) {
  657. dst = src;
  658. } else {
  659. ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
  660. AES_BLOCK_SIZE, DMA_FROM_DEVICE);
  661. if (ret)
  662. goto e_src;
  663. }
  664. op.soc = 0;
  665. op.eom = 0;
  666. op.init = 1;
  667. while (src.sg_wa.bytes_left) {
  668. ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
  669. if (!src.sg_wa.bytes_left) {
  670. unsigned int nbytes = ilen % AES_BLOCK_SIZE;
  671. if (nbytes) {
  672. op.eom = 1;
  673. op.u.aes.size = (nbytes * 8) - 1;
  674. }
  675. }
  676. ret = cmd_q->ccp->vdata->perform->aes(&op);
  677. if (ret) {
  678. cmd->engine_error = cmd_q->cmd_error;
  679. goto e_dst;
  680. }
  681. ccp_process_data(&src, &dst, &op);
  682. op.init = 0;
  683. }
  684. }
  685. /* Step 3: Update the IV portion of the context with the original IV */
  686. ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  687. CCP_PASSTHRU_BYTESWAP_256BIT);
  688. if (ret) {
  689. cmd->engine_error = cmd_q->cmd_error;
  690. goto e_dst;
  691. }
  692. ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
  693. if (ret)
  694. goto e_dst;
  695. ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  696. CCP_PASSTHRU_BYTESWAP_256BIT);
  697. if (ret) {
  698. cmd->engine_error = cmd_q->cmd_error;
  699. goto e_dst;
  700. }
  701. /* Step 4: Concatenate the lengths of the AAD and source, and
  702. * hash that 16 byte buffer.
  703. */
  704. ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
  705. DMA_BIDIRECTIONAL);
  706. if (ret)
  707. goto e_dst;
  708. final = (unsigned long long *) final_wa.address;
  709. final[0] = cpu_to_be64(aes->aad_len * 8);
  710. final[1] = cpu_to_be64(ilen * 8);
  711. memset(&op, 0, sizeof(op));
  712. op.cmd_q = cmd_q;
  713. op.jobid = jobid;
  714. op.sb_key = cmd_q->sb_key; /* Pre-allocated */
  715. op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
  716. op.init = 1;
  717. op.u.aes.type = aes->type;
  718. op.u.aes.mode = CCP_AES_MODE_GHASH;
  719. op.u.aes.action = CCP_AES_GHASHFINAL;
  720. op.src.type = CCP_MEMTYPE_SYSTEM;
  721. op.src.u.dma.address = final_wa.dma.address;
  722. op.src.u.dma.length = AES_BLOCK_SIZE;
  723. op.dst.type = CCP_MEMTYPE_SYSTEM;
  724. op.dst.u.dma.address = final_wa.dma.address;
  725. op.dst.u.dma.length = AES_BLOCK_SIZE;
  726. op.eom = 1;
  727. op.u.aes.size = 0;
  728. ret = cmd_q->ccp->vdata->perform->aes(&op);
  729. if (ret)
  730. goto e_dst;
  731. if (aes->action == CCP_AES_ACTION_ENCRYPT) {
  732. /* Put the ciphered tag after the ciphertext. */
  733. ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
  734. } else {
  735. /* Does this ciphered tag match the input? */
  736. ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
  737. DMA_BIDIRECTIONAL);
  738. if (ret)
  739. goto e_tag;
  740. ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
  741. if (ret)
  742. goto e_tag;
  743. ret = crypto_memneq(tag.address, final_wa.address,
  744. authsize) ? -EBADMSG : 0;
  745. ccp_dm_free(&tag);
  746. }
  747. e_tag:
  748. ccp_dm_free(&final_wa);
  749. e_dst:
  750. if (ilen > 0 && !in_place)
  751. ccp_free_data(&dst, cmd_q);
  752. e_src:
  753. if (ilen > 0)
  754. ccp_free_data(&src, cmd_q);
  755. e_aad:
  756. if (aes->aad_len)
  757. ccp_free_data(&aad, cmd_q);
  758. e_ctx:
  759. ccp_dm_free(&ctx);
  760. e_key:
  761. ccp_dm_free(&key);
  762. return ret;
  763. }
  764. static noinline_for_stack int
  765. ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  766. {
  767. struct ccp_aes_engine *aes = &cmd->u.aes;
  768. struct ccp_dm_workarea key, ctx;
  769. struct ccp_data src, dst;
  770. struct ccp_op op;
  771. unsigned int dm_offset;
  772. bool in_place = false;
  773. int ret;
  774. if (!((aes->key_len == AES_KEYSIZE_128) ||
  775. (aes->key_len == AES_KEYSIZE_192) ||
  776. (aes->key_len == AES_KEYSIZE_256)))
  777. return -EINVAL;
  778. if (((aes->mode == CCP_AES_MODE_ECB) ||
  779. (aes->mode == CCP_AES_MODE_CBC) ||
  780. (aes->mode == CCP_AES_MODE_CFB)) &&
  781. (aes->src_len & (AES_BLOCK_SIZE - 1)))
  782. return -EINVAL;
  783. if (!aes->key || !aes->src || !aes->dst)
  784. return -EINVAL;
  785. if (aes->mode != CCP_AES_MODE_ECB) {
  786. if (aes->iv_len != AES_BLOCK_SIZE)
  787. return -EINVAL;
  788. if (!aes->iv)
  789. return -EINVAL;
  790. }
  791. BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
  792. BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
  793. ret = -EIO;
  794. memset(&op, 0, sizeof(op));
  795. op.cmd_q = cmd_q;
  796. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  797. op.sb_key = cmd_q->sb_key;
  798. op.sb_ctx = cmd_q->sb_ctx;
  799. op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
  800. op.u.aes.type = aes->type;
  801. op.u.aes.mode = aes->mode;
  802. op.u.aes.action = aes->action;
  803. /* All supported key sizes fit in a single (32-byte) SB entry
  804. * and must be in little endian format. Use the 256-bit byte
  805. * swap passthru option to convert from big endian to little
  806. * endian.
  807. */
  808. ret = ccp_init_dm_workarea(&key, cmd_q,
  809. CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
  810. DMA_TO_DEVICE);
  811. if (ret)
  812. return ret;
  813. dm_offset = CCP_SB_BYTES - aes->key_len;
  814. ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
  815. if (ret)
  816. goto e_key;
  817. ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
  818. CCP_PASSTHRU_BYTESWAP_256BIT);
  819. if (ret) {
  820. cmd->engine_error = cmd_q->cmd_error;
  821. goto e_key;
  822. }
  823. /* The AES context fits in a single (32-byte) SB entry and
  824. * must be in little endian format. Use the 256-bit byte swap
  825. * passthru option to convert from big endian to little endian.
  826. */
  827. ret = ccp_init_dm_workarea(&ctx, cmd_q,
  828. CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
  829. DMA_BIDIRECTIONAL);
  830. if (ret)
  831. goto e_key;
  832. if (aes->mode != CCP_AES_MODE_ECB) {
  833. /* Load the AES context - convert to LE */
  834. dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
  835. ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
  836. if (ret)
  837. goto e_ctx;
  838. ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  839. CCP_PASSTHRU_BYTESWAP_256BIT);
  840. if (ret) {
  841. cmd->engine_error = cmd_q->cmd_error;
  842. goto e_ctx;
  843. }
  844. }
  845. switch (aes->mode) {
  846. case CCP_AES_MODE_CFB: /* CFB128 only */
  847. case CCP_AES_MODE_CTR:
  848. op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
  849. break;
  850. default:
  851. op.u.aes.size = 0;
  852. }
  853. /* Prepare the input and output data workareas. For in-place
  854. * operations we need to set the dma direction to BIDIRECTIONAL
  855. * and copy the src workarea to the dst workarea.
  856. */
  857. if (sg_virt(aes->src) == sg_virt(aes->dst))
  858. in_place = true;
  859. ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
  860. AES_BLOCK_SIZE,
  861. in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  862. if (ret)
  863. goto e_ctx;
  864. if (in_place) {
  865. dst = src;
  866. } else {
  867. ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
  868. AES_BLOCK_SIZE, DMA_FROM_DEVICE);
  869. if (ret)
  870. goto e_src;
  871. }
  872. /* Send data to the CCP AES engine */
  873. while (src.sg_wa.bytes_left) {
  874. ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
  875. if (!src.sg_wa.bytes_left) {
  876. op.eom = 1;
  877. /* Since we don't retrieve the AES context in ECB
  878. * mode we have to wait for the operation to complete
  879. * on the last piece of data
  880. */
  881. if (aes->mode == CCP_AES_MODE_ECB)
  882. op.soc = 1;
  883. }
  884. ret = cmd_q->ccp->vdata->perform->aes(&op);
  885. if (ret) {
  886. cmd->engine_error = cmd_q->cmd_error;
  887. goto e_dst;
  888. }
  889. ccp_process_data(&src, &dst, &op);
  890. }
  891. if (aes->mode != CCP_AES_MODE_ECB) {
  892. /* Retrieve the AES context - convert from LE to BE using
  893. * 32-byte (256-bit) byteswapping
  894. */
  895. ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  896. CCP_PASSTHRU_BYTESWAP_256BIT);
  897. if (ret) {
  898. cmd->engine_error = cmd_q->cmd_error;
  899. goto e_dst;
  900. }
  901. /* ...but we only need AES_BLOCK_SIZE bytes */
  902. dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
  903. ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
  904. }
  905. e_dst:
  906. if (!in_place)
  907. ccp_free_data(&dst, cmd_q);
  908. e_src:
  909. ccp_free_data(&src, cmd_q);
  910. e_ctx:
  911. ccp_dm_free(&ctx);
  912. e_key:
  913. ccp_dm_free(&key);
  914. return ret;
  915. }
  916. static noinline_for_stack int
  917. ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  918. {
  919. struct ccp_xts_aes_engine *xts = &cmd->u.xts;
  920. struct ccp_dm_workarea key, ctx;
  921. struct ccp_data src, dst;
  922. struct ccp_op op;
  923. unsigned int unit_size, dm_offset;
  924. bool in_place = false;
  925. unsigned int sb_count;
  926. enum ccp_aes_type aestype;
  927. int ret;
  928. switch (xts->unit_size) {
  929. case CCP_XTS_AES_UNIT_SIZE_16:
  930. unit_size = 16;
  931. break;
  932. case CCP_XTS_AES_UNIT_SIZE_512:
  933. unit_size = 512;
  934. break;
  935. case CCP_XTS_AES_UNIT_SIZE_1024:
  936. unit_size = 1024;
  937. break;
  938. case CCP_XTS_AES_UNIT_SIZE_2048:
  939. unit_size = 2048;
  940. break;
  941. case CCP_XTS_AES_UNIT_SIZE_4096:
  942. unit_size = 4096;
  943. break;
  944. default:
  945. return -EINVAL;
  946. }
  947. if (xts->key_len == AES_KEYSIZE_128)
  948. aestype = CCP_AES_TYPE_128;
  949. else if (xts->key_len == AES_KEYSIZE_256)
  950. aestype = CCP_AES_TYPE_256;
  951. else
  952. return -EINVAL;
  953. if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
  954. return -EINVAL;
  955. if (xts->iv_len != AES_BLOCK_SIZE)
  956. return -EINVAL;
  957. if (!xts->key || !xts->iv || !xts->src || !xts->dst)
  958. return -EINVAL;
  959. BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
  960. BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
  961. ret = -EIO;
  962. memset(&op, 0, sizeof(op));
  963. op.cmd_q = cmd_q;
  964. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  965. op.sb_key = cmd_q->sb_key;
  966. op.sb_ctx = cmd_q->sb_ctx;
  967. op.init = 1;
  968. op.u.xts.type = aestype;
  969. op.u.xts.action = xts->action;
  970. op.u.xts.unit_size = xts->unit_size;
  971. /* A version 3 device only supports 128-bit keys, which fits into a
  972. * single SB entry. A version 5 device uses a 512-bit vector, so two
  973. * SB entries.
  974. */
  975. if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
  976. sb_count = CCP_XTS_AES_KEY_SB_COUNT;
  977. else
  978. sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
  979. ret = ccp_init_dm_workarea(&key, cmd_q,
  980. sb_count * CCP_SB_BYTES,
  981. DMA_TO_DEVICE);
  982. if (ret)
  983. return ret;
  984. if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
  985. /* All supported key sizes must be in little endian format.
  986. * Use the 256-bit byte swap passthru option to convert from
  987. * big endian to little endian.
  988. */
  989. dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
  990. ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
  991. if (ret)
  992. goto e_key;
  993. ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
  994. if (ret)
  995. goto e_key;
  996. } else {
  997. /* Version 5 CCPs use a 512-bit space for the key: each portion
  998. * occupies 256 bits, or one entire slot, and is zero-padded.
  999. */
  1000. unsigned int pad;
  1001. dm_offset = CCP_SB_BYTES;
  1002. pad = dm_offset - xts->key_len;
  1003. ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
  1004. if (ret)
  1005. goto e_key;
  1006. ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
  1007. xts->key_len, xts->key_len);
  1008. if (ret)
  1009. goto e_key;
  1010. }
  1011. ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
  1012. CCP_PASSTHRU_BYTESWAP_256BIT);
  1013. if (ret) {
  1014. cmd->engine_error = cmd_q->cmd_error;
  1015. goto e_key;
  1016. }
  1017. /* The AES context fits in a single (32-byte) SB entry and
  1018. * for XTS is already in little endian format so no byte swapping
  1019. * is needed.
  1020. */
  1021. ret = ccp_init_dm_workarea(&ctx, cmd_q,
  1022. CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
  1023. DMA_BIDIRECTIONAL);
  1024. if (ret)
  1025. goto e_key;
  1026. ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
  1027. if (ret)
  1028. goto e_ctx;
  1029. ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  1030. CCP_PASSTHRU_BYTESWAP_NOOP);
  1031. if (ret) {
  1032. cmd->engine_error = cmd_q->cmd_error;
  1033. goto e_ctx;
  1034. }
  1035. /* Prepare the input and output data workareas. For in-place
  1036. * operations we need to set the dma direction to BIDIRECTIONAL
  1037. * and copy the src workarea to the dst workarea.
  1038. */
  1039. if (sg_virt(xts->src) == sg_virt(xts->dst))
  1040. in_place = true;
  1041. ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
  1042. unit_size,
  1043. in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  1044. if (ret)
  1045. goto e_ctx;
  1046. if (in_place) {
  1047. dst = src;
  1048. } else {
  1049. ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
  1050. unit_size, DMA_FROM_DEVICE);
  1051. if (ret)
  1052. goto e_src;
  1053. }
  1054. /* Send data to the CCP AES engine */
  1055. while (src.sg_wa.bytes_left) {
  1056. ccp_prepare_data(&src, &dst, &op, unit_size, true);
  1057. if (!src.sg_wa.bytes_left)
  1058. op.eom = 1;
  1059. ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
  1060. if (ret) {
  1061. cmd->engine_error = cmd_q->cmd_error;
  1062. goto e_dst;
  1063. }
  1064. ccp_process_data(&src, &dst, &op);
  1065. }
  1066. /* Retrieve the AES context - convert from LE to BE using
  1067. * 32-byte (256-bit) byteswapping
  1068. */
  1069. ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  1070. CCP_PASSTHRU_BYTESWAP_256BIT);
  1071. if (ret) {
  1072. cmd->engine_error = cmd_q->cmd_error;
  1073. goto e_dst;
  1074. }
  1075. /* ...but we only need AES_BLOCK_SIZE bytes */
  1076. dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
  1077. ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
  1078. e_dst:
  1079. if (!in_place)
  1080. ccp_free_data(&dst, cmd_q);
  1081. e_src:
  1082. ccp_free_data(&src, cmd_q);
  1083. e_ctx:
  1084. ccp_dm_free(&ctx);
  1085. e_key:
  1086. ccp_dm_free(&key);
  1087. return ret;
  1088. }
  1089. static noinline_for_stack int
  1090. ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  1091. {
  1092. struct ccp_des3_engine *des3 = &cmd->u.des3;
  1093. struct ccp_dm_workarea key, ctx;
  1094. struct ccp_data src, dst;
  1095. struct ccp_op op;
  1096. unsigned int dm_offset;
  1097. unsigned int len_singlekey;
  1098. bool in_place = false;
  1099. int ret;
  1100. /* Error checks */
  1101. if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
  1102. return -EINVAL;
  1103. if (!cmd_q->ccp->vdata->perform->des3)
  1104. return -EINVAL;
  1105. if (des3->key_len != DES3_EDE_KEY_SIZE)
  1106. return -EINVAL;
  1107. if (((des3->mode == CCP_DES3_MODE_ECB) ||
  1108. (des3->mode == CCP_DES3_MODE_CBC)) &&
  1109. (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
  1110. return -EINVAL;
  1111. if (!des3->key || !des3->src || !des3->dst)
  1112. return -EINVAL;
  1113. if (des3->mode != CCP_DES3_MODE_ECB) {
  1114. if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
  1115. return -EINVAL;
  1116. if (!des3->iv)
  1117. return -EINVAL;
  1118. }
  1119. ret = -EIO;
  1120. /* Zero out all the fields of the command desc */
  1121. memset(&op, 0, sizeof(op));
  1122. /* Set up the Function field */
  1123. op.cmd_q = cmd_q;
  1124. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  1125. op.sb_key = cmd_q->sb_key;
  1126. op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
  1127. op.u.des3.type = des3->type;
  1128. op.u.des3.mode = des3->mode;
  1129. op.u.des3.action = des3->action;
  1130. /*
  1131. * All supported key sizes fit in a single (32-byte) KSB entry and
  1132. * (like AES) must be in little endian format. Use the 256-bit byte
  1133. * swap passthru option to convert from big endian to little endian.
  1134. */
  1135. ret = ccp_init_dm_workarea(&key, cmd_q,
  1136. CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
  1137. DMA_TO_DEVICE);
  1138. if (ret)
  1139. return ret;
  1140. /*
  1141. * The contents of the key triplet are in the reverse order of what
  1142. * is required by the engine. Copy the 3 pieces individually to put
  1143. * them where they belong.
  1144. */
  1145. dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
  1146. len_singlekey = des3->key_len / 3;
  1147. ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
  1148. des3->key, 0, len_singlekey);
  1149. if (ret)
  1150. goto e_key;
  1151. ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
  1152. des3->key, len_singlekey, len_singlekey);
  1153. if (ret)
  1154. goto e_key;
  1155. ret = ccp_set_dm_area(&key, dm_offset,
  1156. des3->key, 2 * len_singlekey, len_singlekey);
  1157. if (ret)
  1158. goto e_key;
  1159. /* Copy the key to the SB */
  1160. ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
  1161. CCP_PASSTHRU_BYTESWAP_256BIT);
  1162. if (ret) {
  1163. cmd->engine_error = cmd_q->cmd_error;
  1164. goto e_key;
  1165. }
  1166. /*
  1167. * The DES3 context fits in a single (32-byte) KSB entry and
  1168. * must be in little endian format. Use the 256-bit byte swap
  1169. * passthru option to convert from big endian to little endian.
  1170. */
  1171. if (des3->mode != CCP_DES3_MODE_ECB) {
  1172. op.sb_ctx = cmd_q->sb_ctx;
  1173. ret = ccp_init_dm_workarea(&ctx, cmd_q,
  1174. CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
  1175. DMA_BIDIRECTIONAL);
  1176. if (ret)
  1177. goto e_key;
  1178. /* Load the context into the LSB */
  1179. dm_offset = CCP_SB_BYTES - des3->iv_len;
  1180. ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
  1181. des3->iv_len);
  1182. if (ret)
  1183. goto e_ctx;
  1184. ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  1185. CCP_PASSTHRU_BYTESWAP_256BIT);
  1186. if (ret) {
  1187. cmd->engine_error = cmd_q->cmd_error;
  1188. goto e_ctx;
  1189. }
  1190. }
  1191. /*
  1192. * Prepare the input and output data workareas. For in-place
  1193. * operations we need to set the dma direction to BIDIRECTIONAL
  1194. * and copy the src workarea to the dst workarea.
  1195. */
  1196. if (sg_virt(des3->src) == sg_virt(des3->dst))
  1197. in_place = true;
  1198. ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
  1199. DES3_EDE_BLOCK_SIZE,
  1200. in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  1201. if (ret)
  1202. goto e_ctx;
  1203. if (in_place)
  1204. dst = src;
  1205. else {
  1206. ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
  1207. DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
  1208. if (ret)
  1209. goto e_src;
  1210. }
  1211. /* Send data to the CCP DES3 engine */
  1212. while (src.sg_wa.bytes_left) {
  1213. ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
  1214. if (!src.sg_wa.bytes_left) {
  1215. op.eom = 1;
  1216. /* Since we don't retrieve the context in ECB mode
  1217. * we have to wait for the operation to complete
  1218. * on the last piece of data
  1219. */
  1220. op.soc = 0;
  1221. }
  1222. ret = cmd_q->ccp->vdata->perform->des3(&op);
  1223. if (ret) {
  1224. cmd->engine_error = cmd_q->cmd_error;
  1225. goto e_dst;
  1226. }
  1227. ccp_process_data(&src, &dst, &op);
  1228. }
  1229. if (des3->mode != CCP_DES3_MODE_ECB) {
  1230. /* Retrieve the context and make BE */
  1231. ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  1232. CCP_PASSTHRU_BYTESWAP_256BIT);
  1233. if (ret) {
  1234. cmd->engine_error = cmd_q->cmd_error;
  1235. goto e_dst;
  1236. }
  1237. /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
  1238. ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
  1239. DES3_EDE_BLOCK_SIZE);
  1240. }
  1241. e_dst:
  1242. if (!in_place)
  1243. ccp_free_data(&dst, cmd_q);
  1244. e_src:
  1245. ccp_free_data(&src, cmd_q);
  1246. e_ctx:
  1247. if (des3->mode != CCP_DES3_MODE_ECB)
  1248. ccp_dm_free(&ctx);
  1249. e_key:
  1250. ccp_dm_free(&key);
  1251. return ret;
  1252. }
  1253. static noinline_for_stack int
  1254. ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  1255. {
  1256. struct ccp_sha_engine *sha = &cmd->u.sha;
  1257. struct ccp_dm_workarea ctx;
  1258. struct ccp_data src;
  1259. struct ccp_op op;
  1260. unsigned int ioffset, ooffset;
  1261. unsigned int digest_size;
  1262. int sb_count;
  1263. const void *init;
  1264. u64 block_size;
  1265. int ctx_size;
  1266. int ret;
  1267. switch (sha->type) {
  1268. case CCP_SHA_TYPE_1:
  1269. if (sha->ctx_len < SHA1_DIGEST_SIZE)
  1270. return -EINVAL;
  1271. block_size = SHA1_BLOCK_SIZE;
  1272. break;
  1273. case CCP_SHA_TYPE_224:
  1274. if (sha->ctx_len < SHA224_DIGEST_SIZE)
  1275. return -EINVAL;
  1276. block_size = SHA224_BLOCK_SIZE;
  1277. break;
  1278. case CCP_SHA_TYPE_256:
  1279. if (sha->ctx_len < SHA256_DIGEST_SIZE)
  1280. return -EINVAL;
  1281. block_size = SHA256_BLOCK_SIZE;
  1282. break;
  1283. case CCP_SHA_TYPE_384:
  1284. if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
  1285. || sha->ctx_len < SHA384_DIGEST_SIZE)
  1286. return -EINVAL;
  1287. block_size = SHA384_BLOCK_SIZE;
  1288. break;
  1289. case CCP_SHA_TYPE_512:
  1290. if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
  1291. || sha->ctx_len < SHA512_DIGEST_SIZE)
  1292. return -EINVAL;
  1293. block_size = SHA512_BLOCK_SIZE;
  1294. break;
  1295. default:
  1296. return -EINVAL;
  1297. }
  1298. if (!sha->ctx)
  1299. return -EINVAL;
  1300. if (!sha->final && (sha->src_len & (block_size - 1)))
  1301. return -EINVAL;
  1302. /* The version 3 device can't handle zero-length input */
  1303. if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
  1304. if (!sha->src_len) {
  1305. unsigned int digest_len;
  1306. const u8 *sha_zero;
  1307. /* Not final, just return */
  1308. if (!sha->final)
  1309. return 0;
  1310. /* CCP can't do a zero length sha operation so the
  1311. * caller must buffer the data.
  1312. */
  1313. if (sha->msg_bits)
  1314. return -EINVAL;
  1315. /* The CCP cannot perform zero-length sha operations
  1316. * so the caller is required to buffer data for the
  1317. * final operation. However, a sha operation for a
  1318. * message with a total length of zero is valid so
  1319. * known values are required to supply the result.
  1320. */
  1321. switch (sha->type) {
  1322. case CCP_SHA_TYPE_1:
  1323. sha_zero = sha1_zero_message_hash;
  1324. digest_len = SHA1_DIGEST_SIZE;
  1325. break;
  1326. case CCP_SHA_TYPE_224:
  1327. sha_zero = sha224_zero_message_hash;
  1328. digest_len = SHA224_DIGEST_SIZE;
  1329. break;
  1330. case CCP_SHA_TYPE_256:
  1331. sha_zero = sha256_zero_message_hash;
  1332. digest_len = SHA256_DIGEST_SIZE;
  1333. break;
  1334. default:
  1335. return -EINVAL;
  1336. }
  1337. scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
  1338. digest_len, 1);
  1339. return 0;
  1340. }
  1341. }
  1342. /* Set variables used throughout */
  1343. switch (sha->type) {
  1344. case CCP_SHA_TYPE_1:
  1345. digest_size = SHA1_DIGEST_SIZE;
  1346. init = (void *) ccp_sha1_init;
  1347. ctx_size = SHA1_DIGEST_SIZE;
  1348. sb_count = 1;
  1349. if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
  1350. ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
  1351. else
  1352. ooffset = ioffset = 0;
  1353. break;
  1354. case CCP_SHA_TYPE_224:
  1355. digest_size = SHA224_DIGEST_SIZE;
  1356. init = (void *) ccp_sha224_init;
  1357. ctx_size = SHA256_DIGEST_SIZE;
  1358. sb_count = 1;
  1359. ioffset = 0;
  1360. if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
  1361. ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
  1362. else
  1363. ooffset = 0;
  1364. break;
  1365. case CCP_SHA_TYPE_256:
  1366. digest_size = SHA256_DIGEST_SIZE;
  1367. init = (void *) ccp_sha256_init;
  1368. ctx_size = SHA256_DIGEST_SIZE;
  1369. sb_count = 1;
  1370. ooffset = ioffset = 0;
  1371. break;
  1372. case CCP_SHA_TYPE_384:
  1373. digest_size = SHA384_DIGEST_SIZE;
  1374. init = (void *) ccp_sha384_init;
  1375. ctx_size = SHA512_DIGEST_SIZE;
  1376. sb_count = 2;
  1377. ioffset = 0;
  1378. ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
  1379. break;
  1380. case CCP_SHA_TYPE_512:
  1381. digest_size = SHA512_DIGEST_SIZE;
  1382. init = (void *) ccp_sha512_init;
  1383. ctx_size = SHA512_DIGEST_SIZE;
  1384. sb_count = 2;
  1385. ooffset = ioffset = 0;
  1386. break;
  1387. default:
  1388. ret = -EINVAL;
  1389. goto e_data;
  1390. }
  1391. /* For zero-length plaintext the src pointer is ignored;
  1392. * otherwise both parts must be valid
  1393. */
  1394. if (sha->src_len && !sha->src)
  1395. return -EINVAL;
  1396. memset(&op, 0, sizeof(op));
  1397. op.cmd_q = cmd_q;
  1398. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  1399. op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
  1400. op.u.sha.type = sha->type;
  1401. op.u.sha.msg_bits = sha->msg_bits;
  1402. /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
  1403. * SHA384/512 require 2 adjacent SB slots, with the right half in the
  1404. * first slot, and the left half in the second. Each portion must then
  1405. * be in little endian format: use the 256-bit byte swap option.
  1406. */
  1407. ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
  1408. DMA_BIDIRECTIONAL);
  1409. if (ret)
  1410. return ret;
  1411. if (sha->first) {
  1412. switch (sha->type) {
  1413. case CCP_SHA_TYPE_1:
  1414. case CCP_SHA_TYPE_224:
  1415. case CCP_SHA_TYPE_256:
  1416. memcpy(ctx.address + ioffset, init, ctx_size);
  1417. break;
  1418. case CCP_SHA_TYPE_384:
  1419. case CCP_SHA_TYPE_512:
  1420. memcpy(ctx.address + ctx_size / 2, init,
  1421. ctx_size / 2);
  1422. memcpy(ctx.address, init + ctx_size / 2,
  1423. ctx_size / 2);
  1424. break;
  1425. default:
  1426. ret = -EINVAL;
  1427. goto e_ctx;
  1428. }
  1429. } else {
  1430. /* Restore the context */
  1431. ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
  1432. sb_count * CCP_SB_BYTES);
  1433. if (ret)
  1434. goto e_ctx;
  1435. }
  1436. ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  1437. CCP_PASSTHRU_BYTESWAP_256BIT);
  1438. if (ret) {
  1439. cmd->engine_error = cmd_q->cmd_error;
  1440. goto e_ctx;
  1441. }
  1442. if (sha->src) {
  1443. /* Send data to the CCP SHA engine; block_size is set above */
  1444. ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
  1445. block_size, DMA_TO_DEVICE);
  1446. if (ret)
  1447. goto e_ctx;
  1448. while (src.sg_wa.bytes_left) {
  1449. ccp_prepare_data(&src, NULL, &op, block_size, false);
  1450. if (sha->final && !src.sg_wa.bytes_left)
  1451. op.eom = 1;
  1452. ret = cmd_q->ccp->vdata->perform->sha(&op);
  1453. if (ret) {
  1454. cmd->engine_error = cmd_q->cmd_error;
  1455. goto e_data;
  1456. }
  1457. ccp_process_data(&src, NULL, &op);
  1458. }
  1459. } else {
  1460. op.eom = 1;
  1461. ret = cmd_q->ccp->vdata->perform->sha(&op);
  1462. if (ret) {
  1463. cmd->engine_error = cmd_q->cmd_error;
  1464. goto e_data;
  1465. }
  1466. }
  1467. /* Retrieve the SHA context - convert from LE to BE using
  1468. * 32-byte (256-bit) byteswapping to BE
  1469. */
  1470. ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
  1471. CCP_PASSTHRU_BYTESWAP_256BIT);
  1472. if (ret) {
  1473. cmd->engine_error = cmd_q->cmd_error;
  1474. goto e_data;
  1475. }
  1476. if (sha->final) {
  1477. /* Finishing up, so get the digest */
  1478. switch (sha->type) {
  1479. case CCP_SHA_TYPE_1:
  1480. case CCP_SHA_TYPE_224:
  1481. case CCP_SHA_TYPE_256:
  1482. ccp_get_dm_area(&ctx, ooffset,
  1483. sha->ctx, 0,
  1484. digest_size);
  1485. break;
  1486. case CCP_SHA_TYPE_384:
  1487. case CCP_SHA_TYPE_512:
  1488. ccp_get_dm_area(&ctx, 0,
  1489. sha->ctx, LSB_ITEM_SIZE - ooffset,
  1490. LSB_ITEM_SIZE);
  1491. ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
  1492. sha->ctx, 0,
  1493. LSB_ITEM_SIZE - ooffset);
  1494. break;
  1495. default:
  1496. ret = -EINVAL;
  1497. goto e_data;
  1498. }
  1499. } else {
  1500. /* Stash the context */
  1501. ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
  1502. sb_count * CCP_SB_BYTES);
  1503. }
  1504. if (sha->final && sha->opad) {
  1505. /* HMAC operation, recursively perform final SHA */
  1506. struct ccp_cmd hmac_cmd;
  1507. struct scatterlist sg;
  1508. u8 *hmac_buf;
  1509. if (sha->opad_len != block_size) {
  1510. ret = -EINVAL;
  1511. goto e_data;
  1512. }
  1513. hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
  1514. if (!hmac_buf) {
  1515. ret = -ENOMEM;
  1516. goto e_data;
  1517. }
  1518. sg_init_one(&sg, hmac_buf, block_size + digest_size);
  1519. scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
  1520. switch (sha->type) {
  1521. case CCP_SHA_TYPE_1:
  1522. case CCP_SHA_TYPE_224:
  1523. case CCP_SHA_TYPE_256:
  1524. memcpy(hmac_buf + block_size,
  1525. ctx.address + ooffset,
  1526. digest_size);
  1527. break;
  1528. case CCP_SHA_TYPE_384:
  1529. case CCP_SHA_TYPE_512:
  1530. memcpy(hmac_buf + block_size,
  1531. ctx.address + LSB_ITEM_SIZE + ooffset,
  1532. LSB_ITEM_SIZE);
  1533. memcpy(hmac_buf + block_size +
  1534. (LSB_ITEM_SIZE - ooffset),
  1535. ctx.address,
  1536. LSB_ITEM_SIZE);
  1537. break;
  1538. default:
  1539. kfree(hmac_buf);
  1540. ret = -EINVAL;
  1541. goto e_data;
  1542. }
  1543. memset(&hmac_cmd, 0, sizeof(hmac_cmd));
  1544. hmac_cmd.engine = CCP_ENGINE_SHA;
  1545. hmac_cmd.u.sha.type = sha->type;
  1546. hmac_cmd.u.sha.ctx = sha->ctx;
  1547. hmac_cmd.u.sha.ctx_len = sha->ctx_len;
  1548. hmac_cmd.u.sha.src = &sg;
  1549. hmac_cmd.u.sha.src_len = block_size + digest_size;
  1550. hmac_cmd.u.sha.opad = NULL;
  1551. hmac_cmd.u.sha.opad_len = 0;
  1552. hmac_cmd.u.sha.first = 1;
  1553. hmac_cmd.u.sha.final = 1;
  1554. hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
  1555. ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
  1556. if (ret)
  1557. cmd->engine_error = hmac_cmd.engine_error;
  1558. kfree(hmac_buf);
  1559. }
  1560. e_data:
  1561. if (sha->src)
  1562. ccp_free_data(&src, cmd_q);
  1563. e_ctx:
  1564. ccp_dm_free(&ctx);
  1565. return ret;
  1566. }
  1567. static noinline_for_stack int
  1568. ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  1569. {
  1570. struct ccp_rsa_engine *rsa = &cmd->u.rsa;
  1571. struct ccp_dm_workarea exp, src, dst;
  1572. struct ccp_op op;
  1573. unsigned int sb_count, i_len, o_len;
  1574. int ret;
  1575. /* Check against the maximum allowable size, in bits */
  1576. if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
  1577. return -EINVAL;
  1578. if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
  1579. return -EINVAL;
  1580. memset(&op, 0, sizeof(op));
  1581. op.cmd_q = cmd_q;
  1582. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  1583. /* The RSA modulus must precede the message being acted upon, so
  1584. * it must be copied to a DMA area where the message and the
  1585. * modulus can be concatenated. Therefore the input buffer
  1586. * length required is twice the output buffer length (which
  1587. * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
  1588. * Buffer sizes must be a multiple of 32 bytes; rounding up may be
  1589. * required.
  1590. */
  1591. o_len = 32 * ((rsa->key_size + 255) / 256);
  1592. i_len = o_len * 2;
  1593. sb_count = 0;
  1594. if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
  1595. /* sb_count is the number of storage block slots required
  1596. * for the modulus.
  1597. */
  1598. sb_count = o_len / CCP_SB_BYTES;
  1599. op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
  1600. sb_count);
  1601. if (!op.sb_key)
  1602. return -EIO;
  1603. } else {
  1604. /* A version 5 device allows a modulus size that will not fit
  1605. * in the LSB, so the command will transfer it from memory.
  1606. * Set the sb key to the default, even though it's not used.
  1607. */
  1608. op.sb_key = cmd_q->sb_key;
  1609. }
  1610. /* The RSA exponent must be in little endian format. Reverse its
  1611. * byte order.
  1612. */
  1613. ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
  1614. if (ret)
  1615. goto e_sb;
  1616. ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
  1617. if (ret)
  1618. goto e_exp;
  1619. if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
  1620. /* Copy the exponent to the local storage block, using
  1621. * as many 32-byte blocks as were allocated above. It's
  1622. * already little endian, so no further change is required.
  1623. */
  1624. ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
  1625. CCP_PASSTHRU_BYTESWAP_NOOP);
  1626. if (ret) {
  1627. cmd->engine_error = cmd_q->cmd_error;
  1628. goto e_exp;
  1629. }
  1630. } else {
  1631. /* The exponent can be retrieved from memory via DMA. */
  1632. op.exp.u.dma.address = exp.dma.address;
  1633. op.exp.u.dma.offset = 0;
  1634. }
  1635. /* Concatenate the modulus and the message. Both the modulus and
  1636. * the operands must be in little endian format. Since the input
  1637. * is in big endian format it must be converted.
  1638. */
  1639. ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
  1640. if (ret)
  1641. goto e_exp;
  1642. ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
  1643. if (ret)
  1644. goto e_src;
  1645. ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
  1646. if (ret)
  1647. goto e_src;
  1648. /* Prepare the output area for the operation */
  1649. ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
  1650. if (ret)
  1651. goto e_src;
  1652. op.soc = 1;
  1653. op.src.u.dma.address = src.dma.address;
  1654. op.src.u.dma.offset = 0;
  1655. op.src.u.dma.length = i_len;
  1656. op.dst.u.dma.address = dst.dma.address;
  1657. op.dst.u.dma.offset = 0;
  1658. op.dst.u.dma.length = o_len;
  1659. op.u.rsa.mod_size = rsa->key_size;
  1660. op.u.rsa.input_len = i_len;
  1661. ret = cmd_q->ccp->vdata->perform->rsa(&op);
  1662. if (ret) {
  1663. cmd->engine_error = cmd_q->cmd_error;
  1664. goto e_dst;
  1665. }
  1666. ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
  1667. e_dst:
  1668. ccp_dm_free(&dst);
  1669. e_src:
  1670. ccp_dm_free(&src);
  1671. e_exp:
  1672. ccp_dm_free(&exp);
  1673. e_sb:
  1674. if (sb_count)
  1675. cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
  1676. return ret;
  1677. }
  1678. static noinline_for_stack int
  1679. ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  1680. {
  1681. struct ccp_passthru_engine *pt = &cmd->u.passthru;
  1682. struct ccp_dm_workarea mask;
  1683. struct ccp_data src, dst;
  1684. struct ccp_op op;
  1685. bool in_place = false;
  1686. unsigned int i;
  1687. int ret = 0;
  1688. if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
  1689. return -EINVAL;
  1690. if (!pt->src || !pt->dst)
  1691. return -EINVAL;
  1692. if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
  1693. if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
  1694. return -EINVAL;
  1695. if (!pt->mask)
  1696. return -EINVAL;
  1697. }
  1698. BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
  1699. memset(&op, 0, sizeof(op));
  1700. op.cmd_q = cmd_q;
  1701. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  1702. if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
  1703. /* Load the mask */
  1704. op.sb_key = cmd_q->sb_key;
  1705. ret = ccp_init_dm_workarea(&mask, cmd_q,
  1706. CCP_PASSTHRU_SB_COUNT *
  1707. CCP_SB_BYTES,
  1708. DMA_TO_DEVICE);
  1709. if (ret)
  1710. return ret;
  1711. ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
  1712. if (ret)
  1713. goto e_mask;
  1714. ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
  1715. CCP_PASSTHRU_BYTESWAP_NOOP);
  1716. if (ret) {
  1717. cmd->engine_error = cmd_q->cmd_error;
  1718. goto e_mask;
  1719. }
  1720. }
  1721. /* Prepare the input and output data workareas. For in-place
  1722. * operations we need to set the dma direction to BIDIRECTIONAL
  1723. * and copy the src workarea to the dst workarea.
  1724. */
  1725. if (sg_virt(pt->src) == sg_virt(pt->dst))
  1726. in_place = true;
  1727. ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
  1728. CCP_PASSTHRU_MASKSIZE,
  1729. in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  1730. if (ret)
  1731. goto e_mask;
  1732. if (in_place) {
  1733. dst = src;
  1734. } else {
  1735. ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
  1736. CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
  1737. if (ret)
  1738. goto e_src;
  1739. }
  1740. /* Send data to the CCP Passthru engine
  1741. * Because the CCP engine works on a single source and destination
  1742. * dma address at a time, each entry in the source scatterlist
  1743. * (after the dma_map_sg call) must be less than or equal to the
  1744. * (remaining) length in the destination scatterlist entry and the
  1745. * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
  1746. */
  1747. dst.sg_wa.sg_used = 0;
  1748. for (i = 1; i <= src.sg_wa.dma_count; i++) {
  1749. if (!dst.sg_wa.sg ||
  1750. (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
  1751. ret = -EINVAL;
  1752. goto e_dst;
  1753. }
  1754. if (i == src.sg_wa.dma_count) {
  1755. op.eom = 1;
  1756. op.soc = 1;
  1757. }
  1758. op.src.type = CCP_MEMTYPE_SYSTEM;
  1759. op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
  1760. op.src.u.dma.offset = 0;
  1761. op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
  1762. op.dst.type = CCP_MEMTYPE_SYSTEM;
  1763. op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
  1764. op.dst.u.dma.offset = dst.sg_wa.sg_used;
  1765. op.dst.u.dma.length = op.src.u.dma.length;
  1766. ret = cmd_q->ccp->vdata->perform->passthru(&op);
  1767. if (ret) {
  1768. cmd->engine_error = cmd_q->cmd_error;
  1769. goto e_dst;
  1770. }
  1771. dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
  1772. if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
  1773. dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
  1774. dst.sg_wa.sg_used = 0;
  1775. }
  1776. src.sg_wa.sg = sg_next(src.sg_wa.sg);
  1777. }
  1778. e_dst:
  1779. if (!in_place)
  1780. ccp_free_data(&dst, cmd_q);
  1781. e_src:
  1782. ccp_free_data(&src, cmd_q);
  1783. e_mask:
  1784. if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
  1785. ccp_dm_free(&mask);
  1786. return ret;
  1787. }
  1788. static noinline_for_stack int
  1789. ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
  1790. struct ccp_cmd *cmd)
  1791. {
  1792. struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
  1793. struct ccp_dm_workarea mask;
  1794. struct ccp_op op;
  1795. int ret;
  1796. if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
  1797. return -EINVAL;
  1798. if (!pt->src_dma || !pt->dst_dma)
  1799. return -EINVAL;
  1800. if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
  1801. if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
  1802. return -EINVAL;
  1803. if (!pt->mask)
  1804. return -EINVAL;
  1805. }
  1806. BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
  1807. memset(&op, 0, sizeof(op));
  1808. op.cmd_q = cmd_q;
  1809. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  1810. if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
  1811. /* Load the mask */
  1812. op.sb_key = cmd_q->sb_key;
  1813. mask.length = pt->mask_len;
  1814. mask.dma.address = pt->mask;
  1815. mask.dma.length = pt->mask_len;
  1816. ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
  1817. CCP_PASSTHRU_BYTESWAP_NOOP);
  1818. if (ret) {
  1819. cmd->engine_error = cmd_q->cmd_error;
  1820. return ret;
  1821. }
  1822. }
  1823. /* Send data to the CCP Passthru engine */
  1824. op.eom = 1;
  1825. op.soc = 1;
  1826. op.src.type = CCP_MEMTYPE_SYSTEM;
  1827. op.src.u.dma.address = pt->src_dma;
  1828. op.src.u.dma.offset = 0;
  1829. op.src.u.dma.length = pt->src_len;
  1830. op.dst.type = CCP_MEMTYPE_SYSTEM;
  1831. op.dst.u.dma.address = pt->dst_dma;
  1832. op.dst.u.dma.offset = 0;
  1833. op.dst.u.dma.length = pt->src_len;
  1834. ret = cmd_q->ccp->vdata->perform->passthru(&op);
  1835. if (ret)
  1836. cmd->engine_error = cmd_q->cmd_error;
  1837. return ret;
  1838. }
  1839. static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  1840. {
  1841. struct ccp_ecc_engine *ecc = &cmd->u.ecc;
  1842. struct ccp_dm_workarea src, dst;
  1843. struct ccp_op op;
  1844. int ret;
  1845. u8 *save;
  1846. if (!ecc->u.mm.operand_1 ||
  1847. (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
  1848. return -EINVAL;
  1849. if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
  1850. if (!ecc->u.mm.operand_2 ||
  1851. (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
  1852. return -EINVAL;
  1853. if (!ecc->u.mm.result ||
  1854. (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
  1855. return -EINVAL;
  1856. memset(&op, 0, sizeof(op));
  1857. op.cmd_q = cmd_q;
  1858. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  1859. /* Concatenate the modulus and the operands. Both the modulus and
  1860. * the operands must be in little endian format. Since the input
  1861. * is in big endian format it must be converted and placed in a
  1862. * fixed length buffer.
  1863. */
  1864. ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
  1865. DMA_TO_DEVICE);
  1866. if (ret)
  1867. return ret;
  1868. /* Save the workarea address since it is updated in order to perform
  1869. * the concatenation
  1870. */
  1871. save = src.address;
  1872. /* Copy the ECC modulus */
  1873. ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
  1874. if (ret)
  1875. goto e_src;
  1876. src.address += CCP_ECC_OPERAND_SIZE;
  1877. /* Copy the first operand */
  1878. ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
  1879. ecc->u.mm.operand_1_len);
  1880. if (ret)
  1881. goto e_src;
  1882. src.address += CCP_ECC_OPERAND_SIZE;
  1883. if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
  1884. /* Copy the second operand */
  1885. ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
  1886. ecc->u.mm.operand_2_len);
  1887. if (ret)
  1888. goto e_src;
  1889. src.address += CCP_ECC_OPERAND_SIZE;
  1890. }
  1891. /* Restore the workarea address */
  1892. src.address = save;
  1893. /* Prepare the output area for the operation */
  1894. ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
  1895. DMA_FROM_DEVICE);
  1896. if (ret)
  1897. goto e_src;
  1898. op.soc = 1;
  1899. op.src.u.dma.address = src.dma.address;
  1900. op.src.u.dma.offset = 0;
  1901. op.src.u.dma.length = src.length;
  1902. op.dst.u.dma.address = dst.dma.address;
  1903. op.dst.u.dma.offset = 0;
  1904. op.dst.u.dma.length = dst.length;
  1905. op.u.ecc.function = cmd->u.ecc.function;
  1906. ret = cmd_q->ccp->vdata->perform->ecc(&op);
  1907. if (ret) {
  1908. cmd->engine_error = cmd_q->cmd_error;
  1909. goto e_dst;
  1910. }
  1911. ecc->ecc_result = le16_to_cpup(
  1912. (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
  1913. if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
  1914. ret = -EIO;
  1915. goto e_dst;
  1916. }
  1917. /* Save the ECC result */
  1918. ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
  1919. CCP_ECC_MODULUS_BYTES);
  1920. e_dst:
  1921. ccp_dm_free(&dst);
  1922. e_src:
  1923. ccp_dm_free(&src);
  1924. return ret;
  1925. }
  1926. static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  1927. {
  1928. struct ccp_ecc_engine *ecc = &cmd->u.ecc;
  1929. struct ccp_dm_workarea src, dst;
  1930. struct ccp_op op;
  1931. int ret;
  1932. u8 *save;
  1933. if (!ecc->u.pm.point_1.x ||
  1934. (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
  1935. !ecc->u.pm.point_1.y ||
  1936. (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
  1937. return -EINVAL;
  1938. if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
  1939. if (!ecc->u.pm.point_2.x ||
  1940. (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
  1941. !ecc->u.pm.point_2.y ||
  1942. (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
  1943. return -EINVAL;
  1944. } else {
  1945. if (!ecc->u.pm.domain_a ||
  1946. (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
  1947. return -EINVAL;
  1948. if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
  1949. if (!ecc->u.pm.scalar ||
  1950. (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
  1951. return -EINVAL;
  1952. }
  1953. if (!ecc->u.pm.result.x ||
  1954. (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
  1955. !ecc->u.pm.result.y ||
  1956. (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
  1957. return -EINVAL;
  1958. memset(&op, 0, sizeof(op));
  1959. op.cmd_q = cmd_q;
  1960. op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
  1961. /* Concatenate the modulus and the operands. Both the modulus and
  1962. * the operands must be in little endian format. Since the input
  1963. * is in big endian format it must be converted and placed in a
  1964. * fixed length buffer.
  1965. */
  1966. ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
  1967. DMA_TO_DEVICE);
  1968. if (ret)
  1969. return ret;
  1970. /* Save the workarea address since it is updated in order to perform
  1971. * the concatenation
  1972. */
  1973. save = src.address;
  1974. /* Copy the ECC modulus */
  1975. ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
  1976. if (ret)
  1977. goto e_src;
  1978. src.address += CCP_ECC_OPERAND_SIZE;
  1979. /* Copy the first point X and Y coordinate */
  1980. ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
  1981. ecc->u.pm.point_1.x_len);
  1982. if (ret)
  1983. goto e_src;
  1984. src.address += CCP_ECC_OPERAND_SIZE;
  1985. ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
  1986. ecc->u.pm.point_1.y_len);
  1987. if (ret)
  1988. goto e_src;
  1989. src.address += CCP_ECC_OPERAND_SIZE;
  1990. /* Set the first point Z coordinate to 1 */
  1991. *src.address = 0x01;
  1992. src.address += CCP_ECC_OPERAND_SIZE;
  1993. if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
  1994. /* Copy the second point X and Y coordinate */
  1995. ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
  1996. ecc->u.pm.point_2.x_len);
  1997. if (ret)
  1998. goto e_src;
  1999. src.address += CCP_ECC_OPERAND_SIZE;
  2000. ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
  2001. ecc->u.pm.point_2.y_len);
  2002. if (ret)
  2003. goto e_src;
  2004. src.address += CCP_ECC_OPERAND_SIZE;
  2005. /* Set the second point Z coordinate to 1 */
  2006. *src.address = 0x01;
  2007. src.address += CCP_ECC_OPERAND_SIZE;
  2008. } else {
  2009. /* Copy the Domain "a" parameter */
  2010. ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
  2011. ecc->u.pm.domain_a_len);
  2012. if (ret)
  2013. goto e_src;
  2014. src.address += CCP_ECC_OPERAND_SIZE;
  2015. if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
  2016. /* Copy the scalar value */
  2017. ret = ccp_reverse_set_dm_area(&src, 0,
  2018. ecc->u.pm.scalar, 0,
  2019. ecc->u.pm.scalar_len);
  2020. if (ret)
  2021. goto e_src;
  2022. src.address += CCP_ECC_OPERAND_SIZE;
  2023. }
  2024. }
  2025. /* Restore the workarea address */
  2026. src.address = save;
  2027. /* Prepare the output area for the operation */
  2028. ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
  2029. DMA_FROM_DEVICE);
  2030. if (ret)
  2031. goto e_src;
  2032. op.soc = 1;
  2033. op.src.u.dma.address = src.dma.address;
  2034. op.src.u.dma.offset = 0;
  2035. op.src.u.dma.length = src.length;
  2036. op.dst.u.dma.address = dst.dma.address;
  2037. op.dst.u.dma.offset = 0;
  2038. op.dst.u.dma.length = dst.length;
  2039. op.u.ecc.function = cmd->u.ecc.function;
  2040. ret = cmd_q->ccp->vdata->perform->ecc(&op);
  2041. if (ret) {
  2042. cmd->engine_error = cmd_q->cmd_error;
  2043. goto e_dst;
  2044. }
  2045. ecc->ecc_result = le16_to_cpup(
  2046. (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
  2047. if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
  2048. ret = -EIO;
  2049. goto e_dst;
  2050. }
  2051. /* Save the workarea address since it is updated as we walk through
  2052. * to copy the point math result
  2053. */
  2054. save = dst.address;
  2055. /* Save the ECC result X and Y coordinates */
  2056. ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
  2057. CCP_ECC_MODULUS_BYTES);
  2058. dst.address += CCP_ECC_OUTPUT_SIZE;
  2059. ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
  2060. CCP_ECC_MODULUS_BYTES);
  2061. dst.address += CCP_ECC_OUTPUT_SIZE;
  2062. /* Restore the workarea address */
  2063. dst.address = save;
  2064. e_dst:
  2065. ccp_dm_free(&dst);
  2066. e_src:
  2067. ccp_dm_free(&src);
  2068. return ret;
  2069. }
  2070. static noinline_for_stack int
  2071. ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  2072. {
  2073. struct ccp_ecc_engine *ecc = &cmd->u.ecc;
  2074. ecc->ecc_result = 0;
  2075. if (!ecc->mod ||
  2076. (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
  2077. return -EINVAL;
  2078. switch (ecc->function) {
  2079. case CCP_ECC_FUNCTION_MMUL_384BIT:
  2080. case CCP_ECC_FUNCTION_MADD_384BIT:
  2081. case CCP_ECC_FUNCTION_MINV_384BIT:
  2082. return ccp_run_ecc_mm_cmd(cmd_q, cmd);
  2083. case CCP_ECC_FUNCTION_PADD_384BIT:
  2084. case CCP_ECC_FUNCTION_PMUL_384BIT:
  2085. case CCP_ECC_FUNCTION_PDBL_384BIT:
  2086. return ccp_run_ecc_pm_cmd(cmd_q, cmd);
  2087. default:
  2088. return -EINVAL;
  2089. }
  2090. }
  2091. int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
  2092. {
  2093. int ret;
  2094. cmd->engine_error = 0;
  2095. cmd_q->cmd_error = 0;
  2096. cmd_q->int_rcvd = 0;
  2097. cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
  2098. switch (cmd->engine) {
  2099. case CCP_ENGINE_AES:
  2100. switch (cmd->u.aes.mode) {
  2101. case CCP_AES_MODE_CMAC:
  2102. ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
  2103. break;
  2104. case CCP_AES_MODE_GCM:
  2105. ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
  2106. break;
  2107. default:
  2108. ret = ccp_run_aes_cmd(cmd_q, cmd);
  2109. break;
  2110. }
  2111. break;
  2112. case CCP_ENGINE_XTS_AES_128:
  2113. ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
  2114. break;
  2115. case CCP_ENGINE_DES3:
  2116. ret = ccp_run_des3_cmd(cmd_q, cmd);
  2117. break;
  2118. case CCP_ENGINE_SHA:
  2119. ret = ccp_run_sha_cmd(cmd_q, cmd);
  2120. break;
  2121. case CCP_ENGINE_RSA:
  2122. ret = ccp_run_rsa_cmd(cmd_q, cmd);
  2123. break;
  2124. case CCP_ENGINE_PASSTHRU:
  2125. if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
  2126. ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
  2127. else
  2128. ret = ccp_run_passthru_cmd(cmd_q, cmd);
  2129. break;
  2130. case CCP_ENGINE_ECC:
  2131. ret = ccp_run_ecc_cmd(cmd_q, cmd);
  2132. break;
  2133. default:
  2134. ret = -EINVAL;
  2135. }
  2136. return ret;
  2137. }