Blame - crypto/tcrypt.c - hafnium/third_party/linux

blob: 6e0a054bb61dac713cf72212b42a18fb6e7b67d7 [file] [log] [blame]

Andrew Scull	b4b6d4a	2019-01-02 15:54:55 +0000	[diff] [blame]	1	/*
				2	* Quick & dirty crypto testing module.
				3	*
				4	* This will only exist until we have a better testing mechanism
				5	* (e.g. a char device).
				6	*
				7	* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
				8	* Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
				9	* Copyright (c) 2007 Nokia Siemens Networks
				10	*
				11	* Updated RFC4106 AES-GCM testing.
				12	* Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
				13	* Adrian Hoban <adrian.hoban@intel.com>
				14	* Gabriele Paoloni <gabriele.paoloni@intel.com>
				15	* Tadeusz Struk (tadeusz.struk@intel.com)
				16	* Copyright (c) 2010, Intel Corporation.
				17	*
				18	* This program is free software; you can redistribute it and/or modify it
				19	* under the terms of the GNU General Public License as published by the Free
				20	* Software Foundation; either version 2 of the License, or (at your option)
				21	* any later version.
				22	*
				23	*/
				24
				25	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
				26
				27	#include <crypto/aead.h>
				28	#include <crypto/hash.h>
				29	#include <crypto/skcipher.h>
				30	#include <linux/err.h>
				31	#include <linux/fips.h>
				32	#include <linux/init.h>
				33	#include <linux/gfp.h>
				34	#include <linux/module.h>
				35	#include <linux/scatterlist.h>
				36	#include <linux/string.h>
				37	#include <linux/moduleparam.h>
				38	#include <linux/jiffies.h>
				39	#include <linux/timex.h>
				40	#include <linux/interrupt.h>
				41	#include "tcrypt.h"
				42
				43	/*
				44	* Need slab memory for testing (size in number of pages).
				45	*/
				46	#define TVMEMSIZE 4
				47
				48	/*
				49	* Used by test_cipher_speed()
				50	*/
				51	#define ENCRYPT 1
				52	#define DECRYPT 0
				53
				54	#define MAX_DIGEST_SIZE 64
				55
				56	/*
				57	* return a string with the driver name
				58	*/
				59	#define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm))
				60
				61	/*
				62	* Used by test_cipher_speed()
				63	*/
				64	static unsigned int sec;
				65
				66	static char *alg = NULL;
				67	static u32 type;
				68	static u32 mask;
				69	static int mode;
				70	static u32 num_mb = 8;
				71	static char *tvmem[TVMEMSIZE];
				72
				73	static char *check[] = {
				74	"des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3",
				75	"blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes",
				76	"cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea",
				77	"khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt",
				78	"camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320",
				79	"lzo", "cts", "zlib", "sha3-224", "sha3-256", "sha3-384", "sha3-512",
				80	NULL
				81	};
				82
				83	static u32 block_sizes[] = { 16, 64, 256, 1024, 8192, 0 };
				84	static u32 aead_sizes[] = { 16, 64, 256, 512, 1024, 2048, 4096, 8192, 0 };
				85
				86	#define XBUFSIZE 8
				87	#define MAX_IVLEN 32
				88
				89	static int testmgr_alloc_buf(char *buf[XBUFSIZE])
				90	{
				91	int i;
				92
				93	for (i = 0; i < XBUFSIZE; i++) {
				94	buf[i] = (void *)__get_free_page(GFP_KERNEL);
				95	if (!buf[i])
				96	goto err_free_buf;
				97	}
				98
				99	return 0;
				100
				101	err_free_buf:
				102	while (i-- > 0)
				103	free_page((unsigned long)buf[i]);
				104
				105	return -ENOMEM;
				106	}
				107
				108	static void testmgr_free_buf(char *buf[XBUFSIZE])
				109	{
				110	int i;
				111
				112	for (i = 0; i < XBUFSIZE; i++)
				113	free_page((unsigned long)buf[i]);
				114	}
				115
				116	static void sg_init_aead(struct scatterlist sg, char xbuf[XBUFSIZE],
				117	unsigned int buflen, const void *assoc,
				118	unsigned int aad_size)
				119	{
				120	int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
				121	int k, rem;
				122
				123	if (np > XBUFSIZE) {
				124	rem = PAGE_SIZE;
				125	np = XBUFSIZE;
				126	} else {
				127	rem = buflen % PAGE_SIZE;
				128	}
				129
				130	sg_init_table(sg, np + 1);
				131
				132	sg_set_buf(&sg[0], assoc, aad_size);
				133
				134	if (rem)
				135	np--;
				136	for (k = 0; k < np; k++)
				137	sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
				138
				139	if (rem)
				140	sg_set_buf(&sg[k + 1], xbuf[k], rem);
				141	}
				142
				143	static inline int do_one_aead_op(struct aead_request *req, int ret)
				144	{
				145	struct crypto_wait *wait = req->base.data;
				146
				147	return crypto_wait_req(ret, wait);
				148	}
				149
				150	struct test_mb_aead_data {
				151	struct scatterlist sg[XBUFSIZE];
				152	struct scatterlist sgout[XBUFSIZE];
				153	struct aead_request *req;
				154	struct crypto_wait wait;
				155	char *xbuf[XBUFSIZE];
				156	char *xoutbuf[XBUFSIZE];
				157	char *axbuf[XBUFSIZE];
				158	};
				159
				160	static int do_mult_aead_op(struct test_mb_aead_data *data, int enc,
				161	u32 num_mb, int *rc)
				162	{
				163	int i, err = 0;
				164
				165	/* Fire up a bunch of concurrent requests */
				166	for (i = 0; i < num_mb; i++) {
				167	if (enc == ENCRYPT)
				168	rc[i] = crypto_aead_encrypt(data[i].req);
				169	else
				170	rc[i] = crypto_aead_decrypt(data[i].req);
				171	}
				172
				173	/* Wait for all requests to finish */
				174	for (i = 0; i < num_mb; i++) {
				175	rc[i] = crypto_wait_req(rc[i], &data[i].wait);
				176
				177	if (rc[i]) {
				178	pr_info("concurrent request %d error %d\n", i, rc[i]);
				179	err = rc[i];
				180	}
				181	}
				182
				183	return err;
				184	}
				185
				186	static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc,
				187	int blen, int secs, u32 num_mb)
				188	{
				189	unsigned long start, end;
				190	int bcount;
				191	int ret = 0;
				192	int *rc;
				193
				194	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
				195	if (!rc)
				196	return -ENOMEM;
				197
				198	for (start = jiffies, end = start + secs * HZ, bcount = 0;
				199	time_before(jiffies, end); bcount++) {
				200	ret = do_mult_aead_op(data, enc, num_mb, rc);
				201	if (ret)
				202	goto out;
				203	}
				204
				205	pr_cont("%d operations in %d seconds (%ld bytes)\n",
				206	bcount * num_mb, secs, (long)bcount * blen * num_mb);
				207
				208	out:
				209	kfree(rc);
				210	return ret;
				211	}
				212
				213	static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
				214	int blen, u32 num_mb)
				215	{
				216	unsigned long cycles = 0;
				217	int ret = 0;
				218	int i;
				219	int *rc;
				220
				221	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
				222	if (!rc)
				223	return -ENOMEM;
				224
				225	/* Warm-up run. */
				226	for (i = 0; i < 4; i++) {
				227	ret = do_mult_aead_op(data, enc, num_mb, rc);
				228	if (ret)
				229	goto out;
				230	}
				231
				232	/* The real thing. */
				233	for (i = 0; i < 8; i++) {
				234	cycles_t start, end;
				235
				236	start = get_cycles();
				237	ret = do_mult_aead_op(data, enc, num_mb, rc);
				238	end = get_cycles();
				239
				240	if (ret)
				241	goto out;
				242
				243	cycles += end - start;
				244	}
				245
				246	pr_cont("1 operation in %lu cycles (%d bytes)\n",
				247	(cycles + 4) / (8 * num_mb), blen);
				248
				249	out:
				250	kfree(rc);
				251	return ret;
				252	}
				253
				254	static void test_mb_aead_speed(const char *algo, int enc, int secs,
				255	struct aead_speed_template *template,
				256	unsigned int tcount, u8 authsize,
				257	unsigned int aad_size, u8 *keysize, u32 num_mb)
				258	{
				259	struct test_mb_aead_data *data;
				260	struct crypto_aead *tfm;
				261	unsigned int i, j, iv_len;
				262	const char *key;
				263	const char *e;
				264	void *assoc;
				265	u32 *b_size;
				266	char *iv;
				267	int ret;
				268
				269
				270	if (aad_size >= PAGE_SIZE) {
				271	pr_err("associate data length (%u) too big\n", aad_size);
				272	return;
				273	}
				274
				275	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
				276	if (!iv)
				277	return;
				278
				279	if (enc == ENCRYPT)
				280	e = "encryption";
				281	else
				282	e = "decryption";
				283
				284	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
				285	if (!data)
				286	goto out_free_iv;
				287
				288	tfm = crypto_alloc_aead(algo, 0, 0);
				289	if (IS_ERR(tfm)) {
				290	pr_err("failed to load transform for %s: %ld\n",
				291	algo, PTR_ERR(tfm));
				292	goto out_free_data;
				293	}
				294
				295	ret = crypto_aead_setauthsize(tfm, authsize);
				296
				297	for (i = 0; i < num_mb; ++i)
				298	if (testmgr_alloc_buf(data[i].xbuf)) {
				299	while (i--)
				300	testmgr_free_buf(data[i].xbuf);
				301	goto out_free_tfm;
				302	}
				303
				304	for (i = 0; i < num_mb; ++i)
				305	if (testmgr_alloc_buf(data[i].axbuf)) {
				306	while (i--)
				307	testmgr_free_buf(data[i].axbuf);
				308	goto out_free_xbuf;
				309	}
				310
				311	for (i = 0; i < num_mb; ++i)
				312	if (testmgr_alloc_buf(data[i].xoutbuf)) {
				313	while (i--)
				314	testmgr_free_buf(data[i].xoutbuf);
				315	goto out_free_axbuf;
				316	}
				317
				318	for (i = 0; i < num_mb; ++i) {
				319	data[i].req = aead_request_alloc(tfm, GFP_KERNEL);
				320	if (!data[i].req) {
				321	pr_err("alg: skcipher: Failed to allocate request for %s\n",
				322	algo);
				323	while (i--)
				324	aead_request_free(data[i].req);
				325	goto out_free_xoutbuf;
				326	}
				327	}
				328
				329	for (i = 0; i < num_mb; ++i) {
				330	crypto_init_wait(&data[i].wait);
				331	aead_request_set_callback(data[i].req,
				332	CRYPTO_TFM_REQ_MAY_BACKLOG,
				333	crypto_req_done, &data[i].wait);
				334	}
				335
				336	pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
				337	get_driver_name(crypto_aead, tfm), e);
				338
				339	i = 0;
				340	do {
				341	b_size = aead_sizes;
				342	do {
				343	if (b_size + authsize > XBUFSIZE PAGE_SIZE) {
				344	pr_err("template (%u) too big for buffer (%lu)\n",
				345	authsize + *b_size,
				346	XBUFSIZE * PAGE_SIZE);
				347	goto out;
				348	}
				349
				350	pr_info("test %u (%d bit key, %d byte blocks): ", i,
				351	keysize 8, *b_size);
				352
				353	/* Set up tfm global state, i.e. the key */
				354
				355	memset(tvmem[0], 0xff, PAGE_SIZE);
				356	key = tvmem[0];
				357	for (j = 0; j < tcount; j++) {
				358	if (template[j].klen == *keysize) {
				359	key = template[j].key;
				360	break;
				361	}
				362	}
				363
				364	crypto_aead_clear_flags(tfm, ~0);
				365
				366	ret = crypto_aead_setkey(tfm, key, *keysize);
				367	if (ret) {
				368	pr_err("setkey() failed flags=%x\n",
				369	crypto_aead_get_flags(tfm));
				370	goto out;
				371	}
				372
				373	iv_len = crypto_aead_ivsize(tfm);
				374	if (iv_len)
				375	memset(iv, 0xff, iv_len);
				376
				377	/* Now setup per request stuff, i.e. buffers */
				378
				379	for (j = 0; j < num_mb; ++j) {
				380	struct test_mb_aead_data *cur = &data[j];
				381
				382	assoc = cur->axbuf[0];
				383	memset(assoc, 0xff, aad_size);
				384
				385	sg_init_aead(cur->sg, cur->xbuf,
				386	*b_size + (enc ? 0 : authsize),
				387	assoc, aad_size);
				388
				389	sg_init_aead(cur->sgout, cur->xoutbuf,
				390	*b_size + (enc ? authsize : 0),
				391	assoc, aad_size);
				392
				393	aead_request_set_ad(cur->req, aad_size);
				394
				395	if (!enc) {
				396
				397	aead_request_set_crypt(cur->req,
				398	cur->sgout,
				399	cur->sg,
				400	*b_size, iv);
				401	ret = crypto_aead_encrypt(cur->req);
				402	ret = do_one_aead_op(cur->req, ret);
				403
				404	if (ret) {
				405	pr_err("calculating auth failed failed (%d)\n",
				406	ret);
				407	break;
				408	}
				409	}
				410
				411	aead_request_set_crypt(cur->req, cur->sg,
				412	cur->sgout, *b_size +
				413	(enc ? 0 : authsize),
				414	iv);
				415
				416	}
				417
				418	if (secs) {
				419	ret = test_mb_aead_jiffies(data, enc, *b_size,
				420	secs, num_mb);
				421	cond_resched();
				422	} else {
				423	ret = test_mb_aead_cycles(data, enc, *b_size,
				424	num_mb);
				425	}
				426
				427	if (ret) {
				428	pr_err("%s() failed return code=%d\n", e, ret);
				429	break;
				430	}
				431	b_size++;
				432	i++;
				433	} while (*b_size);
				434	keysize++;
				435	} while (*keysize);
				436
				437	out:
				438	for (i = 0; i < num_mb; ++i)
				439	aead_request_free(data[i].req);
				440	out_free_xoutbuf:
				441	for (i = 0; i < num_mb; ++i)
				442	testmgr_free_buf(data[i].xoutbuf);
				443	out_free_axbuf:
				444	for (i = 0; i < num_mb; ++i)
				445	testmgr_free_buf(data[i].axbuf);
				446	out_free_xbuf:
				447	for (i = 0; i < num_mb; ++i)
				448	testmgr_free_buf(data[i].xbuf);
				449	out_free_tfm:
				450	crypto_free_aead(tfm);
				451	out_free_data:
				452	kfree(data);
				453	out_free_iv:
				454	kfree(iv);
				455	}
				456
				457	static int test_aead_jiffies(struct aead_request *req, int enc,
				458	int blen, int secs)
				459	{
				460	unsigned long start, end;
				461	int bcount;
				462	int ret;
				463
				464	for (start = jiffies, end = start + secs * HZ, bcount = 0;
				465	time_before(jiffies, end); bcount++) {
				466	if (enc)
				467	ret = do_one_aead_op(req, crypto_aead_encrypt(req));
				468	else
				469	ret = do_one_aead_op(req, crypto_aead_decrypt(req));
				470
				471	if (ret)
				472	return ret;
				473	}
				474
				475	printk("%d operations in %d seconds (%ld bytes)\n",
				476	bcount, secs, (long)bcount * blen);
				477	return 0;
				478	}
				479
				480	static int test_aead_cycles(struct aead_request *req, int enc, int blen)
				481	{
				482	unsigned long cycles = 0;
				483	int ret = 0;
				484	int i;
				485
				486	/* Warm-up run. */
				487	for (i = 0; i < 4; i++) {
				488	if (enc)
				489	ret = do_one_aead_op(req, crypto_aead_encrypt(req));
				490	else
				491	ret = do_one_aead_op(req, crypto_aead_decrypt(req));
				492
				493	if (ret)
				494	goto out;
				495	}
				496
				497	/* The real thing. */
				498	for (i = 0; i < 8; i++) {
				499	cycles_t start, end;
				500
				501	start = get_cycles();
				502	if (enc)
				503	ret = do_one_aead_op(req, crypto_aead_encrypt(req));
				504	else
				505	ret = do_one_aead_op(req, crypto_aead_decrypt(req));
				506	end = get_cycles();
				507
				508	if (ret)
				509	goto out;
				510
				511	cycles += end - start;
				512	}
				513
				514	out:
				515	if (ret == 0)
				516	printk("1 operation in %lu cycles (%d bytes)\n",
				517	(cycles + 4) / 8, blen);
				518
				519	return ret;
				520	}
				521
				522	static void test_aead_speed(const char *algo, int enc, unsigned int secs,
				523	struct aead_speed_template *template,
				524	unsigned int tcount, u8 authsize,
				525	unsigned int aad_size, u8 *keysize)
				526	{
				527	unsigned int i, j;
				528	struct crypto_aead *tfm;
				529	int ret = -ENOMEM;
				530	const char *key;
				531	struct aead_request *req;
				532	struct scatterlist *sg;
				533	struct scatterlist *sgout;
				534	const char *e;
				535	void *assoc;
				536	char *iv;
				537	char *xbuf[XBUFSIZE];
				538	char *xoutbuf[XBUFSIZE];
				539	char *axbuf[XBUFSIZE];
				540	unsigned int *b_size;
				541	unsigned int iv_len;
				542	struct crypto_wait wait;
				543
				544	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
				545	if (!iv)
				546	return;
				547
				548	if (aad_size >= PAGE_SIZE) {
				549	pr_err("associate data length (%u) too big\n", aad_size);
				550	goto out_noxbuf;
				551	}
				552
				553	if (enc == ENCRYPT)
				554	e = "encryption";
				555	else
				556	e = "decryption";
				557
				558	if (testmgr_alloc_buf(xbuf))
				559	goto out_noxbuf;
				560	if (testmgr_alloc_buf(axbuf))
				561	goto out_noaxbuf;
				562	if (testmgr_alloc_buf(xoutbuf))
				563	goto out_nooutbuf;
				564
				565	sg = kmalloc(sizeof(sg) 9 * 2, GFP_KERNEL);
				566	if (!sg)
				567	goto out_nosg;
				568	sgout = &sg[9];
				569
				570	tfm = crypto_alloc_aead(algo, 0, 0);
				571
				572	if (IS_ERR(tfm)) {
				573	pr_err("alg: aead: Failed to load transform for %s: %ld\n", algo,
				574	PTR_ERR(tfm));
				575	goto out_notfm;
				576	}
				577
				578	crypto_init_wait(&wait);
				579	printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo,
				580	get_driver_name(crypto_aead, tfm), e);
				581
				582	req = aead_request_alloc(tfm, GFP_KERNEL);
				583	if (!req) {
				584	pr_err("alg: aead: Failed to allocate request for %s\n",
				585	algo);
				586	goto out_noreq;
				587	}
				588
				589	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
				590	crypto_req_done, &wait);
				591
				592	i = 0;
				593	do {
				594	b_size = aead_sizes;
				595	do {
				596	assoc = axbuf[0];
				597	memset(assoc, 0xff, aad_size);
				598
				599	if ((keysize + b_size) > TVMEMSIZE * PAGE_SIZE) {
				600	pr_err("template (%u) too big for tvmem (%lu)\n",
				601	keysize + b_size,
				602	TVMEMSIZE * PAGE_SIZE);
				603	goto out;
				604	}
				605
				606	key = tvmem[0];
				607	for (j = 0; j < tcount; j++) {
				608	if (template[j].klen == *keysize) {
				609	key = template[j].key;
				610	break;
				611	}
				612	}
				613	ret = crypto_aead_setkey(tfm, key, *keysize);
				614	ret = crypto_aead_setauthsize(tfm, authsize);
				615
				616	iv_len = crypto_aead_ivsize(tfm);
				617	if (iv_len)
				618	memset(iv, 0xff, iv_len);
				619
				620	crypto_aead_clear_flags(tfm, ~0);
				621	printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ",
				622	i, keysize 8, *b_size);
				623
				624
				625	memset(tvmem[0], 0xff, PAGE_SIZE);
				626
				627	if (ret) {
				628	pr_err("setkey() failed flags=%x\n",
				629	crypto_aead_get_flags(tfm));
				630	goto out;
				631	}
				632
				633	sg_init_aead(sg, xbuf, *b_size + (enc ? 0 : authsize),
				634	assoc, aad_size);
				635
				636	sg_init_aead(sgout, xoutbuf,
				637	*b_size + (enc ? authsize : 0), assoc,
				638	aad_size);
				639
				640	aead_request_set_ad(req, aad_size);
				641
				642	if (!enc) {
				643
				644	/*
				645	* For decryption we need a proper auth so
				646	* we do the encryption path once with buffers
				647	* reversed (input <-> output) to calculate it
				648	*/
				649	aead_request_set_crypt(req, sgout, sg,
				650	*b_size, iv);
				651	ret = do_one_aead_op(req,
				652	crypto_aead_encrypt(req));
				653
				654	if (ret) {
				655	pr_err("calculating auth failed failed (%d)\n",
				656	ret);
				657	break;
				658	}
				659	}
				660
				661	aead_request_set_crypt(req, sg, sgout,
				662	*b_size + (enc ? 0 : authsize),
				663	iv);
				664
				665	if (secs) {
				666	ret = test_aead_jiffies(req, enc, *b_size,
				667	secs);
				668	cond_resched();
				669	} else {
				670	ret = test_aead_cycles(req, enc, *b_size);
				671	}
				672
				673	if (ret) {
				674	pr_err("%s() failed return code=%d\n", e, ret);
				675	break;
				676	}
				677	b_size++;
				678	i++;
				679	} while (*b_size);
				680	keysize++;
				681	} while (*keysize);
				682
				683	out:
				684	aead_request_free(req);
				685	out_noreq:
				686	crypto_free_aead(tfm);
				687	out_notfm:
				688	kfree(sg);
				689	out_nosg:
				690	testmgr_free_buf(xoutbuf);
				691	out_nooutbuf:
				692	testmgr_free_buf(axbuf);
				693	out_noaxbuf:
				694	testmgr_free_buf(xbuf);
				695	out_noxbuf:
				696	kfree(iv);
				697	}
				698
				699	static void test_hash_sg_init(struct scatterlist *sg)
				700	{
				701	int i;
				702
				703	sg_init_table(sg, TVMEMSIZE);
				704	for (i = 0; i < TVMEMSIZE; i++) {
				705	sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
				706	memset(tvmem[i], 0xff, PAGE_SIZE);
				707	}
				708	}
				709
				710	static inline int do_one_ahash_op(struct ahash_request *req, int ret)
				711	{
				712	struct crypto_wait *wait = req->base.data;
				713
				714	return crypto_wait_req(ret, wait);
				715	}
				716
				717	struct test_mb_ahash_data {
				718	struct scatterlist sg[XBUFSIZE];
				719	char result[64];
				720	struct ahash_request *req;
				721	struct crypto_wait wait;
				722	char *xbuf[XBUFSIZE];
				723	};
				724
				725	static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb,
				726	int *rc)
				727	{
				728	int i, err = 0;
				729
				730	/* Fire up a bunch of concurrent requests */
				731	for (i = 0; i < num_mb; i++)
				732	rc[i] = crypto_ahash_digest(data[i].req);
				733
				734	/* Wait for all requests to finish */
				735	for (i = 0; i < num_mb; i++) {
				736	rc[i] = crypto_wait_req(rc[i], &data[i].wait);
				737
				738	if (rc[i]) {
				739	pr_info("concurrent request %d error %d\n", i, rc[i]);
				740	err = rc[i];
				741	}
				742	}
				743
				744	return err;
				745	}
				746
				747	static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen,
				748	int secs, u32 num_mb)
				749	{
				750	unsigned long start, end;
				751	int bcount;
				752	int ret = 0;
				753	int *rc;
				754
				755	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
				756	if (!rc)
				757	return -ENOMEM;
				758
				759	for (start = jiffies, end = start + secs * HZ, bcount = 0;
				760	time_before(jiffies, end); bcount++) {
				761	ret = do_mult_ahash_op(data, num_mb, rc);
				762	if (ret)
				763	goto out;
				764	}
				765
				766	pr_cont("%d operations in %d seconds (%ld bytes)\n",
				767	bcount * num_mb, secs, (long)bcount * blen * num_mb);
				768
				769	out:
				770	kfree(rc);
				771	return ret;
				772	}
				773
				774	static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen,
				775	u32 num_mb)
				776	{
				777	unsigned long cycles = 0;
				778	int ret = 0;
				779	int i;
				780	int *rc;
				781
				782	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
				783	if (!rc)
				784	return -ENOMEM;
				785
				786	/* Warm-up run. */
				787	for (i = 0; i < 4; i++) {
				788	ret = do_mult_ahash_op(data, num_mb, rc);
				789	if (ret)
				790	goto out;
				791	}
				792
				793	/* The real thing. */
				794	for (i = 0; i < 8; i++) {
				795	cycles_t start, end;
				796
				797	start = get_cycles();
				798	ret = do_mult_ahash_op(data, num_mb, rc);
				799	end = get_cycles();
				800
				801	if (ret)
				802	goto out;
				803
				804	cycles += end - start;
				805	}
				806
				807	pr_cont("1 operation in %lu cycles (%d bytes)\n",
				808	(cycles + 4) / (8 * num_mb), blen);
				809
				810	out:
				811	kfree(rc);
				812	return ret;
				813	}
				814
				815	static void test_mb_ahash_speed(const char *algo, unsigned int secs,
				816	struct hash_speed *speed, u32 num_mb)
				817	{
				818	struct test_mb_ahash_data *data;
				819	struct crypto_ahash *tfm;
				820	unsigned int i, j, k;
				821	int ret;
				822
				823	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
				824	if (!data)
				825	return;
				826
				827	tfm = crypto_alloc_ahash(algo, 0, 0);
				828	if (IS_ERR(tfm)) {
				829	pr_err("failed to load transform for %s: %ld\n",
				830	algo, PTR_ERR(tfm));
				831	goto free_data;
				832	}
				833
				834	for (i = 0; i < num_mb; ++i) {
				835	if (testmgr_alloc_buf(data[i].xbuf))
				836	goto out;
				837
				838	crypto_init_wait(&data[i].wait);
				839
				840	data[i].req = ahash_request_alloc(tfm, GFP_KERNEL);
				841	if (!data[i].req) {
				842	pr_err("alg: hash: Failed to allocate request for %s\n",
				843	algo);
				844	goto out;
				845	}
				846
				847	ahash_request_set_callback(data[i].req, 0, crypto_req_done,
				848	&data[i].wait);
				849
				850	sg_init_table(data[i].sg, XBUFSIZE);
				851	for (j = 0; j < XBUFSIZE; j++) {
				852	sg_set_buf(data[i].sg + j, data[i].xbuf[j], PAGE_SIZE);
				853	memset(data[i].xbuf[j], 0xff, PAGE_SIZE);
				854	}
				855	}
				856
				857	pr_info("\ntesting speed of multibuffer %s (%s)\n", algo,
				858	get_driver_name(crypto_ahash, tfm));
				859
				860	for (i = 0; speed[i].blen != 0; i++) {
				861	/* For some reason this only tests digests. */
				862	if (speed[i].blen != speed[i].plen)
				863	continue;
				864
				865	if (speed[i].blen > XBUFSIZE * PAGE_SIZE) {
				866	pr_err("template (%u) too big for tvmem (%lu)\n",
				867	speed[i].blen, XBUFSIZE * PAGE_SIZE);
				868	goto out;
				869	}
				870
				871	if (speed[i].klen)
				872	crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen);
				873
				874	for (k = 0; k < num_mb; k++)
				875	ahash_request_set_crypt(data[k].req, data[k].sg,
				876	data[k].result, speed[i].blen);
				877
				878	pr_info("test%3u "
				879	"(%5u byte blocks,%5u bytes per update,%4u updates): ",
				880	i, speed[i].blen, speed[i].plen,
				881	speed[i].blen / speed[i].plen);
				882
				883	if (secs) {
				884	ret = test_mb_ahash_jiffies(data, speed[i].blen, secs,
				885	num_mb);
				886	cond_resched();
				887	} else {
				888	ret = test_mb_ahash_cycles(data, speed[i].blen, num_mb);
				889	}
				890
				891
				892	if (ret) {
				893	pr_err("At least one hashing failed ret=%d\n", ret);
				894	break;
				895	}
				896	}
				897
				898	out:
				899	for (k = 0; k < num_mb; ++k)
				900	ahash_request_free(data[k].req);
				901
				902	for (k = 0; k < num_mb; ++k)
				903	testmgr_free_buf(data[k].xbuf);
				904
				905	crypto_free_ahash(tfm);
				906
				907	free_data:
				908	kfree(data);
				909	}
				910
				911	static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
				912	char *out, int secs)
				913	{
				914	unsigned long start, end;
				915	int bcount;
				916	int ret;
				917
				918	for (start = jiffies, end = start + secs * HZ, bcount = 0;
				919	time_before(jiffies, end); bcount++) {
				920	ret = do_one_ahash_op(req, crypto_ahash_digest(req));
				921	if (ret)
				922	return ret;
				923	}
				924
				925	printk("%6u opers/sec, %9lu bytes/sec\n",
				926	bcount / secs, ((long)bcount * blen) / secs);
				927
				928	return 0;
				929	}
				930
				931	static int test_ahash_jiffies(struct ahash_request *req, int blen,
				932	int plen, char *out, int secs)
				933	{
				934	unsigned long start, end;
				935	int bcount, pcount;
				936	int ret;
				937
				938	if (plen == blen)
				939	return test_ahash_jiffies_digest(req, blen, out, secs);
				940
				941	for (start = jiffies, end = start + secs * HZ, bcount = 0;
				942	time_before(jiffies, end); bcount++) {
				943	ret = do_one_ahash_op(req, crypto_ahash_init(req));
				944	if (ret)
				945	return ret;
				946	for (pcount = 0; pcount < blen; pcount += plen) {
				947	ret = do_one_ahash_op(req, crypto_ahash_update(req));
				948	if (ret)
				949	return ret;
				950	}
				951	/* we assume there is enough space in 'out' for the result */
				952	ret = do_one_ahash_op(req, crypto_ahash_final(req));
				953	if (ret)
				954	return ret;
				955	}
				956
				957	pr_cont("%6u opers/sec, %9lu bytes/sec\n",
				958	bcount / secs, ((long)bcount * blen) / secs);
				959
				960	return 0;
				961	}
				962
				963	static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
				964	char *out)
				965	{
				966	unsigned long cycles = 0;
				967	int ret, i;
				968
				969	/* Warm-up run. */
				970	for (i = 0; i < 4; i++) {
				971	ret = do_one_ahash_op(req, crypto_ahash_digest(req));
				972	if (ret)
				973	goto out;
				974	}
				975
				976	/* The real thing. */
				977	for (i = 0; i < 8; i++) {
				978	cycles_t start, end;
				979
				980	start = get_cycles();
				981
				982	ret = do_one_ahash_op(req, crypto_ahash_digest(req));
				983	if (ret)
				984	goto out;
				985
				986	end = get_cycles();
				987
				988	cycles += end - start;
				989	}
				990
				991	out:
				992	if (ret)
				993	return ret;
				994
				995	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
				996	cycles / 8, cycles / (8 * blen));
				997
				998	return 0;
				999	}
				1000
				1001	static int test_ahash_cycles(struct ahash_request *req, int blen,
				1002	int plen, char *out)
				1003	{
				1004	unsigned long cycles = 0;
				1005	int i, pcount, ret;
				1006
				1007	if (plen == blen)
				1008	return test_ahash_cycles_digest(req, blen, out);
				1009
				1010	/* Warm-up run. */
				1011	for (i = 0; i < 4; i++) {
				1012	ret = do_one_ahash_op(req, crypto_ahash_init(req));
				1013	if (ret)
				1014	goto out;
				1015	for (pcount = 0; pcount < blen; pcount += plen) {
				1016	ret = do_one_ahash_op(req, crypto_ahash_update(req));
				1017	if (ret)
				1018	goto out;
				1019	}
				1020	ret = do_one_ahash_op(req, crypto_ahash_final(req));
				1021	if (ret)
				1022	goto out;
				1023	}
				1024
				1025	/* The real thing. */
				1026	for (i = 0; i < 8; i++) {
				1027	cycles_t start, end;
				1028
				1029	start = get_cycles();
				1030
				1031	ret = do_one_ahash_op(req, crypto_ahash_init(req));
				1032	if (ret)
				1033	goto out;
				1034	for (pcount = 0; pcount < blen; pcount += plen) {
				1035	ret = do_one_ahash_op(req, crypto_ahash_update(req));
				1036	if (ret)
				1037	goto out;
				1038	}
				1039	ret = do_one_ahash_op(req, crypto_ahash_final(req));
				1040	if (ret)
				1041	goto out;
				1042
				1043	end = get_cycles();
				1044
				1045	cycles += end - start;
				1046	}
				1047
				1048	out:
				1049	if (ret)
				1050	return ret;
				1051
				1052	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
				1053	cycles / 8, cycles / (8 * blen));
				1054
				1055	return 0;
				1056	}
				1057
				1058	static void test_ahash_speed_common(const char *algo, unsigned int secs,
				1059	struct hash_speed *speed, unsigned mask)
				1060	{
				1061	struct scatterlist sg[TVMEMSIZE];
				1062	struct crypto_wait wait;
				1063	struct ahash_request *req;
				1064	struct crypto_ahash *tfm;
				1065	char *output;
				1066	int i, ret;
				1067
				1068	tfm = crypto_alloc_ahash(algo, 0, mask);
				1069	if (IS_ERR(tfm)) {
				1070	pr_err("failed to load transform for %s: %ld\n",
				1071	algo, PTR_ERR(tfm));
				1072	return;
				1073	}
				1074
				1075	printk(KERN_INFO "\ntesting speed of async %s (%s)\n", algo,
				1076	get_driver_name(crypto_ahash, tfm));
				1077
				1078	if (crypto_ahash_digestsize(tfm) > MAX_DIGEST_SIZE) {
				1079	pr_err("digestsize(%u) > %d\n", crypto_ahash_digestsize(tfm),
				1080	MAX_DIGEST_SIZE);
				1081	goto out;
				1082	}
				1083
				1084	test_hash_sg_init(sg);
				1085	req = ahash_request_alloc(tfm, GFP_KERNEL);
				1086	if (!req) {
				1087	pr_err("ahash request allocation failure\n");
				1088	goto out;
				1089	}
				1090
				1091	crypto_init_wait(&wait);
				1092	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
				1093	crypto_req_done, &wait);
				1094
				1095	output = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL);
				1096	if (!output)
				1097	goto out_nomem;
				1098
				1099	for (i = 0; speed[i].blen != 0; i++) {
				1100	if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
				1101	pr_err("template (%u) too big for tvmem (%lu)\n",
				1102	speed[i].blen, TVMEMSIZE * PAGE_SIZE);
				1103	break;
				1104	}
				1105
				1106	if (speed[i].klen)
				1107	crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen);
				1108
				1109	pr_info("test%3u "
				1110	"(%5u byte blocks,%5u bytes per update,%4u updates): ",
				1111	i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
				1112
				1113	ahash_request_set_crypt(req, sg, output, speed[i].plen);
				1114
				1115	if (secs) {
				1116	ret = test_ahash_jiffies(req, speed[i].blen,
				1117	speed[i].plen, output, secs);
				1118	cond_resched();
				1119	} else {
				1120	ret = test_ahash_cycles(req, speed[i].blen,
				1121	speed[i].plen, output);
				1122	}
				1123
				1124	if (ret) {
				1125	pr_err("hashing failed ret=%d\n", ret);
				1126	break;
				1127	}
				1128	}
				1129
				1130	kfree(output);
				1131
				1132	out_nomem:
				1133	ahash_request_free(req);
				1134
				1135	out:
				1136	crypto_free_ahash(tfm);
				1137	}
				1138
				1139	static void test_ahash_speed(const char *algo, unsigned int secs,
				1140	struct hash_speed *speed)
				1141	{
				1142	return test_ahash_speed_common(algo, secs, speed, 0);
				1143	}
				1144
				1145	static void test_hash_speed(const char *algo, unsigned int secs,
				1146	struct hash_speed *speed)
				1147	{
				1148	return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
				1149	}
				1150
				1151	struct test_mb_skcipher_data {
				1152	struct scatterlist sg[XBUFSIZE];
				1153	struct skcipher_request *req;
				1154	struct crypto_wait wait;
				1155	char *xbuf[XBUFSIZE];
				1156	};
				1157
				1158	static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc,
				1159	u32 num_mb, int *rc)
				1160	{
				1161	int i, err = 0;
				1162
				1163	/* Fire up a bunch of concurrent requests */
				1164	for (i = 0; i < num_mb; i++) {
				1165	if (enc == ENCRYPT)
				1166	rc[i] = crypto_skcipher_encrypt(data[i].req);
				1167	else
				1168	rc[i] = crypto_skcipher_decrypt(data[i].req);
				1169	}
				1170
				1171	/* Wait for all requests to finish */
				1172	for (i = 0; i < num_mb; i++) {
				1173	rc[i] = crypto_wait_req(rc[i], &data[i].wait);
				1174
				1175	if (rc[i]) {
				1176	pr_info("concurrent request %d error %d\n", i, rc[i]);
				1177	err = rc[i];
				1178	}
				1179	}
				1180
				1181	return err;
				1182	}
				1183
				1184	static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc,
				1185	int blen, int secs, u32 num_mb)
				1186	{
				1187	unsigned long start, end;
				1188	int bcount;
				1189	int ret = 0;
				1190	int *rc;
				1191
				1192	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
				1193	if (!rc)
				1194	return -ENOMEM;
				1195
				1196	for (start = jiffies, end = start + secs * HZ, bcount = 0;
				1197	time_before(jiffies, end); bcount++) {
				1198	ret = do_mult_acipher_op(data, enc, num_mb, rc);
				1199	if (ret)
				1200	goto out;
				1201	}
				1202
				1203	pr_cont("%d operations in %d seconds (%ld bytes)\n",
				1204	bcount * num_mb, secs, (long)bcount * blen * num_mb);
				1205
				1206	out:
				1207	kfree(rc);
				1208	return ret;
				1209	}
				1210
				1211	static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
				1212	int blen, u32 num_mb)
				1213	{
				1214	unsigned long cycles = 0;
				1215	int ret = 0;
				1216	int i;
				1217	int *rc;
				1218
				1219	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
				1220	if (!rc)
				1221	return -ENOMEM;
				1222
				1223	/* Warm-up run. */
				1224	for (i = 0; i < 4; i++) {
				1225	ret = do_mult_acipher_op(data, enc, num_mb, rc);
				1226	if (ret)
				1227	goto out;
				1228	}
				1229
				1230	/* The real thing. */
				1231	for (i = 0; i < 8; i++) {
				1232	cycles_t start, end;
				1233
				1234	start = get_cycles();
				1235	ret = do_mult_acipher_op(data, enc, num_mb, rc);
				1236	end = get_cycles();
				1237
				1238	if (ret)
				1239	goto out;
				1240
				1241	cycles += end - start;
				1242	}
				1243
				1244	pr_cont("1 operation in %lu cycles (%d bytes)\n",
				1245	(cycles + 4) / (8 * num_mb), blen);
				1246
				1247	out:
				1248	kfree(rc);
				1249	return ret;
				1250	}
				1251
				1252	static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
				1253	struct cipher_speed_template *template,
				1254	unsigned int tcount, u8 *keysize, u32 num_mb)
				1255	{
				1256	struct test_mb_skcipher_data *data;
				1257	struct crypto_skcipher *tfm;
				1258	unsigned int i, j, iv_len;
				1259	const char *key;
				1260	const char *e;
				1261	u32 *b_size;
				1262	char iv[128];
				1263	int ret;
				1264
				1265	if (enc == ENCRYPT)
				1266	e = "encryption";
				1267	else
				1268	e = "decryption";
				1269
				1270	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
				1271	if (!data)
				1272	return;
				1273
				1274	tfm = crypto_alloc_skcipher(algo, 0, 0);
				1275	if (IS_ERR(tfm)) {
				1276	pr_err("failed to load transform for %s: %ld\n",
				1277	algo, PTR_ERR(tfm));
				1278	goto out_free_data;
				1279	}
				1280
				1281	for (i = 0; i < num_mb; ++i)
				1282	if (testmgr_alloc_buf(data[i].xbuf)) {
				1283	while (i--)
				1284	testmgr_free_buf(data[i].xbuf);
				1285	goto out_free_tfm;
				1286	}
				1287
				1288
				1289	for (i = 0; i < num_mb; ++i)
				1290	if (testmgr_alloc_buf(data[i].xbuf)) {
				1291	while (i--)
				1292	testmgr_free_buf(data[i].xbuf);
				1293	goto out_free_tfm;
				1294	}
				1295
				1296
				1297	for (i = 0; i < num_mb; ++i) {
				1298	data[i].req = skcipher_request_alloc(tfm, GFP_KERNEL);
				1299	if (!data[i].req) {
				1300	pr_err("alg: skcipher: Failed to allocate request for %s\n",
				1301	algo);
				1302	while (i--)
				1303	skcipher_request_free(data[i].req);
				1304	goto out_free_xbuf;
				1305	}
				1306	}
				1307
				1308	for (i = 0; i < num_mb; ++i) {
				1309	skcipher_request_set_callback(data[i].req,
				1310	CRYPTO_TFM_REQ_MAY_BACKLOG,
				1311	crypto_req_done, &data[i].wait);
				1312	crypto_init_wait(&data[i].wait);
				1313	}
				1314
				1315	pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
				1316	get_driver_name(crypto_skcipher, tfm), e);
				1317
				1318	i = 0;
				1319	do {
				1320	b_size = block_sizes;
				1321	do {
				1322	if (b_size > XBUFSIZE PAGE_SIZE) {
				1323	pr_err("template (%u) too big for buffer (%lu)\n",
				1324	b_size, XBUFSIZE PAGE_SIZE);
				1325	goto out;
				1326	}
				1327
				1328	pr_info("test %u (%d bit key, %d byte blocks): ", i,
				1329	keysize 8, *b_size);
				1330
				1331	/* Set up tfm global state, i.e. the key */
				1332
				1333	memset(tvmem[0], 0xff, PAGE_SIZE);
				1334	key = tvmem[0];
				1335	for (j = 0; j < tcount; j++) {
				1336	if (template[j].klen == *keysize) {
				1337	key = template[j].key;
				1338	break;
				1339	}
				1340	}
				1341
				1342	crypto_skcipher_clear_flags(tfm, ~0);
				1343
				1344	ret = crypto_skcipher_setkey(tfm, key, *keysize);
				1345	if (ret) {
				1346	pr_err("setkey() failed flags=%x\n",
				1347	crypto_skcipher_get_flags(tfm));
				1348	goto out;
				1349	}
				1350
				1351	iv_len = crypto_skcipher_ivsize(tfm);
				1352	if (iv_len)
				1353	memset(&iv, 0xff, iv_len);
				1354
				1355	/* Now setup per request stuff, i.e. buffers */
				1356
				1357	for (j = 0; j < num_mb; ++j) {
				1358	struct test_mb_skcipher_data *cur = &data[j];
				1359	unsigned int k = *b_size;
				1360	unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE);
				1361	unsigned int p = 0;
				1362
				1363	sg_init_table(cur->sg, pages);
				1364
				1365	while (k > PAGE_SIZE) {
				1366	sg_set_buf(cur->sg + p, cur->xbuf[p],
				1367	PAGE_SIZE);
				1368	memset(cur->xbuf[p], 0xff, PAGE_SIZE);
				1369	p++;
				1370	k -= PAGE_SIZE;
				1371	}
				1372
				1373	sg_set_buf(cur->sg + p, cur->xbuf[p], k);
				1374	memset(cur->xbuf[p], 0xff, k);
				1375
				1376	skcipher_request_set_crypt(cur->req, cur->sg,
				1377	cur->sg, *b_size,
				1378	iv);
				1379	}
				1380
				1381	if (secs) {
				1382	ret = test_mb_acipher_jiffies(data, enc,
				1383	*b_size, secs,
				1384	num_mb);
				1385	cond_resched();
				1386	} else {
				1387	ret = test_mb_acipher_cycles(data, enc,
				1388	*b_size, num_mb);
				1389	}
				1390
				1391	if (ret) {
				1392	pr_err("%s() failed flags=%x\n", e,
				1393	crypto_skcipher_get_flags(tfm));
				1394	break;
				1395	}
				1396	b_size++;
				1397	i++;
				1398	} while (*b_size);
				1399	keysize++;
				1400	} while (*keysize);
				1401
				1402	out:
				1403	for (i = 0; i < num_mb; ++i)
				1404	skcipher_request_free(data[i].req);
				1405	out_free_xbuf:
				1406	for (i = 0; i < num_mb; ++i)
				1407	testmgr_free_buf(data[i].xbuf);
				1408	out_free_tfm:
				1409	crypto_free_skcipher(tfm);
				1410	out_free_data:
				1411	kfree(data);
				1412	}
				1413
				1414	static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
				1415	{
				1416	struct crypto_wait *wait = req->base.data;
				1417
				1418	return crypto_wait_req(ret, wait);
				1419	}
				1420
				1421	static int test_acipher_jiffies(struct skcipher_request *req, int enc,
				1422	int blen, int secs)
				1423	{
				1424	unsigned long start, end;
				1425	int bcount;
				1426	int ret;
				1427
				1428	for (start = jiffies, end = start + secs * HZ, bcount = 0;
				1429	time_before(jiffies, end); bcount++) {
				1430	if (enc)
				1431	ret = do_one_acipher_op(req,
				1432	crypto_skcipher_encrypt(req));
				1433	else
				1434	ret = do_one_acipher_op(req,
				1435	crypto_skcipher_decrypt(req));
				1436
				1437	if (ret)
				1438	return ret;
				1439	}
				1440
				1441	pr_cont("%d operations in %d seconds (%ld bytes)\n",
				1442	bcount, secs, (long)bcount * blen);
				1443	return 0;
				1444	}
				1445
				1446	static int test_acipher_cycles(struct skcipher_request *req, int enc,
				1447	int blen)
				1448	{
				1449	unsigned long cycles = 0;
				1450	int ret = 0;
				1451	int i;
				1452
				1453	/* Warm-up run. */
				1454	for (i = 0; i < 4; i++) {
				1455	if (enc)
				1456	ret = do_one_acipher_op(req,
				1457	crypto_skcipher_encrypt(req));
				1458	else
				1459	ret = do_one_acipher_op(req,
				1460	crypto_skcipher_decrypt(req));
				1461
				1462	if (ret)
				1463	goto out;
				1464	}
				1465
				1466	/* The real thing. */
				1467	for (i = 0; i < 8; i++) {
				1468	cycles_t start, end;
				1469
				1470	start = get_cycles();
				1471	if (enc)
				1472	ret = do_one_acipher_op(req,
				1473	crypto_skcipher_encrypt(req));
				1474	else
				1475	ret = do_one_acipher_op(req,
				1476	crypto_skcipher_decrypt(req));
				1477	end = get_cycles();
				1478
				1479	if (ret)
				1480	goto out;
				1481
				1482	cycles += end - start;
				1483	}
				1484
				1485	out:
				1486	if (ret == 0)
				1487	pr_cont("1 operation in %lu cycles (%d bytes)\n",
				1488	(cycles + 4) / 8, blen);
				1489
				1490	return ret;
				1491	}
				1492
				1493	static void test_skcipher_speed(const char *algo, int enc, unsigned int secs,
				1494	struct cipher_speed_template *template,
				1495	unsigned int tcount, u8 *keysize, bool async)
				1496	{
				1497	unsigned int ret, i, j, k, iv_len;
				1498	struct crypto_wait wait;
				1499	const char *key;
				1500	char iv[128];
				1501	struct skcipher_request *req;
				1502	struct crypto_skcipher *tfm;
				1503	const char *e;
				1504	u32 *b_size;
				1505
				1506	if (enc == ENCRYPT)
				1507	e = "encryption";
				1508	else
				1509	e = "decryption";
				1510
				1511	crypto_init_wait(&wait);
				1512
				1513	tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC);
				1514
				1515	if (IS_ERR(tfm)) {
				1516	pr_err("failed to load transform for %s: %ld\n", algo,
				1517	PTR_ERR(tfm));
				1518	return;
				1519	}
				1520
				1521	pr_info("\ntesting speed of async %s (%s) %s\n", algo,
				1522	get_driver_name(crypto_skcipher, tfm), e);
				1523
				1524	req = skcipher_request_alloc(tfm, GFP_KERNEL);
				1525	if (!req) {
				1526	pr_err("tcrypt: skcipher: Failed to allocate request for %s\n",
				1527	algo);
				1528	goto out;
				1529	}
				1530
				1531	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
				1532	crypto_req_done, &wait);
				1533
				1534	i = 0;
				1535	do {
				1536	b_size = block_sizes;
				1537
				1538	do {
				1539	struct scatterlist sg[TVMEMSIZE];
				1540
				1541	if ((keysize + b_size) > TVMEMSIZE * PAGE_SIZE) {
				1542	pr_err("template (%u) too big for "
				1543	"tvmem (%lu)\n", keysize + b_size,
				1544	TVMEMSIZE * PAGE_SIZE);
				1545	goto out_free_req;
				1546	}
				1547
				1548	pr_info("test %u (%d bit key, %d byte blocks): ", i,
				1549	keysize 8, *b_size);
				1550
				1551	memset(tvmem[0], 0xff, PAGE_SIZE);
				1552
				1553	/* set key, plain text and IV */
				1554	key = tvmem[0];
				1555	for (j = 0; j < tcount; j++) {
				1556	if (template[j].klen == *keysize) {
				1557	key = template[j].key;
				1558	break;
				1559	}
				1560	}
				1561
				1562	crypto_skcipher_clear_flags(tfm, ~0);
				1563
				1564	ret = crypto_skcipher_setkey(tfm, key, *keysize);
				1565	if (ret) {
				1566	pr_err("setkey() failed flags=%x\n",
				1567	crypto_skcipher_get_flags(tfm));
				1568	goto out_free_req;
				1569	}
				1570
				1571	k = keysize + b_size;
				1572	sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE));
				1573
				1574	if (k > PAGE_SIZE) {
				1575	sg_set_buf(sg, tvmem[0] + *keysize,
				1576	PAGE_SIZE - *keysize);
				1577	k -= PAGE_SIZE;
				1578	j = 1;
				1579	while (k > PAGE_SIZE) {
				1580	sg_set_buf(sg + j, tvmem[j], PAGE_SIZE);
				1581	memset(tvmem[j], 0xff, PAGE_SIZE);
				1582	j++;
				1583	k -= PAGE_SIZE;
				1584	}
				1585	sg_set_buf(sg + j, tvmem[j], k);
				1586	memset(tvmem[j], 0xff, k);
				1587	} else {
				1588	sg_set_buf(sg, tvmem[0] + keysize, b_size);
				1589	}
				1590
				1591	iv_len = crypto_skcipher_ivsize(tfm);
				1592	if (iv_len)
				1593	memset(&iv, 0xff, iv_len);
				1594
				1595	skcipher_request_set_crypt(req, sg, sg, *b_size, iv);
				1596
				1597	if (secs) {
				1598	ret = test_acipher_jiffies(req, enc,
				1599	*b_size, secs);
				1600	cond_resched();
				1601	} else {
				1602	ret = test_acipher_cycles(req, enc,
				1603	*b_size);
				1604	}
				1605
				1606	if (ret) {
				1607	pr_err("%s() failed flags=%x\n", e,
				1608	crypto_skcipher_get_flags(tfm));
				1609	break;
				1610	}
				1611	b_size++;
				1612	i++;
				1613	} while (*b_size);
				1614	keysize++;
				1615	} while (*keysize);
				1616
				1617	out_free_req:
				1618	skcipher_request_free(req);
				1619	out:
				1620	crypto_free_skcipher(tfm);
				1621	}
				1622
				1623	static void test_acipher_speed(const char *algo, int enc, unsigned int secs,
				1624	struct cipher_speed_template *template,
				1625	unsigned int tcount, u8 *keysize)
				1626	{
				1627	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
				1628	true);
				1629	}
				1630
				1631	static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
				1632	struct cipher_speed_template *template,
				1633	unsigned int tcount, u8 *keysize)
				1634	{
				1635	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
				1636	false);
				1637	}
				1638
				1639	static void test_available(void)
				1640	{
				1641	char **name = check;
				1642
				1643	while (*name) {
				1644	printk("alg %s ", *name);
				1645	printk(crypto_has_alg(*name, 0, 0) ?
				1646	"found\n" : "not found\n");
				1647	name++;
				1648	}
				1649	}
				1650
				1651	static inline int tcrypt_test(const char *alg)
				1652	{
				1653	int ret;
				1654
				1655	pr_debug("testing %s\n", alg);
				1656
				1657	ret = alg_test(alg, alg, 0, 0);
				1658	/* non-fips algs return -EINVAL in fips mode */
				1659	if (fips_enabled && ret == -EINVAL)
				1660	ret = 0;
				1661	return ret;
				1662	}
				1663
				1664	static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
				1665	{
				1666	int i;
				1667	int ret = 0;
				1668
				1669	switch (m) {
				1670	case 0:
				1671	if (alg) {
				1672	if (!crypto_has_alg(alg, type,
				1673	mask ?: CRYPTO_ALG_TYPE_MASK))
				1674	ret = -ENOENT;
				1675	break;
				1676	}
				1677
				1678	for (i = 1; i < 200; i++)
				1679	ret += do_test(NULL, 0, 0, i, num_mb);
				1680	break;
				1681
				1682	case 1:
				1683	ret += tcrypt_test("md5");
				1684	break;
				1685
				1686	case 2:
				1687	ret += tcrypt_test("sha1");
				1688	break;
				1689
				1690	case 3:
				1691	ret += tcrypt_test("ecb(des)");
				1692	ret += tcrypt_test("cbc(des)");
				1693	ret += tcrypt_test("ctr(des)");
				1694	break;
				1695
				1696	case 4:
				1697	ret += tcrypt_test("ecb(des3_ede)");
				1698	ret += tcrypt_test("cbc(des3_ede)");
				1699	ret += tcrypt_test("ctr(des3_ede)");
				1700	break;
				1701
				1702	case 5:
				1703	ret += tcrypt_test("md4");
				1704	break;
				1705
				1706	case 6:
				1707	ret += tcrypt_test("sha256");
				1708	break;
				1709
				1710	case 7:
				1711	ret += tcrypt_test("ecb(blowfish)");
				1712	ret += tcrypt_test("cbc(blowfish)");
				1713	ret += tcrypt_test("ctr(blowfish)");
				1714	break;
				1715
				1716	case 8:
				1717	ret += tcrypt_test("ecb(twofish)");
				1718	ret += tcrypt_test("cbc(twofish)");
				1719	ret += tcrypt_test("ctr(twofish)");
				1720	ret += tcrypt_test("lrw(twofish)");
				1721	ret += tcrypt_test("xts(twofish)");
				1722	break;
				1723
				1724	case 9:
				1725	ret += tcrypt_test("ecb(serpent)");
				1726	ret += tcrypt_test("cbc(serpent)");
				1727	ret += tcrypt_test("ctr(serpent)");
				1728	ret += tcrypt_test("lrw(serpent)");
				1729	ret += tcrypt_test("xts(serpent)");
				1730	break;
				1731
				1732	case 10:
				1733	ret += tcrypt_test("ecb(aes)");
				1734	ret += tcrypt_test("cbc(aes)");
				1735	ret += tcrypt_test("lrw(aes)");
				1736	ret += tcrypt_test("xts(aes)");
				1737	ret += tcrypt_test("ctr(aes)");
				1738	ret += tcrypt_test("rfc3686(ctr(aes))");
				1739	break;
				1740
				1741	case 11:
				1742	ret += tcrypt_test("sha384");
				1743	break;
				1744
				1745	case 12:
				1746	ret += tcrypt_test("sha512");
				1747	break;
				1748
				1749	case 13:
				1750	ret += tcrypt_test("deflate");
				1751	break;
				1752
				1753	case 14:
				1754	ret += tcrypt_test("ecb(cast5)");
				1755	ret += tcrypt_test("cbc(cast5)");
				1756	ret += tcrypt_test("ctr(cast5)");
				1757	break;
				1758
				1759	case 15:
				1760	ret += tcrypt_test("ecb(cast6)");
				1761	ret += tcrypt_test("cbc(cast6)");
				1762	ret += tcrypt_test("ctr(cast6)");
				1763	ret += tcrypt_test("lrw(cast6)");
				1764	ret += tcrypt_test("xts(cast6)");
				1765	break;
				1766
				1767	case 16:
				1768	ret += tcrypt_test("ecb(arc4)");
				1769	break;
				1770
				1771	case 17:
				1772	ret += tcrypt_test("michael_mic");
				1773	break;
				1774
				1775	case 18:
				1776	ret += tcrypt_test("crc32c");
				1777	break;
				1778
				1779	case 19:
				1780	ret += tcrypt_test("ecb(tea)");
				1781	break;
				1782
				1783	case 20:
				1784	ret += tcrypt_test("ecb(xtea)");
				1785	break;
				1786
				1787	case 21:
				1788	ret += tcrypt_test("ecb(khazad)");
				1789	break;
				1790
				1791	case 22:
				1792	ret += tcrypt_test("wp512");
				1793	break;
				1794
				1795	case 23:
				1796	ret += tcrypt_test("wp384");
				1797	break;
				1798
				1799	case 24:
				1800	ret += tcrypt_test("wp256");
				1801	break;
				1802
				1803	case 25:
				1804	ret += tcrypt_test("ecb(tnepres)");
				1805	break;
				1806
				1807	case 26:
				1808	ret += tcrypt_test("ecb(anubis)");
				1809	ret += tcrypt_test("cbc(anubis)");
				1810	break;
				1811
				1812	case 27:
				1813	ret += tcrypt_test("tgr192");
				1814	break;
				1815
				1816	case 28:
				1817	ret += tcrypt_test("tgr160");
				1818	break;
				1819
				1820	case 29:
				1821	ret += tcrypt_test("tgr128");
				1822	break;
				1823
				1824	case 30:
				1825	ret += tcrypt_test("ecb(xeta)");
				1826	break;
				1827
				1828	case 31:
				1829	ret += tcrypt_test("pcbc(fcrypt)");
				1830	break;
				1831
				1832	case 32:
				1833	ret += tcrypt_test("ecb(camellia)");
				1834	ret += tcrypt_test("cbc(camellia)");
				1835	ret += tcrypt_test("ctr(camellia)");
				1836	ret += tcrypt_test("lrw(camellia)");
				1837	ret += tcrypt_test("xts(camellia)");
				1838	break;
				1839
				1840	case 33:
				1841	ret += tcrypt_test("sha224");
				1842	break;
				1843
				1844	case 34:
				1845	ret += tcrypt_test("salsa20");
				1846	break;
				1847
				1848	case 35:
				1849	ret += tcrypt_test("gcm(aes)");
				1850	break;
				1851
				1852	case 36:
				1853	ret += tcrypt_test("lzo");
				1854	break;
				1855
				1856	case 37:
				1857	ret += tcrypt_test("ccm(aes)");
				1858	break;
				1859
				1860	case 38:
				1861	ret += tcrypt_test("cts(cbc(aes))");
				1862	break;
				1863
				1864	case 39:
				1865	ret += tcrypt_test("rmd128");
				1866	break;
				1867
				1868	case 40:
				1869	ret += tcrypt_test("rmd160");
				1870	break;
				1871
				1872	case 41:
				1873	ret += tcrypt_test("rmd256");
				1874	break;
				1875
				1876	case 42:
				1877	ret += tcrypt_test("rmd320");
				1878	break;
				1879
				1880	case 43:
				1881	ret += tcrypt_test("ecb(seed)");
				1882	break;
				1883
				1884	case 44:
				1885	ret += tcrypt_test("zlib");
				1886	break;
				1887
				1888	case 45:
				1889	ret += tcrypt_test("rfc4309(ccm(aes))");
				1890	break;
				1891
				1892	case 46:
				1893	ret += tcrypt_test("ghash");
				1894	break;
				1895
				1896	case 47:
				1897	ret += tcrypt_test("crct10dif");
				1898	break;
				1899
				1900	case 48:
				1901	ret += tcrypt_test("sha3-224");
				1902	break;
				1903
				1904	case 49:
				1905	ret += tcrypt_test("sha3-256");
				1906	break;
				1907
				1908	case 50:
				1909	ret += tcrypt_test("sha3-384");
				1910	break;
				1911
				1912	case 51:
				1913	ret += tcrypt_test("sha3-512");
				1914	break;
				1915
				1916	case 52:
				1917	ret += tcrypt_test("sm3");
				1918	break;
				1919
				1920	case 100:
				1921	ret += tcrypt_test("hmac(md5)");
				1922	break;
				1923
				1924	case 101:
				1925	ret += tcrypt_test("hmac(sha1)");
				1926	break;
				1927
				1928	case 102:
				1929	ret += tcrypt_test("hmac(sha256)");
				1930	break;
				1931
				1932	case 103:
				1933	ret += tcrypt_test("hmac(sha384)");
				1934	break;
				1935
				1936	case 104:
				1937	ret += tcrypt_test("hmac(sha512)");
				1938	break;
				1939
				1940	case 105:
				1941	ret += tcrypt_test("hmac(sha224)");
				1942	break;
				1943
				1944	case 106:
				1945	ret += tcrypt_test("xcbc(aes)");
				1946	break;
				1947
				1948	case 107:
				1949	ret += tcrypt_test("hmac(rmd128)");
				1950	break;
				1951
				1952	case 108:
				1953	ret += tcrypt_test("hmac(rmd160)");
				1954	break;
				1955
				1956	case 109:
				1957	ret += tcrypt_test("vmac64(aes)");
				1958	break;
				1959
				1960	case 111:
				1961	ret += tcrypt_test("hmac(sha3-224)");
				1962	break;
				1963
				1964	case 112:
				1965	ret += tcrypt_test("hmac(sha3-256)");
				1966	break;
				1967
				1968	case 113:
				1969	ret += tcrypt_test("hmac(sha3-384)");
				1970	break;
				1971
				1972	case 114:
				1973	ret += tcrypt_test("hmac(sha3-512)");
				1974	break;
				1975
				1976	case 150:
				1977	ret += tcrypt_test("ansi_cprng");
				1978	break;
				1979
				1980	case 151:
				1981	ret += tcrypt_test("rfc4106(gcm(aes))");
				1982	break;
				1983
				1984	case 152:
				1985	ret += tcrypt_test("rfc4543(gcm(aes))");
				1986	break;
				1987
				1988	case 153:
				1989	ret += tcrypt_test("cmac(aes)");
				1990	break;
				1991
				1992	case 154:
				1993	ret += tcrypt_test("cmac(des3_ede)");
				1994	break;
				1995
				1996	case 155:
				1997	ret += tcrypt_test("authenc(hmac(sha1),cbc(aes))");
				1998	break;
				1999
				2000	case 156:
				2001	ret += tcrypt_test("authenc(hmac(md5),ecb(cipher_null))");
				2002	break;
				2003
				2004	case 157:
				2005	ret += tcrypt_test("authenc(hmac(sha1),ecb(cipher_null))");
				2006	break;
				2007	case 181:
				2008	ret += tcrypt_test("authenc(hmac(sha1),cbc(des))");
				2009	break;
				2010	case 182:
				2011	ret += tcrypt_test("authenc(hmac(sha1),cbc(des3_ede))");
				2012	break;
				2013	case 183:
				2014	ret += tcrypt_test("authenc(hmac(sha224),cbc(des))");
				2015	break;
				2016	case 184:
				2017	ret += tcrypt_test("authenc(hmac(sha224),cbc(des3_ede))");
				2018	break;
				2019	case 185:
				2020	ret += tcrypt_test("authenc(hmac(sha256),cbc(des))");
				2021	break;
				2022	case 186:
				2023	ret += tcrypt_test("authenc(hmac(sha256),cbc(des3_ede))");
				2024	break;
				2025	case 187:
				2026	ret += tcrypt_test("authenc(hmac(sha384),cbc(des))");
				2027	break;
				2028	case 188:
				2029	ret += tcrypt_test("authenc(hmac(sha384),cbc(des3_ede))");
				2030	break;
				2031	case 189:
				2032	ret += tcrypt_test("authenc(hmac(sha512),cbc(des))");
				2033	break;
				2034	case 190:
				2035	ret += tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))");
				2036	break;
				2037	case 191:
				2038	ret += tcrypt_test("ecb(sm4)");
				2039	break;
				2040	case 200:
				2041	test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
				2042	speed_template_16_24_32);
				2043	test_cipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
				2044	speed_template_16_24_32);
				2045	test_cipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
				2046	speed_template_16_24_32);
				2047	test_cipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
				2048	speed_template_16_24_32);
				2049	test_cipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
				2050	speed_template_32_40_48);
				2051	test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
				2052	speed_template_32_40_48);
				2053	test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
				2054	speed_template_32_64);
				2055	test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
				2056	speed_template_32_64);
				2057	test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
				2058	speed_template_16_24_32);
				2059	test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
				2060	speed_template_16_24_32);
				2061	test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
				2062	speed_template_16_24_32);
				2063	test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
				2064	speed_template_16_24_32);
				2065	break;
				2066
				2067	case 201:
				2068	test_cipher_speed("ecb(des3_ede)", ENCRYPT, sec,
				2069	des3_speed_template, DES3_SPEED_VECTORS,
				2070	speed_template_24);
				2071	test_cipher_speed("ecb(des3_ede)", DECRYPT, sec,
				2072	des3_speed_template, DES3_SPEED_VECTORS,
				2073	speed_template_24);
				2074	test_cipher_speed("cbc(des3_ede)", ENCRYPT, sec,
				2075	des3_speed_template, DES3_SPEED_VECTORS,
				2076	speed_template_24);
				2077	test_cipher_speed("cbc(des3_ede)", DECRYPT, sec,
				2078	des3_speed_template, DES3_SPEED_VECTORS,
				2079	speed_template_24);
				2080	test_cipher_speed("ctr(des3_ede)", ENCRYPT, sec,
				2081	des3_speed_template, DES3_SPEED_VECTORS,
				2082	speed_template_24);
				2083	test_cipher_speed("ctr(des3_ede)", DECRYPT, sec,
				2084	des3_speed_template, DES3_SPEED_VECTORS,
				2085	speed_template_24);
				2086	break;
				2087
				2088	case 202:
				2089	test_cipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
				2090	speed_template_16_24_32);
				2091	test_cipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
				2092	speed_template_16_24_32);
				2093	test_cipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
				2094	speed_template_16_24_32);
				2095	test_cipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
				2096	speed_template_16_24_32);
				2097	test_cipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
				2098	speed_template_16_24_32);
				2099	test_cipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
				2100	speed_template_16_24_32);
				2101	test_cipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
				2102	speed_template_32_40_48);
				2103	test_cipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
				2104	speed_template_32_40_48);
				2105	test_cipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
				2106	speed_template_32_48_64);
				2107	test_cipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
				2108	speed_template_32_48_64);
				2109	break;
				2110
				2111	case 203:
				2112	test_cipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
				2113	speed_template_8_32);
				2114	test_cipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
				2115	speed_template_8_32);
				2116	test_cipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
				2117	speed_template_8_32);
				2118	test_cipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
				2119	speed_template_8_32);
				2120	test_cipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
				2121	speed_template_8_32);
				2122	test_cipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
				2123	speed_template_8_32);
				2124	break;
				2125
				2126	case 204:
				2127	test_cipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
				2128	speed_template_8);
				2129	test_cipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
				2130	speed_template_8);
				2131	test_cipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
				2132	speed_template_8);
				2133	test_cipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
				2134	speed_template_8);
				2135	break;
				2136
				2137	case 205:
				2138	test_cipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
				2139	speed_template_16_24_32);
				2140	test_cipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
				2141	speed_template_16_24_32);
				2142	test_cipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
				2143	speed_template_16_24_32);
				2144	test_cipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
				2145	speed_template_16_24_32);
				2146	test_cipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
				2147	speed_template_16_24_32);
				2148	test_cipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
				2149	speed_template_16_24_32);
				2150	test_cipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
				2151	speed_template_32_40_48);
				2152	test_cipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
				2153	speed_template_32_40_48);
				2154	test_cipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
				2155	speed_template_32_48_64);
				2156	test_cipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
				2157	speed_template_32_48_64);
				2158	break;
				2159
				2160	case 206:
				2161	test_cipher_speed("salsa20", ENCRYPT, sec, NULL, 0,
				2162	speed_template_16_32);
				2163	break;
				2164
				2165	case 207:
				2166	test_cipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
				2167	speed_template_16_32);
				2168	test_cipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
				2169	speed_template_16_32);
				2170	test_cipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
				2171	speed_template_16_32);
				2172	test_cipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
				2173	speed_template_16_32);
				2174	test_cipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
				2175	speed_template_16_32);
				2176	test_cipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
				2177	speed_template_16_32);
				2178	test_cipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
				2179	speed_template_32_48);
				2180	test_cipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
				2181	speed_template_32_48);
				2182	test_cipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
				2183	speed_template_32_64);
				2184	test_cipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
				2185	speed_template_32_64);
				2186	break;
				2187
				2188	case 208:
				2189	test_cipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
				2190	speed_template_8);
				2191	break;
				2192
				2193	case 209:
				2194	test_cipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
				2195	speed_template_8_16);
				2196	test_cipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
				2197	speed_template_8_16);
				2198	test_cipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
				2199	speed_template_8_16);
				2200	test_cipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
				2201	speed_template_8_16);
				2202	test_cipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
				2203	speed_template_8_16);
				2204	test_cipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
				2205	speed_template_8_16);
				2206	break;
				2207
				2208	case 210:
				2209	test_cipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
				2210	speed_template_16_32);
				2211	test_cipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
				2212	speed_template_16_32);
				2213	test_cipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
				2214	speed_template_16_32);
				2215	test_cipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
				2216	speed_template_16_32);
				2217	test_cipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
				2218	speed_template_16_32);
				2219	test_cipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
				2220	speed_template_16_32);
				2221	test_cipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
				2222	speed_template_32_48);
				2223	test_cipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
				2224	speed_template_32_48);
				2225	test_cipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
				2226	speed_template_32_64);
				2227	test_cipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
				2228	speed_template_32_64);
				2229	break;
				2230
				2231	case 211:
				2232	test_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec,
				2233	NULL, 0, 16, 16, aead_speed_template_20);
				2234	test_aead_speed("gcm(aes)", ENCRYPT, sec,
				2235	NULL, 0, 16, 8, speed_template_16_24_32);
				2236	test_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec,
				2237	NULL, 0, 16, 16, aead_speed_template_20);
				2238	test_aead_speed("gcm(aes)", DECRYPT, sec,
				2239	NULL, 0, 16, 8, speed_template_16_24_32);
				2240	break;
				2241
				2242	case 212:
				2243	test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec,
				2244	NULL, 0, 16, 16, aead_speed_template_19);
				2245	test_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec,
				2246	NULL, 0, 16, 16, aead_speed_template_19);
				2247	break;
				2248
				2249	case 213:
				2250	test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec,
				2251	NULL, 0, 16, 8, aead_speed_template_36);
				2252	test_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, sec,
				2253	NULL, 0, 16, 8, aead_speed_template_36);
				2254	break;
				2255
				2256	case 214:
				2257	test_cipher_speed("chacha20", ENCRYPT, sec, NULL, 0,
				2258	speed_template_32);
				2259	break;
				2260
				2261	case 215:
				2262	test_mb_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, NULL,
				2263	0, 16, 16, aead_speed_template_20, num_mb);
				2264	test_mb_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8,
				2265	speed_template_16_24_32, num_mb);
				2266	test_mb_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, NULL,
				2267	0, 16, 16, aead_speed_template_20, num_mb);
				2268	test_mb_aead_speed("gcm(aes)", DECRYPT, sec, NULL, 0, 16, 8,
				2269	speed_template_16_24_32, num_mb);
				2270	break;
				2271
				2272	case 216:
				2273	test_mb_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0,
				2274	16, 16, aead_speed_template_19, num_mb);
				2275	test_mb_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, NULL, 0,
				2276	16, 16, aead_speed_template_19, num_mb);
				2277	break;
				2278
				2279	case 217:
				2280	test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT,
				2281	sec, NULL, 0, 16, 8, aead_speed_template_36,
				2282	num_mb);
				2283	test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT,
				2284	sec, NULL, 0, 16, 8, aead_speed_template_36,
				2285	num_mb);
				2286	break;
				2287
				2288	case 300:
				2289	if (alg) {
				2290	test_hash_speed(alg, sec, generic_hash_speed_template);
				2291	break;
				2292	}
				2293	/* fall through */
				2294	case 301:
				2295	test_hash_speed("md4", sec, generic_hash_speed_template);
				2296	if (mode > 300 && mode < 400) break;
				2297	/* fall through */
				2298	case 302:
				2299	test_hash_speed("md5", sec, generic_hash_speed_template);
				2300	if (mode > 300 && mode < 400) break;
				2301	/* fall through */
				2302	case 303:
				2303	test_hash_speed("sha1", sec, generic_hash_speed_template);
				2304	if (mode > 300 && mode < 400) break;
				2305	/* fall through */
				2306	case 304:
				2307	test_hash_speed("sha256", sec, generic_hash_speed_template);
				2308	if (mode > 300 && mode < 400) break;
				2309	/* fall through */
				2310	case 305:
				2311	test_hash_speed("sha384", sec, generic_hash_speed_template);
				2312	if (mode > 300 && mode < 400) break;
				2313	/* fall through */
				2314	case 306:
				2315	test_hash_speed("sha512", sec, generic_hash_speed_template);
				2316	if (mode > 300 && mode < 400) break;
				2317	/* fall through */
				2318	case 307:
				2319	test_hash_speed("wp256", sec, generic_hash_speed_template);
				2320	if (mode > 300 && mode < 400) break;
				2321	/* fall through */
				2322	case 308:
				2323	test_hash_speed("wp384", sec, generic_hash_speed_template);
				2324	if (mode > 300 && mode < 400) break;
				2325	/* fall through */
				2326	case 309:
				2327	test_hash_speed("wp512", sec, generic_hash_speed_template);
				2328	if (mode > 300 && mode < 400) break;
				2329	/* fall through */
				2330	case 310:
				2331	test_hash_speed("tgr128", sec, generic_hash_speed_template);
				2332	if (mode > 300 && mode < 400) break;
				2333	/* fall through */
				2334	case 311:
				2335	test_hash_speed("tgr160", sec, generic_hash_speed_template);
				2336	if (mode > 300 && mode < 400) break;
				2337	/* fall through */
				2338	case 312:
				2339	test_hash_speed("tgr192", sec, generic_hash_speed_template);
				2340	if (mode > 300 && mode < 400) break;
				2341	/* fall through */
				2342	case 313:
				2343	test_hash_speed("sha224", sec, generic_hash_speed_template);
				2344	if (mode > 300 && mode < 400) break;
				2345	/* fall through */
				2346	case 314:
				2347	test_hash_speed("rmd128", sec, generic_hash_speed_template);
				2348	if (mode > 300 && mode < 400) break;
				2349	/* fall through */
				2350	case 315:
				2351	test_hash_speed("rmd160", sec, generic_hash_speed_template);
				2352	if (mode > 300 && mode < 400) break;
				2353	/* fall through */
				2354	case 316:
				2355	test_hash_speed("rmd256", sec, generic_hash_speed_template);
				2356	if (mode > 300 && mode < 400) break;
				2357	/* fall through */
				2358	case 317:
				2359	test_hash_speed("rmd320", sec, generic_hash_speed_template);
				2360	if (mode > 300 && mode < 400) break;
				2361	/* fall through */
				2362	case 318:
				2363	test_hash_speed("ghash-generic", sec, hash_speed_template_16);
				2364	if (mode > 300 && mode < 400) break;
				2365	/* fall through */
				2366	case 319:
				2367	test_hash_speed("crc32c", sec, generic_hash_speed_template);
				2368	if (mode > 300 && mode < 400) break;
				2369	/* fall through */
				2370	case 320:
				2371	test_hash_speed("crct10dif", sec, generic_hash_speed_template);
				2372	if (mode > 300 && mode < 400) break;
				2373	/* fall through */
				2374	case 321:
				2375	test_hash_speed("poly1305", sec, poly1305_speed_template);
				2376	if (mode > 300 && mode < 400) break;
				2377	/* fall through */
				2378	case 322:
				2379	test_hash_speed("sha3-224", sec, generic_hash_speed_template);
				2380	if (mode > 300 && mode < 400) break;
				2381	/* fall through */
				2382	case 323:
				2383	test_hash_speed("sha3-256", sec, generic_hash_speed_template);
				2384	if (mode > 300 && mode < 400) break;
				2385	/* fall through */
				2386	case 324:
				2387	test_hash_speed("sha3-384", sec, generic_hash_speed_template);
				2388	if (mode > 300 && mode < 400) break;
				2389	/* fall through */
				2390	case 325:
				2391	test_hash_speed("sha3-512", sec, generic_hash_speed_template);
				2392	if (mode > 300 && mode < 400) break;
				2393	/* fall through */
				2394	case 326:
				2395	test_hash_speed("sm3", sec, generic_hash_speed_template);
				2396	if (mode > 300 && mode < 400) break;
				2397	/* fall through */
				2398	case 399:
				2399	break;
				2400
				2401	case 400:
				2402	if (alg) {
				2403	test_ahash_speed(alg, sec, generic_hash_speed_template);
				2404	break;
				2405	}
				2406	/* fall through */
				2407	case 401:
				2408	test_ahash_speed("md4", sec, generic_hash_speed_template);
				2409	if (mode > 400 && mode < 500) break;
				2410	/* fall through */
				2411	case 402:
				2412	test_ahash_speed("md5", sec, generic_hash_speed_template);
				2413	if (mode > 400 && mode < 500) break;
				2414	/* fall through */
				2415	case 403:
				2416	test_ahash_speed("sha1", sec, generic_hash_speed_template);
				2417	if (mode > 400 && mode < 500) break;
				2418	/* fall through */
				2419	case 404:
				2420	test_ahash_speed("sha256", sec, generic_hash_speed_template);
				2421	if (mode > 400 && mode < 500) break;
				2422	/* fall through */
				2423	case 405:
				2424	test_ahash_speed("sha384", sec, generic_hash_speed_template);
				2425	if (mode > 400 && mode < 500) break;
				2426	/* fall through */
				2427	case 406:
				2428	test_ahash_speed("sha512", sec, generic_hash_speed_template);
				2429	if (mode > 400 && mode < 500) break;
				2430	/* fall through */
				2431	case 407:
				2432	test_ahash_speed("wp256", sec, generic_hash_speed_template);
				2433	if (mode > 400 && mode < 500) break;
				2434	/* fall through */
				2435	case 408:
				2436	test_ahash_speed("wp384", sec, generic_hash_speed_template);
				2437	if (mode > 400 && mode < 500) break;
				2438	/* fall through */
				2439	case 409:
				2440	test_ahash_speed("wp512", sec, generic_hash_speed_template);
				2441	if (mode > 400 && mode < 500) break;
				2442	/* fall through */
				2443	case 410:
				2444	test_ahash_speed("tgr128", sec, generic_hash_speed_template);
				2445	if (mode > 400 && mode < 500) break;
				2446	/* fall through */
				2447	case 411:
				2448	test_ahash_speed("tgr160", sec, generic_hash_speed_template);
				2449	if (mode > 400 && mode < 500) break;
				2450	/* fall through */
				2451	case 412:
				2452	test_ahash_speed("tgr192", sec, generic_hash_speed_template);
				2453	if (mode > 400 && mode < 500) break;
				2454	/* fall through */
				2455	case 413:
				2456	test_ahash_speed("sha224", sec, generic_hash_speed_template);
				2457	if (mode > 400 && mode < 500) break;
				2458	/* fall through */
				2459	case 414:
				2460	test_ahash_speed("rmd128", sec, generic_hash_speed_template);
				2461	if (mode > 400 && mode < 500) break;
				2462	/* fall through */
				2463	case 415:
				2464	test_ahash_speed("rmd160", sec, generic_hash_speed_template);
				2465	if (mode > 400 && mode < 500) break;
				2466	/* fall through */
				2467	case 416:
				2468	test_ahash_speed("rmd256", sec, generic_hash_speed_template);
				2469	if (mode > 400 && mode < 500) break;
				2470	/* fall through */
				2471	case 417:
				2472	test_ahash_speed("rmd320", sec, generic_hash_speed_template);
				2473	if (mode > 400 && mode < 500) break;
				2474	/* fall through */
				2475	case 418:
				2476	test_ahash_speed("sha3-224", sec, generic_hash_speed_template);
				2477	if (mode > 400 && mode < 500) break;
				2478	/* fall through */
				2479	case 419:
				2480	test_ahash_speed("sha3-256", sec, generic_hash_speed_template);
				2481	if (mode > 400 && mode < 500) break;
				2482	/* fall through */
				2483	case 420:
				2484	test_ahash_speed("sha3-384", sec, generic_hash_speed_template);
				2485	if (mode > 400 && mode < 500) break;
				2486	/* fall through */
				2487	case 421:
				2488	test_ahash_speed("sha3-512", sec, generic_hash_speed_template);
				2489	if (mode > 400 && mode < 500) break;
				2490	/* fall through */
				2491	case 422:
				2492	test_mb_ahash_speed("sha1", sec, generic_hash_speed_template,
				2493	num_mb);
				2494	if (mode > 400 && mode < 500) break;
				2495	/* fall through */
				2496	case 423:
				2497	test_mb_ahash_speed("sha256", sec, generic_hash_speed_template,
				2498	num_mb);
				2499	if (mode > 400 && mode < 500) break;
				2500	/* fall through */
				2501	case 424:
				2502	test_mb_ahash_speed("sha512", sec, generic_hash_speed_template,
				2503	num_mb);
				2504	if (mode > 400 && mode < 500) break;
				2505	/* fall through */
				2506	case 425:
				2507	test_mb_ahash_speed("sm3", sec, generic_hash_speed_template,
				2508	num_mb);
				2509	if (mode > 400 && mode < 500) break;
				2510	/* fall through */
				2511	case 499:
				2512	break;
				2513
				2514	case 500:
				2515	test_acipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
				2516	speed_template_16_24_32);
				2517	test_acipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
				2518	speed_template_16_24_32);
				2519	test_acipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
				2520	speed_template_16_24_32);
				2521	test_acipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
				2522	speed_template_16_24_32);
				2523	test_acipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
				2524	speed_template_32_40_48);
				2525	test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
				2526	speed_template_32_40_48);
				2527	test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
				2528	speed_template_32_64);
				2529	test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
				2530	speed_template_32_64);
				2531	test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
				2532	speed_template_16_24_32);
				2533	test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
				2534	speed_template_16_24_32);
				2535	test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
				2536	speed_template_16_24_32);
				2537	test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
				2538	speed_template_16_24_32);
				2539	test_acipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
				2540	speed_template_16_24_32);
				2541	test_acipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
				2542	speed_template_16_24_32);
				2543	test_acipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
				2544	speed_template_16_24_32);
				2545	test_acipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
				2546	speed_template_16_24_32);
				2547	test_acipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, 0,
				2548	speed_template_20_28_36);
				2549	test_acipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, 0,
				2550	speed_template_20_28_36);
				2551	break;
				2552
				2553	case 501:
				2554	test_acipher_speed("ecb(des3_ede)", ENCRYPT, sec,
				2555	des3_speed_template, DES3_SPEED_VECTORS,
				2556	speed_template_24);
				2557	test_acipher_speed("ecb(des3_ede)", DECRYPT, sec,
				2558	des3_speed_template, DES3_SPEED_VECTORS,
				2559	speed_template_24);
				2560	test_acipher_speed("cbc(des3_ede)", ENCRYPT, sec,
				2561	des3_speed_template, DES3_SPEED_VECTORS,
				2562	speed_template_24);
				2563	test_acipher_speed("cbc(des3_ede)", DECRYPT, sec,
				2564	des3_speed_template, DES3_SPEED_VECTORS,
				2565	speed_template_24);
				2566	test_acipher_speed("cfb(des3_ede)", ENCRYPT, sec,
				2567	des3_speed_template, DES3_SPEED_VECTORS,
				2568	speed_template_24);
				2569	test_acipher_speed("cfb(des3_ede)", DECRYPT, sec,
				2570	des3_speed_template, DES3_SPEED_VECTORS,
				2571	speed_template_24);
				2572	test_acipher_speed("ofb(des3_ede)", ENCRYPT, sec,
				2573	des3_speed_template, DES3_SPEED_VECTORS,
				2574	speed_template_24);
				2575	test_acipher_speed("ofb(des3_ede)", DECRYPT, sec,
				2576	des3_speed_template, DES3_SPEED_VECTORS,
				2577	speed_template_24);
				2578	break;
				2579
				2580	case 502:
				2581	test_acipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
				2582	speed_template_8);
				2583	test_acipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
				2584	speed_template_8);
				2585	test_acipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
				2586	speed_template_8);
				2587	test_acipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
				2588	speed_template_8);
				2589	test_acipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
				2590	speed_template_8);
				2591	test_acipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
				2592	speed_template_8);
				2593	test_acipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
				2594	speed_template_8);
				2595	test_acipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
				2596	speed_template_8);
				2597	break;
				2598
				2599	case 503:
				2600	test_acipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
				2601	speed_template_16_32);
				2602	test_acipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
				2603	speed_template_16_32);
				2604	test_acipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
				2605	speed_template_16_32);
				2606	test_acipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
				2607	speed_template_16_32);
				2608	test_acipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
				2609	speed_template_16_32);
				2610	test_acipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
				2611	speed_template_16_32);
				2612	test_acipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
				2613	speed_template_32_48);
				2614	test_acipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
				2615	speed_template_32_48);
				2616	test_acipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
				2617	speed_template_32_64);
				2618	test_acipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
				2619	speed_template_32_64);
				2620	break;
				2621
				2622	case 504:
				2623	test_acipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
				2624	speed_template_16_24_32);
				2625	test_acipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
				2626	speed_template_16_24_32);
				2627	test_acipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
				2628	speed_template_16_24_32);
				2629	test_acipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
				2630	speed_template_16_24_32);
				2631	test_acipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
				2632	speed_template_16_24_32);
				2633	test_acipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
				2634	speed_template_16_24_32);
				2635	test_acipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
				2636	speed_template_32_40_48);
				2637	test_acipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
				2638	speed_template_32_40_48);
				2639	test_acipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
				2640	speed_template_32_48_64);
				2641	test_acipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
				2642	speed_template_32_48_64);
				2643	break;
				2644
				2645	case 505:
				2646	test_acipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
				2647	speed_template_8);
				2648	break;
				2649
				2650	case 506:
				2651	test_acipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
				2652	speed_template_8_16);
				2653	test_acipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
				2654	speed_template_8_16);
				2655	test_acipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
				2656	speed_template_8_16);
				2657	test_acipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
				2658	speed_template_8_16);
				2659	test_acipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
				2660	speed_template_8_16);
				2661	test_acipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
				2662	speed_template_8_16);
				2663	break;
				2664
				2665	case 507:
				2666	test_acipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
				2667	speed_template_16_32);
				2668	test_acipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
				2669	speed_template_16_32);
				2670	test_acipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
				2671	speed_template_16_32);
				2672	test_acipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
				2673	speed_template_16_32);
				2674	test_acipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
				2675	speed_template_16_32);
				2676	test_acipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
				2677	speed_template_16_32);
				2678	test_acipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
				2679	speed_template_32_48);
				2680	test_acipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
				2681	speed_template_32_48);
				2682	test_acipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
				2683	speed_template_32_64);
				2684	test_acipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
				2685	speed_template_32_64);
				2686	break;
				2687
				2688	case 508:
				2689	test_acipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
				2690	speed_template_16_32);
				2691	test_acipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
				2692	speed_template_16_32);
				2693	test_acipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
				2694	speed_template_16_32);
				2695	test_acipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
				2696	speed_template_16_32);
				2697	test_acipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
				2698	speed_template_16_32);
				2699	test_acipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
				2700	speed_template_16_32);
				2701	test_acipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
				2702	speed_template_32_48);
				2703	test_acipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
				2704	speed_template_32_48);
				2705	test_acipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
				2706	speed_template_32_64);
				2707	test_acipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
				2708	speed_template_32_64);
				2709	break;
				2710
				2711	case 509:
				2712	test_acipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
				2713	speed_template_8_32);
				2714	test_acipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
				2715	speed_template_8_32);
				2716	test_acipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
				2717	speed_template_8_32);
				2718	test_acipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
				2719	speed_template_8_32);
				2720	test_acipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
				2721	speed_template_8_32);
				2722	test_acipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
				2723	speed_template_8_32);
				2724	break;
				2725
				2726	case 600:
				2727	test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
				2728	speed_template_16_24_32, num_mb);
				2729	test_mb_skcipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
				2730	speed_template_16_24_32, num_mb);
				2731	test_mb_skcipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
				2732	speed_template_16_24_32, num_mb);
				2733	test_mb_skcipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
				2734	speed_template_16_24_32, num_mb);
				2735	test_mb_skcipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
				2736	speed_template_32_40_48, num_mb);
				2737	test_mb_skcipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
				2738	speed_template_32_40_48, num_mb);
				2739	test_mb_skcipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
				2740	speed_template_32_64, num_mb);
				2741	test_mb_skcipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
				2742	speed_template_32_64, num_mb);
				2743	test_mb_skcipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
				2744	speed_template_16_24_32, num_mb);
				2745	test_mb_skcipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
				2746	speed_template_16_24_32, num_mb);
				2747	test_mb_skcipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
				2748	speed_template_16_24_32, num_mb);
				2749	test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
				2750	speed_template_16_24_32, num_mb);
				2751	test_mb_skcipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
				2752	speed_template_16_24_32, num_mb);
				2753	test_mb_skcipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
				2754	speed_template_16_24_32, num_mb);
				2755	test_mb_skcipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
				2756	speed_template_16_24_32, num_mb);
				2757	test_mb_skcipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
				2758	speed_template_16_24_32, num_mb);
				2759	test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL,
				2760	0, speed_template_20_28_36, num_mb);
				2761	test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL,
				2762	0, speed_template_20_28_36, num_mb);
				2763	break;
				2764
				2765	case 601:
				2766	test_mb_skcipher_speed("ecb(des3_ede)", ENCRYPT, sec,
				2767	des3_speed_template, DES3_SPEED_VECTORS,
				2768	speed_template_24, num_mb);
				2769	test_mb_skcipher_speed("ecb(des3_ede)", DECRYPT, sec,
				2770	des3_speed_template, DES3_SPEED_VECTORS,
				2771	speed_template_24, num_mb);
				2772	test_mb_skcipher_speed("cbc(des3_ede)", ENCRYPT, sec,
				2773	des3_speed_template, DES3_SPEED_VECTORS,
				2774	speed_template_24, num_mb);
				2775	test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec,
				2776	des3_speed_template, DES3_SPEED_VECTORS,
				2777	speed_template_24, num_mb);
				2778	test_mb_skcipher_speed("cfb(des3_ede)", ENCRYPT, sec,
				2779	des3_speed_template, DES3_SPEED_VECTORS,
				2780	speed_template_24, num_mb);
				2781	test_mb_skcipher_speed("cfb(des3_ede)", DECRYPT, sec,
				2782	des3_speed_template, DES3_SPEED_VECTORS,
				2783	speed_template_24, num_mb);
				2784	test_mb_skcipher_speed("ofb(des3_ede)", ENCRYPT, sec,
				2785	des3_speed_template, DES3_SPEED_VECTORS,
				2786	speed_template_24, num_mb);
				2787	test_mb_skcipher_speed("ofb(des3_ede)", DECRYPT, sec,
				2788	des3_speed_template, DES3_SPEED_VECTORS,
				2789	speed_template_24, num_mb);
				2790	break;
				2791
				2792	case 602:
				2793	test_mb_skcipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
				2794	speed_template_8, num_mb);
				2795	test_mb_skcipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
				2796	speed_template_8, num_mb);
				2797	test_mb_skcipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
				2798	speed_template_8, num_mb);
				2799	test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
				2800	speed_template_8, num_mb);
				2801	test_mb_skcipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
				2802	speed_template_8, num_mb);
				2803	test_mb_skcipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
				2804	speed_template_8, num_mb);
				2805	test_mb_skcipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
				2806	speed_template_8, num_mb);
				2807	test_mb_skcipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
				2808	speed_template_8, num_mb);
				2809	break;
				2810
				2811	case 603:
				2812	test_mb_skcipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
				2813	speed_template_16_32, num_mb);
				2814	test_mb_skcipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
				2815	speed_template_16_32, num_mb);
				2816	test_mb_skcipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
				2817	speed_template_16_32, num_mb);
				2818	test_mb_skcipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
				2819	speed_template_16_32, num_mb);
				2820	test_mb_skcipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
				2821	speed_template_16_32, num_mb);
				2822	test_mb_skcipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
				2823	speed_template_16_32, num_mb);
				2824	test_mb_skcipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
				2825	speed_template_32_48, num_mb);
				2826	test_mb_skcipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
				2827	speed_template_32_48, num_mb);
				2828	test_mb_skcipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
				2829	speed_template_32_64, num_mb);
				2830	test_mb_skcipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
				2831	speed_template_32_64, num_mb);
				2832	break;
				2833
				2834	case 604:
				2835	test_mb_skcipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
				2836	speed_template_16_24_32, num_mb);
				2837	test_mb_skcipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
				2838	speed_template_16_24_32, num_mb);
				2839	test_mb_skcipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
				2840	speed_template_16_24_32, num_mb);
				2841	test_mb_skcipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
				2842	speed_template_16_24_32, num_mb);
				2843	test_mb_skcipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
				2844	speed_template_16_24_32, num_mb);
				2845	test_mb_skcipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
				2846	speed_template_16_24_32, num_mb);
				2847	test_mb_skcipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
				2848	speed_template_32_40_48, num_mb);
				2849	test_mb_skcipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
				2850	speed_template_32_40_48, num_mb);
				2851	test_mb_skcipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
				2852	speed_template_32_48_64, num_mb);
				2853	test_mb_skcipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
				2854	speed_template_32_48_64, num_mb);
				2855	break;
				2856
				2857	case 605:
				2858	test_mb_skcipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
				2859	speed_template_8, num_mb);
				2860	break;
				2861
				2862	case 606:
				2863	test_mb_skcipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
				2864	speed_template_8_16, num_mb);
				2865	test_mb_skcipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
				2866	speed_template_8_16, num_mb);
				2867	test_mb_skcipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
				2868	speed_template_8_16, num_mb);
				2869	test_mb_skcipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
				2870	speed_template_8_16, num_mb);
				2871	test_mb_skcipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
				2872	speed_template_8_16, num_mb);
				2873	test_mb_skcipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
				2874	speed_template_8_16, num_mb);
				2875	break;
				2876
				2877	case 607:
				2878	test_mb_skcipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
				2879	speed_template_16_32, num_mb);
				2880	test_mb_skcipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
				2881	speed_template_16_32, num_mb);
				2882	test_mb_skcipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
				2883	speed_template_16_32, num_mb);
				2884	test_mb_skcipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
				2885	speed_template_16_32, num_mb);
				2886	test_mb_skcipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
				2887	speed_template_16_32, num_mb);
				2888	test_mb_skcipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
				2889	speed_template_16_32, num_mb);
				2890	test_mb_skcipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
				2891	speed_template_32_48, num_mb);
				2892	test_mb_skcipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
				2893	speed_template_32_48, num_mb);
				2894	test_mb_skcipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
				2895	speed_template_32_64, num_mb);
				2896	test_mb_skcipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
				2897	speed_template_32_64, num_mb);
				2898	break;
				2899
				2900	case 608:
				2901	test_mb_skcipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
				2902	speed_template_16_32, num_mb);
				2903	test_mb_skcipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
				2904	speed_template_16_32, num_mb);
				2905	test_mb_skcipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
				2906	speed_template_16_32, num_mb);
				2907	test_mb_skcipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
				2908	speed_template_16_32, num_mb);
				2909	test_mb_skcipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
				2910	speed_template_16_32, num_mb);
				2911	test_mb_skcipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
				2912	speed_template_16_32, num_mb);
				2913	test_mb_skcipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
				2914	speed_template_32_48, num_mb);
				2915	test_mb_skcipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
				2916	speed_template_32_48, num_mb);
				2917	test_mb_skcipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
				2918	speed_template_32_64, num_mb);
				2919	test_mb_skcipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
				2920	speed_template_32_64, num_mb);
				2921	break;
				2922
				2923	case 609:
				2924	test_mb_skcipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
				2925	speed_template_8_32, num_mb);
				2926	test_mb_skcipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
				2927	speed_template_8_32, num_mb);
				2928	test_mb_skcipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
				2929	speed_template_8_32, num_mb);
				2930	test_mb_skcipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
				2931	speed_template_8_32, num_mb);
				2932	test_mb_skcipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
				2933	speed_template_8_32, num_mb);
				2934	test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
				2935	speed_template_8_32, num_mb);
				2936	break;
				2937
				2938	case 1000:
				2939	test_available();
				2940	break;
				2941	}
				2942
				2943	return ret;
				2944	}
				2945
				2946	static int __init tcrypt_mod_init(void)
				2947	{
				2948	int err = -ENOMEM;
				2949	int i;
				2950
				2951	for (i = 0; i < TVMEMSIZE; i++) {
				2952	tvmem[i] = (void *)__get_free_page(GFP_KERNEL);
				2953	if (!tvmem[i])
				2954	goto err_free_tv;
				2955	}
				2956
				2957	err = do_test(alg, type, mask, mode, num_mb);
				2958
				2959	if (err) {
				2960	printk(KERN_ERR "tcrypt: one or more tests failed!\n");
				2961	goto err_free_tv;
				2962	} else {
				2963	pr_debug("all tests passed\n");
				2964	}
				2965
				2966	/* We intentionaly return -EAGAIN to prevent keeping the module,
				2967	* unless we're running in fips mode. It does all its work from
				2968	* init() and doesn't offer any runtime functionality, but in
				2969	* the fips case, checking for a successful load is helpful.
				2970	* => we don't need it in the memory, do we?
				2971	* -- mludvig
				2972	*/
				2973	if (!fips_enabled)
				2974	err = -EAGAIN;
				2975
				2976	err_free_tv:
				2977	for (i = 0; i < TVMEMSIZE && tvmem[i]; i++)
				2978	free_page((unsigned long)tvmem[i]);
				2979
				2980	return err;
				2981	}
				2982
				2983	/*
				2984	* If an init function is provided, an exit function must also be provided
				2985	* to allow module unload.
				2986	*/
				2987	static void __exit tcrypt_mod_fini(void) { }
				2988
				2989	module_init(tcrypt_mod_init);
				2990	module_exit(tcrypt_mod_fini);
				2991
				2992	module_param(alg, charp, 0);
				2993	module_param(type, uint, 0);
				2994	module_param(mask, uint, 0);
				2995	module_param(mode, int, 0);
				2996	module_param(sec, uint, 0);
				2997	MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
				2998	"(defaults to zero which uses CPU cycles instead)");
				2999	module_param(num_mb, uint, 0000);
				3000	MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)");
				3001
				3002	MODULE_LICENSE("GPL");
				3003	MODULE_DESCRIPTION("Quick & dirty crypto testing module");
				3004	MODULE_AUTHOR("James Morris <jmorris@intercode.com.au>");