werunplugged/unplugged-kernel
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: Linux kernel 4.19 for Unplugged OS

Browse Source
This commit is contained in:
ziv committed 2025-10-06 13:52:42 +00:00
commit d30235b04c
80519 files changed
+35800681
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/842/842.h
+130
View File
@@ -0,0 +1,130 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __842_H__
#define __842_H__
/* The 842 compressed format is made up of multiple blocks, each of
* which have the format:
*
* <template>[arg1][arg2][arg3][arg4]
*
* where there are between 0 and 4 template args, depending on the specific
* template operation. For normal operations, each arg is either a specific
* number of data bytes to add to the output buffer, or an index pointing
* to a previously-written number of data bytes to copy to the output buffer.
*
* The template code is a 5-bit value. This code indicates what to do with
* the following data. Template codes from 0 to 0x19 should use the template
* table, the static "decomp_ops" table used in decompress. For each template
* (table row), there are between 1 and 4 actions; each action corresponds to
* an arg following the template code bits. Each action is either a "data"
* type action, or a "index" type action, and each action results in 2, 4, or 8
* bytes being written to the output buffer. Each template (i.e. all actions
* in the table row) will add up to 8 bytes being written to the output buffer.
* Any row with less than 4 actions is padded with noop actions, indicated by
* N0 (for which there is no corresponding arg in the compressed data buffer).
*
* "Data" actions, indicated in the table by D2, D4, and D8, mean that the
* corresponding arg is 2, 4, or 8 bytes, respectively, in the compressed data
* buffer should be copied directly to the output buffer.
*
* "Index" actions, indicated in the table by I2, I4, and I8, mean the
* corresponding arg is an index parameter that points to, respectively, a 2,
* 4, or 8 byte value already in the output buffer, that should be copied to
* the end of the output buffer. Essentially, the index points to a position
* in a ring buffer that contains the last N bytes of output buffer data.
* The number of bits for each index's arg are: 8 bits for I2, 9 bits for I4,
* and 8 bits for I8. Since each index points to a 2, 4, or 8 byte section,
* this means that I2 can reference 512 bytes ((2^8 bits = 256) * 2 bytes), I4
* can reference 2048 bytes ((2^9 = 512) * 4 bytes), and I8 can reference 2048
* bytes ((2^8 = 256) * 8 bytes). Think of it as a kind-of ring buffer for
* each of I2, I4, and I8 that are updated for each byte written to the output
* buffer. In this implementation, the output buffer is directly used for each
* index; there is no additional memory required. Note that the index is into
* a ring buffer, not a sliding window; for example, if there have been 260
* bytes written to the output buffer, an I2 index of 0 would index to byte 256
* in the output buffer, while an I2 index of 16 would index to byte 16 in the
* output buffer.
*
* There are also 3 special template codes; 0x1b for "repeat", 0x1c for
* "zeros", and 0x1e for "end". The "repeat" operation is followed by a 6 bit
* arg N indicating how many times to repeat. The last 8 bytes written to the
* output buffer are written again to the output buffer, N + 1 times. The
* "zeros" operation, which has no arg bits, writes 8 zeros to the output
* buffer. The "end" operation, which also has no arg bits, signals the end
* of the compressed data. There may be some number of padding (don't care,
* but usually 0) bits after the "end" operation bits, to fill the buffer
* length to a specific byte multiple (usually a multiple of 8, 16, or 32
* bytes).
*
* This software implementation also uses one of the undefined template values,
* 0x1d as a special "short data" template code, to represent less than 8 bytes
* of uncompressed data. It is followed by a 3 bit arg N indicating how many
* data bytes will follow, and then N bytes of data, which should be copied to
* the output buffer. This allows the software 842 compressor to accept input
* buffers that are not an exact multiple of 8 bytes long. However, those
* compressed buffers containing this sw-only template will be rejected by
* the 842 hardware decompressor, and must be decompressed with this software
* library. The 842 software compression module includes a parameter to
* disable using this sw-only "short data" template, and instead simply
* reject any input buffer that is not a multiple of 8 bytes long.
*
* After all actions for each operation code are processed, another template
* code is in the next 5 bits. The decompression ends once the "end" template
* code is detected.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/bitops.h>
#include <linux/crc32.h>
#include <asm/unaligned.h>
#include <linux/sw842.h>
/* special templates */
#define OP_REPEAT (0x1B)
#define OP_ZEROS (0x1C)
#define OP_END (0x1E)
/* sw only template - this is not in the hw design; it's used only by this
* software compressor and decompressor, to allow input buffers that aren't
* a multiple of 8.
*/
#define OP_SHORT_DATA (0x1D)
/* additional bits of each op param */
#define OP_BITS (5)
#define REPEAT_BITS (6)
#define SHORT_DATA_BITS (3)
#define I2_BITS (8)
#define I4_BITS (9)
#define I8_BITS (8)
#define CRC_BITS (32)
#define REPEAT_BITS_MAX (0x3f)
#define SHORT_DATA_BITS_MAX (0x7)
/* Arbitrary values used to indicate action */
#define OP_ACTION (0x70)
#define OP_ACTION_INDEX (0x10)
#define OP_ACTION_DATA (0x20)
#define OP_ACTION_NOOP (0x40)
#define OP_AMOUNT (0x0f)
#define OP_AMOUNT_0 (0x00)
#define OP_AMOUNT_2 (0x02)
#define OP_AMOUNT_4 (0x04)
#define OP_AMOUNT_8 (0x08)
#define D2 (OP_ACTION_DATA | OP_AMOUNT_2)
#define D4 (OP_ACTION_DATA | OP_AMOUNT_4)
#define D8 (OP_ACTION_DATA | OP_AMOUNT_8)
#define I2 (OP_ACTION_INDEX | OP_AMOUNT_2)
#define I4 (OP_ACTION_INDEX | OP_AMOUNT_4)
#define I8 (OP_ACTION_INDEX | OP_AMOUNT_8)
#define N0 (OP_ACTION_NOOP | OP_AMOUNT_0)
/* the max of the regular templates - not including the special templates */
#define OPS_MAX (0x1a)
#endif
lib/842/842_compress.c
+639
View File
@@ -0,0 +1,639 @@
/*
* 842 Software Compression
*
* Copyright (C) 2015 Dan Streetman, IBM Corp
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* See 842.h for details of the 842 compressed format.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#define MODULE_NAME "842_compress"
#include <linux/hashtable.h>
#include "842.h"
#include "842_debugfs.h"
#define SW842_HASHTABLE8_BITS (10)
#define SW842_HASHTABLE4_BITS (11)
#define SW842_HASHTABLE2_BITS (10)
/* By default, we allow compressing input buffers of any length, but we must
* use the non-standard "short data" template so the decompressor can correctly
* reproduce the uncompressed data buffer at the right length. However the
* hardware 842 compressor will not recognize the "short data" template, and
* will fail to decompress any compressed buffer containing it (I have no idea
* why anyone would want to use software to compress and hardware to decompress
* but that's beside the point). This parameter forces the compression
* function to simply reject any input buffer that isn't a multiple of 8 bytes
* long, instead of using the "short data" template, so that all compressed
* buffers produced by this function will be decompressable by the 842 hardware
* decompressor. Unless you have a specific need for that, leave this disabled
* so that any length buffer can be compressed.
*/
static bool sw842_strict;
module_param_named(strict, sw842_strict, bool, 0644);
static u8 comp_ops[OPS_MAX][5] = { /* params size in bits */
{ I8, N0, N0, N0, 0x19 }, /* 8 */
{ I4, I4, N0, N0, 0x18 }, /* 18 */
{ I4, I2, I2, N0, 0x17 }, /* 25 */
{ I2, I2, I4, N0, 0x13 }, /* 25 */
{ I2, I2, I2, I2, 0x12 }, /* 32 */
{ I4, I2, D2, N0, 0x16 }, /* 33 */
{ I4, D2, I2, N0, 0x15 }, /* 33 */
{ I2, D2, I4, N0, 0x0e }, /* 33 */
{ D2, I2, I4, N0, 0x09 }, /* 33 */
{ I2, I2, I2, D2, 0x11 }, /* 40 */
{ I2, I2, D2, I2, 0x10 }, /* 40 */
{ I2, D2, I2, I2, 0x0d }, /* 40 */
{ D2, I2, I2, I2, 0x08 }, /* 40 */
{ I4, D4, N0, N0, 0x14 }, /* 41 */
{ D4, I4, N0, N0, 0x04 }, /* 41 */
{ I2, I2, D4, N0, 0x0f }, /* 48 */
{ I2, D2, I2, D2, 0x0c }, /* 48 */
{ I2, D4, I2, N0, 0x0b }, /* 48 */
{ D2, I2, I2, D2, 0x07 }, /* 48 */
{ D2, I2, D2, I2, 0x06 }, /* 48 */
{ D4, I2, I2, N0, 0x03 }, /* 48 */
{ I2, D2, D4, N0, 0x0a }, /* 56 */
{ D2, I2, D4, N0, 0x05 }, /* 56 */
{ D4, I2, D2, N0, 0x02 }, /* 56 */
{ D4, D2, I2, N0, 0x01 }, /* 56 */
{ D8, N0, N0, N0, 0x00 }, /* 64 */
};
struct sw842_hlist_node8 {
struct hlist_node node;
u64 data;
u8 index;
};
struct sw842_hlist_node4 {
struct hlist_node node;
u32 data;
u16 index;
};
struct sw842_hlist_node2 {
struct hlist_node node;
u16 data;
u8 index;
};
#define INDEX_NOT_FOUND (-1)
#define INDEX_NOT_CHECKED (-2)
struct sw842_param {
u8 *in;
u8 *instart;
u64 ilen;
u8 *out;
u64 olen;
u8 bit;
u64 data8[1];
u32 data4[2];
u16 data2[4];
int index8[1];
int index4[2];
int index2[4];
DECLARE_HASHTABLE(htable8, SW842_HASHTABLE8_BITS);
DECLARE_HASHTABLE(htable4, SW842_HASHTABLE4_BITS);
DECLARE_HASHTABLE(htable2, SW842_HASHTABLE2_BITS);
struct sw842_hlist_node8 node8[1 << I8_BITS];
struct sw842_hlist_node4 node4[1 << I4_BITS];
struct sw842_hlist_node2 node2[1 << I2_BITS];
};
#define get_input_data(p, o, b) \
be##b##_to_cpu(get_unaligned((__be##b *)((p)->in + (o))))
#define init_hashtable_nodes(p, b) do { \
int _i; \
hash_init((p)->htable##b); \
for (_i = 0; _i < ARRAY_SIZE((p)->node##b); _i++) { \
(p)->node##b[_i].index = _i; \
(p)->node##b[_i].data = 0; \
INIT_HLIST_NODE(&(p)->node##b[_i].node); \
} \
} while (0)
#define find_index(p, b, n) ({ \
struct sw842_hlist_node##b *_n; \
p->index##b[n] = INDEX_NOT_FOUND; \
hash_for_each_possible(p->htable##b, _n, node, p->data##b[n]) { \
if (p->data##b[n] == _n->data) { \
p->index##b[n] = _n->index; \
break; \
} \
} \
p->index##b[n] >= 0; \
})
#define check_index(p, b, n) \
((p)->index##b[n] == INDEX_NOT_CHECKED \
? find_index(p, b, n) \
: (p)->index##b[n] >= 0)
#define replace_hash(p, b, i, d) do { \
struct sw842_hlist_node##b *_n = &(p)->node##b[(i)+(d)]; \
hash_del(&_n->node); \
_n->data = (p)->data##b[d]; \
pr_debug("add hash index%x %x pos %x data %lx\n", b, \
(unsigned int)_n->index, \
(unsigned int)((p)->in - (p)->instart), \
(unsigned long)_n->data); \
hash_add((p)->htable##b, &_n->node, _n->data); \
} while (0)
static u8 bmask[8] = { 0x00, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe };
static int add_bits(struct sw842_param *p, u64 d, u8 n);
static int __split_add_bits(struct sw842_param *p, u64 d, u8 n, u8 s)
{
int ret;
if (n <= s)
return -EINVAL;
ret = add_bits(p, d >> s, n - s);
if (ret)
return ret;
return add_bits(p, d & GENMASK_ULL(s - 1, 0), s);
}
static int add_bits(struct sw842_param *p, u64 d, u8 n)
{
int b = p->bit, bits = b + n, s = round_up(bits, 8) - bits;
u64 o;
u8 *out = p->out;
pr_debug("add %u bits %lx\n", (unsigned char)n, (unsigned long)d);
if (n > 64)
return -EINVAL;
/* split this up if writing to > 8 bytes (i.e. n == 64 && p->bit > 0),
* or if we're at the end of the output buffer and would write past end
*/
if (bits > 64)
return __split_add_bits(p, d, n, 32);
else if (p->olen < 8 && bits > 32 && bits <= 56)
return __split_add_bits(p, d, n, 16);
else if (p->olen < 4 && bits > 16 && bits <= 24)
return __split_add_bits(p, d, n, 8);
if (DIV_ROUND_UP(bits, 8) > p->olen)
return -ENOSPC;
o = *out & bmask[b];
d <<= s;
if (bits <= 8)
*out = o | d;
else if (bits <= 16)
put_unaligned(cpu_to_be16(o << 8 | d), (__be16 *)out);
else if (bits <= 24)
put_unaligned(cpu_to_be32(o << 24 | d << 8), (__be32 *)out);
else if (bits <= 32)
put_unaligned(cpu_to_be32(o << 24 | d), (__be32 *)out);
else if (bits <= 40)
put_unaligned(cpu_to_be64(o << 56 | d << 24), (__be64 *)out);
else if (bits <= 48)
put_unaligned(cpu_to_be64(o << 56 | d << 16), (__be64 *)out);
else if (bits <= 56)
put_unaligned(cpu_to_be64(o << 56 | d << 8), (__be64 *)out);
else
put_unaligned(cpu_to_be64(o << 56 | d), (__be64 *)out);
p->bit += n;
if (p->bit > 7) {
p->out += p->bit / 8;
p->olen -= p->bit / 8;
p->bit %= 8;
}
return 0;
}
static int add_template(struct sw842_param *p, u8 c)
{
int ret, i, b = 0;
u8 *t = comp_ops[c];
bool inv = false;
if (c >= OPS_MAX)
return -EINVAL;
pr_debug("template %x\n", t[4]);
ret = add_bits(p, t[4], OP_BITS);
if (ret)
return ret;
for (i = 0; i < 4; i++) {
pr_debug("op %x\n", t[i]);
switch (t[i] & OP_AMOUNT) {
case OP_AMOUNT_8:
if (b)
inv = true;
else if (t[i] & OP_ACTION_INDEX)
ret = add_bits(p, p->index8[0], I8_BITS);
else if (t[i] & OP_ACTION_DATA)
ret = add_bits(p, p->data8[0], 64);
else
inv = true;
break;
case OP_AMOUNT_4:
if (b == 2 && t[i] & OP_ACTION_DATA)
ret = add_bits(p, get_input_data(p, 2, 32), 32);
else if (b != 0 && b != 4)
inv = true;
else if (t[i] & OP_ACTION_INDEX)
ret = add_bits(p, p->index4[b >> 2], I4_BITS);
else if (t[i] & OP_ACTION_DATA)
ret = add_bits(p, p->data4[b >> 2], 32);
else
inv = true;
break;
case OP_AMOUNT_2:
if (b != 0 && b != 2 && b != 4 && b != 6)
inv = true;
if (t[i] & OP_ACTION_INDEX)
ret = add_bits(p, p->index2[b >> 1], I2_BITS);
else if (t[i] & OP_ACTION_DATA)
ret = add_bits(p, p->data2[b >> 1], 16);
else
inv = true;
break;
case OP_AMOUNT_0:
inv = (b != 8) || !(t[i] & OP_ACTION_NOOP);
break;
default:
inv = true;
break;
}
if (ret)
return ret;
if (inv) {
pr_err("Invalid templ %x op %d : %x %x %x %x\n",
c, i, t[0], t[1], t[2], t[3]);
return -EINVAL;
}
b += t[i] & OP_AMOUNT;
}
if (b != 8) {
pr_err("Invalid template %x len %x : %x %x %x %x\n",
c, b, t[0], t[1], t[2], t[3]);
return -EINVAL;
}
if (sw842_template_counts)
atomic_inc(&template_count[t[4]]);
return 0;
}
static int add_repeat_template(struct sw842_param *p, u8 r)
{
int ret;
/* repeat param is 0-based */
if (!r || --r > REPEAT_BITS_MAX)
return -EINVAL;
ret = add_bits(p, OP_REPEAT, OP_BITS);
if (ret)
return ret;
ret = add_bits(p, r, REPEAT_BITS);
if (ret)
return ret;
if (sw842_template_counts)
atomic_inc(&template_repeat_count);
return 0;
}
static int add_short_data_template(struct sw842_param *p, u8 b)
{
int ret, i;
if (!b || b > SHORT_DATA_BITS_MAX)
return -EINVAL;
ret = add_bits(p, OP_SHORT_DATA, OP_BITS);
if (ret)
return ret;
ret = add_bits(p, b, SHORT_DATA_BITS);
if (ret)
return ret;
for (i = 0; i < b; i++) {
ret = add_bits(p, p->in[i], 8);
if (ret)
return ret;
}
if (sw842_template_counts)
atomic_inc(&template_short_data_count);
return 0;
}
static int add_zeros_template(struct sw842_param *p)
{
int ret = add_bits(p, OP_ZEROS, OP_BITS);
if (ret)
return ret;
if (sw842_template_counts)
atomic_inc(&template_zeros_count);
return 0;
}
static int add_end_template(struct sw842_param *p)
{
int ret = add_bits(p, OP_END, OP_BITS);
if (ret)
return ret;
if (sw842_template_counts)
atomic_inc(&template_end_count);
return 0;
}
static bool check_template(struct sw842_param *p, u8 c)
{
u8 *t = comp_ops[c];
int i, match, b = 0;
if (c >= OPS_MAX)
return false;
for (i = 0; i < 4; i++) {
if (t[i] & OP_ACTION_INDEX) {
if (t[i] & OP_AMOUNT_2)
match = check_index(p, 2, b >> 1);
else if (t[i] & OP_AMOUNT_4)
match = check_index(p, 4, b >> 2);
else if (t[i] & OP_AMOUNT_8)
match = check_index(p, 8, 0);
else
return false;
if (!match)
return false;
}
b += t[i] & OP_AMOUNT;
}
return true;
}
static void get_next_data(struct sw842_param *p)
{
p->data8[0] = get_input_data(p, 0, 64);
p->data4[0] = get_input_data(p, 0, 32);
p->data4[1] = get_input_data(p, 4, 32);
p->data2[0] = get_input_data(p, 0, 16);
p->data2[1] = get_input_data(p, 2, 16);
p->data2[2] = get_input_data(p, 4, 16);
p->data2[3] = get_input_data(p, 6, 16);
}
/* update the hashtable entries.
* only call this after finding/adding the current template
* the dataN fields for the current 8 byte block must be already updated
*/
static void update_hashtables(struct sw842_param *p)
{
u64 pos = p->in - p->instart;
u64 n8 = (pos >> 3) % (1 << I8_BITS);
u64 n4 = (pos >> 2) % (1 << I4_BITS);
u64 n2 = (pos >> 1) % (1 << I2_BITS);
replace_hash(p, 8, n8, 0);
replace_hash(p, 4, n4, 0);
replace_hash(p, 4, n4, 1);
replace_hash(p, 2, n2, 0);
replace_hash(p, 2, n2, 1);
replace_hash(p, 2, n2, 2);
replace_hash(p, 2, n2, 3);
}
/* find the next template to use, and add it
* the p->dataN fields must already be set for the current 8 byte block
*/
static int process_next(struct sw842_param *p)
{
int ret, i;
p->index8[0] = INDEX_NOT_CHECKED;
p->index4[0] = INDEX_NOT_CHECKED;
p->index4[1] = INDEX_NOT_CHECKED;
p->index2[0] = INDEX_NOT_CHECKED;
p->index2[1] = INDEX_NOT_CHECKED;
p->index2[2] = INDEX_NOT_CHECKED;
p->index2[3] = INDEX_NOT_CHECKED;
/* check up to OPS_MAX - 1; last op is our fallback */
for (i = 0; i < OPS_MAX - 1; i++) {
if (check_template(p, i))
break;
}
ret = add_template(p, i);
if (ret)
return ret;
return 0;
}
/**
* sw842_compress
*
* Compress the uncompressed buffer of length @ilen at @in to the output buffer
* @out, using no more than @olen bytes, using the 842 compression format.
*
* Returns: 0 on success, error on failure. The @olen parameter
* will contain the number of output bytes written on success, or
* 0 on error.
*/
int sw842_compress(const u8 *in, unsigned int ilen,
u8 *out, unsigned int *olen, void *wmem)
{
struct sw842_param *p = (struct sw842_param *)wmem;
int ret;
u64 last, next, pad, total;
u8 repeat_count = 0;
u32 crc;
BUILD_BUG_ON(sizeof(*p) > SW842_MEM_COMPRESS);
init_hashtable_nodes(p, 8);
init_hashtable_nodes(p, 4);
init_hashtable_nodes(p, 2);
p->in = (u8 *)in;
p->instart = p->in;
p->ilen = ilen;
p->out = out;
p->olen = *olen;
p->bit = 0;
total = p->olen;
*olen = 0;
/* if using strict mode, we can only compress a multiple of 8 */
if (sw842_strict && (ilen % 8)) {
pr_err("Using strict mode, can't compress len %d\n", ilen);
return -EINVAL;
}
/* let's compress at least 8 bytes, mkay? */
if (unlikely(ilen < 8))
goto skip_comp;
/* make initial 'last' different so we don't match the first time */
last = ~get_unaligned((u64 *)p->in);
while (p->ilen > 7) {
next = get_unaligned((u64 *)p->in);
/* must get the next data, as we need to update the hashtable
* entries with the new data every time
*/
get_next_data(p);
/* we don't care about endianness in last or next;
* we're just comparing 8 bytes to another 8 bytes,
* they're both the same endianness
*/
if (next == last) {
/* repeat count bits are 0-based, so we stop at +1 */
if (++repeat_count <= REPEAT_BITS_MAX)
goto repeat;
}
if (repeat_count) {
ret = add_repeat_template(p, repeat_count);
repeat_count = 0;
if (next == last) /* reached max repeat bits */
goto repeat;
}
if (next == 0)
ret = add_zeros_template(p);
else
ret = process_next(p);
if (ret)
return ret;
repeat:
last = next;
update_hashtables(p);
p->in += 8;
p->ilen -= 8;
}
if (repeat_count) {
ret = add_repeat_template(p, repeat_count);
if (ret)
return ret;
}
skip_comp:
if (p->ilen > 0) {
ret = add_short_data_template(p, p->ilen);
if (ret)
return ret;
p->in += p->ilen;
p->ilen = 0;
}
ret = add_end_template(p);
if (ret)
return ret;
/*
* crc(0:31) is appended to target data starting with the next
* bit after End of stream template.
* nx842 calculates CRC for data in big-endian format. So doing
* same here so that sw842 decompression can be used for both
* compressed data.
*/
crc = crc32_be(0, in, ilen);
ret = add_bits(p, crc, CRC_BITS);
if (ret)
return ret;
if (p->bit) {
p->out++;
p->olen--;
p->bit = 0;
}
/* pad compressed length to multiple of 8 */
pad = (8 - ((total - p->olen) % 8)) % 8;
if (pad) {
if (pad > p->olen) /* we were so close! */
return -ENOSPC;
memset(p->out, 0, pad);
p->out += pad;
p->olen -= pad;
}
if (unlikely((total - p->olen) > UINT_MAX))
return -ENOSPC;
*olen = total - p->olen;
return 0;
}
EXPORT_SYMBOL_GPL(sw842_compress);
static int __init sw842_init(void)
{
if (sw842_template_counts)
sw842_debugfs_create();
return 0;
}
module_init(sw842_init);
static void __exit sw842_exit(void)
{
if (sw842_template_counts)
sw842_debugfs_remove();
}
module_exit(sw842_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Software 842 Compressor");
MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
lib/842/842_debugfs.h
+53
View File
@@ -0,0 +1,53 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __842_DEBUGFS_H__
#define __842_DEBUGFS_H__
#include <linux/debugfs.h>
static bool sw842_template_counts;
module_param_named(template_counts, sw842_template_counts, bool, 0444);
static atomic_t template_count[OPS_MAX], template_repeat_count,
template_zeros_count, template_short_data_count, template_end_count;
static struct dentry *sw842_debugfs_root;
static int __init sw842_debugfs_create(void)
{
umode_t m = S_IRUGO | S_IWUSR;
int i;
if (!debugfs_initialized())
return -ENODEV;
sw842_debugfs_root = debugfs_create_dir(MODULE_NAME, NULL);
if (IS_ERR(sw842_debugfs_root))
return PTR_ERR(sw842_debugfs_root);
for (i = 0; i < ARRAY_SIZE(template_count); i++) {
char name[32];
snprintf(name, 32, "template_%02x", i);
debugfs_create_atomic_t(name, m, sw842_debugfs_root,
&template_count[i]);
}
debugfs_create_atomic_t("template_repeat", m, sw842_debugfs_root,
&template_repeat_count);
debugfs_create_atomic_t("template_zeros", m, sw842_debugfs_root,
&template_zeros_count);
debugfs_create_atomic_t("template_short_data", m, sw842_debugfs_root,
&template_short_data_count);
debugfs_create_atomic_t("template_end", m, sw842_debugfs_root,
&template_end_count);
return 0;
}
static void __exit sw842_debugfs_remove(void)
{
if (sw842_debugfs_root && !IS_ERR(sw842_debugfs_root))
debugfs_remove_recursive(sw842_debugfs_root);
}
#endif
lib/842/842_decompress.c
+426
View File
@@ -0,0 +1,426 @@
/*
* 842 Software Decompression
*
* Copyright (C) 2015 Dan Streetman, IBM Corp
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* See 842.h for details of the 842 compressed format.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#define MODULE_NAME "842_decompress"
#include "842.h"
#include "842_debugfs.h"
/* rolling fifo sizes */
#define I2_FIFO_SIZE (2 * (1 << I2_BITS))
#define I4_FIFO_SIZE (4 * (1 << I4_BITS))
#define I8_FIFO_SIZE (8 * (1 << I8_BITS))
static u8 decomp_ops[OPS_MAX][4] = {
{ D8, N0, N0, N0 },
{ D4, D2, I2, N0 },
{ D4, I2, D2, N0 },
{ D4, I2, I2, N0 },
{ D4, I4, N0, N0 },
{ D2, I2, D4, N0 },
{ D2, I2, D2, I2 },
{ D2, I2, I2, D2 },
{ D2, I2, I2, I2 },
{ D2, I2, I4, N0 },
{ I2, D2, D4, N0 },
{ I2, D4, I2, N0 },
{ I2, D2, I2, D2 },
{ I2, D2, I2, I2 },
{ I2, D2, I4, N0 },
{ I2, I2, D4, N0 },
{ I2, I2, D2, I2 },
{ I2, I2, I2, D2 },
{ I2, I2, I2, I2 },
{ I2, I2, I4, N0 },
{ I4, D4, N0, N0 },
{ I4, D2, I2, N0 },
{ I4, I2, D2, N0 },
{ I4, I2, I2, N0 },
{ I4, I4, N0, N0 },
{ I8, N0, N0, N0 }
};
struct sw842_param {
u8 *in;
u8 bit;
u64 ilen;
u8 *out;
u8 *ostart;
u64 olen;
};
#define beN_to_cpu(d, s) \
((s) == 2 ? be16_to_cpu(get_unaligned((__be16 *)d)) : \
(s) == 4 ? be32_to_cpu(get_unaligned((__be32 *)d)) : \
(s) == 8 ? be64_to_cpu(get_unaligned((__be64 *)d)) : \
0)
static int next_bits(struct sw842_param *p, u64 *d, u8 n);
static int __split_next_bits(struct sw842_param *p, u64 *d, u8 n, u8 s)
{
u64 tmp = 0;
int ret;
if (n <= s) {
pr_debug("split_next_bits invalid n %u s %u\n", n, s);
return -EINVAL;
}
ret = next_bits(p, &tmp, n - s);
if (ret)
return ret;
ret = next_bits(p, d, s);
if (ret)
return ret;
*d |= tmp << s;
return 0;
}
static int next_bits(struct sw842_param *p, u64 *d, u8 n)
{
u8 *in = p->in, b = p->bit, bits = b + n;
if (n > 64) {
pr_debug("next_bits invalid n %u\n", n);
return -EINVAL;
}
/* split this up if reading > 8 bytes, or if we're at the end of
* the input buffer and would read past the end
*/
if (bits > 64)
return __split_next_bits(p, d, n, 32);
else if (p->ilen < 8 && bits > 32 && bits <= 56)
return __split_next_bits(p, d, n, 16);
else if (p->ilen < 4 && bits > 16 && bits <= 24)
return __split_next_bits(p, d, n, 8);
if (DIV_ROUND_UP(bits, 8) > p->ilen)
return -EOVERFLOW;
if (bits <= 8)
*d = *in >> (8 - bits);
else if (bits <= 16)
*d = be16_to_cpu(get_unaligned((__be16 *)in)) >> (16 - bits);
else if (bits <= 32)
*d = be32_to_cpu(get_unaligned((__be32 *)in)) >> (32 - bits);
else
*d = be64_to_cpu(get_unaligned((__be64 *)in)) >> (64 - bits);
*d &= GENMASK_ULL(n - 1, 0);
p->bit += n;
if (p->bit > 7) {
p->in += p->bit / 8;
p->ilen -= p->bit / 8;
p->bit %= 8;
}
return 0;
}
static int do_data(struct sw842_param *p, u8 n)
{
u64 v;
int ret;
if (n > p->olen)
return -ENOSPC;
ret = next_bits(p, &v, n * 8);
if (ret)
return ret;
switch (n) {
case 2:
put_unaligned(cpu_to_be16((u16)v), (__be16 *)p->out);
break;
case 4:
put_unaligned(cpu_to_be32((u32)v), (__be32 *)p->out);
break;
case 8:
put_unaligned(cpu_to_be64((u64)v), (__be64 *)p->out);
break;
default:
return -EINVAL;
}
p->out += n;
p->olen -= n;
return 0;
}
static int __do_index(struct sw842_param *p, u8 size, u8 bits, u64 fsize)
{
u64 index, offset, total = round_down(p->out - p->ostart, 8);
int ret;
ret = next_bits(p, &index, bits);
if (ret)
return ret;
offset = index * size;
/* a ring buffer of fsize is used; correct the offset */
if (total > fsize) {
/* this is where the current fifo is */
u64 section = round_down(total, fsize);
/* the current pos in the fifo */
u64 pos = total - section;
/* if the offset is past/at the pos, we need to
* go back to the last fifo section
*/
if (offset >= pos)
section -= fsize;
offset += section;
}
if (offset + size > total) {
pr_debug("index%x %lx points past end %lx\n", size,
(unsigned long)offset, (unsigned long)total);
return -EINVAL;
}
if (size != 2 && size != 4 && size != 8)
WARN(1, "__do_index invalid size %x\n", size);
else
pr_debug("index%x to %lx off %lx adjoff %lx tot %lx data %lx\n",
size, (unsigned long)index,
(unsigned long)(index * size), (unsigned long)offset,
(unsigned long)total,
(unsigned long)beN_to_cpu(&p->ostart[offset], size));
memcpy(p->out, &p->ostart[offset], size);
p->out += size;
p->olen -= size;
return 0;
}
static int do_index(struct sw842_param *p, u8 n)
{
switch (n) {
case 2:
return __do_index(p, 2, I2_BITS, I2_FIFO_SIZE);
case 4:
return __do_index(p, 4, I4_BITS, I4_FIFO_SIZE);
case 8:
return __do_index(p, 8, I8_BITS, I8_FIFO_SIZE);
default:
return -EINVAL;
}
}
static int do_op(struct sw842_param *p, u8 o)
{
int i, ret = 0;
if (o >= OPS_MAX)
return -EINVAL;
for (i = 0; i < 4; i++) {
u8 op = decomp_ops[o][i];
pr_debug("op is %x\n", op);
switch (op & OP_ACTION) {
case OP_ACTION_DATA:
ret = do_data(p, op & OP_AMOUNT);
break;
case OP_ACTION_INDEX:
ret = do_index(p, op & OP_AMOUNT);
break;
case OP_ACTION_NOOP:
break;
default:
pr_err("Internal error, invalid op %x\n", op);
return -EINVAL;
}
if (ret)
return ret;
}
if (sw842_template_counts)
atomic_inc(&template_count[o]);
return 0;
}
/**
* sw842_decompress
*
* Decompress the 842-compressed buffer of length @ilen at @in
* to the output buffer @out, using no more than @olen bytes.
*
* The compressed buffer must be only a single 842-compressed buffer,
* with the standard format described in the comments in 842.h
* Processing will stop when the 842 "END" template is detected,
* not the end of the buffer.
*
* Returns: 0 on success, error on failure. The @olen parameter
* will contain the number of output bytes written on success, or
* 0 on error.
*/
int sw842_decompress(const u8 *in, unsigned int ilen,
u8 *out, unsigned int *olen)
{
struct sw842_param p;
int ret;
u64 op, rep, tmp, bytes, total;
u64 crc;
p.in = (u8 *)in;
p.bit = 0;
p.ilen = ilen;
p.out = out;
p.ostart = out;
p.olen = *olen;
total = p.olen;
*olen = 0;
do {
ret = next_bits(&p, &op, OP_BITS);
if (ret)
return ret;
pr_debug("template is %lx\n", (unsigned long)op);
switch (op) {
case OP_REPEAT:
ret = next_bits(&p, &rep, REPEAT_BITS);
if (ret)
return ret;
if (p.out == out) /* no previous bytes */
return -EINVAL;
/* copy rep + 1 */
rep++;
if (rep * 8 > p.olen)
return -ENOSPC;
while (rep-- > 0) {
memcpy(p.out, p.out - 8, 8);
p.out += 8;
p.olen -= 8;
}
if (sw842_template_counts)
atomic_inc(&template_repeat_count);
break;
case OP_ZEROS:
if (8 > p.olen)
return -ENOSPC;
memset(p.out, 0, 8);
p.out += 8;
p.olen -= 8;
if (sw842_template_counts)
atomic_inc(&template_zeros_count);
break;
case OP_SHORT_DATA:
ret = next_bits(&p, &bytes, SHORT_DATA_BITS);
if (ret)
return ret;
if (!bytes || bytes > SHORT_DATA_BITS_MAX)
return -EINVAL;
while (bytes-- > 0) {
ret = next_bits(&p, &tmp, 8);
if (ret)
return ret;
*p.out = (u8)tmp;
p.out++;
p.olen--;
}
if (sw842_template_counts)
atomic_inc(&template_short_data_count);
break;
case OP_END:
if (sw842_template_counts)
atomic_inc(&template_end_count);
break;
default: /* use template */
ret = do_op(&p, op);
if (ret)
return ret;
break;
}
} while (op != OP_END);
/*
* crc(0:31) is saved in compressed data starting with the
* next bit after End of stream template.
*/
ret = next_bits(&p, &crc, CRC_BITS);
if (ret)
return ret;
/*
* Validate CRC saved in compressed data.
*/
if (crc != (u64)crc32_be(0, out, total - p.olen)) {
pr_debug("CRC mismatch for decompression\n");
return -EINVAL;
}
if (unlikely((total - p.olen) > UINT_MAX))
return -ENOSPC;
*olen = total - p.olen;
return 0;
}
EXPORT_SYMBOL_GPL(sw842_decompress);
static int __init sw842_init(void)
{
if (sw842_template_counts)
sw842_debugfs_create();
return 0;
}
module_init(sw842_init);
static void __exit sw842_exit(void)
{
if (sw842_template_counts)
sw842_debugfs_remove();
}
module_exit(sw842_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Software 842 Decompressor");
MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
lib/842/Makefile
+2
View File
@@ -0,0 +1,2 @@
obj-$(CONFIG_842_COMPRESS) += 842_compress.o
obj-$(CONFIG_842_DECOMPRESS) += 842_decompress.o
lib/Kconfig
+628
View File
@@ -0,0 +1,628 @@
#
# Library configuration
#
config BINARY_PRINTF
def_bool n
menu "Library routines"
config RAID6_PQ
tristate
config BITREVERSE
tristate
config HAVE_ARCH_BITREVERSE
bool
default n
depends on BITREVERSE
help
This option enables the use of hardware bit-reversal instructions on
architectures which support such operations.
config RATIONAL
bool
config GENERIC_STRNCPY_FROM_USER
bool
config GENERIC_STRNLEN_USER
bool
config GENERIC_NET_UTILS
bool
config GENERIC_FIND_FIRST_BIT
bool
config NO_GENERIC_PCI_IOPORT_MAP
bool
config GENERIC_PCI_IOMAP
bool
config GENERIC_IOMAP
bool
select GENERIC_PCI_IOMAP
config STMP_DEVICE
bool
config ARCH_USE_CMPXCHG_LOCKREF
bool
config ARCH_HAS_FAST_MULTIPLIER
bool
config INDIRECT_PIO
bool "Access I/O in non-MMIO mode"
depends on ARM64
help
On some platforms where no separate I/O space exists, there are I/O
hosts which can not be accessed in MMIO mode. Using the logical PIO
mechanism, the host-local I/O resource can be mapped into system
logic PIO space shared with MMIO hosts, such as PCI/PCIe, then the
system can access the I/O devices with the mapped-logic PIO through
I/O accessors.
This way has relatively little I/O performance cost. Please make
sure your devices really need this configure item enabled.
When in doubt, say N.
config CRC_CCITT
tristate "CRC-CCITT functions"
help
This option is provided for the case where no in-kernel-tree
modules require CRC-CCITT functions, but a module built outside
the kernel tree does. Such modules that use library CRC-CCITT
functions require M here.
config CRC16
tristate "CRC16 functions"
help
This option is provided for the case where no in-kernel-tree
modules require CRC16 functions, but a module built outside
the kernel tree does. Such modules that use library CRC16
functions require M here.
config CRC_T10DIF
tristate "CRC calculation for the T10 Data Integrity Field"
select CRYPTO
select CRYPTO_CRCT10DIF
help
This option is only needed if a module that's not in the
kernel tree needs to calculate CRC checks for use with the
SCSI data integrity subsystem.
config CRC_ITU_T
tristate "CRC ITU-T V.41 functions"
help
This option is provided for the case where no in-kernel-tree
modules require CRC ITU-T V.41 functions, but a module built outside
the kernel tree does. Such modules that use library CRC ITU-T V.41
functions require M here.
config CRC32
tristate "CRC32/CRC32c functions"
default y
select BITREVERSE
help
This option is provided for the case where no in-kernel-tree
modules require CRC32/CRC32c functions, but a module built outside
the kernel tree does. Such modules that use library CRC32/CRC32c
functions require M here.
config CRC32_SELFTEST
tristate "CRC32 perform self test on init"
depends on CRC32
help
This option enables the CRC32 library functions to perform a
self test on initialization. The self test computes crc32_le
and crc32_be over byte strings with random alignment and length
and computes the total elapsed time and number of bytes processed.
choice
prompt "CRC32 implementation"
depends on CRC32
default CRC32_SLICEBY8
help
This option allows a kernel builder to override the default choice
of CRC32 algorithm. Choose the default ("slice by 8") unless you
know that you need one of the others.
config CRC32_SLICEBY8
bool "Slice by 8 bytes"
help
Calculate checksum 8 bytes at a time with a clever slicing algorithm.
This is the fastest algorithm, but comes with a 8KiB lookup table.
Most modern processors have enough cache to hold this table without
thrashing the cache.
This is the default implementation choice. Choose this one unless
you have a good reason not to.
config CRC32_SLICEBY4
bool "Slice by 4 bytes"
help
Calculate checksum 4 bytes at a time with a clever slicing algorithm.
This is a bit slower than slice by 8, but has a smaller 4KiB lookup
table.
Only choose this option if you know what you are doing.
config CRC32_SARWATE
bool "Sarwate's Algorithm (one byte at a time)"
help
Calculate checksum a byte at a time using Sarwate's algorithm. This
is not particularly fast, but has a small 256 byte lookup table.
Only choose this option if you know what you are doing.
config CRC32_BIT
bool "Classic Algorithm (one bit at a time)"
help
Calculate checksum one bit at a time. This is VERY slow, but has
no lookup table. This is provided as a debugging option.
Only choose this option if you are debugging crc32.
endchoice
config CRC64
tristate "CRC64 functions"
help
This option is provided for the case where no in-kernel-tree
modules require CRC64 functions, but a module built outside
the kernel tree does. Such modules that use library CRC64
functions require M here.
config CRC4
tristate "CRC4 functions"
help
This option is provided for the case where no in-kernel-tree
modules require CRC4 functions, but a module built outside
the kernel tree does. Such modules that use library CRC4
functions require M here.
config CRC7
tristate "CRC7 functions"
help
This option is provided for the case where no in-kernel-tree
modules require CRC7 functions, but a module built outside
the kernel tree does. Such modules that use library CRC7
functions require M here.
config LIBCRC32C
tristate "CRC32c (Castagnoli, et al) Cyclic Redundancy-Check"
select CRYPTO
select CRYPTO_CRC32C
help
This option is provided for the case where no in-kernel-tree
modules require CRC32c functions, but a module built outside the
kernel tree does. Such modules that use library CRC32c functions
require M here. See Castagnoli93.
Module will be libcrc32c.
config CRC8
tristate "CRC8 function"
help
This option provides CRC8 function. Drivers may select this
when they need to do cyclic redundancy check according CRC8
algorithm. Module will be called crc8.
config XXHASH
tristate
config AUDIT_GENERIC
bool
depends on AUDIT && !AUDIT_ARCH
default y
config AUDIT_ARCH_COMPAT_GENERIC
bool
default n
config AUDIT_COMPAT_GENERIC
bool
depends on AUDIT_GENERIC && AUDIT_ARCH_COMPAT_GENERIC && COMPAT
default y
config RANDOM32_SELFTEST
bool "PRNG perform self test on init"
help
This option enables the 32 bit PRNG library functions to perform a
self test on initialization.
#
# compression support is select'ed if needed
#
config 842_COMPRESS
select CRC32
tristate
config 842_DECOMPRESS
select CRC32
tristate
config ZLIB_INFLATE
tristate
config ZLIB_DEFLATE
tristate
select BITREVERSE
config LZO_COMPRESS
tristate
config LZO_DECOMPRESS
tristate
config LZ4_COMPRESS
tristate
config LZ4HC_COMPRESS
tristate
config LZ4_DECOMPRESS
tristate
config ZSTD_COMPRESS
select XXHASH
tristate
config ZSTD_DECOMPRESS
select XXHASH
tristate
source "lib/xz/Kconfig"
#
# These all provide a common interface (hence the apparent duplication with
# ZLIB_INFLATE; DECOMPRESS_GZIP is just a wrapper.)
#
config DECOMPRESS_GZIP
select ZLIB_INFLATE
tristate
config DECOMPRESS_BZIP2
tristate
config DECOMPRESS_LZMA
tristate
config DECOMPRESS_XZ
select XZ_DEC
tristate
config DECOMPRESS_LZO
select LZO_DECOMPRESS
tristate
config DECOMPRESS_LZ4
select LZ4_DECOMPRESS
tristate
#
# Generic allocator support is selected if needed
#
config GENERIC_ALLOCATOR
bool
#
# reed solomon support is select'ed if needed
#
config REED_SOLOMON
tristate
config REED_SOLOMON_ENC8
bool
config REED_SOLOMON_DEC8
bool
config REED_SOLOMON_ENC16
bool
config REED_SOLOMON_DEC16
bool
#
# BCH support is selected if needed
#
config BCH
tristate
config BCH_CONST_PARAMS
bool
help
Drivers may select this option to force specific constant
values for parameters 'm' (Galois field order) and 't'
(error correction capability). Those specific values must
be set by declaring default values for symbols BCH_CONST_M
and BCH_CONST_T.
Doing so will enable extra compiler optimizations,
improving encoding and decoding performance up to 2x for
usual (m,t) values (typically such that m*t < 200).
When this option is selected, the BCH library supports
only a single (m,t) configuration. This is mainly useful
for NAND flash board drivers requiring known, fixed BCH
parameters.
config BCH_CONST_M
int
range 5 15
help
Constant value for Galois field order 'm'. If 'k' is the
number of data bits to protect, 'm' should be chosen such
that (k + m*t) <= 2**m - 1.
Drivers should declare a default value for this symbol if
they select option BCH_CONST_PARAMS.
config BCH_CONST_T
int
help
Constant value for error correction capability in bits 't'.
Drivers should declare a default value for this symbol if
they select option BCH_CONST_PARAMS.
#
# Textsearch support is select'ed if needed
#
config TEXTSEARCH
bool
config TEXTSEARCH_KMP
tristate
config TEXTSEARCH_BM
tristate
config TEXTSEARCH_FSM
tristate
config BTREE
bool
config INTERVAL_TREE
bool
help
Simple, embeddable, interval-tree. Can find the start of an
overlapping range in log(n) time and then iterate over all
overlapping nodes. The algorithm is implemented as an
augmented rbtree.
See:
Documentation/rbtree.txt
for more information.
config RADIX_TREE_MULTIORDER
bool
config ASSOCIATIVE_ARRAY
bool
help
Generic associative array. Can be searched and iterated over whilst
it is being modified. It is also reasonably quick to search and
modify. The algorithms are non-recursive, and the trees are highly
capacious.
See:
Documentation/core-api/assoc_array.rst
for more information.
config HAS_IOMEM
bool
depends on !NO_IOMEM
select GENERIC_IO
default y
config HAS_IOPORT_MAP
bool
depends on HAS_IOMEM && !NO_IOPORT_MAP
default y
source "kernel/dma/Kconfig"
config SGL_ALLOC
bool
default n
config IOMMU_HELPER
bool
config CHECK_SIGNATURE
bool
config CPUMASK_OFFSTACK
bool "Force CPU masks off stack" if DEBUG_PER_CPU_MAPS
help
Use dynamic allocation for cpumask_var_t, instead of putting
them on the stack. This is a bit more expensive, but avoids
stack overflow.
config CPU_RMAP
bool
depends on SMP
config DQL
bool
config GLOB
bool
# This actually supports modular compilation, but the module overhead
# is ridiculous for the amount of code involved. Until an out-of-tree
# driver asks for it, we'll just link it directly it into the kernel
# when required. Since we're ignoring out-of-tree users, there's also
# no need bother prompting for a manual decision:
# prompt "glob_match() function"
help
This option provides a glob_match function for performing
simple text pattern matching. It originated in the ATA code
to blacklist particular drive models, but other device drivers
may need similar functionality.
All drivers in the Linux kernel tree that require this function
should automatically select this option. Say N unless you
are compiling an out-of tree driver which tells you that it
depends on this.
config GLOB_SELFTEST
tristate "glob self-test on init"
depends on GLOB
help
This option enables a simple self-test of the glob_match
function on startup. It is primarily useful for people
working on the code to ensure they haven't introduced any
regressions.
It only adds a little bit of code and slows kernel boot (or
module load) by a small amount, so you're welcome to play with
it, but you probably don't need it.
#
# Netlink attribute parsing support is select'ed if needed
#
config NLATTR
bool
#
# Generic 64-bit atomic support is selected if needed
#
config GENERIC_ATOMIC64
bool
config LRU_CACHE
tristate
config CLZ_TAB
bool
config CORDIC
tristate "CORDIC algorithm"
help
This option provides an implementation of the CORDIC algorithm;
calculations are in fixed point. Module will be called cordic.
config DDR
bool "JEDEC DDR data"
help
Data from JEDEC specs for DDR SDRAM memories,
particularly the AC timing parameters and addressing
information. This data is useful for drivers handling
DDR SDRAM controllers.
config IRQ_POLL
bool "IRQ polling library"
help
Helper library to poll interrupt mitigation using polling.
config MPILIB
tristate
select CLZ_TAB
help
Multiprecision maths library from GnuPG.
It is used to implement RSA digital signature verification,
which is used by IMA/EVM digital signature extension.
config SIGNATURE
tristate
depends on KEYS
select CRYPTO
select CRYPTO_SHA1
select MPILIB
help
Digital signature verification. Currently only RSA is supported.
Implementation is done using GnuPG MPI library
#
# libfdt files, only selected if needed.
#
config LIBFDT
bool
config OID_REGISTRY
tristate
help
Enable fast lookup object identifier registry.
config UCS2_STRING
tristate
#
# generic vdso
#
source "lib/vdso/Kconfig"
source "lib/fonts/Kconfig"
config SG_SPLIT
def_bool n
help
Provides a helper to split scatterlists into chunks, each chunk being
a scatterlist. This should be selected by a driver or an API which
whishes to split a scatterlist amongst multiple DMA channels.
config SG_POOL
def_bool n
help
Provides a helper to allocate chained scatterlists. This should be
selected by a driver or an API which whishes to allocate chained
scatterlist.
#
# sg chaining option
#
config ARCH_HAS_SG_CHAIN
def_bool n
config ARCH_HAS_PMEM_API
bool
config ARCH_HAS_UACCESS_FLUSHCACHE
bool
config ARCH_HAS_UACCESS_MCSAFE
bool
config STACKDEPOT
bool
select STACKTRACE
config SBITMAP
bool
config PARMAN
tristate "parman" if COMPILE_TEST
config PRIME_NUMBERS
tristate
config STRING_SELFTEST
tristate "Test string functions"
endmenu
config GENERIC_LIB_ASHLDI3
bool
config GENERIC_LIB_ASHRDI3
bool
config GENERIC_LIB_LSHRDI3
bool
config GENERIC_LIB_MULDI3
bool
config GENERIC_LIB_CMPDI2
bool
config GENERIC_LIB_UCMPDI2
bool
lib/Kconfig.debug
+2078
View File
File diff suppressed because it is too large. Load diff
lib/Kconfig.kasan
+141
View File
@@ -0,0 +1,141 @@
# This config refers to the generic KASAN mode.
config HAVE_ARCH_KASAN
bool
config HAVE_ARCH_KASAN_SW_TAGS
bool
config CC_HAS_KASAN_GENERIC
def_bool $(cc-option, -fsanitize=kernel-address)
config CC_HAS_KASAN_SW_TAGS
def_bool $(cc-option, -fsanitize=kernel-hwaddress)
config KASAN
bool "KASAN: runtime memory debugger"
depends on (HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC) || \
(HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS)
depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB)
help
Enables KASAN (KernelAddressSANitizer) - runtime memory debugger,
designed to find out-of-bounds accesses and use-after-free bugs.
See Documentation/dev-tools/kasan.rst for details.
choice
prompt "KASAN mode"
depends on KASAN
default KASAN_GENERIC
help
KASAN has two modes: generic KASAN (similar to userspace ASan,
x86_64/arm64/xtensa, enabled with CONFIG_KASAN_GENERIC) and
software tag-based KASAN (a version based on software memory
tagging, arm64 only, similar to userspace HWASan, enabled with
CONFIG_KASAN_SW_TAGS).
Both generic and tag-based KASAN are strictly debugging features.
config KASAN_GENERIC
bool "Generic mode"
depends on HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC
depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB)
select SLUB_DEBUG if SLUB
select CONSTRUCTORS
select STACKDEPOT
help
Enables generic KASAN mode.
Supported in both GCC and Clang. With GCC it requires version 4.9.2
or later for basic support and version 5.0 or later for detection of
out-of-bounds accesses for stack and global variables and for inline
instrumentation mode (CONFIG_KASAN_INLINE). With Clang it requires
version 3.7.0 or later and it doesn't support detection of
out-of-bounds accesses for global variables yet.
This mode consumes about 1/8th of available memory at kernel start
and introduces an overhead of ~x1.5 for the rest of the allocations.
The performance slowdown is ~x3.
For better error detection enable CONFIG_STACKTRACE.
Currently CONFIG_KASAN_GENERIC doesn't work with CONFIG_DEBUG_SLAB
(the resulting kernel does not boot).
config KASAN_SW_TAGS
bool "Software tag-based mode"
depends on HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS
depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB)
select SLUB_DEBUG if SLUB
select CONSTRUCTORS
select STACKDEPOT
help
Enables software tag-based KASAN mode.
This mode requires Top Byte Ignore support by the CPU and therefore
is only supported for arm64.
This mode requires Clang version 7.0.0 or later.
This mode consumes about 1/16th of available memory at kernel start
and introduces an overhead of ~20% for the rest of the allocations.
This mode may potentially introduce problems relating to pointer
casting and comparison, as it embeds tags into the top byte of each
pointer.
For better error detection enable CONFIG_STACKTRACE.
Currently CONFIG_KASAN_SW_TAGS doesn't work with CONFIG_DEBUG_SLAB
(the resulting kernel does not boot).
endchoice
choice
prompt "Instrumentation type"
depends on KASAN
default KASAN_OUTLINE
config KASAN_OUTLINE
bool "Outline instrumentation"
help
Before every memory access compiler insert function call
__asan_load*/__asan_store*. These functions performs check
of shadow memory. This is slower than inline instrumentation,
however it doesn't bloat size of kernel's .text section so
much as inline does.
config KASAN_INLINE
bool "Inline instrumentation"
help
Compiler directly inserts code checking shadow memory before
memory accesses. This is faster than outline (in some workloads
it gives about x2 boost over outline instrumentation), but
make kernel's .text size much bigger.
For CONFIG_KASAN_GENERIC this requires GCC 5.0 or later.
endchoice
config KASAN_STACK_ENABLE
bool "Enable stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST
depends on KASAN
help
The LLVM stack address sanitizer has a know problem that
causes excessive stack usage in a lot of functions, see
https://bugs.llvm.org/show_bug.cgi?id=38809
Disabling asan-stack makes it safe to run kernels build
with clang-8 with KASAN enabled, though it loses some of
the functionality.
This feature is always disabled when compile-testing with clang
to avoid cluttering the output in stack overflow warnings,
but clang users can still enable it for builds without
CONFIG_COMPILE_TEST. On gcc it is assumed to always be safe
to use and enabled by default.
config KASAN_STACK
int
default 1 if KASAN_STACK_ENABLE || CC_IS_GCC
default 0
config KASAN_SW_TAGS_IDENTIFY
bool "Enable memory corruption identification"
depends on KASAN_SW_TAGS
help
This option enables best-effort identification of bug type
(use-after-free or out-of-bounds) at the cost of increased
memory consumption.
config TEST_KASAN
tristate "Module for testing KASAN for bug detection"
depends on m && KASAN
help
This is a test module doing various nasty things like
out of bounds accesses, use after free. It is useful for testing
kernel debugging features like KASAN.
lib/Kconfig.kgdb
+126
View File
@@ -0,0 +1,126 @@
config HAVE_ARCH_KGDB
bool
menuconfig KGDB
bool "KGDB: kernel debugger"
depends on HAVE_ARCH_KGDB
depends on DEBUG_KERNEL
help
If you say Y here, it will be possible to remotely debug the
kernel using gdb. It is recommended but not required, that
you also turn on the kernel config option
CONFIG_FRAME_POINTER to aid in producing more reliable stack
backtraces in the external debugger. Documentation of
kernel debugger is available at http://kgdb.sourceforge.net
as well as in Documentation/dev-tools/kgdb.rst. If
unsure, say N.
if KGDB
config KGDB_SERIAL_CONSOLE
tristate "KGDB: use kgdb over the serial console"
select CONSOLE_POLL
select MAGIC_SYSRQ
depends on TTY && HW_CONSOLE
default y
help
Share a serial console with kgdb. Sysrq-g must be used
to break in initially.
config KGDB_TESTS
bool "KGDB: internal test suite"
default n
help
This is a kgdb I/O module specifically designed to test
kgdb's internal functions. This kgdb I/O module is
intended to for the development of new kgdb stubs
as well as regression testing the kgdb internals.
See the drivers/misc/kgdbts.c for the details about
the tests. The most basic of this I/O module is to boot
a kernel boot arguments "kgdbwait kgdbts=V1F100"
config KGDB_TESTS_ON_BOOT
bool "KGDB: Run tests on boot"
depends on KGDB_TESTS
default n
help
Run the kgdb tests on boot up automatically without the need
to pass in a kernel parameter
config KGDB_TESTS_BOOT_STRING
string "KGDB: which internal kgdb tests to run"
depends on KGDB_TESTS_ON_BOOT
default "V1F100"
help
This is the command string to send the kgdb test suite on
boot. See the drivers/misc/kgdbts.c for detailed
information about other strings you could use beyond the
default of V1F100.
config KGDB_LOW_LEVEL_TRAP
bool "KGDB: Allow debugging with traps in notifiers"
depends on X86 || MIPS
default n
help
This will add an extra call back to kgdb for the breakpoint
exception handler which will allow kgdb to step through a
notify handler.
config KGDB_KDB
bool "KGDB_KDB: include kdb frontend for kgdb"
default n
help
KDB frontend for kernel
config KDB_DEFAULT_ENABLE
hex "KDB: Select kdb command functions to be enabled by default"
depends on KGDB_KDB
default 0x1
help
Specifiers which kdb commands are enabled by default. This may
be set to 1 or 0 to enable all commands or disable almost all
commands.
Alternatively the following bitmask applies:
0x0002 - allow arbitrary reads from memory and symbol lookup
0x0004 - allow arbitrary writes to memory
0x0008 - allow current register state to be inspected
0x0010 - allow current register state to be modified
0x0020 - allow passive inspection (backtrace, process list, lsmod)
0x0040 - allow flow control management (breakpoint, single step)
0x0080 - enable signalling of processes
0x0100 - allow machine to be rebooted
The config option merely sets the default at boot time. Both
issuing 'echo X > /sys/module/kdb/parameters/cmd_enable' or
setting with kdb.cmd_enable=X kernel command line option will
override the default settings.
config KDB_KEYBOARD
bool "KGDB_KDB: keyboard as input device"
depends on VT && KGDB_KDB
default n
help
KDB can use a PS/2 type keyboard for an input device
config KDB_CONTINUE_CATASTROPHIC
int "KDB: continue after catastrophic errors"
depends on KGDB_KDB
default "0"
help
This integer controls the behaviour of kdb when the kernel gets a
catastrophic error, i.e. for a panic or oops.
When KDB is active and a catastrophic error occurs, nothing extra
will happen until you type 'go'.
CONFIG_KDB_CONTINUE_CATASTROPHIC == 0 (default). The first time
you type 'go', you will be warned by kdb. The secend time you type
'go', KDB tries to continue. No guarantees that the
kernel is still usable in this situation.
CONFIG_KDB_CONTINUE_CATASTROPHIC == 1. KDB tries to continue.
No guarantees that the kernel is still usable in this situation.
CONFIG_KDB_CONTINUE_CATASTROPHIC == 2. KDB forces a reboot.
If you are not sure, say 0.
endif # KGDB
lib/Kconfig.ubsan
+44
View File
@@ -0,0 +1,44 @@
config ARCH_HAS_UBSAN_SANITIZE_ALL
bool
config UBSAN
bool "Undefined behaviour sanity checker"
help
This option enables undefined behaviour sanity checker
Compile-time instrumentation is used to detect various undefined
behaviours in runtime. Various types of checks may be enabled
via boot parameter ubsan_handle
(see: Documentation/dev-tools/ubsan.rst).
config UBSAN_SANITIZE_ALL
bool "Enable instrumentation for the entire kernel"
depends on UBSAN
depends on ARCH_HAS_UBSAN_SANITIZE_ALL
# We build with -Wno-maybe-uninitilzed, but we still want to
# use -Wmaybe-uninitilized in allmodconfig builds.
# So dependsy bellow used to disable this option in allmodconfig
depends on !COMPILE_TEST
default y
help
This option activates instrumentation for the entire kernel.
If you don't enable this option, you have to explicitly specify
UBSAN_SANITIZE := y for the files/directories you want to check for UB.
Enabling this option will get kernel image size increased
significantly.
config UBSAN_ALIGNMENT
bool "Enable checking of pointers alignment"
depends on UBSAN
default y if !HAVE_EFFICIENT_UNALIGNED_ACCESS
help
This option enables detection of unaligned memory accesses.
Enabling this option on architectures that support unaligned
accesses may produce a lot of false positives.
config TEST_UBSAN
tristate "Module for testing for undefined behavior detection"
depends on m && UBSAN
help
This is a test module for UBSAN.
It triggers various undefined behavior, and detect it.
lib/Makefile
+292
View File
@@ -0,0 +1,292 @@
# SPDX-License-Identifier: GPL-2.0
#
# Makefile for some libs needed in the kernel.
#
ifdef CONFIG_FUNCTION_TRACER
ORIG_CFLAGS := $(KBUILD_CFLAGS)
KBUILD_CFLAGS = $(subst $(CC_FLAGS_FTRACE),,$(ORIG_CFLAGS))
endif
# These files are disabled because they produce lots of non-interesting and/or
# flaky coverage that is not a function of syscall inputs. For example,
# rbtree can be global and individual rotations don't correlate with inputs.
KCOV_INSTRUMENT_string.o := n
KCOV_INSTRUMENT_rbtree.o := n
KCOV_INSTRUMENT_list_debug.o := n
KCOV_INSTRUMENT_debugobjects.o := n
KCOV_INSTRUMENT_dynamic_debug.o := n
# Early boot use of cmdline, don't instrument it
ifdef CONFIG_AMD_MEM_ENCRYPT
KASAN_SANITIZE_string.o := n
ifdef CONFIG_FUNCTION_TRACER
CFLAGS_REMOVE_string.o = -pg
endif
CFLAGS_string.o := $(call cc-option, -fno-stack-protector)
endif
lib-y := ctype.o string.o vsprintf.o cmdline.o \
rbtree.o radix-tree.o timerqueue.o\
idr.o int_sqrt.o extable.o \
sha1.o irq_regs.o argv_split.o \
flex_proportions.o ratelimit.o show_mem.o \
is_single_threaded.o plist.o decompress.o kobject_uevent.o \
earlycpio.o seq_buf.o siphash.o dec_and_lock.o \
nmi_backtrace.o nodemask.o win_minmax.o
lib-$(CONFIG_PRINTK) += dump_stack.o
lib-$(CONFIG_MMU) += ioremap.o
lib-$(CONFIG_SMP) += cpumask.o
lib-y += kobject.o klist.o
obj-y += lockref.o
obj-y += bcd.o div64.o sort.o parser.o debug_locks.o random32.o \
bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \
gcd.o lcm.o list_sort.o uuid.o flex_array.o iov_iter.o clz_ctz.o \
bsearch.o find_bit.o llist.o memweight.o kfifo.o \
percpu-refcount.o rhashtable.o reciprocal_div.o \
once.o refcount.o usercopy.o errseq.o bucket_locks.o
obj-$(CONFIG_STRING_SELFTEST) += test_string.o
obj-y += string_helpers.o
obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o
obj-y += hexdump.o
obj-$(CONFIG_TEST_HEXDUMP) += test_hexdump.o
obj-y += kstrtox.o
obj-$(CONFIG_FIND_BIT_BENCHMARK) += find_bit_benchmark.o
obj-$(CONFIG_TEST_BPF) += test_bpf.o
obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o
obj-$(CONFIG_TEST_SYSCTL) += test_sysctl.o
obj-$(CONFIG_TEST_HASH) += test_hash.o test_siphash.o
obj-$(CONFIG_TEST_IDA) += test_ida.o
obj-$(CONFIG_TEST_KASAN) += test_kasan.o
CFLAGS_test_kasan.o += -fno-builtin
obj-$(CONFIG_TEST_UBSAN) += test_ubsan.o
UBSAN_SANITIZE_test_ubsan.o := y
obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o
obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o
obj-$(CONFIG_TEST_LKM) += test_module.o
obj-$(CONFIG_TEST_OVERFLOW) += test_overflow.o
obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o
obj-$(CONFIG_TEST_SORT) += test_sort.o
obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o
obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o
obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o
obj-$(CONFIG_TEST_PRINTF) += test_printf.o
obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o
obj-$(CONFIG_TEST_BITFIELD) += test_bitfield.o
obj-$(CONFIG_TEST_UUID) += test_uuid.o
obj-$(CONFIG_TEST_PARMAN) += test_parman.o
obj-$(CONFIG_TEST_KMOD) += test_kmod.o
obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o
obj-$(CONFIG_TEST_MEMCAT_P) += test_memcat_p.o
obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o
CFLAGS_test_stackinit.o += $(call cc-disable-warning, switch-unreachable)
obj-$(CONFIG_TEST_STACKINIT) += test_stackinit.o
obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o
ifeq ($(CONFIG_DEBUG_KOBJECT),y)
CFLAGS_kobject.o += -DDEBUG
CFLAGS_kobject_uevent.o += -DDEBUG
endif
obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o
CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any)
obj-y += crypto/
obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o
obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o
obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o
obj-y += logic_pio.o
obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o
obj-$(CONFIG_BTREE) += btree.o
obj-$(CONFIG_INTERVAL_TREE) += interval_tree.o
obj-$(CONFIG_ASSOCIATIVE_ARRAY) += assoc_array.o
obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o
obj-$(CONFIG_DEBUG_LIST) += list_debug.o
obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o
obj-$(CONFIG_BITREVERSE) += bitrev.o
obj-$(CONFIG_RATIONAL) += rational.o
obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o
obj-$(CONFIG_CRC16) += crc16.o
obj-$(CONFIG_CRC_T10DIF)+= crc-t10dif.o
obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
obj-$(CONFIG_CRC32) += crc32.o
obj-$(CONFIG_CRC64) += crc64.o
obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o
obj-$(CONFIG_CRC4) += crc4.o
obj-$(CONFIG_CRC7) += crc7.o
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
obj-$(CONFIG_CRC8) += crc8.o
obj-$(CONFIG_XXHASH) += xxhash.o
obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
obj-$(CONFIG_842_COMPRESS) += 842/
obj-$(CONFIG_842_DECOMPRESS) += 842/
obj-$(CONFIG_ZLIB_INFLATE) += zlib_inflate/
obj-$(CONFIG_ZLIB_DEFLATE) += zlib_deflate/
obj-$(CONFIG_REED_SOLOMON) += reed_solomon/
obj-$(CONFIG_BCH) += bch.o
obj-$(CONFIG_LZO_COMPRESS) += lzo/
obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
obj-$(CONFIG_LZ4_COMPRESS) += lz4/
obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/
obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
obj-$(CONFIG_ZSTD_COMPRESS) += zstd/
obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/
obj-$(CONFIG_XZ_DEC) += xz/
obj-$(CONFIG_RAID6_PQ) += raid6/
lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o
lib-$(CONFIG_DECOMPRESS_XZ) += decompress_unxz.o
lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
lib-$(CONFIG_DECOMPRESS_LZ4) += decompress_unlz4.o
obj-$(CONFIG_TEXTSEARCH) += textsearch.o
obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
obj-$(CONFIG_TEXTSEARCH_BM) += ts_bm.o
obj-$(CONFIG_TEXTSEARCH_FSM) += ts_fsm.o
obj-$(CONFIG_SMP) += percpu_counter.o
obj-$(CONFIG_AUDIT_GENERIC) += audit.o
obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o
obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o
obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o
obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o
obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o
obj-$(CONFIG_NETDEV_NOTIFIER_ERROR_INJECT) += netdev-notifier-error-inject.o
obj-$(CONFIG_MEMORY_NOTIFIER_ERROR_INJECT) += memory-notifier-error-inject.o
obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \
of-reconfig-notifier-error-inject.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
lib-$(CONFIG_GENERIC_BUG) += bug.o
obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o
obj-$(CONFIG_DYNAMIC_DEBUG) += dynamic_debug.o
obj-$(CONFIG_NLATTR) += nlattr.o
obj-$(CONFIG_LRU_CACHE) += lru_cache.o
obj-$(CONFIG_GENERIC_CSUM) += checksum.o
obj-$(CONFIG_GENERIC_ATOMIC64) += atomic64.o
obj-$(CONFIG_ATOMIC64_SELFTEST) += atomic64_test.o
obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o
obj-$(CONFIG_CORDIC) += cordic.o
obj-$(CONFIG_DQL) += dynamic_queue_limits.o
obj-$(CONFIG_GLOB) += glob.o
obj-$(CONFIG_GLOB_SELFTEST) += globtest.o
obj-$(CONFIG_MPILIB) += mpi/
obj-$(CONFIG_SIGNATURE) += digsig.o
lib-$(CONFIG_CLZ_TAB) += clz_tab.o
obj-$(CONFIG_DDR) += jedec_ddr_data.o
obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o
obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o
obj-$(CONFIG_GENERIC_NET_UTILS) += net_utils.o
obj-$(CONFIG_SG_SPLIT) += sg_split.o
obj-$(CONFIG_SG_POOL) += sg_pool.o
obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
obj-$(CONFIG_IRQ_POLL) += irq_poll.o
obj-$(CONFIG_STACKDEPOT) += stackdepot.o
KASAN_SANITIZE_stackdepot.o := n
KCOV_INSTRUMENT_stackdepot.o := n
libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \
fdt_empty_tree.o
$(foreach file, $(libfdt_files), \
$(eval CFLAGS_$(file) = -I$(src)/../scripts/dtc/libfdt))
lib-$(CONFIG_LIBFDT) += $(libfdt_files)
obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o
obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o
obj-$(CONFIG_PERCPU_TEST) += percpu_test.o
obj-$(CONFIG_ASN1) += asn1_decoder.o
obj-$(CONFIG_FONT_SUPPORT) += fonts/
obj-$(CONFIG_PRIME_NUMBERS) += prime_numbers.o
hostprogs-y := gen_crc32table
hostprogs-y += gen_crc64table
clean-files := crc32table.h
clean-files += crc64table.h
$(obj)/crc32.o: $(obj)/crc32table.h
quiet_cmd_crc32 = GEN $@
cmd_crc32 = $< > $@
$(obj)/crc32table.h: $(obj)/gen_crc32table
$(call cmd,crc32)
$(obj)/crc64.o: $(obj)/crc64table.h
quiet_cmd_crc64 = GEN $@
cmd_crc64 = $< > $@
$(obj)/crc64table.h: $(obj)/gen_crc64table
$(call cmd,crc64)
#
# Build a fast OID lookip registry from include/linux/oid_registry.h
#
obj-$(CONFIG_OID_REGISTRY) += oid_registry.o
$(obj)/oid_registry.o: $(obj)/oid_registry_data.c
$(obj)/oid_registry_data.c: $(srctree)/include/linux/oid_registry.h \
$(src)/build_OID_registry
$(call cmd,build_OID_registry)
quiet_cmd_build_OID_registry = GEN $@
cmd_build_OID_registry = perl $(srctree)/$(src)/build_OID_registry $< $@
clean-files += oid_registry_data.c
obj-$(CONFIG_UCS2_STRING) += ucs2_string.o
obj-$(CONFIG_UBSAN) += ubsan.o
UBSAN_SANITIZE_ubsan.o := n
KASAN_SANITIZE_ubsan.o := n
CFLAGS_ubsan.o := $(call cc-option, -fno-stack-protector) $(DISABLE_STACKLEAK_PLUGIN)
obj-$(CONFIG_SBITMAP) += sbitmap.o
obj-$(CONFIG_PARMAN) += parman.o
# GCC library routines
obj-$(CONFIG_GENERIC_LIB_ASHLDI3) += ashldi3.o
obj-$(CONFIG_GENERIC_LIB_ASHRDI3) += ashrdi3.o
obj-$(CONFIG_GENERIC_LIB_LSHRDI3) += lshrdi3.o
obj-$(CONFIG_GENERIC_LIB_MULDI3) += muldi3.o
obj-$(CONFIG_GENERIC_LIB_CMPDI2) += cmpdi2.o
obj-$(CONFIG_GENERIC_LIB_UCMPDI2) += ucmpdi2.o
lib/argv_split.c
+95
View File
@@ -0,0 +1,95 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Helper function for splitting a string into an argv-like array.
*/
#include <linux/kernel.h>
#include <linux/ctype.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/export.h>
static int count_argc(const char *str)
{
int count = 0;
bool was_space;
for (was_space = true; *str; str++) {
if (isspace(*str)) {
was_space = true;
} else if (was_space) {
was_space = false;
count++;
}
}
return count;
}
/**
* argv_free - free an argv
* @argv - the argument vector to be freed
*
* Frees an argv and the strings it points to.
*/
void argv_free(char **argv)
{
argv--;
kfree(argv[0]);
kfree(argv);
}
EXPORT_SYMBOL(argv_free);
/**
* argv_split - split a string at whitespace, returning an argv
* @gfp: the GFP mask used to allocate memory
* @str: the string to be split
* @argcp: returned argument count
*
* Returns an array of pointers to strings which are split out from
* @str. This is performed by strictly splitting on white-space; no
* quote processing is performed. Multiple whitespace characters are
* considered to be a single argument separator. The returned array
* is always NULL-terminated. Returns NULL on memory allocation
* failure.
*
* The source string at `str' may be undergoing concurrent alteration via
* userspace sysctl activity (at least). The argv_split() implementation
* attempts to handle this gracefully by taking a local copy to work on.
*/
char **argv_split(gfp_t gfp, const char *str, int *argcp)
{
char *argv_str;
bool was_space;
char **argv, **argv_ret;
int argc;
argv_str = kstrndup(str, KMALLOC_MAX_SIZE - 1, gfp);
if (!argv_str)
return NULL;
argc = count_argc(argv_str);
argv = kmalloc_array(argc + 2, sizeof(*argv), gfp);
if (!argv) {
kfree(argv_str);
return NULL;
}
*argv = argv_str;
argv_ret = ++argv;
for (was_space = true; *argv_str; argv_str++) {
if (isspace(*argv_str)) {
was_space = true;
*argv_str = 0;
} else if (was_space) {
was_space = false;
*argv++ = argv_str;
}
}
*argv = NULL;
if (argcp)
*argcp = argc;
return argv_ret;
}
EXPORT_SYMBOL(argv_split);
lib/ashldi3.c
+44
View File
@@ -0,0 +1,44 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see the file COPYING, or write
* to the Free Software Foundation, Inc.
*/
#include <linux/export.h>
#include <linux/libgcc.h>
long long notrace __ashldi3(long long u, word_type b)
{
DWunion uu, w;
word_type bm;
if (b == 0)
return u;
uu.ll = u;
bm = 32 - b;
if (bm <= 0) {
w.s.low = 0;
w.s.high = (unsigned int) uu.s.low << -bm;
} else {
const unsigned int carries = (unsigned int) uu.s.low >> bm;
w.s.low = (unsigned int) uu.s.low << b;
w.s.high = ((unsigned int) uu.s.high << b) | carries;
}
return w.ll;
}
EXPORT_SYMBOL(__ashldi3);
lib/ashrdi3.c
+46
View File
@@ -0,0 +1,46 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see the file COPYING, or write
* to the Free Software Foundation, Inc.
*/
#include <linux/export.h>
#include <linux/libgcc.h>
long long notrace __ashrdi3(long long u, word_type b)
{
DWunion uu, w;
word_type bm;
if (b == 0)
return u;
uu.ll = u;
bm = 32 - b;
if (bm <= 0) {
/* w.s.high = 1..1 or 0..0 */
w.s.high =
uu.s.high >> 31;
w.s.low = uu.s.high >> -bm;
} else {
const unsigned int carries = (unsigned int) uu.s.high << bm;
w.s.high = uu.s.high >> b;
w.s.low = ((unsigned int) uu.s.low >> b) | carries;
}
return w.ll;
}
EXPORT_SYMBOL(__ashrdi3);
lib/asn1_decoder.c
+521
View File
@@ -0,0 +1,521 @@
/* Decoder for ASN.1 BER/DER/CER encoded bytestream
*
* Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
* Written by David Howells (dhowells@redhat.com)
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public Licence
* as published by the Free Software Foundation; either version
* 2 of the Licence, or (at your option) any later version.
*/
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/asn1_decoder.h>
#include <linux/asn1_ber_bytecode.h>
static const unsigned char asn1_op_lengths[ASN1_OP__NR] = {
/* OPC TAG JMP ACT */
[ASN1_OP_MATCH] = 1 + 1,
[ASN1_OP_MATCH_OR_SKIP] = 1 + 1,
[ASN1_OP_MATCH_ACT] = 1 + 1 + 1,
[ASN1_OP_MATCH_ACT_OR_SKIP] = 1 + 1 + 1,
[ASN1_OP_MATCH_JUMP] = 1 + 1 + 1,
[ASN1_OP_MATCH_JUMP_OR_SKIP] = 1 + 1 + 1,
[ASN1_OP_MATCH_ANY] = 1,
[ASN1_OP_MATCH_ANY_OR_SKIP] = 1,
[ASN1_OP_MATCH_ANY_ACT] = 1 + 1,
[ASN1_OP_MATCH_ANY_ACT_OR_SKIP] = 1 + 1,
[ASN1_OP_COND_MATCH_OR_SKIP] = 1 + 1,
[ASN1_OP_COND_MATCH_ACT_OR_SKIP] = 1 + 1 + 1,
[ASN1_OP_COND_MATCH_JUMP_OR_SKIP] = 1 + 1 + 1,
[ASN1_OP_COND_MATCH_ANY] = 1,
[ASN1_OP_COND_MATCH_ANY_OR_SKIP] = 1,
[ASN1_OP_COND_MATCH_ANY_ACT] = 1 + 1,
[ASN1_OP_COND_MATCH_ANY_ACT_OR_SKIP] = 1 + 1,
[ASN1_OP_COND_FAIL] = 1,
[ASN1_OP_COMPLETE] = 1,
[ASN1_OP_ACT] = 1 + 1,
[ASN1_OP_MAYBE_ACT] = 1 + 1,
[ASN1_OP_RETURN] = 1,
[ASN1_OP_END_SEQ] = 1,
[ASN1_OP_END_SEQ_OF] = 1 + 1,
[ASN1_OP_END_SET] = 1,
[ASN1_OP_END_SET_OF] = 1 + 1,
[ASN1_OP_END_SEQ_ACT] = 1 + 1,
[ASN1_OP_END_SEQ_OF_ACT] = 1 + 1 + 1,
[ASN1_OP_END_SET_ACT] = 1 + 1,
[ASN1_OP_END_SET_OF_ACT] = 1 + 1 + 1,
};
/*
* Find the length of an indefinite length object
* @data: The data buffer
* @datalen: The end of the innermost containing element in the buffer
* @_dp: The data parse cursor (updated before returning)
* @_len: Where to return the size of the element.
* @_errmsg: Where to return a pointer to an error message on error
*/
static int asn1_find_indefinite_length(const unsigned char *data, size_t datalen,
size_t *_dp, size_t *_len,
const char **_errmsg)
{
unsigned char tag, tmp;
size_t dp = *_dp, len, n;
int indef_level = 1;
next_tag:
if (unlikely(datalen - dp < 2)) {
if (datalen == dp)
goto missing_eoc;
goto data_overrun_error;
}
/* Extract a tag from the data */
tag = data[dp++];
if (tag == ASN1_EOC) {
/* It appears to be an EOC. */
if (data[dp++] != 0)
goto invalid_eoc;
if (--indef_level <= 0) {
*_len = dp - *_dp;
*_dp = dp;
return 0;
}
goto next_tag;
}
if (unlikely((tag & 0x1f) == ASN1_LONG_TAG)) {
do {
if (unlikely(datalen - dp < 2))
goto data_overrun_error;
tmp = data[dp++];
} while (tmp & 0x80);
}
/* Extract the length */
len = data[dp++];
if (len <= 0x7f)
goto check_length;
if (unlikely(len == ASN1_INDEFINITE_LENGTH)) {
/* Indefinite length */
if (unlikely((tag & ASN1_CONS_BIT) == ASN1_PRIM << 5))
goto indefinite_len_primitive;
indef_level++;
goto next_tag;
}
n = len - 0x80;
if (unlikely(n > sizeof(len) - 1))
goto length_too_long;
if (unlikely(n > datalen - dp))
goto data_overrun_error;
len = 0;
for (; n > 0; n--) {
len <<= 8;
len |= data[dp++];
}
check_length:
if (len > datalen - dp)
goto data_overrun_error;
dp += len;
goto next_tag;
length_too_long:
*_errmsg = "Unsupported length";
goto error;
indefinite_len_primitive:
*_errmsg = "Indefinite len primitive not permitted";
goto error;
invalid_eoc:
*_errmsg = "Invalid length EOC";
goto error;
data_overrun_error:
*_errmsg = "Data overrun error";
goto error;
missing_eoc:
*_errmsg = "Missing EOC in indefinite len cons";
error:
*_dp = dp;
return -1;
}
/**
* asn1_ber_decoder - Decoder BER/DER/CER ASN.1 according to pattern
* @decoder: The decoder definition (produced by asn1_compiler)
* @context: The caller's context (to be passed to the action functions)
* @data: The encoded data
* @datalen: The size of the encoded data
*
* Decode BER/DER/CER encoded ASN.1 data according to a bytecode pattern
* produced by asn1_compiler. Action functions are called on marked tags to
* allow the caller to retrieve significant data.
*
* LIMITATIONS:
*
* To keep down the amount of stack used by this function, the following limits
* have been imposed:
*
* (1) This won't handle datalen > 65535 without increasing the size of the
* cons stack elements and length_too_long checking.
*
* (2) The stack of constructed types is 10 deep. If the depth of non-leaf
* constructed types exceeds this, the decode will fail.
*
* (3) The SET type (not the SET OF type) isn't really supported as tracking
* what members of the set have been seen is a pain.
*/
int asn1_ber_decoder(const struct asn1_decoder *decoder,
void *context,
const unsigned char *data,
size_t datalen)
{
const unsigned char *machine = decoder->machine;
const asn1_action_t *actions = decoder->actions;
size_t machlen = decoder->machlen;
enum asn1_opcode op;
unsigned char tag = 0, csp = 0, jsp = 0, optag = 0, hdr = 0;
const char *errmsg;
size_t pc = 0, dp = 0, tdp = 0, len = 0;
int ret;
unsigned char flags = 0;
#define FLAG_INDEFINITE_LENGTH 0x01
#define FLAG_MATCHED 0x02
#define FLAG_LAST_MATCHED 0x04 /* Last tag matched */
#define FLAG_CONS 0x20 /* Corresponds to CONS bit in the opcode tag
* - ie. whether or not we are going to parse
* a compound type.
*/
#define NR_CONS_STACK 10
unsigned short cons_dp_stack[NR_CONS_STACK];
unsigned short cons_datalen_stack[NR_CONS_STACK];
unsigned char cons_hdrlen_stack[NR_CONS_STACK];
#define NR_JUMP_STACK 10
unsigned char jump_stack[NR_JUMP_STACK];
if (datalen > 65535)
return -EMSGSIZE;
next_op:
pr_debug("next_op: pc=\e[32m%zu\e[m/%zu dp=\e[33m%zu\e[m/%zu C=%d J=%d\n",
pc, machlen, dp, datalen, csp, jsp);
if (unlikely(pc >= machlen))
goto machine_overrun_error;
op = machine[pc];
if (unlikely(pc + asn1_op_lengths[op] > machlen))
goto machine_overrun_error;
/* If this command is meant to match a tag, then do that before
* evaluating the command.
*/
if (op <= ASN1_OP__MATCHES_TAG) {
unsigned char tmp;
/* Skip conditional matches if possible */
if ((op & ASN1_OP_MATCH__COND && flags & FLAG_MATCHED) ||
(op & ASN1_OP_MATCH__SKIP && dp == datalen)) {
flags &= ~FLAG_LAST_MATCHED;
pc += asn1_op_lengths[op];
goto next_op;
}
flags = 0;
hdr = 2;
/* Extract a tag from the data */
if (unlikely(datalen - dp < 2))
goto data_overrun_error;
tag = data[dp++];
if (unlikely((tag & 0x1f) == ASN1_LONG_TAG))
goto long_tag_not_supported;
if (op & ASN1_OP_MATCH__ANY) {
pr_debug("- any %02x\n", tag);
} else {
/* Extract the tag from the machine
* - Either CONS or PRIM are permitted in the data if
* CONS is not set in the op stream, otherwise CONS
* is mandatory.
*/
optag = machine[pc + 1];
flags |= optag & FLAG_CONS;
/* Determine whether the tag matched */
tmp = optag ^ tag;
tmp &= ~(optag & ASN1_CONS_BIT);
pr_debug("- match? %02x %02x %02x\n", tag, optag, tmp);
if (tmp != 0) {
/* All odd-numbered tags are MATCH_OR_SKIP. */
if (op & ASN1_OP_MATCH__SKIP) {
pc += asn1_op_lengths[op];
dp--;
goto next_op;
}
goto tag_mismatch;
}
}
flags |= FLAG_MATCHED;
len = data[dp++];
if (len > 0x7f) {
if (unlikely(len == ASN1_INDEFINITE_LENGTH)) {
/* Indefinite length */
if (unlikely(!(tag & ASN1_CONS_BIT)))
goto indefinite_len_primitive;
flags |= FLAG_INDEFINITE_LENGTH;
if (unlikely(2 > datalen - dp))
goto data_overrun_error;
} else {
int n = len - 0x80;
if (unlikely(n > 2))
goto length_too_long;
if (unlikely(n > datalen - dp))
goto data_overrun_error;
hdr += n;
for (len = 0; n > 0; n--) {
len <<= 8;
len |= data[dp++];
}
if (unlikely(len > datalen - dp))
goto data_overrun_error;
}
} else {
if (unlikely(len > datalen - dp))
goto data_overrun_error;
}
if (flags & FLAG_CONS) {
/* For expected compound forms, we stack the positions
* of the start and end of the data.
*/
if (unlikely(csp >= NR_CONS_STACK))
goto cons_stack_overflow;
cons_dp_stack[csp] = dp;
cons_hdrlen_stack[csp] = hdr;
if (!(flags & FLAG_INDEFINITE_LENGTH)) {
cons_datalen_stack[csp] = datalen;
datalen = dp + len;
} else {
cons_datalen_stack[csp] = 0;
}
csp++;
}
pr_debug("- TAG: %02x %zu%s\n",
tag, len, flags & FLAG_CONS ? " CONS" : "");
tdp = dp;
}
/* Decide how to handle the operation */
switch (op) {
case ASN1_OP_MATCH:
case ASN1_OP_MATCH_OR_SKIP:
case ASN1_OP_MATCH_ACT:
case ASN1_OP_MATCH_ACT_OR_SKIP:
case ASN1_OP_MATCH_ANY:
case ASN1_OP_MATCH_ANY_OR_SKIP:
case ASN1_OP_MATCH_ANY_ACT:
case ASN1_OP_MATCH_ANY_ACT_OR_SKIP:
case ASN1_OP_COND_MATCH_OR_SKIP:
case ASN1_OP_COND_MATCH_ACT_OR_SKIP:
case ASN1_OP_COND_MATCH_ANY:
case ASN1_OP_COND_MATCH_ANY_OR_SKIP:
case ASN1_OP_COND_MATCH_ANY_ACT:
case ASN1_OP_COND_MATCH_ANY_ACT_OR_SKIP:
if (!(flags & FLAG_CONS)) {
if (flags & FLAG_INDEFINITE_LENGTH) {
size_t tmp = dp;
ret = asn1_find_indefinite_length(
data, datalen, &tmp, &len, &errmsg);
if (ret < 0)
goto error;
}
pr_debug("- LEAF: %zu\n", len);
}
if (op & ASN1_OP_MATCH__ACT) {
unsigned char act;
if (op & ASN1_OP_MATCH__ANY)
act = machine[pc + 1];
else
act = machine[pc + 2];
ret = actions[act](context, hdr, tag, data + dp, len);
if (ret < 0)
return ret;
}
if (!(flags & FLAG_CONS))
dp += len;
pc += asn1_op_lengths[op];
goto next_op;
case ASN1_OP_MATCH_JUMP:
case ASN1_OP_MATCH_JUMP_OR_SKIP:
case ASN1_OP_COND_MATCH_JUMP_OR_SKIP:
pr_debug("- MATCH_JUMP\n");
if (unlikely(jsp == NR_JUMP_STACK))
goto jump_stack_overflow;
jump_stack[jsp++] = pc + asn1_op_lengths[op];
pc = machine[pc + 2];
goto next_op;
case ASN1_OP_COND_FAIL:
if (unlikely(!(flags & FLAG_MATCHED)))
goto tag_mismatch;
pc += asn1_op_lengths[op];
goto next_op;
case ASN1_OP_COMPLETE:
if (unlikely(jsp != 0 || csp != 0)) {
pr_err("ASN.1 decoder error: Stacks not empty at completion (%u, %u)\n",
jsp, csp);
return -EBADMSG;
}
return 0;
case ASN1_OP_END_SET:
case ASN1_OP_END_SET_ACT:
if (unlikely(!(flags & FLAG_MATCHED)))
goto tag_mismatch;
case ASN1_OP_END_SEQ:
case ASN1_OP_END_SET_OF:
case ASN1_OP_END_SEQ_OF:
case ASN1_OP_END_SEQ_ACT:
case ASN1_OP_END_SET_OF_ACT:
case ASN1_OP_END_SEQ_OF_ACT:
if (unlikely(csp <= 0))
goto cons_stack_underflow;
csp--;
tdp = cons_dp_stack[csp];
hdr = cons_hdrlen_stack[csp];
len = datalen;
datalen = cons_datalen_stack[csp];
pr_debug("- end cons t=%zu dp=%zu l=%zu/%zu\n",
tdp, dp, len, datalen);
if (datalen == 0) {
/* Indefinite length - check for the EOC. */
datalen = len;
if (unlikely(datalen - dp < 2))
goto data_overrun_error;
if (data[dp++] != 0) {
if (op & ASN1_OP_END__OF) {
dp--;
csp++;
pc = machine[pc + 1];
pr_debug("- continue\n");
goto next_op;
}
goto missing_eoc;
}
if (data[dp++] != 0)
goto invalid_eoc;
len = dp - tdp - 2;
} else {
if (dp < len && (op & ASN1_OP_END__OF)) {
datalen = len;
csp++;
pc = machine[pc + 1];
pr_debug("- continue\n");
goto next_op;
}
if (dp != len)
goto cons_length_error;
len -= tdp;
pr_debug("- cons len l=%zu d=%zu\n", len, dp - tdp);
}
if (op & ASN1_OP_END__ACT) {
unsigned char act;
if (op & ASN1_OP_END__OF)
act = machine[pc + 2];
else
act = machine[pc + 1];
ret = actions[act](context, hdr, 0, data + tdp, len);
if (ret < 0)
return ret;
}
pc += asn1_op_lengths[op];
goto next_op;
case ASN1_OP_MAYBE_ACT:
if (!(flags & FLAG_LAST_MATCHED)) {
pc += asn1_op_lengths[op];
goto next_op;
}
case ASN1_OP_ACT:
ret = actions[machine[pc + 1]](context, hdr, tag, data + tdp, len);
if (ret < 0)
return ret;
pc += asn1_op_lengths[op];
goto next_op;
case ASN1_OP_RETURN:
if (unlikely(jsp <= 0))
goto jump_stack_underflow;
pc = jump_stack[--jsp];
flags |= FLAG_MATCHED | FLAG_LAST_MATCHED;
goto next_op;
default:
break;
}
/* Shouldn't reach here */
pr_err("ASN.1 decoder error: Found reserved opcode (%u) pc=%zu\n",
op, pc);
return -EBADMSG;
data_overrun_error:
errmsg = "Data overrun error";
goto error;
machine_overrun_error:
errmsg = "Machine overrun error";
goto error;
jump_stack_underflow:
errmsg = "Jump stack underflow";
goto error;
jump_stack_overflow:
errmsg = "Jump stack overflow";
goto error;
cons_stack_underflow:
errmsg = "Cons stack underflow";
goto error;
cons_stack_overflow:
errmsg = "Cons stack overflow";
goto error;
cons_length_error:
errmsg = "Cons length error";
goto error;
missing_eoc:
errmsg = "Missing EOC in indefinite len cons";
goto error;
invalid_eoc:
errmsg = "Invalid length EOC";
goto error;
length_too_long:
errmsg = "Unsupported length";
goto error;
indefinite_len_primitive:
errmsg = "Indefinite len primitive not permitted";
goto error;
tag_mismatch:
errmsg = "Unexpected tag";
goto error;
long_tag_not_supported:
errmsg = "Long tag not supported";
error:
pr_debug("\nASN1: %s [m=%zu d=%zu ot=%02x t=%02x l=%zu]\n",
errmsg, pc, dp, optag, tag, len);
return -EBADMSG;
}
EXPORT_SYMBOL_GPL(asn1_ber_decoder);
MODULE_LICENSE("GPL");
lib/assoc_array.c
+1723
View File
File diff suppressed because it is too large. Load diff
lib/atomic64.c
+195
View File
@@ -0,0 +1,195 @@
/*
* Generic implementation of 64-bit atomics using spinlocks,
* useful on processors that don't have 64-bit atomic instructions.
*
* Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#include <linux/types.h>
#include <linux/cache.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/atomic.h>
/*
* We use a hashed array of spinlocks to provide exclusive access
* to each atomic64_t variable. Since this is expected to used on
* systems with small numbers of CPUs (<= 4 or so), we use a
* relatively small array of 16 spinlocks to avoid wasting too much
* memory on the spinlock array.
*/
#define NR_LOCKS 16
/*
* Ensure each lock is in a separate cacheline.
*/
static union {
raw_spinlock_t lock;
char pad[L1_CACHE_BYTES];
} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
[0 ... (NR_LOCKS - 1)] = {
.lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
},
};
static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
{
unsigned long addr = (unsigned long) v;
addr >>= L1_CACHE_SHIFT;
addr ^= (addr >> 8) ^ (addr >> 16);
return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
}
long long atomic64_read(const atomic64_t *v)
{
unsigned long flags;
raw_spinlock_t *lock = lock_addr(v);
long long val;
raw_spin_lock_irqsave(lock, flags);
val = v->counter;
raw_spin_unlock_irqrestore(lock, flags);
return val;
}
EXPORT_SYMBOL(atomic64_read);
void atomic64_set(atomic64_t *v, long long i)
{
unsigned long flags;
raw_spinlock_t *lock = lock_addr(v);
raw_spin_lock_irqsave(lock, flags);
v->counter = i;
raw_spin_unlock_irqrestore(lock, flags);
}
EXPORT_SYMBOL(atomic64_set);
#define ATOMIC64_OP(op, c_op) \
void atomic64_##op(long long a, atomic64_t *v) \
{ \
unsigned long flags; \
raw_spinlock_t *lock = lock_addr(v); \
\
raw_spin_lock_irqsave(lock, flags); \
v->counter c_op a; \
raw_spin_unlock_irqrestore(lock, flags); \
} \
EXPORT_SYMBOL(atomic64_##op);
#define ATOMIC64_OP_RETURN(op, c_op) \
long long atomic64_##op##_return(long long a, atomic64_t *v) \
{ \
unsigned long flags; \
raw_spinlock_t *lock = lock_addr(v); \
long long val; \
\
raw_spin_lock_irqsave(lock, flags); \
val = (v->counter c_op a); \
raw_spin_unlock_irqrestore(lock, flags); \
return val; \
} \
EXPORT_SYMBOL(atomic64_##op##_return);
#define ATOMIC64_FETCH_OP(op, c_op) \
long long atomic64_fetch_##op(long long a, atomic64_t *v) \
{ \
unsigned long flags; \
raw_spinlock_t *lock = lock_addr(v); \
long long val; \
\
raw_spin_lock_irqsave(lock, flags); \
val = v->counter; \
v->counter c_op a; \
raw_spin_unlock_irqrestore(lock, flags); \
return val; \
} \
EXPORT_SYMBOL(atomic64_fetch_##op);
#define ATOMIC64_OPS(op, c_op) \
ATOMIC64_OP(op, c_op) \
ATOMIC64_OP_RETURN(op, c_op) \
ATOMIC64_FETCH_OP(op, c_op)
ATOMIC64_OPS(add, +=)
ATOMIC64_OPS(sub, -=)
#undef ATOMIC64_OPS
#define ATOMIC64_OPS(op, c_op) \
ATOMIC64_OP(op, c_op) \
ATOMIC64_OP_RETURN(op, c_op) \
ATOMIC64_FETCH_OP(op, c_op)
ATOMIC64_OPS(and, &=)
ATOMIC64_OPS(or, |=)
ATOMIC64_OPS(xor, ^=)
#undef ATOMIC64_OPS
#undef ATOMIC64_FETCH_OP
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP
long long atomic64_dec_if_positive(atomic64_t *v)
{
unsigned long flags;
raw_spinlock_t *lock = lock_addr(v);
long long val;
raw_spin_lock_irqsave(lock, flags);
val = v->counter - 1;
if (val >= 0)
v->counter = val;
raw_spin_unlock_irqrestore(lock, flags);
return val;
}
EXPORT_SYMBOL(atomic64_dec_if_positive);
long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
{
unsigned long flags;
raw_spinlock_t *lock = lock_addr(v);
long long val;
raw_spin_lock_irqsave(lock, flags);
val = v->counter;
if (val == o)
v->counter = n;
raw_spin_unlock_irqrestore(lock, flags);
return val;
}
EXPORT_SYMBOL(atomic64_cmpxchg);
long long atomic64_xchg(atomic64_t *v, long long new)
{
unsigned long flags;
raw_spinlock_t *lock = lock_addr(v);
long long val;
raw_spin_lock_irqsave(lock, flags);
val = v->counter;
v->counter = new;
raw_spin_unlock_irqrestore(lock, flags);
return val;
}
EXPORT_SYMBOL(atomic64_xchg);
long long atomic64_fetch_add_unless(atomic64_t *v, long long a, long long u)
{
unsigned long flags;
raw_spinlock_t *lock = lock_addr(v);
long long val;
raw_spin_lock_irqsave(lock, flags);
val = v->counter;
if (val != u)
v->counter += a;
raw_spin_unlock_irqrestore(lock, flags);
return val;
}
EXPORT_SYMBOL(atomic64_fetch_add_unless);
lib/atomic64_test.c
+280
View File
@@ -0,0 +1,280 @@
/*
* Testsuite for atomic64_t functions
*
* Copyright © 2010 Luca Barbieri
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/init.h>
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/atomic.h>
#include <linux/module.h>
#ifdef CONFIG_X86
#include <asm/cpufeature.h> /* for boot_cpu_has below */
#endif
#define TEST(bit, op, c_op, val) \
do { \
atomic##bit##_set(&v, v0); \
r = v0; \
atomic##bit##_##op(val, &v); \
r c_op val; \
WARN(atomic##bit##_read(&v) != r, "%Lx != %Lx\n", \
(unsigned long long)atomic##bit##_read(&v), \
(unsigned long long)r); \
} while (0)
/*
* Test for a atomic operation family,
* @test should be a macro accepting parameters (bit, op, ...)
*/
#define FAMILY_TEST(test, bit, op, args...) \
do { \
test(bit, op, ##args); \
test(bit, op##_acquire, ##args); \
test(bit, op##_release, ##args); \
test(bit, op##_relaxed, ##args); \
} while (0)
#define TEST_RETURN(bit, op, c_op, val) \
do { \
atomic##bit##_set(&v, v0); \
r = v0; \
r c_op val; \
BUG_ON(atomic##bit##_##op(val, &v) != r); \
BUG_ON(atomic##bit##_read(&v) != r); \
} while (0)
#define TEST_FETCH(bit, op, c_op, val) \
do { \
atomic##bit##_set(&v, v0); \
r = v0; \
r c_op val; \
BUG_ON(atomic##bit##_##op(val, &v) != v0); \
BUG_ON(atomic##bit##_read(&v) != r); \
} while (0)
#define RETURN_FAMILY_TEST(bit, op, c_op, val) \
do { \
FAMILY_TEST(TEST_RETURN, bit, op, c_op, val); \
} while (0)
#define FETCH_FAMILY_TEST(bit, op, c_op, val) \
do { \
FAMILY_TEST(TEST_FETCH, bit, op, c_op, val); \
} while (0)
#define TEST_ARGS(bit, op, init, ret, expect, args...) \
do { \
atomic##bit##_set(&v, init); \
BUG_ON(atomic##bit##_##op(&v, ##args) != ret); \
BUG_ON(atomic##bit##_read(&v) != expect); \
} while (0)
#define XCHG_FAMILY_TEST(bit, init, new) \
do { \
FAMILY_TEST(TEST_ARGS, bit, xchg, init, init, new, new); \
} while (0)
#define CMPXCHG_FAMILY_TEST(bit, init, new, wrong) \
do { \
FAMILY_TEST(TEST_ARGS, bit, cmpxchg, \
init, init, new, init, new); \
FAMILY_TEST(TEST_ARGS, bit, cmpxchg, \
init, init, init, wrong, new); \
} while (0)
#define INC_RETURN_FAMILY_TEST(bit, i) \
do { \
FAMILY_TEST(TEST_ARGS, bit, inc_return, \
i, (i) + one, (i) + one); \
} while (0)
#define DEC_RETURN_FAMILY_TEST(bit, i) \
do { \
FAMILY_TEST(TEST_ARGS, bit, dec_return, \
i, (i) - one, (i) - one); \
} while (0)
static __init void test_atomic(void)
{
int v0 = 0xaaa31337;
int v1 = 0xdeadbeef;
int onestwos = 0x11112222;
int one = 1;
atomic_t v;
int r;
TEST(, add, +=, onestwos);
TEST(, add, +=, -one);
TEST(, sub, -=, onestwos);
TEST(, sub, -=, -one);
TEST(, or, |=, v1);
TEST(, and, &=, v1);
TEST(, xor, ^=, v1);
TEST(, andnot, &= ~, v1);
RETURN_FAMILY_TEST(, add_return, +=, onestwos);
RETURN_FAMILY_TEST(, add_return, +=, -one);
RETURN_FAMILY_TEST(, sub_return, -=, onestwos);
RETURN_FAMILY_TEST(, sub_return, -=, -one);
FETCH_FAMILY_TEST(, fetch_add, +=, onestwos);
FETCH_FAMILY_TEST(, fetch_add, +=, -one);
FETCH_FAMILY_TEST(, fetch_sub, -=, onestwos);
FETCH_FAMILY_TEST(, fetch_sub, -=, -one);
FETCH_FAMILY_TEST(, fetch_or, |=, v1);
FETCH_FAMILY_TEST(, fetch_and, &=, v1);
FETCH_FAMILY_TEST(, fetch_andnot, &= ~, v1);
FETCH_FAMILY_TEST(, fetch_xor, ^=, v1);
INC_RETURN_FAMILY_TEST(, v0);
DEC_RETURN_FAMILY_TEST(, v0);
XCHG_FAMILY_TEST(, v0, v1);
CMPXCHG_FAMILY_TEST(, v0, v1, onestwos);
}
#define INIT(c) do { atomic64_set(&v, c); r = c; } while (0)
static __init void test_atomic64(void)
{
long long v0 = 0xaaa31337c001d00dLL;
long long v1 = 0xdeadbeefdeafcafeLL;
long long v2 = 0xfaceabadf00df001LL;
long long v3 = 0x8000000000000000LL;
long long onestwos = 0x1111111122222222LL;
long long one = 1LL;
int r_int;
atomic64_t v = ATOMIC64_INIT(v0);
long long r = v0;
BUG_ON(v.counter != r);
atomic64_set(&v, v1);
r = v1;
BUG_ON(v.counter != r);
BUG_ON(atomic64_read(&v) != r);
TEST(64, add, +=, onestwos);
TEST(64, add, +=, -one);
TEST(64, sub, -=, onestwos);
TEST(64, sub, -=, -one);
TEST(64, or, |=, v1);
TEST(64, and, &=, v1);
TEST(64, xor, ^=, v1);
TEST(64, andnot, &= ~, v1);
RETURN_FAMILY_TEST(64, add_return, +=, onestwos);
RETURN_FAMILY_TEST(64, add_return, +=, -one);
RETURN_FAMILY_TEST(64, sub_return, -=, onestwos);
RETURN_FAMILY_TEST(64, sub_return, -=, -one);
FETCH_FAMILY_TEST(64, fetch_add, +=, onestwos);
FETCH_FAMILY_TEST(64, fetch_add, +=, -one);
FETCH_FAMILY_TEST(64, fetch_sub, -=, onestwos);
FETCH_FAMILY_TEST(64, fetch_sub, -=, -one);
FETCH_FAMILY_TEST(64, fetch_or, |=, v1);
FETCH_FAMILY_TEST(64, fetch_and, &=, v1);
FETCH_FAMILY_TEST(64, fetch_andnot, &= ~, v1);
FETCH_FAMILY_TEST(64, fetch_xor, ^=, v1);
INIT(v0);
atomic64_inc(&v);
r += one;
BUG_ON(v.counter != r);
INIT(v0);
atomic64_dec(&v);
r -= one;
BUG_ON(v.counter != r);
INC_RETURN_FAMILY_TEST(64, v0);
DEC_RETURN_FAMILY_TEST(64, v0);
XCHG_FAMILY_TEST(64, v0, v1);
CMPXCHG_FAMILY_TEST(64, v0, v1, v2);
INIT(v0);
BUG_ON(atomic64_add_unless(&v, one, v0));
BUG_ON(v.counter != r);
INIT(v0);
BUG_ON(!atomic64_add_unless(&v, one, v1));
r += one;
BUG_ON(v.counter != r);
INIT(onestwos);
BUG_ON(atomic64_dec_if_positive(&v) != (onestwos - 1));
r -= one;
BUG_ON(v.counter != r);
INIT(0);
BUG_ON(atomic64_dec_if_positive(&v) != -one);
BUG_ON(v.counter != r);
INIT(-one);
BUG_ON(atomic64_dec_if_positive(&v) != (-one - one));
BUG_ON(v.counter != r);
INIT(onestwos);
BUG_ON(!atomic64_inc_not_zero(&v));
r += one;
BUG_ON(v.counter != r);
INIT(0);
BUG_ON(atomic64_inc_not_zero(&v));
BUG_ON(v.counter != r);
INIT(-one);
BUG_ON(!atomic64_inc_not_zero(&v));
r += one;
BUG_ON(v.counter != r);
/* Confirm the return value fits in an int, even if the value doesn't */
INIT(v3);
r_int = atomic64_inc_not_zero(&v);
BUG_ON(!r_int);
}
static __init int test_atomics_init(void)
{
test_atomic();
test_atomic64();
#ifdef CONFIG_X86
pr_info("passed for %s platform %s CX8 and %s SSE\n",
#ifdef CONFIG_X86_64
"x86-64",
#elif defined(CONFIG_X86_CMPXCHG64)
"i586+",
#else
"i386+",
#endif
boot_cpu_has(X86_FEATURE_CX8) ? "with" : "without",
boot_cpu_has(X86_FEATURE_XMM) ? "with" : "without");
#else
pr_info("passed\n");
#endif
return 0;
}
static __exit void test_atomics_exit(void) {}
module_init(test_atomics_init);
module_exit(test_atomics_exit);
MODULE_LICENSE("GPL");
lib/audit.c
+85
View File
@@ -0,0 +1,85 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/init.h>
#include <linux/types.h>
#include <linux/audit.h>
#include <asm/unistd.h>
static unsigned dir_class[] = {
#include <asm-generic/audit_dir_write.h>
~0U
};
static unsigned read_class[] = {
#include <asm-generic/audit_read.h>
~0U
};
static unsigned write_class[] = {
#include <asm-generic/audit_write.h>
~0U
};
static unsigned chattr_class[] = {
#include <asm-generic/audit_change_attr.h>
~0U
};
static unsigned signal_class[] = {
#include <asm-generic/audit_signal.h>
~0U
};
int audit_classify_arch(int arch)
{
if (audit_is_compat(arch))
return 1;
else
return 0;
}
int audit_classify_syscall(int abi, unsigned syscall)
{
if (audit_is_compat(abi))
return audit_classify_compat_syscall(abi, syscall);
switch(syscall) {
#ifdef __NR_open
case __NR_open:
return 2;
#endif
#ifdef __NR_openat
case __NR_openat:
return 3;
#endif
#ifdef __NR_socketcall
case __NR_socketcall:
return 4;
#endif
#ifdef __NR_execveat
case __NR_execveat:
#endif
case __NR_execve:
return 5;
default:
return 0;
}
}
static int __init audit_classes_init(void)
{
#ifdef CONFIG_AUDIT_COMPAT_GENERIC
audit_register_class(AUDIT_CLASS_WRITE_32, compat_write_class);
audit_register_class(AUDIT_CLASS_READ_32, compat_read_class);
audit_register_class(AUDIT_CLASS_DIR_WRITE_32, compat_dir_class);
audit_register_class(AUDIT_CLASS_CHATTR_32, compat_chattr_class);
audit_register_class(AUDIT_CLASS_SIGNAL_32, compat_signal_class);
#endif
audit_register_class(AUDIT_CLASS_WRITE, write_class);
audit_register_class(AUDIT_CLASS_READ, read_class);
audit_register_class(AUDIT_CLASS_DIR_WRITE, dir_class);
audit_register_class(AUDIT_CLASS_CHATTR, chattr_class);
audit_register_class(AUDIT_CLASS_SIGNAL, signal_class);
return 0;
}
__initcall(audit_classes_init);
lib/bcd.c
+15
View File
@@ -0,0 +1,15 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bcd.h>
#include <linux/export.h>
unsigned _bcd2bin(unsigned char val)
{
return (val & 0x0f) + (val >> 4) * 10;
}
EXPORT_SYMBOL(_bcd2bin);
unsigned char _bin2bcd(unsigned val)
{
return ((val / 10) << 4) + val % 10;
}
EXPORT_SYMBOL(_bin2bcd);
lib/bch.c
+1384
View File
File diff suppressed because it is too large. Load diff
lib/bitmap.c
+1197
View File
File diff suppressed because it is too large. Load diff
lib/bitrev.c
+46
View File
@@ -0,0 +1,46 @@
#ifndef CONFIG_HAVE_ARCH_BITREVERSE
#include <linux/types.h>
#include <linux/module.h>
#include <linux/bitrev.h>
MODULE_AUTHOR("Akinobu Mita <akinobu.mita@gmail.com>");
MODULE_DESCRIPTION("Bit ordering reversal functions");
MODULE_LICENSE("GPL");
const u8 byte_rev_table[256] = {
0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0,
0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4,
0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec,
0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea,
0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6,
0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1,
0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9,
0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed,
0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3,
0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7,
0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef,
0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff,
};
EXPORT_SYMBOL_GPL(byte_rev_table);
#endif /* CONFIG_HAVE_ARCH_BITREVERSE */
lib/bsearch.c
+57
View File
@@ -0,0 +1,57 @@
/*
* A generic implementation of binary search for the Linux kernel
*
* Copyright (C) 2008-2009 Ksplice, Inc.
* Author: Tim Abbott <tabbott@ksplice.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; version 2.
*/
#include <linux/export.h>
#include <linux/bsearch.h>
#include <linux/kprobes.h>
/*
* bsearch - binary search an array of elements
* @key: pointer to item being searched for
* @base: pointer to first element to search
* @num: number of elements
* @size: size of each element
* @cmp: pointer to comparison function
*
* This function does a binary search on the given array. The
* contents of the array should already be in ascending sorted order
* under the provided comparison function.
*
* Note that the key need not have the same type as the elements in
* the array, e.g. key could be a string and the comparison function
* could compare the string with the struct's name field. However, if
* the key and elements in the array are of the same type, you can use
* the same comparison function for both sort() and bsearch().
*/
void *bsearch(const void *key, const void *base, size_t num, size_t size,
int (*cmp)(const void *key, const void *elt))
{
const char *pivot;
int result;
while (num > 0) {
pivot = base + (num >> 1) * size;
result = cmp(key, pivot);
if (result == 0)
return (void *)pivot;
if (result > 0) {
base = pivot + size;
num--;
}
num >>= 1;
}
return NULL;
}
EXPORT_SYMBOL(bsearch);
NOKPROBE_SYMBOL(bsearch);
lib/btree.c
+803
View File
@@ -0,0 +1,803 @@
/*
* lib/btree.c - Simple In-memory B+Tree
*
* As should be obvious for Linux kernel code, license is GPLv2
*
* Copyright (c) 2007-2008 Joern Engel <joern@purestorage.com>
* Bits and pieces stolen from Peter Zijlstra's code, which is
* Copyright 2007, Red Hat Inc. Peter Zijlstra
* GPLv2
*
* see http://programming.kicks-ass.net/kernel-patches/vma_lookup/btree.patch
*
* A relatively simple B+Tree implementation. I have written it as a learning
* exercise to understand how B+Trees work. Turned out to be useful as well.
*
* B+Trees can be used similar to Linux radix trees (which don't have anything
* in common with textbook radix trees, beware). Prerequisite for them working
* well is that access to a random tree node is much faster than a large number
* of operations within each node.
*
* Disks have fulfilled the prerequisite for a long time. More recently DRAM
* has gained similar properties, as memory access times, when measured in cpu
* cycles, have increased. Cacheline sizes have increased as well, which also
* helps B+Trees.
*
* Compared to radix trees, B+Trees are more efficient when dealing with a
* sparsely populated address space. Between 25% and 50% of the memory is
* occupied with valid pointers. When densely populated, radix trees contain
* ~98% pointers - hard to beat. Very sparse radix trees contain only ~2%
* pointers.
*
* This particular implementation stores pointers identified by a long value.
* Storing NULL pointers is illegal, lookup will return NULL when no entry
* was found.
*
* A tricks was used that is not commonly found in textbooks. The lowest
* values are to the right, not to the left. All used slots within a node
* are on the left, all unused slots contain NUL values. Most operations
* simply loop once over all slots and terminate on the first NUL.
*/
#include <linux/btree.h>
#include <linux/cache.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
#define MAX(a, b) ((a) > (b) ? (a) : (b))
#define NODESIZE MAX(L1_CACHE_BYTES, 128)
struct btree_geo {
int keylen;
int no_pairs;
int no_longs;
};
struct btree_geo btree_geo32 = {
.keylen = 1,
.no_pairs = NODESIZE / sizeof(long) / 2,
.no_longs = NODESIZE / sizeof(long) / 2,
};
EXPORT_SYMBOL_GPL(btree_geo32);
#define LONG_PER_U64 (64 / BITS_PER_LONG)
struct btree_geo btree_geo64 = {
.keylen = LONG_PER_U64,
.no_pairs = NODESIZE / sizeof(long) / (1 + LONG_PER_U64),
.no_longs = LONG_PER_U64 * (NODESIZE / sizeof(long) / (1 + LONG_PER_U64)),
};
EXPORT_SYMBOL_GPL(btree_geo64);
struct btree_geo btree_geo128 = {
.keylen = 2 * LONG_PER_U64,
.no_pairs = NODESIZE / sizeof(long) / (1 + 2 * LONG_PER_U64),
.no_longs = 2 * LONG_PER_U64 * (NODESIZE / sizeof(long) / (1 + 2 * LONG_PER_U64)),
};
EXPORT_SYMBOL_GPL(btree_geo128);
#define MAX_KEYLEN (2 * LONG_PER_U64)
static struct kmem_cache *btree_cachep;
void *btree_alloc(gfp_t gfp_mask, void *pool_data)
{
return kmem_cache_alloc(btree_cachep, gfp_mask);
}
EXPORT_SYMBOL_GPL(btree_alloc);
void btree_free(void *element, void *pool_data)
{
kmem_cache_free(btree_cachep, element);
}
EXPORT_SYMBOL_GPL(btree_free);
static unsigned long *btree_node_alloc(struct btree_head *head, gfp_t gfp)
{
unsigned long *node;
node = mempool_alloc(head->mempool, gfp);
if (likely(node))
memset(node, 0, NODESIZE);
return node;
}
static int longcmp(const unsigned long *l1, const unsigned long *l2, size_t n)
{
size_t i;
for (i = 0; i < n; i++) {
if (l1[i] < l2[i])
return -1;
if (l1[i] > l2[i])
return 1;
}
return 0;
}
static unsigned long *longcpy(unsigned long *dest, const unsigned long *src,
size_t n)
{
size_t i;
for (i = 0; i < n; i++)
dest[i] = src[i];
return dest;
}
static unsigned long *longset(unsigned long *s, unsigned long c, size_t n)
{
size_t i;
for (i = 0; i < n; i++)
s[i] = c;
return s;
}
static void dec_key(struct btree_geo *geo, unsigned long *key)
{
unsigned long val;
int i;
for (i = geo->keylen - 1; i >= 0; i--) {
val = key[i];
key[i] = val - 1;
if (val)
break;
}
}
static unsigned long *bkey(struct btree_geo *geo, unsigned long *node, int n)
{
return &node[n * geo->keylen];
}
static void *bval(struct btree_geo *geo, unsigned long *node, int n)
{
return (void *)node[geo->no_longs + n];
}
static void setkey(struct btree_geo *geo, unsigned long *node, int n,
unsigned long *key)
{
longcpy(bkey(geo, node, n), key, geo->keylen);
}
static void setval(struct btree_geo *geo, unsigned long *node, int n,
void *val)
{
node[geo->no_longs + n] = (unsigned long) val;
}
static void clearpair(struct btree_geo *geo, unsigned long *node, int n)
{
longset(bkey(geo, node, n), 0, geo->keylen);
node[geo->no_longs + n] = 0;
}
static inline void __btree_init(struct btree_head *head)
{
head->node = NULL;
head->height = 0;
}
void btree_init_mempool(struct btree_head *head, mempool_t *mempool)
{
__btree_init(head);
head->mempool = mempool;
}
EXPORT_SYMBOL_GPL(btree_init_mempool);
int btree_init(struct btree_head *head)
{
__btree_init(head);
head->mempool = mempool_create(0, btree_alloc, btree_free, NULL);
if (!head->mempool)
return -ENOMEM;
return 0;
}
EXPORT_SYMBOL_GPL(btree_init);
void btree_destroy(struct btree_head *head)
{
mempool_free(head->node, head->mempool);
mempool_destroy(head->mempool);
head->mempool = NULL;
}
EXPORT_SYMBOL_GPL(btree_destroy);
void *btree_last(struct btree_head *head, struct btree_geo *geo,
unsigned long *key)
{
int height = head->height;
unsigned long *node = head->node;
if (height == 0)
return NULL;
for ( ; height > 1; height--)
node = bval(geo, node, 0);
longcpy(key, bkey(geo, node, 0), geo->keylen);
return bval(geo, node, 0);
}
EXPORT_SYMBOL_GPL(btree_last);
static int keycmp(struct btree_geo *geo, unsigned long *node, int pos,
unsigned long *key)
{
return longcmp(bkey(geo, node, pos), key, geo->keylen);
}
static int keyzero(struct btree_geo *geo, unsigned long *key)
{
int i;
for (i = 0; i < geo->keylen; i++)
if (key[i])
return 0;
return 1;
}
void *btree_lookup(struct btree_head *head, struct btree_geo *geo,
unsigned long *key)
{
int i, height = head->height;
unsigned long *node = head->node;
if (height == 0)
return NULL;
for ( ; height > 1; height--) {
for (i = 0; i < geo->no_pairs; i++)
if (keycmp(geo, node, i, key) <= 0)
break;
if (i == geo->no_pairs)
return NULL;
node = bval(geo, node, i);
if (!node)
return NULL;
}
if (!node)
return NULL;
for (i = 0; i < geo->no_pairs; i++)
if (keycmp(geo, node, i, key) == 0)
return bval(geo, node, i);
return NULL;
}
EXPORT_SYMBOL_GPL(btree_lookup);
int btree_update(struct btree_head *head, struct btree_geo *geo,
unsigned long *key, void *val)
{
int i, height = head->height;
unsigned long *node = head->node;
if (height == 0)
return -ENOENT;
for ( ; height > 1; height--) {
for (i = 0; i < geo->no_pairs; i++)
if (keycmp(geo, node, i, key) <= 0)
break;
if (i == geo->no_pairs)
return -ENOENT;
node = bval(geo, node, i);
if (!node)
return -ENOENT;
}
if (!node)
return -ENOENT;
for (i = 0; i < geo->no_pairs; i++)
if (keycmp(geo, node, i, key) == 0) {
setval(geo, node, i, val);
return 0;
}
return -ENOENT;
}
EXPORT_SYMBOL_GPL(btree_update);
/*
* Usually this function is quite similar to normal lookup. But the key of
* a parent node may be smaller than the smallest key of all its siblings.
* In such a case we cannot just return NULL, as we have only proven that no
* key smaller than __key, but larger than this parent key exists.
* So we set __key to the parent key and retry. We have to use the smallest
* such parent key, which is the last parent key we encountered.
*/
void *btree_get_prev(struct btree_head *head, struct btree_geo *geo,
unsigned long *__key)
{
int i, height;
unsigned long *node, *oldnode;
unsigned long *retry_key = NULL, key[MAX_KEYLEN];
if (keyzero(geo, __key))
return NULL;
if (head->height == 0)
return NULL;
longcpy(key, __key, geo->keylen);
retry:
dec_key(geo, key);
node = head->node;
for (height = head->height ; height > 1; height--) {
for (i = 0; i < geo->no_pairs; i++)
if (keycmp(geo, node, i, key) <= 0)
break;
if (i == geo->no_pairs)
goto miss;
oldnode = node;
node = bval(geo, node, i);
if (!node)
goto miss;
retry_key = bkey(geo, oldnode, i);
}
if (!node)
goto miss;
for (i = 0; i < geo->no_pairs; i++) {
if (keycmp(geo, node, i, key) <= 0) {
if (bval(geo, node, i)) {
longcpy(__key, bkey(geo, node, i), geo->keylen);
return bval(geo, node, i);
} else
goto miss;
}
}
miss:
if (retry_key) {
longcpy(key, retry_key, geo->keylen);
retry_key = NULL;
goto retry;
}
return NULL;
}
EXPORT_SYMBOL_GPL(btree_get_prev);
static int getpos(struct btree_geo *geo, unsigned long *node,
unsigned long *key)
{
int i;
for (i = 0; i < geo->no_pairs; i++) {
if (keycmp(geo, node, i, key) <= 0)
break;
}
return i;
}
static int getfill(struct btree_geo *geo, unsigned long *node, int start)
{
int i;
for (i = start; i < geo->no_pairs; i++)
if (!bval(geo, node, i))
break;
return i;
}
/*
* locate the correct leaf node in the btree
*/
static unsigned long *find_level(struct btree_head *head, struct btree_geo *geo,
unsigned long *key, int level)
{
unsigned long *node = head->node;
int i, height;
for (height = head->height; height > level; height--) {
for (i = 0; i < geo->no_pairs; i++)
if (keycmp(geo, node, i, key) <= 0)
break;
if ((i == geo->no_pairs) || !bval(geo, node, i)) {
/* right-most key is too large, update it */
/* FIXME: If the right-most key on higher levels is
* always zero, this wouldn't be necessary. */
i--;
setkey(geo, node, i, key);
}
BUG_ON(i < 0);
node = bval(geo, node, i);
}
BUG_ON(!node);
return node;
}
static int btree_grow(struct btree_head *head, struct btree_geo *geo,
gfp_t gfp)
{
unsigned long *node;
int fill;
node = btree_node_alloc(head, gfp);
if (!node)
return -ENOMEM;
if (head->node) {
fill = getfill(geo, head->node, 0);
setkey(geo, node, 0, bkey(geo, head->node, fill - 1));
setval(geo, node, 0, head->node);
}
head->node = node;
head->height++;
return 0;
}
static void btree_shrink(struct btree_head *head, struct btree_geo *geo)
{
unsigned long *node;
int fill;
if (head->height <= 1)
return;
node = head->node;
fill = getfill(geo, node, 0);
BUG_ON(fill > 1);
head->node = bval(geo, node, 0);
head->height--;
mempool_free(node, head->mempool);
}
static int btree_insert_level(struct btree_head *head, struct btree_geo *geo,
unsigned long *key, void *val, int level,
gfp_t gfp)
{
unsigned long *node;
int i, pos, fill, err;
BUG_ON(!val);
if (head->height < level) {
err = btree_grow(head, geo, gfp);
if (err)
return err;
}
retry:
node = find_level(head, geo, key, level);
pos = getpos(geo, node, key);
fill = getfill(geo, node, pos);
/* two identical keys are not allowed */
BUG_ON(pos < fill && keycmp(geo, node, pos, key) == 0);
if (fill == geo->no_pairs) {
/* need to split node */
unsigned long *new;
new = btree_node_alloc(head, gfp);
if (!new)
return -ENOMEM;
err = btree_insert_level(head, geo,
bkey(geo, node, fill / 2 - 1),
new, level + 1, gfp);
if (err) {
mempool_free(new, head->mempool);
return err;
}
for (i = 0; i < fill / 2; i++) {
setkey(geo, new, i, bkey(geo, node, i));
setval(geo, new, i, bval(geo, node, i));
setkey(geo, node, i, bkey(geo, node, i + fill / 2));
setval(geo, node, i, bval(geo, node, i + fill / 2));
clearpair(geo, node, i + fill / 2);
}
if (fill & 1) {
setkey(geo, node, i, bkey(geo, node, fill - 1));
setval(geo, node, i, bval(geo, node, fill - 1));
clearpair(geo, node, fill - 1);
}
goto retry;
}
BUG_ON(fill >= geo->no_pairs);
/* shift and insert */
for (i = fill; i > pos; i--) {
setkey(geo, node, i, bkey(geo, node, i - 1));
setval(geo, node, i, bval(geo, node, i - 1));
}
setkey(geo, node, pos, key);
setval(geo, node, pos, val);
return 0;
}
int btree_insert(struct btree_head *head, struct btree_geo *geo,
unsigned long *key, void *val, gfp_t gfp)
{
BUG_ON(!val);
return btree_insert_level(head, geo, key, val, 1, gfp);
}
EXPORT_SYMBOL_GPL(btree_insert);
static void *btree_remove_level(struct btree_head *head, struct btree_geo *geo,
unsigned long *key, int level);
static void merge(struct btree_head *head, struct btree_geo *geo, int level,
unsigned long *left, int lfill,
unsigned long *right, int rfill,
unsigned long *parent, int lpos)
{
int i;
for (i = 0; i < rfill; i++) {
/* Move all keys to the left */
setkey(geo, left, lfill + i, bkey(geo, right, i));
setval(geo, left, lfill + i, bval(geo, right, i));
}
/* Exchange left and right child in parent */
setval(geo, parent, lpos, right);
setval(geo, parent, lpos + 1, left);
/* Remove left (formerly right) child from parent */
btree_remove_level(head, geo, bkey(geo, parent, lpos), level + 1);
mempool_free(right, head->mempool);
}
static void rebalance(struct btree_head *head, struct btree_geo *geo,
unsigned long *key, int level, unsigned long *child, int fill)
{
unsigned long *parent, *left = NULL, *right = NULL;
int i, no_left, no_right;
if (fill == 0) {
/* Because we don't steal entries from a neighbour, this case
* can happen. Parent node contains a single child, this
* node, so merging with a sibling never happens.
*/
btree_remove_level(head, geo, key, level + 1);
mempool_free(child, head->mempool);
return;
}
parent = find_level(head, geo, key, level + 1);
i = getpos(geo, parent, key);
BUG_ON(bval(geo, parent, i) != child);
if (i > 0) {
left = bval(geo, parent, i - 1);
no_left = getfill(geo, left, 0);
if (fill + no_left <= geo->no_pairs) {
merge(head, geo, level,
left, no_left,
child, fill,
parent, i - 1);
return;
}
}
if (i + 1 < getfill(geo, parent, i)) {
right = bval(geo, parent, i + 1);
no_right = getfill(geo, right, 0);
if (fill + no_right <= geo->no_pairs) {
merge(head, geo, level,
child, fill,
right, no_right,
parent, i);
return;
}
}
/*
* We could also try to steal one entry from the left or right
* neighbor. By not doing so we changed the invariant from
* "all nodes are at least half full" to "no two neighboring
* nodes can be merged". Which means that the average fill of
* all nodes is still half or better.
*/
}
static void *btree_remove_level(struct btree_head *head, struct btree_geo *geo,
unsigned long *key, int level)
{
unsigned long *node;
int i, pos, fill;
void *ret;
if (level > head->height) {
/* we recursed all the way up */
head->height = 0;
head->node = NULL;
return NULL;
}
node = find_level(head, geo, key, level);
pos = getpos(geo, node, key);
fill = getfill(geo, node, pos);
if ((level == 1) && (keycmp(geo, node, pos, key) != 0))
return NULL;
ret = bval(geo, node, pos);
/* remove and shift */
for (i = pos; i < fill - 1; i++) {
setkey(geo, node, i, bkey(geo, node, i + 1));
setval(geo, node, i, bval(geo, node, i + 1));
}
clearpair(geo, node, fill - 1);
if (fill - 1 < geo->no_pairs / 2) {
if (level < head->height)
rebalance(head, geo, key, level, node, fill - 1);
else if (fill - 1 == 1)
btree_shrink(head, geo);
}
return ret;
}
void *btree_remove(struct btree_head *head, struct btree_geo *geo,
unsigned long *key)
{
if (head->height == 0)
return NULL;
return btree_remove_level(head, geo, key, 1);
}
EXPORT_SYMBOL_GPL(btree_remove);
int btree_merge(struct btree_head *target, struct btree_head *victim,
struct btree_geo *geo, gfp_t gfp)
{
unsigned long key[MAX_KEYLEN];
unsigned long dup[MAX_KEYLEN];
void *val;
int err;
BUG_ON(target == victim);
if (!(target->node)) {
/* target is empty, just copy fields over */
target->node = victim->node;
target->height = victim->height;
__btree_init(victim);
return 0;
}
/* TODO: This needs some optimizations. Currently we do three tree
* walks to remove a single object from the victim.
*/
for (;;) {
if (!btree_last(victim, geo, key))
break;
val = btree_lookup(victim, geo, key);
err = btree_insert(target, geo, key, val, gfp);
if (err)
return err;
/* We must make a copy of the key, as the original will get
* mangled inside btree_remove. */
longcpy(dup, key, geo->keylen);
btree_remove(victim, geo, dup);
}
return 0;
}
EXPORT_SYMBOL_GPL(btree_merge);
static size_t __btree_for_each(struct btree_head *head, struct btree_geo *geo,
unsigned long *node, unsigned long opaque,
void (*func)(void *elem, unsigned long opaque,
unsigned long *key, size_t index,
void *func2),
void *func2, int reap, int height, size_t count)
{
int i;
unsigned long *child;
for (i = 0; i < geo->no_pairs; i++) {
child = bval(geo, node, i);
if (!child)
break;
if (height > 1)
count = __btree_for_each(head, geo, child, opaque,
func, func2, reap, height - 1, count);
else
func(child, opaque, bkey(geo, node, i), count++,
func2);
}
if (reap)
mempool_free(node, head->mempool);
return count;
}
static void empty(void *elem, unsigned long opaque, unsigned long *key,
size_t index, void *func2)
{
}
void visitorl(void *elem, unsigned long opaque, unsigned long *key,
size_t index, void *__func)
{
visitorl_t func = __func;
func(elem, opaque, *key, index);
}
EXPORT_SYMBOL_GPL(visitorl);
void visitor32(void *elem, unsigned long opaque, unsigned long *__key,
size_t index, void *__func)
{
visitor32_t func = __func;
u32 *key = (void *)__key;
func(elem, opaque, *key, index);
}
EXPORT_SYMBOL_GPL(visitor32);
void visitor64(void *elem, unsigned long opaque, unsigned long *__key,
size_t index, void *__func)
{
visitor64_t func = __func;
u64 *key = (void *)__key;
func(elem, opaque, *key, index);
}
EXPORT_SYMBOL_GPL(visitor64);
void visitor128(void *elem, unsigned long opaque, unsigned long *__key,
size_t index, void *__func)
{
visitor128_t func = __func;
u64 *key = (void *)__key;
func(elem, opaque, key[0], key[1], index);
}
EXPORT_SYMBOL_GPL(visitor128);
size_t btree_visitor(struct btree_head *head, struct btree_geo *geo,
unsigned long opaque,
void (*func)(void *elem, unsigned long opaque,
unsigned long *key,
size_t index, void *func2),
void *func2)
{
size_t count = 0;
if (!func2)
func = empty;
if (head->node)
count = __btree_for_each(head, geo, head->node, opaque, func,
func2, 0, head->height, 0);
return count;
}
EXPORT_SYMBOL_GPL(btree_visitor);
size_t btree_grim_visitor(struct btree_head *head, struct btree_geo *geo,
unsigned long opaque,
void (*func)(void *elem, unsigned long opaque,
unsigned long *key,
size_t index, void *func2),
void *func2)
{
size_t count = 0;
if (!func2)
func = empty;
if (head->node)
count = __btree_for_each(head, geo, head->node, opaque, func,
func2, 1, head->height, 0);
__btree_init(head);
return count;
}
EXPORT_SYMBOL_GPL(btree_grim_visitor);
static int __init btree_module_init(void)
{
btree_cachep = kmem_cache_create("btree_node", NODESIZE, 0,
SLAB_HWCACHE_ALIGN, NULL);
return 0;
}
static void __exit btree_module_exit(void)
{
kmem_cache_destroy(btree_cachep);
}
/* If core code starts using btree, initialization should happen even earlier */
module_init(btree_module_init);
module_exit(btree_module_exit);
MODULE_AUTHOR("Joern Engel <joern@logfs.org>");
MODULE_AUTHOR("Johannes Berg <johannes@sipsolutions.net>");
MODULE_LICENSE("GPL");
lib/bucket_locks.c
+54
View File
@@ -0,0 +1,54 @@
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
/* Allocate an array of spinlocks to be accessed by a hash. Two arguments
* indicate the number of elements to allocate in the array. max_size
* gives the maximum number of elements to allocate. cpu_mult gives
* the number of locks per CPU to allocate. The size is rounded up
* to a power of 2 to be suitable as a hash table.
*/
int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *locks_mask,
size_t max_size, unsigned int cpu_mult, gfp_t gfp,
const char *name, struct lock_class_key *key)
{
spinlock_t *tlocks = NULL;
unsigned int i, size;
#if defined(CONFIG_PROVE_LOCKING)
unsigned int nr_pcpus = 2;
#else
unsigned int nr_pcpus = num_possible_cpus();
#endif
if (cpu_mult) {
nr_pcpus = min_t(unsigned int, nr_pcpus, 64UL);
size = min_t(unsigned int, nr_pcpus * cpu_mult, max_size);
} else {
size = max_size;
}
if (sizeof(spinlock_t) != 0) {
tlocks = kvmalloc_array(size, sizeof(spinlock_t), gfp);
if (!tlocks)
return -ENOMEM;
for (i = 0; i < size; i++) {
spin_lock_init(&tlocks[i]);
lockdep_init_map(&tlocks[i].dep_map, name, key, 0);
}
}
*locks = tlocks;
*locks_mask = size - 1;
return 0;
}
EXPORT_SYMBOL(__alloc_bucket_spinlocks);
void free_bucket_spinlocks(spinlock_t *locks)
{
kvfree(locks);
}
EXPORT_SYMBOL(free_bucket_spinlocks);
lib/bug.c
+220
View File
@@ -0,0 +1,220 @@
// SPDX-License-Identifier: GPL-2.0
/*
Generic support for BUG()
This respects the following config options:
CONFIG_BUG - emit BUG traps. Nothing happens without this.
CONFIG_GENERIC_BUG - enable this code.
CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit pointers relative to
the containing struct bug_entry for bug_addr and file.
CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG
CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable
(though they're generally always on).
CONFIG_GENERIC_BUG is set by each architecture using this code.
To use this, your architecture must:
1. Set up the config options:
- Enable CONFIG_GENERIC_BUG if CONFIG_BUG
2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON)
- Define HAVE_ARCH_BUG
- Implement BUG() to generate a faulting instruction
- NOTE: struct bug_entry does not have "file" or "line" entries
when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate
the values accordingly.
3. Implement the trap
- In the illegal instruction trap handler (typically), verify
that the fault was in kernel mode, and call report_bug()
- report_bug() will return whether it was a false alarm, a warning,
or an actual bug.
- You must implement the is_valid_bugaddr(bugaddr) callback which
returns true if the eip is a real kernel address, and it points
to the expected BUG trap instruction.
Jeremy Fitzhardinge <jeremy@goop.org> 2006
*/
#define pr_fmt(fmt) fmt
#include <linux/list.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/sched.h>
#include <linux/rculist.h>
extern struct bug_entry __start___bug_table[], __stop___bug_table[];
static inline unsigned long bug_addr(const struct bug_entry *bug)
{
#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
return bug->bug_addr;
#else
return (unsigned long)bug + bug->bug_addr_disp;
#endif
}
#ifdef CONFIG_MODULES
/* Updates are protected by module mutex */
static LIST_HEAD(module_bug_list);
static struct bug_entry *module_find_bug(unsigned long bugaddr)
{
struct module *mod;
struct bug_entry *bug = NULL;
rcu_read_lock_sched();
list_for_each_entry_rcu(mod, &module_bug_list, bug_list) {
unsigned i;
bug = mod->bug_table;
for (i = 0; i < mod->num_bugs; ++i, ++bug)
if (bugaddr == bug_addr(bug))
goto out;
}
bug = NULL;
out:
rcu_read_unlock_sched();
return bug;
}
void module_bug_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
struct module *mod)
{
char *secstrings;
unsigned int i;
lockdep_assert_held(&module_mutex);
mod->bug_table = NULL;
mod->num_bugs = 0;
/* Find the __bug_table section, if present */
secstrings = (char *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
for (i = 1; i < hdr->e_shnum; i++) {
if (strcmp(secstrings+sechdrs[i].sh_name, "__bug_table"))
continue;
mod->bug_table = (void *) sechdrs[i].sh_addr;
mod->num_bugs = sechdrs[i].sh_size / sizeof(struct bug_entry);
break;
}
/*
* Strictly speaking this should have a spinlock to protect against
* traversals, but since we only traverse on BUG()s, a spinlock
* could potentially lead to deadlock and thus be counter-productive.
* Thus, this uses RCU to safely manipulate the bug list, since BUG
* must run in non-interruptive state.
*/
list_add_rcu(&mod->bug_list, &module_bug_list);
}
void module_bug_cleanup(struct module *mod)
{
lockdep_assert_held(&module_mutex);
list_del_rcu(&mod->bug_list);
}
#else
static inline struct bug_entry *module_find_bug(unsigned long bugaddr)
{
return NULL;
}
#endif
struct bug_entry *find_bug(unsigned long bugaddr)
{
struct bug_entry *bug;
for (bug = __start___bug_table; bug < __stop___bug_table; ++bug)
if (bugaddr == bug_addr(bug))
return bug;
return module_find_bug(bugaddr);
}
enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs)
{
struct bug_entry *bug;
const char *file;
unsigned line, warning, once, done;
if (!is_valid_bugaddr(bugaddr))
return BUG_TRAP_TYPE_NONE;
bug = find_bug(bugaddr);
if (!bug)
return BUG_TRAP_TYPE_NONE;
file = NULL;
line = 0;
#ifdef CONFIG_DEBUG_BUGVERBOSE
#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
file = bug->file;
#else
file = (const char *)bug + bug->file_disp;
#endif
line = bug->line;
#endif
warning = (bug->flags & BUGFLAG_WARNING) != 0;
once = (bug->flags & BUGFLAG_ONCE) != 0;
done = (bug->flags & BUGFLAG_DONE) != 0;
if (warning && once) {
if (done)
return BUG_TRAP_TYPE_WARN;
/*
* Since this is the only store, concurrency is not an issue.
*/
bug->flags |= BUGFLAG_DONE;
}
if (warning) {
/* this is a WARN_ON rather than BUG/BUG_ON */
__warn(file, line, (void *)bugaddr, BUG_GET_TAINT(bug), regs,
NULL);
return BUG_TRAP_TYPE_WARN;
}
printk(KERN_DEFAULT CUT_HERE);
if (file)
pr_crit("kernel BUG at %s:%u!\n", file, line);
else
pr_crit("Kernel BUG at %pB [verbose debug info unavailable]\n",
(void *)bugaddr);
return BUG_TRAP_TYPE_BUG;
}
static void clear_once_table(struct bug_entry *start, struct bug_entry *end)
{
struct bug_entry *bug;
for (bug = start; bug < end; bug++)
bug->flags &= ~BUGFLAG_DONE;
}
void generic_bug_clear_once(void)
{
#ifdef CONFIG_MODULES
struct module *mod;
rcu_read_lock_sched();
list_for_each_entry_rcu(mod, &module_bug_list, bug_list)
clear_once_table(mod->bug_table,
mod->bug_table + mod->num_bugs);
rcu_read_unlock_sched();
#endif
clear_once_table(__start___bug_table, __stop___bug_table);
}
lib/build_OID_registry
Executable
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/perl -w
#
# Build a static ASN.1 Object Identified (OID) registry
#
# Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
# Written by David Howells (dhowells@redhat.com)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public Licence
# as published by the Free Software Foundation; either version
# 2 of the Licence, or (at your option) any later version.
#
use strict;
my @names = ();
my @oids = ();
if ($#ARGV != 1) {
print STDERR "Format: ", $0, " <in-h-file> <out-c-file>\n";
exit(2);
}
#
# Open the file to read from
#
open IN_FILE, "<$ARGV[0]" || die;
while (<IN_FILE>) {
chomp;
if (m!\s+OID_([a-zA-z][a-zA-Z0-9_]+),\s+/[*]\s+([012][.0-9]*)\s+[*]/!) {
push @names, $1;
push @oids, $2;
}
}
close IN_FILE || die;
#
# Open the files to write into
#
open C_FILE, ">$ARGV[1]" or die;
print C_FILE "/*\n";
print C_FILE " * Automatically generated by ", $0, ". Do not edit\n";
print C_FILE " */\n";
#
# Split the data up into separate lists and also determine the lengths of the
# encoded data arrays.
#
my @indices = ();
my @lengths = ();
my $total_length = 0;
for (my $i = 0; $i <= $#names; $i++) {
my $name = $names[$i];
my $oid = $oids[$i];
my @components = split(/[.]/, $oid);
# Determine the encoded length of this OID
my $size = $#components;
for (my $loop = 2; $loop <= $#components; $loop++) {
my $c = $components[$loop];
# We will base128 encode the number
my $tmp = ($c == 0) ? 0 : int(log($c)/log(2));
$tmp = int($tmp / 7);
$size += $tmp;
}
push @lengths, $size;
push @indices, $total_length;
$total_length += $size;
}
#
# Emit the look-up-by-OID index table
#
print C_FILE "\n";
if ($total_length <= 255) {
print C_FILE "static const unsigned char oid_index[OID__NR + 1] = {\n";
} else {
print C_FILE "static const unsigned short oid_index[OID__NR + 1] = {\n";
}
for (my $i = 0; $i <= $#names; $i++) {
print C_FILE "\t[OID_", $names[$i], "] = ", $indices[$i], ",\n"
}
print C_FILE "\t[OID__NR] = ", $total_length, "\n";
print C_FILE "};\n";
#
# Encode the OIDs
#
my @encoded_oids = ();
for (my $i = 0; $i <= $#names; $i++) {
my @octets = ();
my @components = split(/[.]/, $oids[$i]);
push @octets, $components[0] * 40 + $components[1];
for (my $loop = 2; $loop <= $#components; $loop++) {
my $c = $components[$loop];
# Base128 encode the number
my $tmp = ($c == 0) ? 0 : int(log($c)/log(2));
$tmp = int($tmp / 7);
for (; $tmp > 0; $tmp--) {
push @octets, (($c >> $tmp * 7) & 0x7f) | 0x80;
}
push @octets, $c & 0x7f;
}
push @encoded_oids, \@octets;
}
#
# Create a hash value for each OID
#
my @hash_values = ();
for (my $i = 0; $i <= $#names; $i++) {
my @octets = @{$encoded_oids[$i]};
my $hash = $#octets;
foreach (@octets) {
$hash += $_ * 33;
}
$hash = ($hash >> 24) ^ ($hash >> 16) ^ ($hash >> 8) ^ ($hash);
push @hash_values, $hash & 0xff;
}
#
# Emit the OID data
#
print C_FILE "\n";
print C_FILE "static const unsigned char oid_data[", $total_length, "] = {\n";
for (my $i = 0; $i <= $#names; $i++) {
my @octets = @{$encoded_oids[$i]};
print C_FILE "\t";
print C_FILE $_, ", " foreach (@octets);
print C_FILE "\t// ", $names[$i];
print C_FILE "\n";
}
print C_FILE "};\n";
#
# Build the search index table (ordered by length then hash then content)
#
my @index_table = ( 0 .. $#names );
@index_table = sort {
my @octets_a = @{$encoded_oids[$a]};
my @octets_b = @{$encoded_oids[$b]};
return $hash_values[$a] <=> $hash_values[$b]
if ($hash_values[$a] != $hash_values[$b]);
return $#octets_a <=> $#octets_b
if ($#octets_a != $#octets_b);
for (my $i = $#octets_a; $i >= 0; $i--) {
return $octets_a[$i] <=> $octets_b[$i]
if ($octets_a[$i] != $octets_b[$i]);
}
return 0;
} @index_table;
#
# Emit the search index and hash value table
#
print C_FILE "\n";
print C_FILE "static const struct {\n";
print C_FILE "\tunsigned char hash;\n";
if ($#names <= 255) {
print C_FILE "\tenum OID oid : 8;\n";
} else {
print C_FILE "\tenum OID oid : 16;\n";
}
print C_FILE "} oid_search_table[OID__NR] = {\n";
for (my $i = 0; $i <= $#names; $i++) {
my @octets = @{$encoded_oids[$index_table[$i]]};
printf(C_FILE "\t[%3u] = { %3u, OID_%-35s }, // ",
$i,
$hash_values[$index_table[$i]],
$names[$index_table[$i]]);
printf C_FILE "%02x", $_ foreach (@octets);
print C_FILE "\n";
}
print C_FILE "};\n";
#
# Emit the OID debugging name table
#
#print C_FILE "\n";
#print C_FILE "const char *const oid_name_table[OID__NR + 1] = {\n";
#
#for (my $i = 0; $i <= $#names; $i++) {
# print C_FILE "\t\"", $names[$i], "\",\n"
#}
#print C_FILE "\t\"Unknown-OID\"\n";
#print C_FILE "};\n";
#
# Polish off
#
close C_FILE or die;
lib/bust_spinlocks.c
+32
View File
@@ -0,0 +1,32 @@
// SPDX-License-Identifier: GPL-2.0
/*
* lib/bust_spinlocks.c
*
* Provides a minimal bust_spinlocks for architectures which don't have one of their own.
*
* bust_spinlocks() clears any spinlocks which would prevent oops, die(), BUG()
* and panic() information from reaching the user.
*/
#include <linux/kernel.h>
#include <linux/printk.h>
#include <linux/spinlock.h>
#include <linux/tty.h>
#include <linux/wait.h>
#include <linux/vt_kern.h>
#include <linux/console.h>
void __attribute__((weak)) bust_spinlocks(int yes)
{
if (yes) {
++oops_in_progress;
} else {
#ifdef CONFIG_VT
unblank_screen();
#endif
console_unblank();
if (--oops_in_progress == 0)
wake_up_klogd();
}
}
lib/check_signature.c
+27
View File
@@ -0,0 +1,27 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/io.h>
#include <linux/export.h>
/**
* check_signature - find BIOS signatures
* @io_addr: mmio address to check
* @signature: signature block
* @length: length of signature
*
* Perform a signature comparison with the mmio address io_addr. This
* address should have been obtained by ioremap.
* Returns 1 on a match.
*/
int check_signature(const volatile void __iomem *io_addr,
const unsigned char *signature, int length)
{
while (length--) {
if (readb(io_addr) != *signature)
return 0;
io_addr++;
signature++;
}
return 1;
}
EXPORT_SYMBOL(check_signature);
lib/checksum.c
+208
View File
@@ -0,0 +1,208 @@
/*
*
* INET An implementation of the TCP/IP protocol suite for the LINUX
* operating system. INET is implemented using the BSD Socket
* interface as the means of communication with the user level.
*
* IP/TCP/UDP checksumming routines
*
* Authors: Jorge Cwik, <jorge@laser.satlink.net>
* Arnt Gulbrandsen, <agulbra@nvg.unit.no>
* Tom May, <ftom@netcom.com>
* Andreas Schwab, <schwab@issan.informatik.uni-dortmund.de>
* Lots of code moved from tcp.c and ip.c; see those files
* for more names.
*
* 03/02/96 Jes Sorensen, Andreas Schwab, Roman Hodek:
* Fixed some nasty bugs, causing some horrible crashes.
* A: At some points, the sum (%0) was used as
* length-counter instead of the length counter
* (%1). Thanks to Roman Hodek for pointing this out.
* B: GCC seems to mess up if one uses too many
* data-registers to hold input values and one tries to
* specify d0 and d1 as scratch registers. Letting gcc
* choose these registers itself solves the problem.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
/* Revised by Kenneth Albanowski for m68knommu. Basic problem: unaligned access
kills, so most of the assembly has to go. */
#include <linux/export.h>
#include <net/checksum.h>
#include <asm/byteorder.h>
#ifndef do_csum
static inline unsigned short from32to16(unsigned int x)
{
/* add up 16-bit and 16-bit for 16+c bit */
x = (x & 0xffff) + (x >> 16);
/* add up carry.. */
x = (x & 0xffff) + (x >> 16);
return x;
}
static unsigned int do_csum(const unsigned char *buff, int len)
{
int odd;
unsigned int result = 0;
if (len <= 0)
goto out;
odd = 1 & (unsigned long) buff;
if (odd) {
#ifdef __LITTLE_ENDIAN
result += (*buff << 8);
#else
result = *buff;
#endif
len--;
buff++;
}
if (len >= 2) {
if (2 & (unsigned long) buff) {
result += *(unsigned short *) buff;
len -= 2;
buff += 2;
}
if (len >= 4) {
const unsigned char *end = buff + ((unsigned)len & ~3);
unsigned int carry = 0;
do {
unsigned int w = *(unsigned int *) buff;
buff += 4;
result += carry;
result += w;
carry = (w > result);
} while (buff < end);
result += carry;
result = (result & 0xffff) + (result >> 16);
}
if (len & 2) {
result += *(unsigned short *) buff;
buff += 2;
}
}
if (len & 1)
#ifdef __LITTLE_ENDIAN
result += *buff;
#else
result += (*buff << 8);
#endif
result = from32to16(result);
if (odd)
result = ((result >> 8) & 0xff) | ((result & 0xff) << 8);
out:
return result;
}
#endif
#ifndef ip_fast_csum
/*
* This is a version of ip_compute_csum() optimized for IP headers,
* which always checksum on 4 octet boundaries.
*/
__sum16 ip_fast_csum(const void *iph, unsigned int ihl)
{
return (__force __sum16)~do_csum(iph, ihl*4);
}
EXPORT_SYMBOL(ip_fast_csum);
#endif
/*
* computes the checksum of a memory block at buff, length len,
* and adds in "sum" (32-bit)
*
* returns a 32-bit number suitable for feeding into itself
* or csum_tcpudp_magic
*
* this function must be called with even lengths, except
* for the last fragment, which may be odd
*
* it's best to have buff aligned on a 32-bit boundary
*/
__wsum csum_partial(const void *buff, int len, __wsum wsum)
{
unsigned int sum = (__force unsigned int)wsum;
unsigned int result = do_csum(buff, len);
/* add in old sum, and carry.. */
result += sum;
if (sum > result)
result += 1;
return (__force __wsum)result;
}
EXPORT_SYMBOL(csum_partial);
/*
* this routine is used for miscellaneous IP-like checksums, mainly
* in icmp.c
*/
__sum16 ip_compute_csum(const void *buff, int len)
{
return (__force __sum16)~do_csum(buff, len);
}
EXPORT_SYMBOL(ip_compute_csum);
/*
* copy from fs while checksumming, otherwise like csum_partial
*/
__wsum
csum_partial_copy_from_user(const void __user *src, void *dst, int len,
__wsum sum, int *csum_err)
{
int missing;
missing = __copy_from_user(dst, src, len);
if (missing) {
memset(dst + len - missing, 0, missing);
*csum_err = -EFAULT;
} else
*csum_err = 0;
return csum_partial(dst, len, sum);
}
EXPORT_SYMBOL(csum_partial_copy_from_user);
/*
* copy from ds while checksumming, otherwise like csum_partial
*/
__wsum
csum_partial_copy(const void *src, void *dst, int len, __wsum sum)
{
memcpy(dst, src, len);
return csum_partial(dst, len, sum);
}
EXPORT_SYMBOL(csum_partial_copy);
#ifndef csum_tcpudp_nofold
static inline u32 from64to32(u64 x)
{
/* add up 32-bit and 32-bit for 32+c bit */
x = (x & 0xffffffff) + (x >> 32);
/* add up carry.. */
x = (x & 0xffffffff) + (x >> 32);
return (u32)x;
}
__wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr,
__u32 len, __u8 proto, __wsum sum)
{
unsigned long long s = (__force u32)sum;
s += (__force u32)saddr;
s += (__force u32)daddr;
#ifdef __BIG_ENDIAN
s += proto + len;
#else
s += (proto + len) << 8;
#endif
return (__force __wsum)from64to32(s);
}
EXPORT_SYMBOL(csum_tcpudp_nofold);
#endif
lib/clz_ctz.c
+65
View File
@@ -0,0 +1,65 @@
/*
* lib/clz_ctz.c
*
* Copyright (C) 2013 Chanho Min <chanho.min@lge.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
* The functions in this file aren't called directly, but are required by
* GCC builtins such as __builtin_ctz, and therefore they can't be removed
* despite appearing unreferenced in kernel source.
*
* __c[lt]z[sd]i2 can be overridden by linking arch-specific versions.
*/
#include <linux/export.h>
#include <linux/kernel.h>
int __weak __ctzsi2(int val);
int __weak __ctzsi2(int val)
{
return __ffs(val);
}
EXPORT_SYMBOL(__ctzsi2);
int __weak __clzsi2(int val);
int __weak __clzsi2(int val)
{
return 32 - fls(val);
}
EXPORT_SYMBOL(__clzsi2);
int __weak __clzdi2(long val);
int __weak __ctzdi2(long val);
#if BITS_PER_LONG == 32
int __weak __clzdi2(long val)
{
return 32 - fls((int)val);
}
EXPORT_SYMBOL(__clzdi2);
int __weak __ctzdi2(long val)
{
return __ffs((u32)val);
}
EXPORT_SYMBOL(__ctzdi2);
#elif BITS_PER_LONG == 64
int __weak __clzdi2(long val)
{
return 64 - fls64((u64)val);
}
EXPORT_SYMBOL(__clzdi2);
int __weak __ctzdi2(long val)
{
return __ffs64((u64)val);
}
EXPORT_SYMBOL(__ctzdi2);
#else
#error BITS_PER_LONG not 32 or 64
#endif
lib/clz_tab.c
+19
View File
@@ -0,0 +1,19 @@
// SPDX-License-Identifier: GPL-2.0
const unsigned char __clz_tab[] = {
0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7,
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
8, 8, 8, 8, 8, 8, 8, 8,
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
8, 8, 8, 8, 8, 8, 8, 8,
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
8, 8, 8, 8, 8, 8, 8, 8,
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
8, 8, 8, 8, 8, 8, 8, 8,
};
lib/cmdline.c
+247
View File
@@ -0,0 +1,247 @@
/*
* linux/lib/cmdline.c
* Helper functions generally used for parsing kernel command line
* and module options.
*
* Code and copyrights come from init/main.c and arch/i386/kernel/setup.c.
*
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*
* GNU Indent formatting options for this file: -kr -i8 -npsl -pcs
*
*/
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/ctype.h>
/*
* If a hyphen was found in get_option, this will handle the
* range of numbers, M-N. This will expand the range and insert
* the values[M, M+1, ..., N] into the ints array in get_options.
*/
static int get_range(char **str, int *pint, int n)
{
int x, inc_counter, upper_range;
(*str)++;
upper_range = simple_strtol((*str), NULL, 0);
inc_counter = upper_range - *pint;
for (x = *pint; n && x < upper_range; x++, n--)
*pint++ = x;
return inc_counter;
}
/**
* get_option - Parse integer from an option string
* @str: option string
* @pint: (output) integer value parsed from @str
*
* Read an int from an option string; if available accept a subsequent
* comma as well.
*
* Return values:
* 0 - no int in string
* 1 - int found, no subsequent comma
* 2 - int found including a subsequent comma
* 3 - hyphen found to denote a range
*/
int get_option(char **str, int *pint)
{
char *cur = *str;
if (!cur || !(*cur))
return 0;
*pint = simple_strtol(cur, str, 0);
if (cur == *str)
return 0;
if (**str == ',') {
(*str)++;
return 2;
}
if (**str == '-')
return 3;
return 1;
}
EXPORT_SYMBOL(get_option);
/**
* get_options - Parse a string into a list of integers
* @str: String to be parsed
* @nints: size of integer array
* @ints: integer array
*
* This function parses a string containing a comma-separated
* list of integers, a hyphen-separated range of _positive_ integers,
* or a combination of both. The parse halts when the array is
* full, or when no more numbers can be retrieved from the
* string.
*
* Return value is the character in the string which caused
* the parse to end (typically a null terminator, if @str is
* completely parseable).
*/
char *get_options(const char *str, int nints, int *ints)
{
int res, i = 1;
while (i < nints) {
res = get_option((char **)&str, ints + i);
if (res == 0)
break;
if (res == 3) {
int range_nums;
range_nums = get_range((char **)&str, ints + i, nints - i);
if (range_nums < 0)
break;
/*
* Decrement the result by one to leave out the
* last number in the range. The next iteration
* will handle the upper number in the range
*/
i += (range_nums - 1);
}
i++;
if (res == 1)
break;
}
ints[0] = i - 1;
return (char *)str;
}
EXPORT_SYMBOL(get_options);
/**
* memparse - parse a string with mem suffixes into a number
* @ptr: Where parse begins
* @retptr: (output) Optional pointer to next char after parse completes
*
* Parses a string into a number. The number stored at @ptr is
* potentially suffixed with K, M, G, T, P, E.
*/
unsigned long long memparse(const char *ptr, char **retptr)
{
char *endptr; /* local pointer to end of parsed string */
unsigned long long ret = simple_strtoull(ptr, &endptr, 0);
switch (*endptr) {
case 'E':
case 'e':
ret <<= 10;
case 'P':
case 'p':
ret <<= 10;
case 'T':
case 't':
ret <<= 10;
case 'G':
case 'g':
ret <<= 10;
case 'M':
case 'm':
ret <<= 10;
case 'K':
case 'k':
ret <<= 10;
endptr++;
default:
break;
}
if (retptr)
*retptr = endptr;
return ret;
}
EXPORT_SYMBOL(memparse);
/**
* parse_option_str - Parse a string and check an option is set or not
* @str: String to be parsed
* @option: option name
*
* This function parses a string containing a comma-separated list of
* strings like a=b,c.
*
* Return true if there's such option in the string, or return false.
*/
bool parse_option_str(const char *str, const char *option)
{
while (*str) {
if (!strncmp(str, option, strlen(option))) {
str += strlen(option);
if (!*str || *str == ',')
return true;
}
while (*str && *str != ',')
str++;
if (*str == ',')
str++;
}
return false;
}
/*
* Parse a string to get a param value pair.
* You can use " around spaces, but can't escape ".
* Hyphens and underscores equivalent in parameter names.
*/
char *next_arg(char *args, char **param, char **val)
{
unsigned int i, equals = 0;
int in_quote = 0, quoted = 0;
char *next;
if (*args == '"') {
args++;
in_quote = 1;
quoted = 1;
}
for (i = 0; args[i]; i++) {
if (isspace(args[i]) && !in_quote)
break;
if (equals == 0) {
if (args[i] == '=')
equals = i;
}
if (args[i] == '"')
in_quote = !in_quote;
}
*param = args;
if (!equals)
*val = NULL;
else {
args[equals] = '\0';
*val = args + equals + 1;
/* Don't include quotes in value. */
if (**val == '"') {
(*val)++;
if (args[i-1] == '"')
args[i-1] = '\0';
}
}
if (quoted && args[i-1] == '"')
args[i-1] = '\0';
if (args[i]) {
args[i] = '\0';
next = args + i + 1;
} else
next = args + i;
/* Chew up trailing spaces. */
return skip_spaces(next);
}
lib/cmpdi2.c
+42
View File
@@ -0,0 +1,42 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see the file COPYING, or write
* to the Free Software Foundation, Inc.
*/
#include <linux/export.h>
#include <linux/libgcc.h>
word_type notrace __cmpdi2(long long a, long long b)
{
const DWunion au = {
.ll = a
};
const DWunion bu = {
.ll = b
};
if (au.s.high < bu.s.high)
return 0;
else if (au.s.high > bu.s.high)
return 2;
if ((unsigned int) au.s.low < (unsigned int) bu.s.low)
return 0;
else if ((unsigned int) au.s.low > (unsigned int) bu.s.low)
return 2;
return 1;
}
EXPORT_SYMBOL(__cmpdi2);
lib/compat_audit.c
+51
View File
@@ -0,0 +1,51 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/init.h>
#include <linux/types.h>
#include <asm/unistd32.h>
unsigned compat_dir_class[] = {
#include <asm-generic/audit_dir_write.h>
~0U
};
unsigned compat_read_class[] = {
#include <asm-generic/audit_read.h>
~0U
};
unsigned compat_write_class[] = {
#include <asm-generic/audit_write.h>
~0U
};
unsigned compat_chattr_class[] = {
#include <asm-generic/audit_change_attr.h>
~0U
};
unsigned compat_signal_class[] = {
#include <asm-generic/audit_signal.h>
~0U
};
int audit_classify_compat_syscall(int abi, unsigned syscall)
{
switch (syscall) {
#ifdef __NR_open
case __NR_open:
return 2;
#endif
#ifdef __NR_openat
case __NR_openat:
return 3;
#endif
#ifdef __NR_socketcall
case __NR_socketcall:
return 4;
#endif
case __NR_execve:
return 5;
default:
return 1;
}
}
lib/cordic.c
+101
View File
@@ -0,0 +1,101 @@
/*
* Copyright (c) 2011 Broadcom Corporation
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/module.h>
#include <linux/cordic.h>
#define CORDIC_ANGLE_GEN 39797
#define CORDIC_PRECISION_SHIFT 16
#define CORDIC_NUM_ITER (CORDIC_PRECISION_SHIFT + 2)
#define FIXED(X) ((s32)((X) << CORDIC_PRECISION_SHIFT))
#define FLOAT(X) (((X) >= 0) \
? ((((X) >> (CORDIC_PRECISION_SHIFT - 1)) + 1) >> 1) \
: -((((-(X)) >> (CORDIC_PRECISION_SHIFT - 1)) + 1) >> 1))
static const s32 arctan_table[] = {
2949120,
1740967,
919879,
466945,
234379,
117304,
58666,
29335,
14668,
7334,
3667,
1833,
917,
458,
229,
115,
57,
29
};
/*
* cordic_calc_iq() - calculates the i/q coordinate for given angle
*
* theta: angle in degrees for which i/q coordinate is to be calculated
* coord: function output parameter holding the i/q coordinate
*/
struct cordic_iq cordic_calc_iq(s32 theta)
{
struct cordic_iq coord;
s32 angle, valtmp;
unsigned iter;
int signx = 1;
int signtheta;
coord.i = CORDIC_ANGLE_GEN;
coord.q = 0;
angle = 0;
theta = FIXED(theta);
signtheta = (theta < 0) ? -1 : 1;
theta = ((theta + FIXED(180) * signtheta) % FIXED(360)) -
FIXED(180) * signtheta;
if (FLOAT(theta) > 90) {
theta -= FIXED(180);
signx = -1;
} else if (FLOAT(theta) < -90) {
theta += FIXED(180);
signx = -1;
}
for (iter = 0; iter < CORDIC_NUM_ITER; iter++) {
if (theta > angle) {
valtmp = coord.i - (coord.q >> iter);
coord.q += (coord.i >> iter);
angle += arctan_table[iter];
} else {
valtmp = coord.i + (coord.q >> iter);
coord.q -= (coord.i >> iter);
angle -= arctan_table[iter];
}
coord.i = valtmp;
}
coord.i *= signx;
coord.q *= signx;
return coord;
}
EXPORT_SYMBOL(cordic_calc_iq);
MODULE_DESCRIPTION("CORDIC algorithm");
MODULE_AUTHOR("Broadcom Corporation");
MODULE_LICENSE("Dual BSD/GPL");
lib/cpu_rmap.c
+307
View File
@@ -0,0 +1,307 @@
/*
* cpu_rmap.c: CPU affinity reverse-map support
* Copyright 2011 Solarflare Communications Inc.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation, incorporated herein by reference.
*/
#include <linux/cpu_rmap.h>
#include <linux/interrupt.h>
#include <linux/export.h>
/*
* These functions maintain a mapping from CPUs to some ordered set of
* objects with CPU affinities. This can be seen as a reverse-map of
* CPU affinity. However, we do not assume that the object affinities
* cover all CPUs in the system. For those CPUs not directly covered
* by object affinities, we attempt to find a nearest object based on
* CPU topology.
*/
/**
* alloc_cpu_rmap - allocate CPU affinity reverse-map
* @size: Number of objects to be mapped
* @flags: Allocation flags e.g. %GFP_KERNEL
*/
struct cpu_rmap *alloc_cpu_rmap(unsigned int size, gfp_t flags)
{
struct cpu_rmap *rmap;
unsigned int cpu;
size_t obj_offset;
/* This is a silly number of objects, and we use u16 indices. */
if (size > 0xffff)
return NULL;
/* Offset of object pointer array from base structure */
obj_offset = ALIGN(offsetof(struct cpu_rmap, near[nr_cpu_ids]),
sizeof(void *));
rmap = kzalloc(obj_offset + size * sizeof(rmap->obj[0]), flags);
if (!rmap)
return NULL;
kref_init(&rmap->refcount);
rmap->obj = (void **)((char *)rmap + obj_offset);
/* Initially assign CPUs to objects on a rota, since we have
* no idea where the objects are. Use infinite distance, so
* any object with known distance is preferable. Include the
* CPUs that are not present/online, since we definitely want
* any newly-hotplugged CPUs to have some object assigned.
*/
for_each_possible_cpu(cpu) {
rmap->near[cpu].index = cpu % size;
rmap->near[cpu].dist = CPU_RMAP_DIST_INF;
}
rmap->size = size;
return rmap;
}
EXPORT_SYMBOL(alloc_cpu_rmap);
/**
* cpu_rmap_release - internal reclaiming helper called from kref_put
* @ref: kref to struct cpu_rmap
*/
static void cpu_rmap_release(struct kref *ref)
{
struct cpu_rmap *rmap = container_of(ref, struct cpu_rmap, refcount);
kfree(rmap);
}
/**
* cpu_rmap_get - internal helper to get new ref on a cpu_rmap
* @rmap: reverse-map allocated with alloc_cpu_rmap()
*/
static inline void cpu_rmap_get(struct cpu_rmap *rmap)
{
kref_get(&rmap->refcount);
}
/**
* cpu_rmap_put - release ref on a cpu_rmap
* @rmap: reverse-map allocated with alloc_cpu_rmap()
*/
int cpu_rmap_put(struct cpu_rmap *rmap)
{
return kref_put(&rmap->refcount, cpu_rmap_release);
}
EXPORT_SYMBOL(cpu_rmap_put);
/* Reevaluate nearest object for given CPU, comparing with the given
* neighbours at the given distance.
*/
static bool cpu_rmap_copy_neigh(struct cpu_rmap *rmap, unsigned int cpu,
const struct cpumask *mask, u16 dist)
{
int neigh;
for_each_cpu(neigh, mask) {
if (rmap->near[cpu].dist > dist &&
rmap->near[neigh].dist <= dist) {
rmap->near[cpu].index = rmap->near[neigh].index;
rmap->near[cpu].dist = dist;
return true;
}
}
return false;
}
#ifdef DEBUG
static void debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix)
{
unsigned index;
unsigned int cpu;
pr_info("cpu_rmap %p, %s:\n", rmap, prefix);
for_each_possible_cpu(cpu) {
index = rmap->near[cpu].index;
pr_info("cpu %d -> obj %u (distance %u)\n",
cpu, index, rmap->near[cpu].dist);
}
}
#else
static inline void
debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix)
{
}
#endif
/**
* cpu_rmap_add - add object to a rmap
* @rmap: CPU rmap allocated with alloc_cpu_rmap()
* @obj: Object to add to rmap
*
* Return index of object.
*/
int cpu_rmap_add(struct cpu_rmap *rmap, void *obj)
{
u16 index;
BUG_ON(rmap->used >= rmap->size);
index = rmap->used++;
rmap->obj[index] = obj;
return index;
}
EXPORT_SYMBOL(cpu_rmap_add);
/**
* cpu_rmap_update - update CPU rmap following a change of object affinity
* @rmap: CPU rmap to update
* @index: Index of object whose affinity changed
* @affinity: New CPU affinity of object
*/
int cpu_rmap_update(struct cpu_rmap *rmap, u16 index,
const struct cpumask *affinity)
{
cpumask_var_t update_mask;
unsigned int cpu;
if (unlikely(!zalloc_cpumask_var(&update_mask, GFP_KERNEL)))
return -ENOMEM;
/* Invalidate distance for all CPUs for which this used to be
* the nearest object. Mark those CPUs for update.
*/
for_each_online_cpu(cpu) {
if (rmap->near[cpu].index == index) {
rmap->near[cpu].dist = CPU_RMAP_DIST_INF;
cpumask_set_cpu(cpu, update_mask);
}
}
debug_print_rmap(rmap, "after invalidating old distances");
/* Set distance to 0 for all CPUs in the new affinity mask.
* Mark all CPUs within their NUMA nodes for update.
*/
for_each_cpu(cpu, affinity) {
rmap->near[cpu].index = index;
rmap->near[cpu].dist = 0;
cpumask_or(update_mask, update_mask,
cpumask_of_node(cpu_to_node(cpu)));
}
debug_print_rmap(rmap, "after updating neighbours");
/* Update distances based on topology */
for_each_cpu(cpu, update_mask) {
if (cpu_rmap_copy_neigh(rmap, cpu,
topology_sibling_cpumask(cpu), 1))
continue;
if (cpu_rmap_copy_neigh(rmap, cpu,
topology_core_cpumask(cpu), 2))
continue;
if (cpu_rmap_copy_neigh(rmap, cpu,
cpumask_of_node(cpu_to_node(cpu)), 3))
continue;
/* We could continue into NUMA node distances, but for now
* we give up.
*/
}
debug_print_rmap(rmap, "after copying neighbours");
free_cpumask_var(update_mask);
return 0;
}
EXPORT_SYMBOL(cpu_rmap_update);
/* Glue between IRQ affinity notifiers and CPU rmaps */
struct irq_glue {
struct irq_affinity_notify notify;
struct cpu_rmap *rmap;
u16 index;
};
/**
* free_irq_cpu_rmap - free a CPU affinity reverse-map used for IRQs
* @rmap: Reverse-map allocated with alloc_irq_cpu_map(), or %NULL
*
* Must be called in process context, before freeing the IRQs.
*/
void free_irq_cpu_rmap(struct cpu_rmap *rmap)
{
struct irq_glue *glue;
u16 index;
if (!rmap)
return;
for (index = 0; index < rmap->used; index++) {
glue = rmap->obj[index];
irq_set_affinity_notifier(glue->notify.irq, NULL);
}
cpu_rmap_put(rmap);
}
EXPORT_SYMBOL(free_irq_cpu_rmap);
/**
* irq_cpu_rmap_notify - callback for IRQ subsystem when IRQ affinity updated
* @notify: struct irq_affinity_notify passed by irq/manage.c
* @mask: cpu mask for new SMP affinity
*
* This is executed in workqueue context.
*/
static void
irq_cpu_rmap_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
{
struct irq_glue *glue =
container_of(notify, struct irq_glue, notify);
int rc;
rc = cpu_rmap_update(glue->rmap, glue->index, mask);
if (rc)
pr_warning("irq_cpu_rmap_notify: update failed: %d\n", rc);
}
/**
* irq_cpu_rmap_release - reclaiming callback for IRQ subsystem
* @ref: kref to struct irq_affinity_notify passed by irq/manage.c
*/
static void irq_cpu_rmap_release(struct kref *ref)
{
struct irq_glue *glue =
container_of(ref, struct irq_glue, notify.kref);
cpu_rmap_put(glue->rmap);
kfree(glue);
}
/**
* irq_cpu_rmap_add - add an IRQ to a CPU affinity reverse-map
* @rmap: The reverse-map
* @irq: The IRQ number
*
* This adds an IRQ affinity notifier that will update the reverse-map
* automatically.
*
* Must be called in process context, after the IRQ is allocated but
* before it is bound with request_irq().
*/
int irq_cpu_rmap_add(struct cpu_rmap *rmap, int irq)
{
struct irq_glue *glue = kzalloc(sizeof(*glue), GFP_KERNEL);
int rc;
if (!glue)
return -ENOMEM;
glue->notify.notify = irq_cpu_rmap_notify;
glue->notify.release = irq_cpu_rmap_release;
glue->rmap = rmap;
cpu_rmap_get(rmap);
glue->index = cpu_rmap_add(rmap, glue);
rc = irq_set_affinity_notifier(irq, &glue->notify);
if (rc) {
cpu_rmap_put(glue->rmap);
kfree(glue);
}
return rc;
}
EXPORT_SYMBOL(irq_cpu_rmap_add);
lib/cpumask.c
+230
View File
@@ -0,0 +1,230 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/bitops.h>
#include <linux/cpumask.h>
#include <linux/export.h>
#include <linux/bootmem.h>
/**
* cpumask_next - get the next cpu in a cpumask
* @n: the cpu prior to the place to search (ie. return will be > @n)
* @srcp: the cpumask pointer
*
* Returns >= nr_cpu_ids if no further cpus set.
*/
unsigned int cpumask_next(int n, const struct cpumask *srcp)
{
/* -1 is a legal arg here. */
if (n != -1)
cpumask_check(n);
return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
}
EXPORT_SYMBOL(cpumask_next);
/**
* cpumask_next_and - get the next cpu in *src1p & *src2p
* @n: the cpu prior to the place to search (ie. return will be > @n)
* @src1p: the first cpumask pointer
* @src2p: the second cpumask pointer
*
* Returns >= nr_cpu_ids if no further cpus set in both.
*/
int cpumask_next_and(int n, const struct cpumask *src1p,
const struct cpumask *src2p)
{
/* -1 is a legal arg here. */
if (n != -1)
cpumask_check(n);
return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
nr_cpumask_bits, n + 1);
}
EXPORT_SYMBOL(cpumask_next_and);
/**
* cpumask_any_but - return a "random" in a cpumask, but not this one.
* @mask: the cpumask to search
* @cpu: the cpu to ignore.
*
* Often used to find any cpu but smp_processor_id() in a mask.
* Returns >= nr_cpu_ids if no cpus set.
*/
int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
{
unsigned int i;
cpumask_check(cpu);
for_each_cpu(i, mask)
if (i != cpu)
break;
return i;
}
EXPORT_SYMBOL(cpumask_any_but);
/**
* cpumask_next_wrap - helper to implement for_each_cpu_wrap
* @n: the cpu prior to the place to search
* @mask: the cpumask pointer
* @start: the start point of the iteration
* @wrap: assume @n crossing @start terminates the iteration
*
* Returns >= nr_cpu_ids on completion
*
* Note: the @wrap argument is required for the start condition when
* we cannot assume @start is set in @mask.
*/
int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
{
int next;
again:
next = cpumask_next(n, mask);
if (wrap && n < start && next >= start) {
return nr_cpumask_bits;
} else if (next >= nr_cpumask_bits) {
wrap = true;
n = -1;
goto again;
}
return next;
}
EXPORT_SYMBOL(cpumask_next_wrap);
/* These are not inline because of header tangles. */
#ifdef CONFIG_CPUMASK_OFFSTACK
/**
* alloc_cpumask_var_node - allocate a struct cpumask on a given node
* @mask: pointer to cpumask_var_t where the cpumask is returned
* @flags: GFP_ flags
*
* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
* a nop returning a constant 1 (in <linux/cpumask.h>)
* Returns TRUE if memory allocation succeeded, FALSE otherwise.
*
* In addition, mask will be NULL if this fails. Note that gcc is
* usually smart enough to know that mask can never be NULL if
* CONFIG_CPUMASK_OFFSTACK=n, so does code elimination in that case
* too.
*/
bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
{
*mask = kmalloc_node(cpumask_size(), flags, node);
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
if (!*mask) {
printk(KERN_ERR "=> alloc_cpumask_var: failed!\n");
dump_stack();
}
#endif
return *mask != NULL;
}
EXPORT_SYMBOL(alloc_cpumask_var_node);
bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
{
return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
}
EXPORT_SYMBOL(zalloc_cpumask_var_node);
/**
* alloc_cpumask_var - allocate a struct cpumask
* @mask: pointer to cpumask_var_t where the cpumask is returned
* @flags: GFP_ flags
*
* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
* a nop returning a constant 1 (in <linux/cpumask.h>).
*
* See alloc_cpumask_var_node.
*/
bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{
return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
}
EXPORT_SYMBOL(alloc_cpumask_var);
bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{
return alloc_cpumask_var(mask, flags | __GFP_ZERO);
}
EXPORT_SYMBOL(zalloc_cpumask_var);
/**
* alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena.
* @mask: pointer to cpumask_var_t where the cpumask is returned
*
* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
* a nop (in <linux/cpumask.h>).
* Either returns an allocated (zero-filled) cpumask, or causes the
* system to panic.
*/
void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask)
{
*mask = memblock_virt_alloc(cpumask_size(), 0);
}
/**
* free_cpumask_var - frees memory allocated for a struct cpumask.
* @mask: cpumask to free
*
* This is safe on a NULL mask.
*/
void free_cpumask_var(cpumask_var_t mask)
{
kfree(mask);
}
EXPORT_SYMBOL(free_cpumask_var);
/**
* free_bootmem_cpumask_var - frees result of alloc_bootmem_cpumask_var
* @mask: cpumask to free
*/
void __init free_bootmem_cpumask_var(cpumask_var_t mask)
{
memblock_free_early(__pa(mask), cpumask_size());
}
#endif
/**
* cpumask_local_spread - select the i'th cpu with local numa cpu's first
* @i: index number
* @node: local numa_node
*
* This function selects an online CPU according to a numa aware policy;
* local cpus are returned first, followed by non-local ones, then it
* wraps around.
*
* It's not very efficient, but useful for setup.
*/
unsigned int cpumask_local_spread(unsigned int i, int node)
{
int cpu;
/* Wrap: we always want a cpu. */
i %= num_online_cpus();
if (node == -1) {
for_each_cpu(cpu, cpu_online_mask)
if (i-- == 0)
return cpu;
} else {
/* NUMA first. */
for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask)
if (i-- == 0)
return cpu;
for_each_cpu(cpu, cpu_online_mask) {
/* Skip NUMA nodes, done above. */
if (cpumask_test_cpu(cpu, cpumask_of_node(node)))
continue;
if (i-- == 0)
return cpu;
}
}
BUG();
}
EXPORT_SYMBOL(cpumask_local_spread);
lib/crc-ccitt.c
+125
View File
@@ -0,0 +1,125 @@
/*
* linux/lib/crc-ccitt.c
*
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-ccitt.h>
/*
* This mysterious table is just the CRC of each possible byte. It can be
* computed using the standard bit-at-a-time methods. The polynomial can
* be seen in entry 128, 0x8408. This corresponds to x^0 + x^5 + x^12.
* Add the implicit x^16, and you have the standard CRC-CCITT.
*/
u16 const crc_ccitt_table[256] = {
0x0000, 0x1189, 0x2312, 0x329b, 0x4624, 0x57ad, 0x6536, 0x74bf,
0x8c48, 0x9dc1, 0xaf5a, 0xbed3, 0xca6c, 0xdbe5, 0xe97e, 0xf8f7,
0x1081, 0x0108, 0x3393, 0x221a, 0x56a5, 0x472c, 0x75b7, 0x643e,
0x9cc9, 0x8d40, 0xbfdb, 0xae52, 0xdaed, 0xcb64, 0xf9ff, 0xe876,
0x2102, 0x308b, 0x0210, 0x1399, 0x6726, 0x76af, 0x4434, 0x55bd,
0xad4a, 0xbcc3, 0x8e58, 0x9fd1, 0xeb6e, 0xfae7, 0xc87c, 0xd9f5,
0x3183, 0x200a, 0x1291, 0x0318, 0x77a7, 0x662e, 0x54b5, 0x453c,
0xbdcb, 0xac42, 0x9ed9, 0x8f50, 0xfbef, 0xea66, 0xd8fd, 0xc974,
0x4204, 0x538d, 0x6116, 0x709f, 0x0420, 0x15a9, 0x2732, 0x36bb,
0xce4c, 0xdfc5, 0xed5e, 0xfcd7, 0x8868, 0x99e1, 0xab7a, 0xbaf3,
0x5285, 0x430c, 0x7197, 0x601e, 0x14a1, 0x0528, 0x37b3, 0x263a,
0xdecd, 0xcf44, 0xfddf, 0xec56, 0x98e9, 0x8960, 0xbbfb, 0xaa72,
0x6306, 0x728f, 0x4014, 0x519d, 0x2522, 0x34ab, 0x0630, 0x17b9,
0xef4e, 0xfec7, 0xcc5c, 0xddd5, 0xa96a, 0xb8e3, 0x8a78, 0x9bf1,
0x7387, 0x620e, 0x5095, 0x411c, 0x35a3, 0x242a, 0x16b1, 0x0738,
0xffcf, 0xee46, 0xdcdd, 0xcd54, 0xb9eb, 0xa862, 0x9af9, 0x8b70,
0x8408, 0x9581, 0xa71a, 0xb693, 0xc22c, 0xd3a5, 0xe13e, 0xf0b7,
0x0840, 0x19c9, 0x2b52, 0x3adb, 0x4e64, 0x5fed, 0x6d76, 0x7cff,
0x9489, 0x8500, 0xb79b, 0xa612, 0xd2ad, 0xc324, 0xf1bf, 0xe036,
0x18c1, 0x0948, 0x3bd3, 0x2a5a, 0x5ee5, 0x4f6c, 0x7df7, 0x6c7e,
0xa50a, 0xb483, 0x8618, 0x9791, 0xe32e, 0xf2a7, 0xc03c, 0xd1b5,
0x2942, 0x38cb, 0x0a50, 0x1bd9, 0x6f66, 0x7eef, 0x4c74, 0x5dfd,
0xb58b, 0xa402, 0x9699, 0x8710, 0xf3af, 0xe226, 0xd0bd, 0xc134,
0x39c3, 0x284a, 0x1ad1, 0x0b58, 0x7fe7, 0x6e6e, 0x5cf5, 0x4d7c,
0xc60c, 0xd785, 0xe51e, 0xf497, 0x8028, 0x91a1, 0xa33a, 0xb2b3,
0x4a44, 0x5bcd, 0x6956, 0x78df, 0x0c60, 0x1de9, 0x2f72, 0x3efb,
0xd68d, 0xc704, 0xf59f, 0xe416, 0x90a9, 0x8120, 0xb3bb, 0xa232,
0x5ac5, 0x4b4c, 0x79d7, 0x685e, 0x1ce1, 0x0d68, 0x3ff3, 0x2e7a,
0xe70e, 0xf687, 0xc41c, 0xd595, 0xa12a, 0xb0a3, 0x8238, 0x93b1,
0x6b46, 0x7acf, 0x4854, 0x59dd, 0x2d62, 0x3ceb, 0x0e70, 0x1ff9,
0xf78f, 0xe606, 0xd49d, 0xc514, 0xb1ab, 0xa022, 0x92b9, 0x8330,
0x7bc7, 0x6a4e, 0x58d5, 0x495c, 0x3de3, 0x2c6a, 0x1ef1, 0x0f78
};
EXPORT_SYMBOL(crc_ccitt_table);
/*
* Similar table to calculate CRC16 variant known as CRC-CCITT-FALSE
* Reflected bits order, does not augment final value.
*/
u16 const crc_ccitt_false_table[256] = {
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50A5, 0x60C6, 0x70E7,
0x8108, 0x9129, 0xA14A, 0xB16B, 0xC18C, 0xD1AD, 0xE1CE, 0xF1EF,
0x1231, 0x0210, 0x3273, 0x2252, 0x52B5, 0x4294, 0x72F7, 0x62D6,
0x9339, 0x8318, 0xB37B, 0xA35A, 0xD3BD, 0xC39C, 0xF3FF, 0xE3DE,
0x2462, 0x3443, 0x0420, 0x1401, 0x64E6, 0x74C7, 0x44A4, 0x5485,
0xA56A, 0xB54B, 0x8528, 0x9509, 0xE5EE, 0xF5CF, 0xC5AC, 0xD58D,
0x3653, 0x2672, 0x1611, 0x0630, 0x76D7, 0x66F6, 0x5695, 0x46B4,
0xB75B, 0xA77A, 0x9719, 0x8738, 0xF7DF, 0xE7FE, 0xD79D, 0xC7BC,
0x48C4, 0x58E5, 0x6886, 0x78A7, 0x0840, 0x1861, 0x2802, 0x3823,
0xC9CC, 0xD9ED, 0xE98E, 0xF9AF, 0x8948, 0x9969, 0xA90A, 0xB92B,
0x5AF5, 0x4AD4, 0x7AB7, 0x6A96, 0x1A71, 0x0A50, 0x3A33, 0x2A12,
0xDBFD, 0xCBDC, 0xFBBF, 0xEB9E, 0x9B79, 0x8B58, 0xBB3B, 0xAB1A,
0x6CA6, 0x7C87, 0x4CE4, 0x5CC5, 0x2C22, 0x3C03, 0x0C60, 0x1C41,
0xEDAE, 0xFD8F, 0xCDEC, 0xDDCD, 0xAD2A, 0xBD0B, 0x8D68, 0x9D49,
0x7E97, 0x6EB6, 0x5ED5, 0x4EF4, 0x3E13, 0x2E32, 0x1E51, 0x0E70,
0xFF9F, 0xEFBE, 0xDFDD, 0xCFFC, 0xBF1B, 0xAF3A, 0x9F59, 0x8F78,
0x9188, 0x81A9, 0xB1CA, 0xA1EB, 0xD10C, 0xC12D, 0xF14E, 0xE16F,
0x1080, 0x00A1, 0x30C2, 0x20E3, 0x5004, 0x4025, 0x7046, 0x6067,
0x83B9, 0x9398, 0xA3FB, 0xB3DA, 0xC33D, 0xD31C, 0xE37F, 0xF35E,
0x02B1, 0x1290, 0x22F3, 0x32D2, 0x4235, 0x5214, 0x6277, 0x7256,
0xB5EA, 0xA5CB, 0x95A8, 0x8589, 0xF56E, 0xE54F, 0xD52C, 0xC50D,
0x34E2, 0x24C3, 0x14A0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
0xA7DB, 0xB7FA, 0x8799, 0x97B8, 0xE75F, 0xF77E, 0xC71D, 0xD73C,
0x26D3, 0x36F2, 0x0691, 0x16B0, 0x6657, 0x7676, 0x4615, 0x5634,
0xD94C, 0xC96D, 0xF90E, 0xE92F, 0x99C8, 0x89E9, 0xB98A, 0xA9AB,
0x5844, 0x4865, 0x7806, 0x6827, 0x18C0, 0x08E1, 0x3882, 0x28A3,
0xCB7D, 0xDB5C, 0xEB3F, 0xFB1E, 0x8BF9, 0x9BD8, 0xABBB, 0xBB9A,
0x4A75, 0x5A54, 0x6A37, 0x7A16, 0x0AF1, 0x1AD0, 0x2AB3, 0x3A92,
0xFD2E, 0xED0F, 0xDD6C, 0xCD4D, 0xBDAA, 0xAD8B, 0x9DE8, 0x8DC9,
0x7C26, 0x6C07, 0x5C64, 0x4C45, 0x3CA2, 0x2C83, 0x1CE0, 0x0CC1,
0xEF1F, 0xFF3E, 0xCF5D, 0xDF7C, 0xAF9B, 0xBFBA, 0x8FD9, 0x9FF8,
0x6E17, 0x7E36, 0x4E55, 0x5E74, 0x2E93, 0x3EB2, 0x0ED1, 0x1EF0
};
EXPORT_SYMBOL(crc_ccitt_false_table);
/**
* crc_ccitt - recompute the CRC (CRC-CCITT variant) for the data
* buffer
* @crc: previous CRC value
* @buffer: data pointer
* @len: number of bytes in the buffer
*/
u16 crc_ccitt(u16 crc, u8 const *buffer, size_t len)
{
while (len--)
crc = crc_ccitt_byte(crc, *buffer++);
return crc;
}
EXPORT_SYMBOL(crc_ccitt);
/**
* crc_ccitt_false - recompute the CRC (CRC-CCITT-FALSE variant)
* for the data buffer
* @crc: previous CRC value
* @buffer: data pointer
* @len: number of bytes in the buffer
*/
u16 crc_ccitt_false(u16 crc, u8 const *buffer, size_t len)
{
while (len--)
crc = crc_ccitt_false_byte(crc, *buffer++);
return crc;
}
EXPORT_SYMBOL(crc_ccitt_false);
MODULE_DESCRIPTION("CRC-CCITT calculations");
MODULE_LICENSE("GPL");
lib/crc-itu-t.c
+69
View File
@@ -0,0 +1,69 @@
/*
* crc-itu-t.c
*
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-itu-t.h>
/** CRC table for the CRC ITU-T V.41 0x1021 (x^16 + x^12 + x^15 + 1) */
const u16 crc_itu_t_table[256] = {
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0
};
EXPORT_SYMBOL(crc_itu_t_table);
/**
* crc_itu_t - Compute the CRC-ITU-T for the data buffer
*
* @crc: previous CRC value
* @buffer: data pointer
* @len: number of bytes in the buffer
*
* Returns the updated CRC value
*/
u16 crc_itu_t(u16 crc, const u8 *buffer, size_t len)
{
while (len--)
crc = crc_itu_t_byte(crc, *buffer++);
return crc;
}
EXPORT_SYMBOL(crc_itu_t);
MODULE_DESCRIPTION("CRC ITU-T V.41 calculations");
MODULE_LICENSE("GPL");
lib/crc-t10dif.c
+70
View File
@@ -0,0 +1,70 @@
/*
* T10 Data Integrity Field CRC16 calculation
*
* Copyright (c) 2007 Oracle Corporation. All rights reserved.
* Written by Martin K. Petersen <martin.petersen@oracle.com>
*
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-t10dif.h>
#include <linux/err.h>
#include <linux/init.h>
#include <crypto/hash.h>
#include <linux/static_key.h>
static struct crypto_shash *crct10dif_tfm;
static struct static_key crct10dif_fallback __read_mostly;
__u16 crc_t10dif_update(__u16 crc, const unsigned char *buffer, size_t len)
{
struct {
struct shash_desc shash;
char ctx[2];
} desc;
int err;
if (static_key_false(&crct10dif_fallback))
return crc_t10dif_generic(crc, buffer, len);
desc.shash.tfm = crct10dif_tfm;
desc.shash.flags = 0;
*(__u16 *)desc.ctx = crc;
err = crypto_shash_update(&desc.shash, buffer, len);
BUG_ON(err);
return *(__u16 *)desc.ctx;
}
EXPORT_SYMBOL(crc_t10dif_update);
__u16 crc_t10dif(const unsigned char *buffer, size_t len)
{
return crc_t10dif_update(0, buffer, len);
}
EXPORT_SYMBOL(crc_t10dif);
static int __init crc_t10dif_mod_init(void)
{
crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0);
if (IS_ERR(crct10dif_tfm)) {
static_key_slow_inc(&crct10dif_fallback);
crct10dif_tfm = NULL;
}
return 0;
}
static void __exit crc_t10dif_mod_fini(void)
{
crypto_free_shash(crct10dif_tfm);
}
module_init(crc_t10dif_mod_init);
module_exit(crc_t10dif_mod_fini);
MODULE_DESCRIPTION("T10 DIF CRC calculation");
MODULE_LICENSE("GPL");
MODULE_SOFTDEP("pre: crct10dif");
lib/crc16.c
+67
View File
@@ -0,0 +1,67 @@
/*
* crc16.c
*
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc16.h>
/** CRC table for the CRC-16. The poly is 0x8005 (x^16 + x^15 + x^2 + 1) */
u16 const crc16_table[256] = {
0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241,
0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440,
0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40,
0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841,
0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40,
0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41,
0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641,
0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040,
0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240,
0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441,
0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41,
0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840,
0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41,
0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40,
0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640,
0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041,
0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240,
0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441,
0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41,
0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840,
0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41,
0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40,
0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640,
0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041,
0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241,
0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440,
0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40,
0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841,
0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40,
0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41,
0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641,
0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040
};
EXPORT_SYMBOL(crc16_table);
/**
* crc16 - compute the CRC-16 for the data buffer
* @crc: previous CRC value
* @buffer: data pointer
* @len: number of bytes in the buffer
*
* Returns the updated CRC value.
*/
u16 crc16(u16 crc, u8 const *buffer, size_t len)
{
while (len--)
crc = crc16_byte(crc, *buffer++);
return crc;
}
EXPORT_SYMBOL(crc16);
MODULE_DESCRIPTION("CRC16 calculations");
MODULE_LICENSE("GPL");
lib/crc32.c
+343
View File
@@ -0,0 +1,343 @@
/*
* Aug 8, 2011 Bob Pearson with help from Joakim Tjernlund and George Spelvin
* cleaned up code to current version of sparse and added the slicing-by-8
* algorithm to the closely similar existing slicing-by-4 algorithm.
*
* Oct 15, 2000 Matt Domsch <Matt_Domsch@dell.com>
* Nicer crc32 functions/docs submitted by linux@horizon.com. Thanks!
* Code was from the public domain, copyright abandoned. Code was
* subsequently included in the kernel, thus was re-licensed under the
* GNU GPL v2.
*
* Oct 12, 2000 Matt Domsch <Matt_Domsch@dell.com>
* Same crc32 function was used in 5 other places in the kernel.
* I made one version, and deleted the others.
* There are various incantations of crc32(). Some use a seed of 0 or ~0.
* Some xor at the end with ~0. The generic crc32() function takes
* seed as an argument, and doesn't xor at the end. Then individual
* users can do whatever they need.
* drivers/net/smc9194.c uses seed ~0, doesn't xor with ~0.
* fs/jffs2 uses seed 0, doesn't xor with ~0.
* fs/partitions/efi.c uses seed ~0, xor's with ~0.
*
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*/
/* see: Documentation/crc32.txt for a description of algorithms */
#include <linux/crc32.h>
#include <linux/crc32poly.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/sched.h>
#include "crc32defs.h"
#if CRC_LE_BITS > 8
# define tole(x) ((__force u32) cpu_to_le32(x))
#else
# define tole(x) (x)
#endif
#if CRC_BE_BITS > 8
# define tobe(x) ((__force u32) cpu_to_be32(x))
#else
# define tobe(x) (x)
#endif
#include "crc32table.h"
MODULE_AUTHOR("Matt Domsch <Matt_Domsch@dell.com>");
MODULE_DESCRIPTION("Various CRC32 calculations");
MODULE_LICENSE("GPL");
#if CRC_LE_BITS > 8 || CRC_BE_BITS > 8
/* implements slicing-by-4 or slicing-by-8 algorithm */
static inline u32 __pure
crc32_body(u32 crc, unsigned char const *buf, size_t len, const u32 (*tab)[256])
{
# ifdef __LITTLE_ENDIAN
# define DO_CRC(x) crc = t0[(crc ^ (x)) & 255] ^ (crc >> 8)
# define DO_CRC4 (t3[(q) & 255] ^ t2[(q >> 8) & 255] ^ \
t1[(q >> 16) & 255] ^ t0[(q >> 24) & 255])
# define DO_CRC8 (t7[(q) & 255] ^ t6[(q >> 8) & 255] ^ \
t5[(q >> 16) & 255] ^ t4[(q >> 24) & 255])
# else
# define DO_CRC(x) crc = t0[((crc >> 24) ^ (x)) & 255] ^ (crc << 8)
# define DO_CRC4 (t0[(q) & 255] ^ t1[(q >> 8) & 255] ^ \
t2[(q >> 16) & 255] ^ t3[(q >> 24) & 255])
# define DO_CRC8 (t4[(q) & 255] ^ t5[(q >> 8) & 255] ^ \
t6[(q >> 16) & 255] ^ t7[(q >> 24) & 255])
# endif
const u32 *b;
size_t rem_len;
# ifdef CONFIG_X86
size_t i;
# endif
const u32 *t0=tab[0], *t1=tab[1], *t2=tab[2], *t3=tab[3];
# if CRC_LE_BITS != 32
const u32 *t4 = tab[4], *t5 = tab[5], *t6 = tab[6], *t7 = tab[7];
# endif
u32 q;
/* Align it */
if (unlikely((long)buf & 3 && len)) {
do {
DO_CRC(*buf++);
} while ((--len) && ((long)buf)&3);
}
# if CRC_LE_BITS == 32
rem_len = len & 3;
len = len >> 2;
# else
rem_len = len & 7;
len = len >> 3;
# endif
b = (const u32 *)buf;
# ifdef CONFIG_X86
--b;
for (i = 0; i < len; i++) {
# else
for (--b; len; --len) {
# endif
q = crc ^ *++b; /* use pre increment for speed */
# if CRC_LE_BITS == 32
crc = DO_CRC4;
# else
crc = DO_CRC8;
q = *++b;
crc ^= DO_CRC4;
# endif
}
len = rem_len;
/* And the last few bytes */
if (len) {
u8 *p = (u8 *)(b + 1) - 1;
# ifdef CONFIG_X86
for (i = 0; i < len; i++)
DO_CRC(*++p); /* use pre increment for speed */
# else
do {
DO_CRC(*++p); /* use pre increment for speed */
} while (--len);
# endif
}
return crc;
#undef DO_CRC
#undef DO_CRC4
#undef DO_CRC8
}
#endif
/**
* crc32_le_generic() - Calculate bitwise little-endian Ethernet AUTODIN II
* CRC32/CRC32C
* @crc: seed value for computation. ~0 for Ethernet, sometimes 0 for other
* uses, or the previous crc32/crc32c value if computing incrementally.
* @p: pointer to buffer over which CRC32/CRC32C is run
* @len: length of buffer @p
* @tab: little-endian Ethernet table
* @polynomial: CRC32/CRC32c LE polynomial
*/
static inline u32 __pure crc32_le_generic(u32 crc, unsigned char const *p,
size_t len, const u32 (*tab)[256],
u32 polynomial)
{
#if CRC_LE_BITS == 1
int i;
while (len--) {
crc ^= *p++;
for (i = 0; i < 8; i++)
crc = (crc >> 1) ^ ((crc & 1) ? polynomial : 0);
}
# elif CRC_LE_BITS == 2
while (len--) {
crc ^= *p++;
crc = (crc >> 2) ^ tab[0][crc & 3];
crc = (crc >> 2) ^ tab[0][crc & 3];
crc = (crc >> 2) ^ tab[0][crc & 3];
crc = (crc >> 2) ^ tab[0][crc & 3];
}
# elif CRC_LE_BITS == 4
while (len--) {
crc ^= *p++;
crc = (crc >> 4) ^ tab[0][crc & 15];
crc = (crc >> 4) ^ tab[0][crc & 15];
}
# elif CRC_LE_BITS == 8
/* aka Sarwate algorithm */
while (len--) {
crc ^= *p++;
crc = (crc >> 8) ^ tab[0][crc & 255];
}
# else
crc = (__force u32) __cpu_to_le32(crc);
crc = crc32_body(crc, p, len, tab);
crc = __le32_to_cpu((__force __le32)crc);
#endif
return crc;
}
#if CRC_LE_BITS == 1
u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len)
{
return crc32_le_generic(crc, p, len, NULL, CRC32_POLY_LE);
}
u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len)
{
return crc32_le_generic(crc, p, len, NULL, CRC32C_POLY_LE);
}
#else
u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len)
{
return crc32_le_generic(crc, p, len,
(const u32 (*)[256])crc32table_le, CRC32_POLY_LE);
}
u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len)
{
return crc32_le_generic(crc, p, len,
(const u32 (*)[256])crc32ctable_le, CRC32C_POLY_LE);
}
#endif
EXPORT_SYMBOL(crc32_le);
EXPORT_SYMBOL(__crc32c_le);
/*
* This multiplies the polynomials x and y modulo the given modulus.
* This follows the "little-endian" CRC convention that the lsbit
* represents the highest power of x, and the msbit represents x^0.
*/
static u32 __attribute_const__ gf2_multiply(u32 x, u32 y, u32 modulus)
{
u32 product = x & 1 ? y : 0;
int i;
for (i = 0; i < 31; i++) {
product = (product >> 1) ^ (product & 1 ? modulus : 0);
x >>= 1;
product ^= x & 1 ? y : 0;
}
return product;
}
/**
* crc32_generic_shift - Append @len 0 bytes to crc, in logarithmic time
* @crc: The original little-endian CRC (i.e. lsbit is x^31 coefficient)
* @len: The number of bytes. @crc is multiplied by x^(8*@len)
* @polynomial: The modulus used to reduce the result to 32 bits.
*
* It's possible to parallelize CRC computations by computing a CRC
* over separate ranges of a buffer, then summing them.
* This shifts the given CRC by 8*len bits (i.e. produces the same effect
* as appending len bytes of zero to the data), in time proportional
* to log(len).
*/
static u32 __attribute_const__ crc32_generic_shift(u32 crc, size_t len,
u32 polynomial)
{
u32 power = polynomial; /* CRC of x^32 */
int i;
/* Shift up to 32 bits in the simple linear way */
for (i = 0; i < 8 * (int)(len & 3); i++)
crc = (crc >> 1) ^ (crc & 1 ? polynomial : 0);
len >>= 2;
if (!len)
return crc;
for (;;) {
/* "power" is x^(2^i), modulo the polynomial */
if (len & 1)
crc = gf2_multiply(crc, power, polynomial);
len >>= 1;
if (!len)
break;
/* Square power, advancing to x^(2^(i+1)) */
power = gf2_multiply(power, power, polynomial);
}
return crc;
}
u32 __attribute_const__ crc32_le_shift(u32 crc, size_t len)
{
return crc32_generic_shift(crc, len, CRC32_POLY_LE);
}
u32 __attribute_const__ __crc32c_le_shift(u32 crc, size_t len)
{
return crc32_generic_shift(crc, len, CRC32C_POLY_LE);
}
EXPORT_SYMBOL(crc32_le_shift);
EXPORT_SYMBOL(__crc32c_le_shift);
/**
* crc32_be_generic() - Calculate bitwise big-endian Ethernet AUTODIN II CRC32
* @crc: seed value for computation. ~0 for Ethernet, sometimes 0 for
* other uses, or the previous crc32 value if computing incrementally.
* @p: pointer to buffer over which CRC32 is run
* @len: length of buffer @p
* @tab: big-endian Ethernet table
* @polynomial: CRC32 BE polynomial
*/
static inline u32 __pure crc32_be_generic(u32 crc, unsigned char const *p,
size_t len, const u32 (*tab)[256],
u32 polynomial)
{
#if CRC_BE_BITS == 1
int i;
while (len--) {
crc ^= *p++ << 24;
for (i = 0; i < 8; i++)
crc =
(crc << 1) ^ ((crc & 0x80000000) ? polynomial :
0);
}
# elif CRC_BE_BITS == 2
while (len--) {
crc ^= *p++ << 24;
crc = (crc << 2) ^ tab[0][crc >> 30];
crc = (crc << 2) ^ tab[0][crc >> 30];
crc = (crc << 2) ^ tab[0][crc >> 30];
crc = (crc << 2) ^ tab[0][crc >> 30];
}
# elif CRC_BE_BITS == 4
while (len--) {
crc ^= *p++ << 24;
crc = (crc << 4) ^ tab[0][crc >> 28];
crc = (crc << 4) ^ tab[0][crc >> 28];
}
# elif CRC_BE_BITS == 8
while (len--) {
crc ^= *p++ << 24;
crc = (crc << 8) ^ tab[0][crc >> 24];
}
# else
crc = (__force u32) __cpu_to_be32(crc);
crc = crc32_body(crc, p, len, tab);
crc = __be32_to_cpu((__force __be32)crc);
# endif
return crc;
}
#if CRC_BE_BITS == 1
u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len)
{
return crc32_be_generic(crc, p, len, NULL, CRC32_POLY_BE);
}
#else
u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len)
{
return crc32_be_generic(crc, p, len,
(const u32 (*)[256])crc32table_be, CRC32_POLY_BE);
}
#endif
EXPORT_SYMBOL(crc32_be);
lib/crc32defs.h
+59
View File
@@ -0,0 +1,59 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Try to choose an implementation variant via Kconfig */
#ifdef CONFIG_CRC32_SLICEBY8
# define CRC_LE_BITS 64
# define CRC_BE_BITS 64
#endif
#ifdef CONFIG_CRC32_SLICEBY4
# define CRC_LE_BITS 32
# define CRC_BE_BITS 32
#endif
#ifdef CONFIG_CRC32_SARWATE
# define CRC_LE_BITS 8
# define CRC_BE_BITS 8
#endif
#ifdef CONFIG_CRC32_BIT
# define CRC_LE_BITS 1
# define CRC_BE_BITS 1
#endif
/*
* How many bits at a time to use. Valid values are 1, 2, 4, 8, 32 and 64.
* For less performance-sensitive, use 4 or 8 to save table size.
* For larger systems choose same as CPU architecture as default.
* This works well on X86_64, SPARC64 systems. This may require some
* elaboration after experiments with other architectures.
*/
#ifndef CRC_LE_BITS
# ifdef CONFIG_64BIT
# define CRC_LE_BITS 64
# else
# define CRC_LE_BITS 32
# endif
#endif
#ifndef CRC_BE_BITS
# ifdef CONFIG_64BIT
# define CRC_BE_BITS 64
# else
# define CRC_BE_BITS 32
# endif
#endif
/*
* Little-endian CRC computation. Used with serial bit streams sent
* lsbit-first. Be sure to use cpu_to_le32() to append the computed CRC.
*/
#if CRC_LE_BITS > 64 || CRC_LE_BITS < 1 || CRC_LE_BITS == 16 || \
CRC_LE_BITS & CRC_LE_BITS-1
# error "CRC_LE_BITS must be one of {1, 2, 4, 8, 32, 64}"
#endif
/*
* Big-endian CRC computation. Used with serial bit streams sent
* msbit-first. Be sure to use cpu_to_be32() to append the computed CRC.
*/
#if CRC_BE_BITS > 64 || CRC_BE_BITS < 1 || CRC_BE_BITS == 16 || \
CRC_BE_BITS & CRC_BE_BITS-1
# error "CRC_BE_BITS must be one of {1, 2, 4, 8, 32, 64}"
#endif
lib/crc32test.c
+852
View File
@@ -0,0 +1,852 @@
/*
* Aug 8, 2011 Bob Pearson with help from Joakim Tjernlund and George Spelvin
* cleaned up code to current version of sparse and added the slicing-by-8
* algorithm to the closely similar existing slicing-by-4 algorithm.
*
* Oct 15, 2000 Matt Domsch <Matt_Domsch@dell.com>
* Nicer crc32 functions/docs submitted by linux@horizon.com. Thanks!
* Code was from the public domain, copyright abandoned. Code was
* subsequently included in the kernel, thus was re-licensed under the
* GNU GPL v2.
*
* Oct 12, 2000 Matt Domsch <Matt_Domsch@dell.com>
* Same crc32 function was used in 5 other places in the kernel.
* I made one version, and deleted the others.
* There are various incantations of crc32(). Some use a seed of 0 or ~0.
* Some xor at the end with ~0. The generic crc32() function takes
* seed as an argument, and doesn't xor at the end. Then individual
* users can do whatever they need.
* drivers/net/smc9194.c uses seed ~0, doesn't xor with ~0.
* fs/jffs2 uses seed 0, doesn't xor with ~0.
* fs/partitions/efi.c uses seed ~0, xor's with ~0.
*
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*/
#include <linux/crc32.h>
#include <linux/module.h>
#include <linux/sched.h>
#include "crc32defs.h"
/* 4096 random bytes */
static u8 const __aligned(8) test_buf[] __initconst =
{
0x5b, 0x85, 0x21, 0xcb, 0x09, 0x68, 0x7d, 0x30,
0xc7, 0x69, 0xd7, 0x30, 0x92, 0xde, 0x59, 0xe4,
0xc9, 0x6e, 0x8b, 0xdb, 0x98, 0x6b, 0xaa, 0x60,
0xa8, 0xb5, 0xbc, 0x6c, 0xa9, 0xb1, 0x5b, 0x2c,
0xea, 0xb4, 0x92, 0x6a, 0x3f, 0x79, 0x91, 0xe4,
0xe9, 0x70, 0x51, 0x8c, 0x7f, 0x95, 0x6f, 0x1a,
0x56, 0xa1, 0x5c, 0x27, 0x03, 0x67, 0x9f, 0x3a,
0xe2, 0x31, 0x11, 0x29, 0x6b, 0x98, 0xfc, 0xc4,
0x53, 0x24, 0xc5, 0x8b, 0xce, 0x47, 0xb2, 0xb9,
0x32, 0xcb, 0xc1, 0xd0, 0x03, 0x57, 0x4e, 0xd4,
0xe9, 0x3c, 0xa1, 0x63, 0xcf, 0x12, 0x0e, 0xca,
0xe1, 0x13, 0xd1, 0x93, 0xa6, 0x88, 0x5c, 0x61,
0x5b, 0xbb, 0xf0, 0x19, 0x46, 0xb4, 0xcf, 0x9e,
0xb6, 0x6b, 0x4c, 0x3a, 0xcf, 0x60, 0xf9, 0x7a,
0x8d, 0x07, 0x63, 0xdb, 0x40, 0xe9, 0x0b, 0x6f,
0xad, 0x97, 0xf1, 0xed, 0xd0, 0x1e, 0x26, 0xfd,
0xbf, 0xb7, 0xc8, 0x04, 0x94, 0xf8, 0x8b, 0x8c,
0xf1, 0xab, 0x7a, 0xd4, 0xdd, 0xf3, 0xe8, 0x88,
0xc3, 0xed, 0x17, 0x8a, 0x9b, 0x40, 0x0d, 0x53,
0x62, 0x12, 0x03, 0x5f, 0x1b, 0x35, 0x32, 0x1f,
0xb4, 0x7b, 0x93, 0x78, 0x0d, 0xdb, 0xce, 0xa4,
0xc0, 0x47, 0xd5, 0xbf, 0x68, 0xe8, 0x5d, 0x74,
0x8f, 0x8e, 0x75, 0x1c, 0xb2, 0x4f, 0x9a, 0x60,
0xd1, 0xbe, 0x10, 0xf4, 0x5c, 0xa1, 0x53, 0x09,
0xa5, 0xe0, 0x09, 0x54, 0x85, 0x5c, 0xdc, 0x07,
0xe7, 0x21, 0x69, 0x7b, 0x8a, 0xfd, 0x90, 0xf1,
0x22, 0xd0, 0xb4, 0x36, 0x28, 0xe6, 0xb8, 0x0f,
0x39, 0xde, 0xc8, 0xf3, 0x86, 0x60, 0x34, 0xd2,
0x5e, 0xdf, 0xfd, 0xcf, 0x0f, 0xa9, 0x65, 0xf0,
0xd5, 0x4d, 0x96, 0x40, 0xe3, 0xdf, 0x3f, 0x95,
0x5a, 0x39, 0x19, 0x93, 0xf4, 0x75, 0xce, 0x22,
0x00, 0x1c, 0x93, 0xe2, 0x03, 0x66, 0xf4, 0x93,
0x73, 0x86, 0x81, 0x8e, 0x29, 0x44, 0x48, 0x86,
0x61, 0x7c, 0x48, 0xa3, 0x43, 0xd2, 0x9c, 0x8d,
0xd4, 0x95, 0xdd, 0xe1, 0x22, 0x89, 0x3a, 0x40,
0x4c, 0x1b, 0x8a, 0x04, 0xa8, 0x09, 0x69, 0x8b,
0xea, 0xc6, 0x55, 0x8e, 0x57, 0xe6, 0x64, 0x35,
0xf0, 0xc7, 0x16, 0x9f, 0x5d, 0x5e, 0x86, 0x40,
0x46, 0xbb, 0xe5, 0x45, 0x88, 0xfe, 0xc9, 0x63,
0x15, 0xfb, 0xf5, 0xbd, 0x71, 0x61, 0xeb, 0x7b,
0x78, 0x70, 0x07, 0x31, 0x03, 0x9f, 0xb2, 0xc8,
0xa7, 0xab, 0x47, 0xfd, 0xdf, 0xa0, 0x78, 0x72,
0xa4, 0x2a, 0xe4, 0xb6, 0xba, 0xc0, 0x1e, 0x86,
0x71, 0xe6, 0x3d, 0x18, 0x37, 0x70, 0xe6, 0xff,
0xe0, 0xbc, 0x0b, 0x22, 0xa0, 0x1f, 0xd3, 0xed,
0xa2, 0x55, 0x39, 0xab, 0xa8, 0x13, 0x73, 0x7c,
0x3f, 0xb2, 0xd6, 0x19, 0xac, 0xff, 0x99, 0xed,
0xe8, 0xe6, 0xa6, 0x22, 0xe3, 0x9c, 0xf1, 0x30,
0xdc, 0x01, 0x0a, 0x56, 0xfa, 0xe4, 0xc9, 0x99,
0xdd, 0xa8, 0xd8, 0xda, 0x35, 0x51, 0x73, 0xb4,
0x40, 0x86, 0x85, 0xdb, 0x5c, 0xd5, 0x85, 0x80,
0x14, 0x9c, 0xfd, 0x98, 0xa9, 0x82, 0xc5, 0x37,
0xff, 0x32, 0x5d, 0xd0, 0x0b, 0xfa, 0xdc, 0x04,
0x5e, 0x09, 0xd2, 0xca, 0x17, 0x4b, 0x1a, 0x8e,
0x15, 0xe1, 0xcc, 0x4e, 0x52, 0x88, 0x35, 0xbd,
0x48, 0xfe, 0x15, 0xa0, 0x91, 0xfd, 0x7e, 0x6c,
0x0e, 0x5d, 0x79, 0x1b, 0x81, 0x79, 0xd2, 0x09,
0x34, 0x70, 0x3d, 0x81, 0xec, 0xf6, 0x24, 0xbb,
0xfb, 0xf1, 0x7b, 0xdf, 0x54, 0xea, 0x80, 0x9b,
0xc7, 0x99, 0x9e, 0xbd, 0x16, 0x78, 0x12, 0x53,
0x5e, 0x01, 0xa7, 0x4e, 0xbd, 0x67, 0xe1, 0x9b,
0x4c, 0x0e, 0x61, 0x45, 0x97, 0xd2, 0xf0, 0x0f,
0xfe, 0x15, 0x08, 0xb7, 0x11, 0x4c, 0xe7, 0xff,
0x81, 0x53, 0xff, 0x91, 0x25, 0x38, 0x7e, 0x40,
0x94, 0xe5, 0xe0, 0xad, 0xe6, 0xd9, 0x79, 0xb6,
0x92, 0xc9, 0xfc, 0xde, 0xc3, 0x1a, 0x23, 0xbb,
0xdd, 0xc8, 0x51, 0x0c, 0x3a, 0x72, 0xfa, 0x73,
0x6f, 0xb7, 0xee, 0x61, 0x39, 0x03, 0x01, 0x3f,
0x7f, 0x94, 0x2e, 0x2e, 0xba, 0x3a, 0xbb, 0xb4,
0xfa, 0x6a, 0x17, 0xfe, 0xea, 0xef, 0x5e, 0x66,
0x97, 0x3f, 0x32, 0x3d, 0xd7, 0x3e, 0xb1, 0xf1,
0x6c, 0x14, 0x4c, 0xfd, 0x37, 0xd3, 0x38, 0x80,
0xfb, 0xde, 0xa6, 0x24, 0x1e, 0xc8, 0xca, 0x7f,
0x3a, 0x93, 0xd8, 0x8b, 0x18, 0x13, 0xb2, 0xe5,
0xe4, 0x93, 0x05, 0x53, 0x4f, 0x84, 0x66, 0xa7,
0x58, 0x5c, 0x7b, 0x86, 0x52, 0x6d, 0x0d, 0xce,
0xa4, 0x30, 0x7d, 0xb6, 0x18, 0x9f, 0xeb, 0xff,
0x22, 0xbb, 0x72, 0x29, 0xb9, 0x44, 0x0b, 0x48,
0x1e, 0x84, 0x71, 0x81, 0xe3, 0x6d, 0x73, 0x26,
0x92, 0xb4, 0x4d, 0x2a, 0x29, 0xb8, 0x1f, 0x72,
0xed, 0xd0, 0xe1, 0x64, 0x77, 0xea, 0x8e, 0x88,
0x0f, 0xef, 0x3f, 0xb1, 0x3b, 0xad, 0xf9, 0xc9,
0x8b, 0xd0, 0xac, 0xc6, 0xcc, 0xa9, 0x40, 0xcc,
0x76, 0xf6, 0x3b, 0x53, 0xb5, 0x88, 0xcb, 0xc8,
0x37, 0xf1, 0xa2, 0xba, 0x23, 0x15, 0x99, 0x09,
0xcc, 0xe7, 0x7a, 0x3b, 0x37, 0xf7, 0x58, 0xc8,
0x46, 0x8c, 0x2b, 0x2f, 0x4e, 0x0e, 0xa6, 0x5c,
0xea, 0x85, 0x55, 0xba, 0x02, 0x0e, 0x0e, 0x48,
0xbc, 0xe1, 0xb1, 0x01, 0x35, 0x79, 0x13, 0x3d,
0x1b, 0xc0, 0x53, 0x68, 0x11, 0xe7, 0x95, 0x0f,
0x9d, 0x3f, 0x4c, 0x47, 0x7b, 0x4d, 0x1c, 0xae,
0x50, 0x9b, 0xcb, 0xdd, 0x05, 0x8d, 0x9a, 0x97,
0xfd, 0x8c, 0xef, 0x0c, 0x1d, 0x67, 0x73, 0xa8,
0x28, 0x36, 0xd5, 0xb6, 0x92, 0x33, 0x40, 0x75,
0x0b, 0x51, 0xc3, 0x64, 0xba, 0x1d, 0xc2, 0xcc,
0xee, 0x7d, 0x54, 0x0f, 0x27, 0x69, 0xa7, 0x27,
0x63, 0x30, 0x29, 0xd9, 0xc8, 0x84, 0xd8, 0xdf,
0x9f, 0x68, 0x8d, 0x04, 0xca, 0xa6, 0xc5, 0xc7,
0x7a, 0x5c, 0xc8, 0xd1, 0xcb, 0x4a, 0xec, 0xd0,
0xd8, 0x20, 0x69, 0xc5, 0x17, 0xcd, 0x78, 0xc8,
0x75, 0x23, 0x30, 0x69, 0xc9, 0xd4, 0xea, 0x5c,
0x4f, 0x6b, 0x86, 0x3f, 0x8b, 0xfe, 0xee, 0x44,
0xc9, 0x7c, 0xb7, 0xdd, 0x3e, 0xe5, 0xec, 0x54,
0x03, 0x3e, 0xaa, 0x82, 0xc6, 0xdf, 0xb2, 0x38,
0x0e, 0x5d, 0xb3, 0x88, 0xd9, 0xd3, 0x69, 0x5f,
0x8f, 0x70, 0x8a, 0x7e, 0x11, 0xd9, 0x1e, 0x7b,
0x38, 0xf1, 0x42, 0x1a, 0xc0, 0x35, 0xf5, 0xc7,
0x36, 0x85, 0xf5, 0xf7, 0xb8, 0x7e, 0xc7, 0xef,
0x18, 0xf1, 0x63, 0xd6, 0x7a, 0xc6, 0xc9, 0x0e,
0x4d, 0x69, 0x4f, 0x84, 0xef, 0x26, 0x41, 0x0c,
0xec, 0xc7, 0xe0, 0x7e, 0x3c, 0x67, 0x01, 0x4c,
0x62, 0x1a, 0x20, 0x6f, 0xee, 0x47, 0x4d, 0xc0,
0x99, 0x13, 0x8d, 0x91, 0x4a, 0x26, 0xd4, 0x37,
0x28, 0x90, 0x58, 0x75, 0x66, 0x2b, 0x0a, 0xdf,
0xda, 0xee, 0x92, 0x25, 0x90, 0x62, 0x39, 0x9e,
0x44, 0x98, 0xad, 0xc1, 0x88, 0xed, 0xe4, 0xb4,
0xaf, 0xf5, 0x8c, 0x9b, 0x48, 0x4d, 0x56, 0x60,
0x97, 0x0f, 0x61, 0x59, 0x9e, 0xa6, 0x27, 0xfe,
0xc1, 0x91, 0x15, 0x38, 0xb8, 0x0f, 0xae, 0x61,
0x7d, 0x26, 0x13, 0x5a, 0x73, 0xff, 0x1c, 0xa3,
0x61, 0x04, 0x58, 0x48, 0x55, 0x44, 0x11, 0xfe,
0x15, 0xca, 0xc3, 0xbd, 0xca, 0xc5, 0xb4, 0x40,
0x5d, 0x1b, 0x7f, 0x39, 0xb5, 0x9c, 0x35, 0xec,
0x61, 0x15, 0x32, 0x32, 0xb8, 0x4e, 0x40, 0x9f,
0x17, 0x1f, 0x0a, 0x4d, 0xa9, 0x91, 0xef, 0xb7,
0xb0, 0xeb, 0xc2, 0x83, 0x9a, 0x6c, 0xd2, 0x79,
0x43, 0x78, 0x5e, 0x2f, 0xe5, 0xdd, 0x1a, 0x3c,
0x45, 0xab, 0x29, 0x40, 0x3a, 0x37, 0x5b, 0x6f,
0xd7, 0xfc, 0x48, 0x64, 0x3c, 0x49, 0xfb, 0x21,
0xbe, 0xc3, 0xff, 0x07, 0xfb, 0x17, 0xe9, 0xc9,
0x0c, 0x4c, 0x5c, 0x15, 0x9e, 0x8e, 0x22, 0x30,
0x0a, 0xde, 0x48, 0x7f, 0xdb, 0x0d, 0xd1, 0x2b,
0x87, 0x38, 0x9e, 0xcc, 0x5a, 0x01, 0x16, 0xee,
0x75, 0x49, 0x0d, 0x30, 0x01, 0x34, 0x6a, 0xb6,
0x9a, 0x5a, 0x2a, 0xec, 0xbb, 0x48, 0xac, 0xd3,
0x77, 0x83, 0xd8, 0x08, 0x86, 0x4f, 0x48, 0x09,
0x29, 0x41, 0x79, 0xa1, 0x03, 0x12, 0xc4, 0xcd,
0x90, 0x55, 0x47, 0x66, 0x74, 0x9a, 0xcc, 0x4f,
0x35, 0x8c, 0xd6, 0x98, 0xef, 0xeb, 0x45, 0xb9,
0x9a, 0x26, 0x2f, 0x39, 0xa5, 0x70, 0x6d, 0xfc,
0xb4, 0x51, 0xee, 0xf4, 0x9c, 0xe7, 0x38, 0x59,
0xad, 0xf4, 0xbc, 0x46, 0xff, 0x46, 0x8e, 0x60,
0x9c, 0xa3, 0x60, 0x1d, 0xf8, 0x26, 0x72, 0xf5,
0x72, 0x9d, 0x68, 0x80, 0x04, 0xf6, 0x0b, 0xa1,
0x0a, 0xd5, 0xa7, 0x82, 0x3a, 0x3e, 0x47, 0xa8,
0x5a, 0xde, 0x59, 0x4f, 0x7b, 0x07, 0xb3, 0xe9,
0x24, 0x19, 0x3d, 0x34, 0x05, 0xec, 0xf1, 0xab,
0x6e, 0x64, 0x8f, 0xd3, 0xe6, 0x41, 0x86, 0x80,
0x70, 0xe3, 0x8d, 0x60, 0x9c, 0x34, 0x25, 0x01,
0x07, 0x4d, 0x19, 0x41, 0x4e, 0x3d, 0x5c, 0x7e,
0xa8, 0xf5, 0xcc, 0xd5, 0x7b, 0xe2, 0x7d, 0x3d,
0x49, 0x86, 0x7d, 0x07, 0xb7, 0x10, 0xe3, 0x35,
0xb8, 0x84, 0x6d, 0x76, 0xab, 0x17, 0xc6, 0x38,
0xb4, 0xd3, 0x28, 0x57, 0xad, 0xd3, 0x88, 0x5a,
0xda, 0xea, 0xc8, 0x94, 0xcc, 0x37, 0x19, 0xac,
0x9c, 0x9f, 0x4b, 0x00, 0x15, 0xc0, 0xc8, 0xca,
0x1f, 0x15, 0xaa, 0xe0, 0xdb, 0xf9, 0x2f, 0x57,
0x1b, 0x24, 0xc7, 0x6f, 0x76, 0x29, 0xfb, 0xed,
0x25, 0x0d, 0xc0, 0xfe, 0xbd, 0x5a, 0xbf, 0x20,
0x08, 0x51, 0x05, 0xec, 0x71, 0xa3, 0xbf, 0xef,
0x5e, 0x99, 0x75, 0xdb, 0x3c, 0x5f, 0x9a, 0x8c,
0xbb, 0x19, 0x5c, 0x0e, 0x93, 0x19, 0xf8, 0x6a,
0xbc, 0xf2, 0x12, 0x54, 0x2f, 0xcb, 0x28, 0x64,
0x88, 0xb3, 0x92, 0x0d, 0x96, 0xd1, 0xa6, 0xe4,
0x1f, 0xf1, 0x4d, 0xa4, 0xab, 0x1c, 0xee, 0x54,
0xf2, 0xad, 0x29, 0x6d, 0x32, 0x37, 0xb2, 0x16,
0x77, 0x5c, 0xdc, 0x2e, 0x54, 0xec, 0x75, 0x26,
0xc6, 0x36, 0xd9, 0x17, 0x2c, 0xf1, 0x7a, 0xdc,
0x4b, 0xf1, 0xe2, 0xd9, 0x95, 0xba, 0xac, 0x87,
0xc1, 0xf3, 0x8e, 0x58, 0x08, 0xd8, 0x87, 0x60,
0xc9, 0xee, 0x6a, 0xde, 0xa4, 0xd2, 0xfc, 0x0d,
0xe5, 0x36, 0xc4, 0x5c, 0x52, 0xb3, 0x07, 0x54,
0x65, 0x24, 0xc1, 0xb1, 0xd1, 0xb1, 0x53, 0x13,
0x31, 0x79, 0x7f, 0x05, 0x76, 0xeb, 0x37, 0x59,
0x15, 0x2b, 0xd1, 0x3f, 0xac, 0x08, 0x97, 0xeb,
0x91, 0x98, 0xdf, 0x6c, 0x09, 0x0d, 0x04, 0x9f,
0xdc, 0x3b, 0x0e, 0x60, 0x68, 0x47, 0x23, 0x15,
0x16, 0xc6, 0x0b, 0x35, 0xf8, 0x77, 0xa2, 0x78,
0x50, 0xd4, 0x64, 0x22, 0x33, 0xff, 0xfb, 0x93,
0x71, 0x46, 0x50, 0x39, 0x1b, 0x9c, 0xea, 0x4e,
0x8d, 0x0c, 0x37, 0xe5, 0x5c, 0x51, 0x3a, 0x31,
0xb2, 0x85, 0x84, 0x3f, 0x41, 0xee, 0xa2, 0xc1,
0xc6, 0x13, 0x3b, 0x54, 0x28, 0xd2, 0x18, 0x37,
0xcc, 0x46, 0x9f, 0x6a, 0x91, 0x3d, 0x5a, 0x15,
0x3c, 0x89, 0xa3, 0x61, 0x06, 0x7d, 0x2e, 0x78,
0xbe, 0x7d, 0x40, 0xba, 0x2f, 0x95, 0xb1, 0x2f,
0x87, 0x3b, 0x8a, 0xbe, 0x6a, 0xf4, 0xc2, 0x31,
0x74, 0xee, 0x91, 0xe0, 0x23, 0xaa, 0x5d, 0x7f,
0xdd, 0xf0, 0x44, 0x8c, 0x0b, 0x59, 0x2b, 0xfc,
0x48, 0x3a, 0xdf, 0x07, 0x05, 0x38, 0x6c, 0xc9,
0xeb, 0x18, 0x24, 0x68, 0x8d, 0x58, 0x98, 0xd3,
0x31, 0xa3, 0xe4, 0x70, 0x59, 0xb1, 0x21, 0xbe,
0x7e, 0x65, 0x7d, 0xb8, 0x04, 0xab, 0xf6, 0xe4,
0xd7, 0xda, 0xec, 0x09, 0x8f, 0xda, 0x6d, 0x24,
0x07, 0xcc, 0x29, 0x17, 0x05, 0x78, 0x1a, 0xc1,
0xb1, 0xce, 0xfc, 0xaa, 0x2d, 0xe7, 0xcc, 0x85,
0x84, 0x84, 0x03, 0x2a, 0x0c, 0x3f, 0xa9, 0xf8,
0xfd, 0x84, 0x53, 0x59, 0x5c, 0xf0, 0xd4, 0x09,
0xf0, 0xd2, 0x6c, 0x32, 0x03, 0xb0, 0xa0, 0x8c,
0x52, 0xeb, 0x23, 0x91, 0x88, 0x43, 0x13, 0x46,
0xf6, 0x1e, 0xb4, 0x1b, 0xf5, 0x8e, 0x3a, 0xb5,
0x3d, 0x00, 0xf6, 0xe5, 0x08, 0x3d, 0x5f, 0x39,
0xd3, 0x21, 0x69, 0xbc, 0x03, 0x22, 0x3a, 0xd2,
0x5c, 0x84, 0xf8, 0x15, 0xc4, 0x80, 0x0b, 0xbc,
0x29, 0x3c, 0xf3, 0x95, 0x98, 0xcd, 0x8f, 0x35,
0xbc, 0xa5, 0x3e, 0xfc, 0xd4, 0x13, 0x9e, 0xde,
0x4f, 0xce, 0x71, 0x9d, 0x09, 0xad, 0xf2, 0x80,
0x6b, 0x65, 0x7f, 0x03, 0x00, 0x14, 0x7c, 0x15,
0x85, 0x40, 0x6d, 0x70, 0xea, 0xdc, 0xb3, 0x63,
0x35, 0x4f, 0x4d, 0xe0, 0xd9, 0xd5, 0x3c, 0x58,
0x56, 0x23, 0x80, 0xe2, 0x36, 0xdd, 0x75, 0x1d,
0x94, 0x11, 0x41, 0x8e, 0xe0, 0x81, 0x8e, 0xcf,
0xe0, 0xe5, 0xf6, 0xde, 0xd1, 0xe7, 0x04, 0x12,
0x79, 0x92, 0x2b, 0x71, 0x2a, 0x79, 0x8b, 0x7c,
0x44, 0x79, 0x16, 0x30, 0x4e, 0xf4, 0xf6, 0x9b,
0xb7, 0x40, 0xa3, 0x5a, 0xa7, 0x69, 0x3e, 0xc1,
0x3a, 0x04, 0xd0, 0x88, 0xa0, 0x3b, 0xdd, 0xc6,
0x9e, 0x7e, 0x1e, 0x1e, 0x8f, 0x44, 0xf7, 0x73,
0x67, 0x1e, 0x1a, 0x78, 0xfa, 0x62, 0xf4, 0xa9,
0xa8, 0xc6, 0x5b, 0xb8, 0xfa, 0x06, 0x7d, 0x5e,
0x38, 0x1c, 0x9a, 0x39, 0xe9, 0x39, 0x98, 0x22,
0x0b, 0xa7, 0xac, 0x0b, 0xf3, 0xbc, 0xf1, 0xeb,
0x8c, 0x81, 0xe3, 0x48, 0x8a, 0xed, 0x42, 0xc2,
0x38, 0xcf, 0x3e, 0xda, 0xd2, 0x89, 0x8d, 0x9c,
0x53, 0xb5, 0x2f, 0x41, 0x01, 0x26, 0x84, 0x9c,
0xa3, 0x56, 0xf6, 0x49, 0xc7, 0xd4, 0x9f, 0x93,
0x1b, 0x96, 0x49, 0x5e, 0xad, 0xb3, 0x84, 0x1f,
0x3c, 0xa4, 0xe0, 0x9b, 0xd1, 0x90, 0xbc, 0x38,
0x6c, 0xdd, 0x95, 0x4d, 0x9d, 0xb1, 0x71, 0x57,
0x2d, 0x34, 0xe8, 0xb8, 0x42, 0xc7, 0x99, 0x03,
0xc7, 0x07, 0x30, 0x65, 0x91, 0x55, 0xd5, 0x90,
0x70, 0x97, 0x37, 0x68, 0xd4, 0x11, 0xf9, 0xe8,
0xce, 0xec, 0xdc, 0x34, 0xd5, 0xd3, 0xb7, 0xc4,
0xb8, 0x97, 0x05, 0x92, 0xad, 0xf8, 0xe2, 0x36,
0x64, 0x41, 0xc9, 0xc5, 0x41, 0x77, 0x52, 0xd7,
0x2c, 0xa5, 0x24, 0x2f, 0xd9, 0x34, 0x0b, 0x47,
0x35, 0xa7, 0x28, 0x8b, 0xc5, 0xcd, 0xe9, 0x46,
0xac, 0x39, 0x94, 0x3c, 0x10, 0xc6, 0x29, 0x73,
0x0e, 0x0e, 0x5d, 0xe0, 0x71, 0x03, 0x8a, 0x72,
0x0e, 0x26, 0xb0, 0x7d, 0x84, 0xed, 0x95, 0x23,
0x49, 0x5a, 0x45, 0x83, 0x45, 0x60, 0x11, 0x4a,
0x46, 0x31, 0xd4, 0xd8, 0x16, 0x54, 0x98, 0x58,
0xed, 0x6d, 0xcc, 0x5d, 0xd6, 0x50, 0x61, 0x9f,
0x9d, 0xc5, 0x3e, 0x9d, 0x32, 0x47, 0xde, 0x96,
0xe1, 0x5d, 0xd8, 0xf8, 0xb4, 0x69, 0x6f, 0xb9,
0x15, 0x90, 0x57, 0x7a, 0xf6, 0xad, 0xb0, 0x5b,
0xf5, 0xa6, 0x36, 0x94, 0xfd, 0x84, 0xce, 0x1c,
0x0f, 0x4b, 0xd0, 0xc2, 0x5b, 0x6b, 0x56, 0xef,
0x73, 0x93, 0x0b, 0xc3, 0xee, 0xd9, 0xcf, 0xd3,
0xa4, 0x22, 0x58, 0xcd, 0x50, 0x6e, 0x65, 0xf4,
0xe9, 0xb7, 0x71, 0xaf, 0x4b, 0xb3, 0xb6, 0x2f,
0x0f, 0x0e, 0x3b, 0xc9, 0x85, 0x14, 0xf5, 0x17,
0xe8, 0x7a, 0x3a, 0xbf, 0x5f, 0x5e, 0xf8, 0x18,
0x48, 0xa6, 0x72, 0xab, 0x06, 0x95, 0xe9, 0xc8,
0xa7, 0xf4, 0x32, 0x44, 0x04, 0x0c, 0x84, 0x98,
0x73, 0xe3, 0x89, 0x8d, 0x5f, 0x7e, 0x4a, 0x42,
0x8f, 0xc5, 0x28, 0xb1, 0x82, 0xef, 0x1c, 0x97,
0x31, 0x3b, 0x4d, 0xe0, 0x0e, 0x10, 0x10, 0x97,
0x93, 0x49, 0x78, 0x2f, 0x0d, 0x86, 0x8b, 0xa1,
0x53, 0xa9, 0x81, 0x20, 0x79, 0xe7, 0x07, 0x77,
0xb6, 0xac, 0x5e, 0xd2, 0x05, 0xcd, 0xe9, 0xdb,
0x8a, 0x94, 0x82, 0x8a, 0x23, 0xb9, 0x3d, 0x1c,
0xa9, 0x7d, 0x72, 0x4a, 0xed, 0x33, 0xa3, 0xdb,
0x21, 0xa7, 0x86, 0x33, 0x45, 0xa5, 0xaa, 0x56,
0x45, 0xb5, 0x83, 0x29, 0x40, 0x47, 0x79, 0x04,
0x6e, 0xb9, 0x95, 0xd0, 0x81, 0x77, 0x2d, 0x48,
0x1e, 0xfe, 0xc3, 0xc2, 0x1e, 0xe5, 0xf2, 0xbe,
0xfd, 0x3b, 0x94, 0x9f, 0xc4, 0xc4, 0x26, 0x9d,
0xe4, 0x66, 0x1e, 0x19, 0xee, 0x6c, 0x79, 0x97,
0x11, 0x31, 0x4b, 0x0d, 0x01, 0xcb, 0xde, 0xa8,
0xf6, 0x6d, 0x7c, 0x39, 0x46, 0x4e, 0x7e, 0x3f,
0x94, 0x17, 0xdf, 0xa1, 0x7d, 0xd9, 0x1c, 0x8e,
0xbc, 0x7d, 0x33, 0x7d, 0xe3, 0x12, 0x40, 0xca,
0xab, 0x37, 0x11, 0x46, 0xd4, 0xae, 0xef, 0x44,
0xa2, 0xb3, 0x6a, 0x66, 0x0e, 0x0c, 0x90, 0x7f,
0xdf, 0x5c, 0x66, 0x5f, 0xf2, 0x94, 0x9f, 0xa6,
0x73, 0x4f, 0xeb, 0x0d, 0xad, 0xbf, 0xc0, 0x63,
0x5c, 0xdc, 0x46, 0x51, 0xe8, 0x8e, 0x90, 0x19,
0xa8, 0xa4, 0x3c, 0x91, 0x79, 0xfa, 0x7e, 0x58,
0x85, 0x13, 0x55, 0xc5, 0x19, 0x82, 0x37, 0x1b,
0x0a, 0x02, 0x1f, 0x99, 0x6b, 0x18, 0xf1, 0x28,
0x08, 0xa2, 0x73, 0xb8, 0x0f, 0x2e, 0xcd, 0xbf,
0xf3, 0x86, 0x7f, 0xea, 0xef, 0xd0, 0xbb, 0xa6,
0x21, 0xdf, 0x49, 0x73, 0x51, 0xcc, 0x36, 0xd3,
0x3e, 0xa0, 0xf8, 0x44, 0xdf, 0xd3, 0xa6, 0xbe,
0x8a, 0xd4, 0x57, 0xdd, 0x72, 0x94, 0x61, 0x0f,
0x82, 0xd1, 0x07, 0xb8, 0x7c, 0x18, 0x83, 0xdf,
0x3a, 0xe5, 0x50, 0x6a, 0x82, 0x20, 0xac, 0xa9,
0xa8, 0xff, 0xd9, 0xf3, 0x77, 0x33, 0x5a, 0x9e,
0x7f, 0x6d, 0xfe, 0x5d, 0x33, 0x41, 0x42, 0xe7,
0x6c, 0x19, 0xe0, 0x44, 0x8a, 0x15, 0xf6, 0x70,
0x98, 0xb7, 0x68, 0x4d, 0xfa, 0x97, 0x39, 0xb0,
0x8e, 0xe8, 0x84, 0x8b, 0x75, 0x30, 0xb7, 0x7d,
0x92, 0x69, 0x20, 0x9c, 0x81, 0xfb, 0x4b, 0xf4,
0x01, 0x50, 0xeb, 0xce, 0x0c, 0x1c, 0x6c, 0xb5,
0x4a, 0xd7, 0x27, 0x0c, 0xce, 0xbb, 0xe5, 0x85,
0xf0, 0xb6, 0xee, 0xd5, 0x70, 0xdd, 0x3b, 0xfc,
0xd4, 0x99, 0xf1, 0x33, 0xdd, 0x8b, 0xc4, 0x2f,
0xae, 0xab, 0x74, 0x96, 0x32, 0xc7, 0x4c, 0x56,
0x3c, 0x89, 0x0f, 0x96, 0x0b, 0x42, 0xc0, 0xcb,
0xee, 0x0f, 0x0b, 0x8c, 0xfb, 0x7e, 0x47, 0x7b,
0x64, 0x48, 0xfd, 0xb2, 0x00, 0x80, 0x89, 0xa5,
0x13, 0x55, 0x62, 0xfc, 0x8f, 0xe2, 0x42, 0x03,
0xb7, 0x4e, 0x2a, 0x79, 0xb4, 0x82, 0xea, 0x23,
0x49, 0xda, 0xaf, 0x52, 0x63, 0x1e, 0x60, 0x03,
0x89, 0x06, 0x44, 0x46, 0x08, 0xc3, 0xc4, 0x87,
0x70, 0x2e, 0xda, 0x94, 0xad, 0x6b, 0xe0, 0xe4,
0xd1, 0x8a, 0x06, 0xc2, 0xa8, 0xc0, 0xa7, 0x43,
0x3c, 0x47, 0x52, 0x0e, 0xc3, 0x77, 0x81, 0x11,
0x67, 0x0e, 0xa0, 0x70, 0x04, 0x47, 0x29, 0x40,
0x86, 0x0d, 0x34, 0x56, 0xa7, 0xc9, 0x35, 0x59,
0x68, 0xdc, 0x93, 0x81, 0x70, 0xee, 0x86, 0xd9,
0x80, 0x06, 0x40, 0x4f, 0x1a, 0x0d, 0x40, 0x30,
0x0b, 0xcb, 0x96, 0x47, 0xc1, 0xb7, 0x52, 0xfd,
0x56, 0xe0, 0x72, 0x4b, 0xfb, 0xbd, 0x92, 0x45,
0x61, 0x71, 0xc2, 0x33, 0x11, 0xbf, 0x52, 0x83,
0x79, 0x26, 0xe0, 0x49, 0x6b, 0xb7, 0x05, 0x8b,
0xe8, 0x0e, 0x87, 0x31, 0xd7, 0x9d, 0x8a, 0xf5,
0xc0, 0x5f, 0x2e, 0x58, 0x4a, 0xdb, 0x11, 0xb3,
0x6c, 0x30, 0x2a, 0x46, 0x19, 0xe3, 0x27, 0x84,
0x1f, 0x63, 0x6e, 0xf6, 0x57, 0xc7, 0xc9, 0xd8,
0x5e, 0xba, 0xb3, 0x87, 0xd5, 0x83, 0x26, 0x34,
0x21, 0x9e, 0x65, 0xde, 0x42, 0xd3, 0xbe, 0x7b,
0xbc, 0x91, 0x71, 0x44, 0x4d, 0x99, 0x3b, 0x31,
0xe5, 0x3f, 0x11, 0x4e, 0x7f, 0x13, 0x51, 0x3b,
0xae, 0x79, 0xc9, 0xd3, 0x81, 0x8e, 0x25, 0x40,
0x10, 0xfc, 0x07, 0x1e, 0xf9, 0x7b, 0x9a, 0x4b,
0x6c, 0xe3, 0xb3, 0xad, 0x1a, 0x0a, 0xdd, 0x9e,
0x59, 0x0c, 0xa2, 0xcd, 0xae, 0x48, 0x4a, 0x38,
0x5b, 0x47, 0x41, 0x94, 0x65, 0x6b, 0xbb, 0xeb,
0x5b, 0xe3, 0xaf, 0x07, 0x5b, 0xd4, 0x4a, 0xa2,
0xc9, 0x5d, 0x2f, 0x64, 0x03, 0xd7, 0x3a, 0x2c,
0x6e, 0xce, 0x76, 0x95, 0xb4, 0xb3, 0xc0, 0xf1,
0xe2, 0x45, 0x73, 0x7a, 0x5c, 0xab, 0xc1, 0xfc,
0x02, 0x8d, 0x81, 0x29, 0xb3, 0xac, 0x07, 0xec,
0x40, 0x7d, 0x45, 0xd9, 0x7a, 0x59, 0xee, 0x34,
0xf0, 0xe9, 0xd5, 0x7b, 0x96, 0xb1, 0x3d, 0x95,
0xcc, 0x86, 0xb5, 0xb6, 0x04, 0x2d, 0xb5, 0x92,
0x7e, 0x76, 0xf4, 0x06, 0xa9, 0xa3, 0x12, 0x0f,
0xb1, 0xaf, 0x26, 0xba, 0x7c, 0xfc, 0x7e, 0x1c,
0xbc, 0x2c, 0x49, 0x97, 0x53, 0x60, 0x13, 0x0b,
0xa6, 0x61, 0x83, 0x89, 0x42, 0xd4, 0x17, 0x0c,
0x6c, 0x26, 0x52, 0xc3, 0xb3, 0xd4, 0x67, 0xf5,
0xe3, 0x04, 0xb7, 0xf4, 0xcb, 0x80, 0xb8, 0xcb,
0x77, 0x56, 0x3e, 0xaa, 0x57, 0x54, 0xee, 0xb4,
0x2c, 0x67, 0xcf, 0xf2, 0xdc, 0xbe, 0x55, 0xf9,
0x43, 0x1f, 0x6e, 0x22, 0x97, 0x67, 0x7f, 0xc4,
0xef, 0xb1, 0x26, 0x31, 0x1e, 0x27, 0xdf, 0x41,
0x80, 0x47, 0x6c, 0xe2, 0xfa, 0xa9, 0x8c, 0x2a,
0xf6, 0xf2, 0xab, 0xf0, 0x15, 0xda, 0x6c, 0xc8,
0xfe, 0xb5, 0x23, 0xde, 0xa9, 0x05, 0x3f, 0x06,
0x54, 0x4c, 0xcd, 0xe1, 0xab, 0xfc, 0x0e, 0x62,
0x33, 0x31, 0x73, 0x2c, 0x76, 0xcb, 0xb4, 0x47,
0x1e, 0x20, 0xad, 0xd8, 0xf2, 0x31, 0xdd, 0xc4,
0x8b, 0x0c, 0x77, 0xbe, 0xe1, 0x8b, 0x26, 0x00,
0x02, 0x58, 0xd6, 0x8d, 0xef, 0xad, 0x74, 0x67,
0xab, 0x3f, 0xef, 0xcb, 0x6f, 0xb0, 0xcc, 0x81,
0x44, 0x4c, 0xaf, 0xe9, 0x49, 0x4f, 0xdb, 0xa0,
0x25, 0xa4, 0xf0, 0x89, 0xf1, 0xbe, 0xd8, 0x10,
0xff, 0xb1, 0x3b, 0x4b, 0xfa, 0x98, 0xf5, 0x79,
0x6d, 0x1e, 0x69, 0x4d, 0x57, 0xb1, 0xc8, 0x19,
0x1b, 0xbd, 0x1e, 0x8c, 0x84, 0xb7, 0x7b, 0xe8,
0xd2, 0x2d, 0x09, 0x41, 0x41, 0x37, 0x3d, 0xb1,
0x6f, 0x26, 0x5d, 0x71, 0x16, 0x3d, 0xb7, 0x83,
0x27, 0x2c, 0xa7, 0xb6, 0x50, 0xbd, 0x91, 0x86,
0xab, 0x24, 0xa1, 0x38, 0xfd, 0xea, 0x71, 0x55,
0x7e, 0x9a, 0x07, 0x77, 0x4b, 0xfa, 0x61, 0x66,
0x20, 0x1e, 0x28, 0x95, 0x18, 0x1b, 0xa4, 0xa0,
0xfd, 0xc0, 0x89, 0x72, 0x43, 0xd9, 0x3b, 0x49,
0x5a, 0x3f, 0x9d, 0xbf, 0xdb, 0xb4, 0x46, 0xea,
0x42, 0x01, 0x77, 0x23, 0x68, 0x95, 0xb6, 0x24,
0xb3, 0xa8, 0x6c, 0x28, 0x3b, 0x11, 0x40, 0x7e,
0x18, 0x65, 0x6d, 0xd8, 0x24, 0x42, 0x7d, 0x88,
0xc0, 0x52, 0xd9, 0x05, 0xe4, 0x95, 0x90, 0x87,
0x8c, 0xf4, 0xd0, 0x6b, 0xb9, 0x83, 0x99, 0x34,
0x6d, 0xfe, 0x54, 0x40, 0x94, 0x52, 0x21, 0x4f,
0x14, 0x25, 0xc5, 0xd6, 0x5e, 0x95, 0xdc, 0x0a,
0x2b, 0x89, 0x20, 0x11, 0x84, 0x48, 0xd6, 0x3a,
0xcd, 0x5c, 0x24, 0xad, 0x62, 0xe3, 0xb1, 0x93,
0x25, 0x8d, 0xcd, 0x7e, 0xfc, 0x27, 0xa3, 0x37,
0xfd, 0x84, 0xfc, 0x1b, 0xb2, 0xf1, 0x27, 0x38,
0x5a, 0xb7, 0xfc, 0xf2, 0xfa, 0x95, 0x66, 0xd4,
0xfb, 0xba, 0xa7, 0xd7, 0xa3, 0x72, 0x69, 0x48,
0x48, 0x8c, 0xeb, 0x28, 0x89, 0xfe, 0x33, 0x65,
0x5a, 0x36, 0x01, 0x7e, 0x06, 0x79, 0x0a, 0x09,
0x3b, 0x74, 0x11, 0x9a, 0x6e, 0xbf, 0xd4, 0x9e,
0x58, 0x90, 0x49, 0x4f, 0x4d, 0x08, 0xd4, 0xe5,
0x4a, 0x09, 0x21, 0xef, 0x8b, 0xb8, 0x74, 0x3b,
0x91, 0xdd, 0x36, 0x85, 0x60, 0x2d, 0xfa, 0xd4,
0x45, 0x7b, 0x45, 0x53, 0xf5, 0x47, 0x87, 0x7e,
0xa6, 0x37, 0xc8, 0x78, 0x7a, 0x68, 0x9d, 0x8d,
0x65, 0x2c, 0x0e, 0x91, 0x5c, 0xa2, 0x60, 0xf0,
0x8e, 0x3f, 0xe9, 0x1a, 0xcd, 0xaa, 0xe7, 0xd5,
0x77, 0x18, 0xaf, 0xc9, 0xbc, 0x18, 0xea, 0x48,
0x1b, 0xfb, 0x22, 0x48, 0x70, 0x16, 0x29, 0x9e,
0x5b, 0xc1, 0x2c, 0x66, 0x23, 0xbc, 0xf0, 0x1f,
0xef, 0xaf, 0xe4, 0xd6, 0x04, 0x19, 0x82, 0x7a,
0x0b, 0xba, 0x4b, 0x46, 0xb1, 0x6a, 0x85, 0x5d,
0xb4, 0x73, 0xd6, 0x21, 0xa1, 0x71, 0x60, 0x14,
0xee, 0x0a, 0x77, 0xc4, 0x66, 0x2e, 0xf9, 0x69,
0x30, 0xaf, 0x41, 0x0b, 0xc8, 0x83, 0x3c, 0x53,
0x99, 0x19, 0x27, 0x46, 0xf7, 0x41, 0x6e, 0x56,
0xdc, 0x94, 0x28, 0x67, 0x4e, 0xb7, 0x25, 0x48,
0x8a, 0xc2, 0xe0, 0x60, 0x96, 0xcc, 0x18, 0xf4,
0x84, 0xdd, 0xa7, 0x5e, 0x3e, 0x05, 0x0b, 0x26,
0x26, 0xb2, 0x5c, 0x1f, 0x57, 0x1a, 0x04, 0x7e,
0x6a, 0xe3, 0x2f, 0xb4, 0x35, 0xb6, 0x38, 0x40,
0x40, 0xcd, 0x6f, 0x87, 0x2e, 0xef, 0xa3, 0xd7,
0xa9, 0xc2, 0xe8, 0x0d, 0x27, 0xdf, 0x44, 0x62,
0x99, 0xa0, 0xfc, 0xcf, 0x81, 0x78, 0xcb, 0xfe,
0xe5, 0xa0, 0x03, 0x4e, 0x6c, 0xd7, 0xf4, 0xaf,
0x7a, 0xbb, 0x61, 0x82, 0xfe, 0x71, 0x89, 0xb2,
0x22, 0x7c, 0x8e, 0x83, 0x04, 0xce, 0xf6, 0x5d,
0x84, 0x8f, 0x95, 0x6a, 0x7f, 0xad, 0xfd, 0x32,
0x9c, 0x5e, 0xe4, 0x9c, 0x89, 0x60, 0x54, 0xaa,
0x96, 0x72, 0xd2, 0xd7, 0x36, 0x85, 0xa9, 0x45,
0xd2, 0x2a, 0xa1, 0x81, 0x49, 0x6f, 0x7e, 0x04,
0xfa, 0xe2, 0xfe, 0x90, 0x26, 0x77, 0x5a, 0x33,
0xb8, 0x04, 0x9a, 0x7a, 0xe6, 0x4c, 0x4f, 0xad,
0x72, 0x96, 0x08, 0x28, 0x58, 0x13, 0xf8, 0xc4,
0x1c, 0xf0, 0xc3, 0x45, 0x95, 0x49, 0x20, 0x8c,
0x9f, 0x39, 0x70, 0xe1, 0x77, 0xfe, 0xd5, 0x4b,
0xaf, 0x86, 0xda, 0xef, 0x22, 0x06, 0x83, 0x36,
0x29, 0x12, 0x11, 0x40, 0xbc, 0x3b, 0x86, 0xaa,
0xaa, 0x65, 0x60, 0xc3, 0x80, 0xca, 0xed, 0xa9,
0xf3, 0xb0, 0x79, 0x96, 0xa2, 0x55, 0x27, 0x28,
0x55, 0x73, 0x26, 0xa5, 0x50, 0xea, 0x92, 0x4b,
0x3c, 0x5c, 0x82, 0x33, 0xf0, 0x01, 0x3f, 0x03,
0xc1, 0x08, 0x05, 0xbf, 0x98, 0xf4, 0x9b, 0x6d,
0xa5, 0xa8, 0xb4, 0x82, 0x0c, 0x06, 0xfa, 0xff,
0x2d, 0x08, 0xf3, 0x05, 0x4f, 0x57, 0x2a, 0x39,
0xd4, 0x83, 0x0d, 0x75, 0x51, 0xd8, 0x5b, 0x1b,
0xd3, 0x51, 0x5a, 0x32, 0x2a, 0x9b, 0x32, 0xb2,
0xf2, 0xa4, 0x96, 0x12, 0xf2, 0xae, 0x40, 0x34,
0x67, 0xa8, 0xf5, 0x44, 0xd5, 0x35, 0x53, 0xfe,
0xa3, 0x60, 0x96, 0x63, 0x0f, 0x1f, 0x6e, 0xb0,
0x5a, 0x42, 0xa6, 0xfc, 0x51, 0x0b, 0x60, 0x27,
0xbc, 0x06, 0x71, 0xed, 0x65, 0x5b, 0x23, 0x86,
0x4a, 0x07, 0x3b, 0x22, 0x07, 0x46, 0xe6, 0x90,
0x3e, 0xf3, 0x25, 0x50, 0x1b, 0x4c, 0x7f, 0x03,
0x08, 0xa8, 0x36, 0x6b, 0x87, 0xe5, 0xe3, 0xdb,
0x9a, 0x38, 0x83, 0xff, 0x9f, 0x1a, 0x9f, 0x57,
0xa4, 0x2a, 0xf6, 0x37, 0xbc, 0x1a, 0xff, 0xc9,
0x1e, 0x35, 0x0c, 0xc3, 0x7c, 0xa3, 0xb2, 0xe5,
0xd2, 0xc6, 0xb4, 0x57, 0x47, 0xe4, 0x32, 0x16,
0x6d, 0xa9, 0xae, 0x64, 0xe6, 0x2d, 0x8d, 0xc5,
0x8d, 0x50, 0x8e, 0xe8, 0x1a, 0x22, 0x34, 0x2a,
0xd9, 0xeb, 0x51, 0x90, 0x4a, 0xb1, 0x41, 0x7d,
0x64, 0xf9, 0xb9, 0x0d, 0xf6, 0x23, 0x33, 0xb0,
0x33, 0xf4, 0xf7, 0x3f, 0x27, 0x84, 0xc6, 0x0f,
0x54, 0xa5, 0xc0, 0x2e, 0xec, 0x0b, 0x3a, 0x48,
0x6e, 0x80, 0x35, 0x81, 0x43, 0x9b, 0x90, 0xb1,
0xd0, 0x2b, 0xea, 0x21, 0xdc, 0xda, 0x5b, 0x09,
0xf4, 0xcc, 0x10, 0xb4, 0xc7, 0xfe, 0x79, 0x51,
0xc3, 0xc5, 0xac, 0x88, 0x74, 0x84, 0x0b, 0x4b,
0xca, 0x79, 0x16, 0x29, 0xfb, 0x69, 0x54, 0xdf,
0x41, 0x7e, 0xe9, 0xc7, 0x8e, 0xea, 0xa5, 0xfe,
0xfc, 0x76, 0x0e, 0x90, 0xc4, 0x92, 0x38, 0xad,
0x7b, 0x48, 0xe6, 0x6e, 0xf7, 0x21, 0xfd, 0x4e,
0x93, 0x0a, 0x7b, 0x41, 0x83, 0x68, 0xfb, 0x57,
0x51, 0x76, 0x34, 0xa9, 0x6c, 0x00, 0xaa, 0x4f,
0x66, 0x65, 0x98, 0x4a, 0x4f, 0xa3, 0xa0, 0xef,
0x69, 0x3f, 0xe3, 0x1c, 0x92, 0x8c, 0xfd, 0xd8,
0xe8, 0xde, 0x7c, 0x7f, 0x3e, 0x84, 0x8e, 0x69,
0x3c, 0xf1, 0xf2, 0x05, 0x46, 0xdc, 0x2f, 0x9d,
0x5e, 0x6e, 0x4c, 0xfb, 0xb5, 0x99, 0x2a, 0x59,
0x63, 0xc1, 0x34, 0xbc, 0x57, 0xc0, 0x0d, 0xb9,
0x61, 0x25, 0xf3, 0x33, 0x23, 0x51, 0xb6, 0x0d,
0x07, 0xa6, 0xab, 0x94, 0x4a, 0xb7, 0x2a, 0xea,
0xee, 0xac, 0xa3, 0xc3, 0x04, 0x8b, 0x0e, 0x56,
0xfe, 0x44, 0xa7, 0x39, 0xe2, 0xed, 0xed, 0xb4,
0x22, 0x2b, 0xac, 0x12, 0x32, 0x28, 0x91, 0xd8,
0xa5, 0xab, 0xff, 0x5f, 0xe0, 0x4b, 0xda, 0x78,
0x17, 0xda, 0xf1, 0x01, 0x5b, 0xcd, 0xe2, 0x5f,
0x50, 0x45, 0x73, 0x2b, 0xe4, 0x76, 0x77, 0xf4,
0x64, 0x1d, 0x43, 0xfb, 0x84, 0x7a, 0xea, 0x91,
0xae, 0xf9, 0x9e, 0xb7, 0xb4, 0xb0, 0x91, 0x5f,
0x16, 0x35, 0x9a, 0x11, 0xb8, 0xc7, 0xc1, 0x8c,
0xc6, 0x10, 0x8d, 0x2f, 0x63, 0x4a, 0xa7, 0x57,
0x3a, 0x51, 0xd6, 0x32, 0x2d, 0x64, 0x72, 0xd4,
0x66, 0xdc, 0x10, 0xa6, 0x67, 0xd6, 0x04, 0x23,
0x9d, 0x0a, 0x11, 0x77, 0xdd, 0x37, 0x94, 0x17,
0x3c, 0xbf, 0x8b, 0x65, 0xb0, 0x2e, 0x5e, 0x66,
0x47, 0x64, 0xac, 0xdd, 0xf0, 0x84, 0xfd, 0x39,
0xfa, 0x15, 0x5d, 0xef, 0xae, 0xca, 0xc1, 0x36,
0xa7, 0x5c, 0xbf, 0xc7, 0x08, 0xc2, 0x66, 0x00,
0x74, 0x74, 0x4e, 0x27, 0x3f, 0x55, 0x8a, 0xb7,
0x38, 0x66, 0x83, 0x6d, 0xcf, 0x99, 0x9e, 0x60,
0x8f, 0xdd, 0x2e, 0x62, 0x22, 0x0e, 0xef, 0x0c,
0x98, 0xa7, 0x85, 0x74, 0x3b, 0x9d, 0xec, 0x9e,
0xa9, 0x19, 0x72, 0xa5, 0x7f, 0x2c, 0x39, 0xb7,
0x7d, 0xb7, 0xf1, 0x12, 0x65, 0x27, 0x4b, 0x5a,
0xde, 0x17, 0xfe, 0xad, 0x44, 0xf3, 0x20, 0x4d,
0xfd, 0xe4, 0x1f, 0xb5, 0x81, 0xb0, 0x36, 0x37,
0x08, 0x6f, 0xc3, 0x0c, 0xe9, 0x85, 0x98, 0x82,
0xa9, 0x62, 0x0c, 0xc4, 0x97, 0xc0, 0x50, 0xc8,
0xa7, 0x3c, 0x50, 0x9f, 0x43, 0xb9, 0xcd, 0x5e,
0x4d, 0xfa, 0x1c, 0x4b, 0x0b, 0xa9, 0x98, 0x85,
0x38, 0x92, 0xac, 0x8d, 0xe4, 0xad, 0x9b, 0x98,
0xab, 0xd9, 0x38, 0xac, 0x62, 0x52, 0xa3, 0x22,
0x63, 0x0f, 0xbf, 0x95, 0x48, 0xdf, 0x69, 0xe7,
0x8b, 0x33, 0xd5, 0xb2, 0xbd, 0x05, 0x49, 0x49,
0x9d, 0x57, 0x73, 0x19, 0x33, 0xae, 0xfa, 0x33,
0xf1, 0x19, 0xa8, 0x80, 0xce, 0x04, 0x9f, 0xbc,
0x1d, 0x65, 0x82, 0x1b, 0xe5, 0x3a, 0x51, 0xc8,
0x1c, 0x21, 0xe3, 0x5d, 0xf3, 0x7d, 0x9b, 0x2f,
0x2c, 0x1d, 0x4a, 0x7f, 0x9b, 0x68, 0x35, 0xa3,
0xb2, 0x50, 0xf7, 0x62, 0x79, 0xcd, 0xf4, 0x98,
0x4f, 0xe5, 0x63, 0x7c, 0x3e, 0x45, 0x31, 0x8c,
0x16, 0xa0, 0x12, 0xc8, 0x58, 0xce, 0x39, 0xa6,
0xbc, 0x54, 0xdb, 0xc5, 0xe0, 0xd5, 0xba, 0xbc,
0xb9, 0x04, 0xf4, 0x8d, 0xe8, 0x2f, 0x15, 0x9d,
};
/* 100 test cases */
static struct crc_test {
u32 crc; /* random starting crc */
u32 start; /* random 6 bit offset in buf */
u32 length; /* random 11 bit length of test */
u32 crc_le; /* expected crc32_le result */
u32 crc_be; /* expected crc32_be result */
u32 crc32c_le; /* expected crc32c_le result */
} const test[] __initconst =
{
{0x674bf11d, 0x00000038, 0x00000542, 0x0af6d466, 0xd8b6e4c1, 0xf6e93d6c},
{0x35c672c6, 0x0000003a, 0x000001aa, 0xc6d3dfba, 0x28aaf3ad, 0x0fe92aca},
{0x496da28e, 0x00000039, 0x000005af, 0xd933660f, 0x5d57e81f, 0x52e1ebb8},
{0x09a9b90e, 0x00000027, 0x000001f8, 0xb45fe007, 0xf45fca9a, 0x0798af9a},
{0xdc97e5a9, 0x00000025, 0x000003b6, 0xf81a3562, 0xe0126ba2, 0x18eb3152},
{0x47c58900, 0x0000000a, 0x000000b9, 0x8e58eccf, 0xf3afc793, 0xd00d08c7},
{0x292561e8, 0x0000000c, 0x00000403, 0xa2ba8aaf, 0x0b797aed, 0x8ba966bc},
{0x415037f6, 0x00000003, 0x00000676, 0xa17d52e8, 0x7f0fdf35, 0x11d694a2},
{0x3466e707, 0x00000026, 0x00000042, 0x258319be, 0x75c484a2, 0x6ab3208d},
{0xafd1281b, 0x00000023, 0x000002ee, 0x4428eaf8, 0x06c7ad10, 0xba4603c5},
{0xd3857b18, 0x00000028, 0x000004a2, 0x5c430821, 0xb062b7cb, 0xe6071c6f},
{0x1d825a8f, 0x0000002b, 0x0000050b, 0xd2c45f0c, 0xd68634e0, 0x179ec30a},
{0x5033e3bc, 0x0000000b, 0x00000078, 0xa3ea4113, 0xac6d31fb, 0x0903beb8},
{0x94f1fb5e, 0x0000000f, 0x000003a2, 0xfbfc50b1, 0x3cfe50ed, 0x6a7cb4fa},
{0xc9a0fe14, 0x00000009, 0x00000473, 0x5fb61894, 0x87070591, 0xdb535801},
{0x88a034b1, 0x0000001c, 0x000005ad, 0xc1b16053, 0x46f95c67, 0x92bed597},
{0xf0f72239, 0x00000020, 0x0000026d, 0xa6fa58f3, 0xf8c2c1dd, 0x192a3f1b},
{0xcc20a5e3, 0x0000003b, 0x0000067a, 0x7740185a, 0x308b979a, 0xccbaec1a},
{0xce589c95, 0x0000002b, 0x00000641, 0xd055e987, 0x40aae25b, 0x7eabae4d},
{0x78edc885, 0x00000035, 0x000005be, 0xa39cb14b, 0x035b0d1f, 0x28c72982},
{0x9d40a377, 0x0000003b, 0x00000038, 0x1f47ccd2, 0x197fbc9d, 0xc3cd4d18},
{0x703d0e01, 0x0000003c, 0x000006f1, 0x88735e7c, 0xfed57c5a, 0xbca8f0e7},
{0x776bf505, 0x0000000f, 0x000005b2, 0x5cc4fc01, 0xf32efb97, 0x713f60b3},
{0x4a3e7854, 0x00000027, 0x000004b8, 0x8d923c82, 0x0cbfb4a2, 0xebd08fd5},
{0x209172dd, 0x0000003b, 0x00000356, 0xb89e9c2b, 0xd7868138, 0x64406c59},
{0x3ba4cc5b, 0x0000002f, 0x00000203, 0xe51601a9, 0x5b2a1032, 0x7421890e},
{0xfc62f297, 0x00000000, 0x00000079, 0x71a8e1a2, 0x5d88685f, 0xe9347603},
{0x64280b8b, 0x00000016, 0x000007ab, 0x0fa7a30c, 0xda3a455f, 0x1bef9060},
{0x97dd724b, 0x00000033, 0x000007ad, 0x5788b2f4, 0xd7326d32, 0x34720072},
{0x61394b52, 0x00000035, 0x00000571, 0xc66525f1, 0xcabe7fef, 0x48310f59},
{0x29b4faff, 0x00000024, 0x0000006e, 0xca13751e, 0x993648e0, 0x783a4213},
{0x29bfb1dc, 0x0000000b, 0x00000244, 0x436c43f7, 0x429f7a59, 0x9e8efd41},
{0x86ae934b, 0x00000035, 0x00000104, 0x0760ec93, 0x9cf7d0f4, 0xfc3d34a5},
{0xc4c1024e, 0x0000002e, 0x000006b1, 0x6516a3ec, 0x19321f9c, 0x17a52ae2},
{0x3287a80a, 0x00000026, 0x00000496, 0x0b257eb1, 0x754ebd51, 0x886d935a},
{0xa4db423e, 0x00000023, 0x0000045d, 0x9b3a66dc, 0x873e9f11, 0xeaaeaeb2},
{0x7a1078df, 0x00000015, 0x0000014a, 0x8c2484c5, 0x6a628659, 0x8e900a4b},
{0x6048bd5b, 0x00000006, 0x0000006a, 0x897e3559, 0xac9961af, 0xd74662b1},
{0xd8f9ea20, 0x0000003d, 0x00000277, 0x60eb905b, 0xed2aaf99, 0xd26752ba},
{0xea5ec3b4, 0x0000002a, 0x000004fe, 0x869965dc, 0x6c1f833b, 0x8b1fcd62},
{0x2dfb005d, 0x00000016, 0x00000345, 0x6a3b117e, 0xf05e8521, 0xf54342fe},
{0x5a214ade, 0x00000020, 0x000005b6, 0x467f70be, 0xcb22ccd3, 0x5b95b988},
{0xf0ab9cca, 0x00000032, 0x00000515, 0xed223df3, 0x7f3ef01d, 0x2e1176be},
{0x91b444f9, 0x0000002e, 0x000007f8, 0x84e9a983, 0x5676756f, 0x66120546},
{0x1b5d2ddb, 0x0000002e, 0x0000012c, 0xba638c4c, 0x3f42047b, 0xf256a5cc},
{0xd824d1bb, 0x0000003a, 0x000007b5, 0x6288653b, 0x3a3ebea0, 0x4af1dd69},
{0x0470180c, 0x00000034, 0x000001f0, 0x9d5b80d6, 0x3de08195, 0x56f0a04a},
{0xffaa3a3f, 0x00000036, 0x00000299, 0xf3a82ab8, 0x53e0c13d, 0x74f6b6b2},
{0x6406cfeb, 0x00000023, 0x00000600, 0xa920b8e8, 0xe4e2acf4, 0x085951fd},
{0xb24aaa38, 0x0000003e, 0x000004a1, 0x657cc328, 0x5077b2c3, 0xc65387eb},
{0x58b2ab7c, 0x00000039, 0x000002b4, 0x3a17ee7e, 0x9dcb3643, 0x1ca9257b},
{0x3db85970, 0x00000006, 0x000002b6, 0x95268b59, 0xb9812c10, 0xfd196d76},
{0x857830c5, 0x00000003, 0x00000590, 0x4ef439d5, 0xf042161d, 0x5ef88339},
{0xe1fcd978, 0x0000003e, 0x000007d8, 0xae8d8699, 0xce0a1ef5, 0x2c3714d9},
{0xb982a768, 0x00000016, 0x000006e0, 0x62fad3df, 0x5f8a067b, 0x58576548},
{0x1d581ce8, 0x0000001e, 0x0000058b, 0xf0f5da53, 0x26e39eee, 0xfd7c57de},
{0x2456719b, 0x00000025, 0x00000503, 0x4296ac64, 0xd50e4c14, 0xd5fedd59},
{0xfae6d8f2, 0x00000000, 0x0000055d, 0x057fdf2e, 0x2a31391a, 0x1cc3b17b},
{0xcba828e3, 0x00000039, 0x000002ce, 0xe3f22351, 0x8f00877b, 0x270eed73},
{0x13d25952, 0x0000000a, 0x0000072d, 0x76d4b4cc, 0x5eb67ec3, 0x91ecbb11},
{0x0342be3f, 0x00000015, 0x00000599, 0xec75d9f1, 0x9d4d2826, 0x05ed8d0c},
{0xeaa344e0, 0x00000014, 0x000004d8, 0x72a4c981, 0x2064ea06, 0x0b09ad5b},
{0xbbb52021, 0x0000003b, 0x00000272, 0x04af99fc, 0xaf042d35, 0xf8d511fb},
{0xb66384dc, 0x0000001d, 0x000007fc, 0xd7629116, 0x782bd801, 0x5ad832cc},
{0x616c01b6, 0x00000022, 0x000002c8, 0x5b1dab30, 0x783ce7d2, 0x1214d196},
{0xce2bdaad, 0x00000016, 0x0000062a, 0x932535c8, 0x3f02926d, 0x5747218a},
{0x00fe84d7, 0x00000005, 0x00000205, 0x850e50aa, 0x753d649c, 0xde8f14de},
{0xbebdcb4c, 0x00000006, 0x0000055d, 0xbeaa37a2, 0x2d8c9eba, 0x3563b7b9},
{0xd8b1a02a, 0x00000010, 0x00000387, 0x5017d2fc, 0x503541a5, 0x071475d0},
{0x3b96cad2, 0x00000036, 0x00000347, 0x1d2372ae, 0x926cd90b, 0x54c79d60},
{0xc94c1ed7, 0x00000005, 0x0000038b, 0x9e9fdb22, 0x144a9178, 0x4c53eee6},
{0x1aad454e, 0x00000025, 0x000002b2, 0xc3f6315c, 0x5c7a35b3, 0x10137a3c},
{0xa4fec9a6, 0x00000000, 0x000006d6, 0x90be5080, 0xa4107605, 0xaa9d6c73},
{0x1bbe71e2, 0x0000001f, 0x000002fd, 0x4e504c3b, 0x284ccaf1, 0xb63d23e7},
{0x4201c7e4, 0x00000002, 0x000002b7, 0x7822e3f9, 0x0cc912a9, 0x7f53e9cf},
{0x23fddc96, 0x00000003, 0x00000627, 0x8a385125, 0x07767e78, 0x13c1cd83},
{0xd82ba25c, 0x00000016, 0x0000063e, 0x98e4148a, 0x283330c9, 0x49ff5867},
{0x786f2032, 0x0000002d, 0x0000060f, 0xf201600a, 0xf561bfcd, 0x8467f211},
{0xfebe4e1f, 0x0000002a, 0x000004f2, 0x95e51961, 0xfd80dcab, 0x3f9683b2},
{0x1a6e0a39, 0x00000008, 0x00000672, 0x8af6c2a5, 0x78dd84cb, 0x76a3f874},
{0x56000ab8, 0x0000000e, 0x000000e5, 0x36bacb8f, 0x22ee1f77, 0x863b702f},
{0x4717fe0c, 0x00000000, 0x000006ec, 0x8439f342, 0x5c8e03da, 0xdc6c58ff},
{0xd5d5d68e, 0x0000003c, 0x000003a3, 0x46fff083, 0x177d1b39, 0x0622cc95},
{0xc25dd6c6, 0x00000024, 0x000006c0, 0x5ceb8eb4, 0x892b0d16, 0xe85605cd},
{0xe9b11300, 0x00000023, 0x00000683, 0x07a5d59a, 0x6c6a3208, 0x31da5f06},
{0x95cd285e, 0x00000001, 0x00000047, 0x7b3a4368, 0x0202c07e, 0xa1f2e784},
{0xd9245a25, 0x0000001e, 0x000003a6, 0xd33c1841, 0x1936c0d5, 0xb07cc616},
{0x103279db, 0x00000006, 0x0000039b, 0xca09b8a0, 0x77d62892, 0xbf943b6c},
{0x1cba3172, 0x00000027, 0x000001c8, 0xcb377194, 0xebe682db, 0x2c01af1c},
{0x8f613739, 0x0000000c, 0x000001df, 0xb4b0bc87, 0x7710bd43, 0x0fe5f56d},
{0x1c6aa90d, 0x0000001b, 0x0000053c, 0x70559245, 0xda7894ac, 0xf8943b2d},
{0xaabe5b93, 0x0000003d, 0x00000715, 0xcdbf42fa, 0x0c3b99e7, 0xe4d89272},
{0xf15dd038, 0x00000006, 0x000006db, 0x6e104aea, 0x8d5967f2, 0x7c2f6bbb},
{0x584dd49c, 0x00000020, 0x000007bc, 0x36b6cfd6, 0xad4e23b2, 0xabbf388b},
{0x5d8c9506, 0x00000020, 0x00000470, 0x4c62378e, 0x31d92640, 0x1dca1f4e},
{0xb80d17b0, 0x00000032, 0x00000346, 0x22a5bb88, 0x9a7ec89f, 0x5c170e23},
{0xdaf0592e, 0x00000023, 0x000007b0, 0x3cab3f99, 0x9b1fdd99, 0xc0e9d672},
{0x4793cc85, 0x0000000d, 0x00000706, 0xe82e04f6, 0xed3db6b7, 0xc18bdc86},
{0x82ebf64e, 0x00000009, 0x000007c3, 0x69d590a9, 0x9efa8499, 0xa874fcdd},
{0xb18a0319, 0x00000026, 0x000007db, 0x1cf98dcc, 0x8fa9ad6a, 0x9dc0bb48},
};
#include <linux/time.h>
static int __init crc32c_test(void)
{
int i;
int errors = 0;
int bytes = 0;
u64 nsec;
unsigned long flags;
/* keep static to prevent cache warming code from
* getting eliminated by the compiler */
static u32 crc;
/* pre-warm the cache */
for (i = 0; i < 100; i++) {
bytes += 2*test[i].length;
crc ^= __crc32c_le(test[i].crc, test_buf +
test[i].start, test[i].length);
}
/* reduce OS noise */
local_irq_save(flags);
nsec = ktime_get_ns();
for (i = 0; i < 100; i++) {
if (test[i].crc32c_le != __crc32c_le(test[i].crc, test_buf +
test[i].start, test[i].length))
errors++;
}
nsec = ktime_get_ns() - nsec;
local_irq_restore(flags);
pr_info("crc32c: CRC_LE_BITS = %d\n", CRC_LE_BITS);
if (errors)
pr_warn("crc32c: %d self tests failed\n", errors);
else {
pr_info("crc32c: self tests passed, processed %d bytes in %lld nsec\n",
bytes, nsec);
}
return 0;
}
static int __init crc32c_combine_test(void)
{
int i, j;
int errors = 0, runs = 0;
for (i = 0; i < 10; i++) {
u32 crc_full;
crc_full = __crc32c_le(test[i].crc, test_buf + test[i].start,
test[i].length);
for (j = 0; j <= test[i].length; ++j) {
u32 crc1, crc2;
u32 len1 = j, len2 = test[i].length - j;
crc1 = __crc32c_le(test[i].crc, test_buf +
test[i].start, len1);
crc2 = __crc32c_le(0, test_buf + test[i].start +
len1, len2);
if (!(crc_full == __crc32c_le_combine(crc1, crc2, len2) &&
crc_full == test[i].crc32c_le))
errors++;
runs++;
cond_resched();
}
}
if (errors)
pr_warn("crc32c_combine: %d/%d self tests failed\n", errors, runs);
else
pr_info("crc32c_combine: %d self tests passed\n", runs);
return 0;
}
static int __init crc32_test(void)
{
int i;
int errors = 0;
int bytes = 0;
u64 nsec;
unsigned long flags;
/* keep static to prevent cache warming code from
* getting eliminated by the compiler */
static u32 crc;
/* pre-warm the cache */
for (i = 0; i < 100; i++) {
bytes += 2*test[i].length;
crc ^= crc32_le(test[i].crc, test_buf +
test[i].start, test[i].length);
crc ^= crc32_be(test[i].crc, test_buf +
test[i].start, test[i].length);
}
/* reduce OS noise */
local_irq_save(flags);
nsec = ktime_get_ns();
for (i = 0; i < 100; i++) {
if (test[i].crc_le != crc32_le(test[i].crc, test_buf +
test[i].start, test[i].length))
errors++;
if (test[i].crc_be != crc32_be(test[i].crc, test_buf +
test[i].start, test[i].length))
errors++;
}
nsec = ktime_get_ns() - nsec;
local_irq_restore(flags);
pr_info("crc32: CRC_LE_BITS = %d, CRC_BE BITS = %d\n",
CRC_LE_BITS, CRC_BE_BITS);
if (errors)
pr_warn("crc32: %d self tests failed\n", errors);
else {
pr_info("crc32: self tests passed, processed %d bytes in %lld nsec\n",
bytes, nsec);
}
return 0;
}
static int __init crc32_combine_test(void)
{
int i, j;
int errors = 0, runs = 0;
for (i = 0; i < 10; i++) {
u32 crc_full;
crc_full = crc32_le(test[i].crc, test_buf + test[i].start,
test[i].length);
for (j = 0; j <= test[i].length; ++j) {
u32 crc1, crc2;
u32 len1 = j, len2 = test[i].length - j;
crc1 = crc32_le(test[i].crc, test_buf +
test[i].start, len1);
crc2 = crc32_le(0, test_buf + test[i].start +
len1, len2);
if (!(crc_full == crc32_le_combine(crc1, crc2, len2) &&
crc_full == test[i].crc_le))
errors++;
runs++;
cond_resched();
}
}
if (errors)
pr_warn("crc32_combine: %d/%d self tests failed\n", errors, runs);
else
pr_info("crc32_combine: %d self tests passed\n", runs);
return 0;
}
static int __init crc32test_init(void)
{
crc32_test();
crc32c_test();
crc32_combine_test();
crc32c_combine_test();
return 0;
}
static void __exit crc32_exit(void)
{
}
module_init(crc32test_init);
module_exit(crc32_exit);
MODULE_AUTHOR("Matt Domsch <Matt_Domsch@dell.com>");
MODULE_DESCRIPTION("CRC32 selftest");
MODULE_LICENSE("GPL");
lib/crc4.c
+46
View File
@@ -0,0 +1,46 @@
/*
* crc4.c - simple crc-4 calculations.
*
* This source code is licensed under the GNU General Public License, Version
* 2. See the file COPYING for more details.
*/
#include <linux/crc4.h>
#include <linux/module.h>
static const uint8_t crc4_tab[] = {
0x0, 0x7, 0xe, 0x9, 0xb, 0xc, 0x5, 0x2,
0x1, 0x6, 0xf, 0x8, 0xa, 0xd, 0x4, 0x3,
};
/**
* crc4 - calculate the 4-bit crc of a value.
* @c: starting crc4
* @x: value to checksum
* @bits: number of bits in @x to checksum
*
* Returns the crc4 value of @x, using polynomial 0b10111.
*
* The @x value is treated as left-aligned, and bits above @bits are ignored
* in the crc calculations.
*/
uint8_t crc4(uint8_t c, uint64_t x, int bits)
{
int i;
/* mask off anything above the top bit */
x &= (1ull << bits) - 1;
/* Align to 4-bits */
bits = (bits + 3) & ~0x3;
/* Calculate crc4 over four-bit nibbles, starting at the MSbit */
for (i = bits - 4; i >= 0; i -= 4)
c = crc4_tab[c ^ ((x >> i) & 0xf)];
return c;
}
EXPORT_SYMBOL_GPL(crc4);
MODULE_DESCRIPTION("CRC4 calculations");
MODULE_LICENSE("GPL");
lib/crc64.c
+56
View File
@@ -0,0 +1,56 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Normal 64-bit CRC calculation.
*
* This is a basic crc64 implementation following ECMA-182 specification,
* which can be found from,
* http://www.ecma-international.org/publications/standards/Ecma-182.htm
*
* Dr. Ross N. Williams has a great document to introduce the idea of CRC
* algorithm, here the CRC64 code is also inspired by the table-driven
* algorithm and detail example from this paper. This paper can be found
* from,
* http://www.ross.net/crc/download/crc_v3.txt
*
* crc64table[256] is the lookup table of a table-driven 64-bit CRC
* calculation, which is generated by gen_crc64table.c in kernel build
* time. The polynomial of crc64 arithmetic is from ECMA-182 specification
* as well, which is defined as,
*
* x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 +
* x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 +
* x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 +
* x^7 + x^4 + x + 1
*
* Copyright 2018 SUSE Linux.
* Author: Coly Li <colyli@suse.de>
*/
#include <linux/module.h>
#include <linux/types.h>
#include "crc64table.h"
MODULE_DESCRIPTION("CRC64 calculations");
MODULE_LICENSE("GPL v2");
/**
* crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
* @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
or the previous crc64 value if computing incrementally.
* @p: pointer to buffer over which CRC64 is run
* @len: length of buffer @p
*/
u64 __pure crc64_be(u64 crc, const void *p, size_t len)
{
size_t i, t;
const unsigned char *_p = p;
for (i = 0; i < len; i++) {
t = ((crc >> 56) ^ (*_p++)) & 0xFF;
crc = crc64table[t] ^ (crc << 8);
}
return crc;
}
EXPORT_SYMBOL_GPL(crc64_be);
lib/crc7.c
+76
View File
@@ -0,0 +1,76 @@
/*
* crc7.c
*
* This source code is licensed under the GNU General Public License,
* Version 2. See the file COPYING for more details.
*/
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc7.h>
/*
* Table for CRC-7 (polynomial x^7 + x^3 + 1).
* This is a big-endian CRC (msbit is highest power of x),
* aligned so the msbit of the byte is the x^6 coefficient
* and the lsbit is not used.
*/
const u8 crc7_be_syndrome_table[256] = {
0x00, 0x12, 0x24, 0x36, 0x48, 0x5a, 0x6c, 0x7e,
0x90, 0x82, 0xb4, 0xa6, 0xd8, 0xca, 0xfc, 0xee,
0x32, 0x20, 0x16, 0x04, 0x7a, 0x68, 0x5e, 0x4c,
0xa2, 0xb0, 0x86, 0x94, 0xea, 0xf8, 0xce, 0xdc,
0x64, 0x76, 0x40, 0x52, 0x2c, 0x3e, 0x08, 0x1a,
0xf4, 0xe6, 0xd0, 0xc2, 0xbc, 0xae, 0x98, 0x8a,
0x56, 0x44, 0x72, 0x60, 0x1e, 0x0c, 0x3a, 0x28,
0xc6, 0xd4, 0xe2, 0xf0, 0x8e, 0x9c, 0xaa, 0xb8,
0xc8, 0xda, 0xec, 0xfe, 0x80, 0x92, 0xa4, 0xb6,
0x58, 0x4a, 0x7c, 0x6e, 0x10, 0x02, 0x34, 0x26,
0xfa, 0xe8, 0xde, 0xcc, 0xb2, 0xa0, 0x96, 0x84,
0x6a, 0x78, 0x4e, 0x5c, 0x22, 0x30, 0x06, 0x14,
0xac, 0xbe, 0x88, 0x9a, 0xe4, 0xf6, 0xc0, 0xd2,
0x3c, 0x2e, 0x18, 0x0a, 0x74, 0x66, 0x50, 0x42,
0x9e, 0x8c, 0xba, 0xa8, 0xd6, 0xc4, 0xf2, 0xe0,
0x0e, 0x1c, 0x2a, 0x38, 0x46, 0x54, 0x62, 0x70,
0x82, 0x90, 0xa6, 0xb4, 0xca, 0xd8, 0xee, 0xfc,
0x12, 0x00, 0x36, 0x24, 0x5a, 0x48, 0x7e, 0x6c,
0xb0, 0xa2, 0x94, 0x86, 0xf8, 0xea, 0xdc, 0xce,
0x20, 0x32, 0x04, 0x16, 0x68, 0x7a, 0x4c, 0x5e,
0xe6, 0xf4, 0xc2, 0xd0, 0xae, 0xbc, 0x8a, 0x98,
0x76, 0x64, 0x52, 0x40, 0x3e, 0x2c, 0x1a, 0x08,
0xd4, 0xc6, 0xf0, 0xe2, 0x9c, 0x8e, 0xb8, 0xaa,
0x44, 0x56, 0x60, 0x72, 0x0c, 0x1e, 0x28, 0x3a,
0x4a, 0x58, 0x6e, 0x7c, 0x02, 0x10, 0x26, 0x34,
0xda, 0xc8, 0xfe, 0xec, 0x92, 0x80, 0xb6, 0xa4,
0x78, 0x6a, 0x5c, 0x4e, 0x30, 0x22, 0x14, 0x06,
0xe8, 0xfa, 0xcc, 0xde, 0xa0, 0xb2, 0x84, 0x96,
0x2e, 0x3c, 0x0a, 0x18, 0x66, 0x74, 0x42, 0x50,
0xbe, 0xac, 0x9a, 0x88, 0xf6, 0xe4, 0xd2, 0xc0,
0x1c, 0x0e, 0x38, 0x2a, 0x54, 0x46, 0x70, 0x62,
0x8c, 0x9e, 0xa8, 0xba, 0xc4, 0xd6, 0xe0, 0xf2
};
EXPORT_SYMBOL(crc7_be_syndrome_table);
/**
* crc7 - update the CRC7 for the data buffer
* @crc: previous CRC7 value
* @buffer: data pointer
* @len: number of bytes in the buffer
* Context: any
*
* Returns the updated CRC7 value.
* The CRC7 is left-aligned in the byte (the lsbit is always 0), as that
* makes the computation easier, and all callers want it in that form.
*
*/
u8 crc7_be(u8 crc, const u8 *buffer, size_t len)
{
while (len--)
crc = crc7_be_byte(crc, *buffer++);
return crc;
}
EXPORT_SYMBOL(crc7_be);
MODULE_DESCRIPTION("CRC7 calculations");
MODULE_LICENSE("GPL");
lib/crc8.c
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright (c) 2011 Broadcom Corporation
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/crc8.h>
#include <linux/printk.h>
/**
* crc8_populate_msb - fill crc table for given polynomial in reverse bit order.
*
* @table: table to be filled.
* @polynomial: polynomial for which table is to be filled.
*/
void crc8_populate_msb(u8 table[CRC8_TABLE_SIZE], u8 polynomial)
{
int i, j;
const u8 msbit = 0x80;
u8 t = msbit;
table[0] = 0;
for (i = 1; i < CRC8_TABLE_SIZE; i *= 2) {
t = (t << 1) ^ (t & msbit ? polynomial : 0);
for (j = 0; j < i; j++)
table[i+j] = table[j] ^ t;
}
}
EXPORT_SYMBOL(crc8_populate_msb);
/**
* crc8_populate_lsb - fill crc table for given polynomial in regular bit order.
*
* @table: table to be filled.
* @polynomial: polynomial for which table is to be filled.
*/
void crc8_populate_lsb(u8 table[CRC8_TABLE_SIZE], u8 polynomial)
{
int i, j;
u8 t = 1;
table[0] = 0;
for (i = (CRC8_TABLE_SIZE >> 1); i; i >>= 1) {
t = (t >> 1) ^ (t & 1 ? polynomial : 0);
for (j = 0; j < CRC8_TABLE_SIZE; j += 2*i)
table[i+j] = table[j] ^ t;
}
}
EXPORT_SYMBOL(crc8_populate_lsb);
/**
* crc8 - calculate a crc8 over the given input data.
*
* @table: crc table used for calculation.
* @pdata: pointer to data buffer.
* @nbytes: number of bytes in data buffer.
* @crc: previous returned crc8 value.
*/
u8 crc8(const u8 table[CRC8_TABLE_SIZE], u8 *pdata, size_t nbytes, u8 crc)
{
/* loop over the buffer data */
while (nbytes-- > 0)
crc = table[(crc ^ *pdata++) & 0xff];
return crc;
}
EXPORT_SYMBOL(crc8);
MODULE_DESCRIPTION("CRC8 (by Williams, Ross N.) function");
MODULE_AUTHOR("Broadcom Corporation");
MODULE_LICENSE("Dual BSD/GPL");
lib/crypto/Kconfig
+118
View File
@@ -0,0 +1,118 @@
# SPDX-License-Identifier: GPL-2.0
comment "Crypto library routines"
config CRYPTO_ARCH_HAVE_LIB_BLAKE2S
tristate
help
Declares whether the architecture provides an arch-specific
accelerated implementation of the Blake2s library interface,
either builtin or as a module.
config CRYPTO_LIB_BLAKE2S_GENERIC
tristate
help
This symbol can be depended upon by arch implementations of the
Blake2s library interface that require the generic code as a
fallback, e.g., for SIMD implementations. If no arch specific
implementation is enabled, this implementation serves the users
of CRYPTO_LIB_BLAKE2S.
config CRYPTO_LIB_BLAKE2S
tristate "BLAKE2s hash function library"
depends on CRYPTO_ARCH_HAVE_LIB_BLAKE2S || !CRYPTO_ARCH_HAVE_LIB_BLAKE2S
select CRYPTO_LIB_BLAKE2S_GENERIC if CRYPTO_ARCH_HAVE_LIB_BLAKE2S=n
help
Enable the Blake2s library interface. This interface may be fulfilled
by either the generic implementation or an arch-specific one, if one
is available and enabled.
config CRYPTO_ARCH_HAVE_LIB_CHACHA
tristate
help
Declares whether the architecture provides an arch-specific
accelerated implementation of the ChaCha library interface,
either builtin or as a module.
config CRYPTO_LIB_CHACHA_GENERIC
tristate
select CRYPTO_ALGAPI
help
This symbol can be depended upon by arch implementations of the
ChaCha library interface that require the generic code as a
fallback, e.g., for SIMD implementations. If no arch specific
implementation is enabled, this implementation serves the users
of CRYPTO_LIB_CHACHA.
config CRYPTO_LIB_CHACHA
tristate "ChaCha library interface"
depends on CRYPTO_ARCH_HAVE_LIB_CHACHA || !CRYPTO_ARCH_HAVE_LIB_CHACHA
select CRYPTO_LIB_CHACHA_GENERIC if CRYPTO_ARCH_HAVE_LIB_CHACHA=n
help
Enable the ChaCha library interface. This interface may be fulfilled
by either the generic implementation or an arch-specific one, if one
is available and enabled.
config CRYPTO_LIB_POLY1305_RSIZE
int
default 2 if MIPS
default 11 if X86_64
default 9 if ARM || ARM64
default 1
config CRYPTO_ARCH_HAVE_LIB_POLY1305
tristate
help
Declares whether the architecture provides an arch-specific
accelerated implementation of the Poly1305 library interface,
either builtin or as a module.
config CRYPTO_LIB_POLY1305_GENERIC
tristate
help
This symbol can be depended upon by arch implementations of the
Poly1305 library interface that require the generic code as a
fallback, e.g., for SIMD implementations. If no arch specific
implementation is enabled, this implementation serves the users
of CRYPTO_LIB_POLY1305.
config CRYPTO_LIB_POLY1305
tristate "Poly1305 library interface"
depends on CRYPTO_ARCH_HAVE_LIB_POLY1305 || !CRYPTO_ARCH_HAVE_LIB_POLY1305
select CRYPTO_LIB_POLY1305_GENERIC if CRYPTO_ARCH_HAVE_LIB_POLY1305=n
help
Enable the Poly1305 library interface. This interface may be fulfilled
by either the generic implementation or an arch-specific one, if one
is available and enabled.
config CRYPTO_ARCH_HAVE_LIB_CURVE25519
tristate
help
Declares whether the architecture provides an arch-specific
accelerated implementation of the Curve25519 library interface,
either builtin or as a module.
config CRYPTO_LIB_CURVE25519_GENERIC
tristate
help
This symbol can be depended upon by arch implementations of the
Curve25519 library interface that require the generic code as a
fallback, e.g., for SIMD implementations. If no arch specific
implementation is enabled, this implementation serves the users
of CRYPTO_LIB_CURVE25519.
config CRYPTO_LIB_CURVE25519
tristate "Curve25519 scalar multiplication library"
depends on CRYPTO_ARCH_HAVE_LIB_CURVE25519 || !CRYPTO_ARCH_HAVE_LIB_CURVE25519
select CRYPTO_LIB_CURVE25519_GENERIC if CRYPTO_ARCH_HAVE_LIB_CURVE25519=n
help
Enable the Curve25519 library interface. This interface may be
fulfilled by either the generic implementation or an arch-specific
one, if one is available and enabled.
config CRYPTO_LIB_CHACHA20POLY1305
tristate "ChaCha20-Poly1305 AEAD support (8-byte nonce library version)"
depends on CRYPTO_ARCH_HAVE_LIB_CHACHA || !CRYPTO_ARCH_HAVE_LIB_CHACHA
depends on CRYPTO_ARCH_HAVE_LIB_POLY1305 || !CRYPTO_ARCH_HAVE_LIB_POLY1305
select CRYPTO_LIB_CHACHA
select CRYPTO_LIB_POLY1305
lib/crypto/Makefile
+33
View File
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: GPL-2.0
# chacha is used by the /dev/random driver which is always builtin
obj-y += chacha.o
obj-$(CONFIG_CRYPTO_LIB_CHACHA_GENERIC) += libchacha.o
obj-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += libpoly1305.o
libpoly1305-y := poly1305-donna32.o
libpoly1305-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o
libpoly1305-y += poly1305.o
obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC) += libblake2s-generic.o
libblake2s-generic-y += blake2s-generic.o
obj-$(CONFIG_CRYPTO_LIB_BLAKE2S) += libblake2s.o
libblake2s-y += blake2s.o
obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o
libchacha20poly1305-y += chacha20poly1305.o
obj-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += libcurve25519-generic.o
libcurve25519-generic-y := curve25519-fiat32.o
libcurve25519-generic-$(CONFIG_ARCH_SUPPORTS_INT128) := curve25519-hacl64.o
libcurve25519-generic-y += curve25519-generic.o
obj-$(CONFIG_CRYPTO_LIB_CURVE25519) += libcurve25519.o
libcurve25519-y += curve25519.o
ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y)
libblake2s-y += blake2s-selftest.o
libchacha20poly1305-y += chacha20poly1305-selftest.o
libcurve25519-y += curve25519-selftest.o
endif
lib/crypto/blake2s-generic.c
+111
View File
@@ -0,0 +1,111 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is an implementation of the BLAKE2s hash and PRF functions.
*
* Information: https://blake2.net/
*
*/
#include <crypto/internal/blake2s.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bug.h>
#include <asm/unaligned.h>
static const u8 blake2s_sigma[10][16] = {
{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
{ 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 },
{ 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 },
{ 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 },
{ 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 },
{ 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 },
{ 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 },
{ 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 },
{ 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 },
{ 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 },
};
static inline void blake2s_increment_counter(struct blake2s_state *state,
const u32 inc)
{
state->t[0] += inc;
state->t[1] += (state->t[0] < inc);
}
void blake2s_compress_generic(struct blake2s_state *state,const u8 *block,
size_t nblocks, const u32 inc)
{
u32 m[16];
u32 v[16];
int i;
WARN_ON(IS_ENABLED(DEBUG) &&
(nblocks > 1 && inc != BLAKE2S_BLOCK_SIZE));
while (nblocks > 0) {
blake2s_increment_counter(state, inc);
memcpy(m, block, BLAKE2S_BLOCK_SIZE);
le32_to_cpu_array(m, ARRAY_SIZE(m));
memcpy(v, state->h, 32);
v[ 8] = BLAKE2S_IV0;
v[ 9] = BLAKE2S_IV1;
v[10] = BLAKE2S_IV2;
v[11] = BLAKE2S_IV3;
v[12] = BLAKE2S_IV4 ^ state->t[0];
v[13] = BLAKE2S_IV5 ^ state->t[1];
v[14] = BLAKE2S_IV6 ^ state->f[0];
v[15] = BLAKE2S_IV7 ^ state->f[1];
#define G(r, i, a, b, c, d) do { \
a += b + m[blake2s_sigma[r][2 * i + 0]]; \
d = ror32(d ^ a, 16); \
c += d; \
b = ror32(b ^ c, 12); \
a += b + m[blake2s_sigma[r][2 * i + 1]]; \
d = ror32(d ^ a, 8); \
c += d; \
b = ror32(b ^ c, 7); \
} while (0)
#define ROUND(r) do { \
G(r, 0, v[0], v[ 4], v[ 8], v[12]); \
G(r, 1, v[1], v[ 5], v[ 9], v[13]); \
G(r, 2, v[2], v[ 6], v[10], v[14]); \
G(r, 3, v[3], v[ 7], v[11], v[15]); \
G(r, 4, v[0], v[ 5], v[10], v[15]); \
G(r, 5, v[1], v[ 6], v[11], v[12]); \
G(r, 6, v[2], v[ 7], v[ 8], v[13]); \
G(r, 7, v[3], v[ 4], v[ 9], v[14]); \
} while (0)
ROUND(0);
ROUND(1);
ROUND(2);
ROUND(3);
ROUND(4);
ROUND(5);
ROUND(6);
ROUND(7);
ROUND(8);
ROUND(9);
#undef G
#undef ROUND
for (i = 0; i < 8; ++i)
state->h[i] ^= v[i] ^ v[i + 8];
block += BLAKE2S_BLOCK_SIZE;
--nblocks;
}
}
EXPORT_SYMBOL(blake2s_compress_generic);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("BLAKE2s hash function");
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
lib/crypto/blake2s-selftest.c
+622
View File
@@ -0,0 +1,622 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*/
#include <crypto/blake2s.h>
#include <linux/string.h>
/*
* blake2s_testvecs[] generated with the program below (using libb2-dev and
* libssl-dev [OpenSSL])
*
* #include <blake2.h>
* #include <stdint.h>
* #include <stdio.h>
*
* #include <openssl/evp.h>
* #include <openssl/hmac.h>
*
* #define BLAKE2S_TESTVEC_COUNT 256
*
* static void print_vec(const uint8_t vec[], int len)
* {
* int i;
*
* printf(" { ");
* for (i = 0; i < len; i++) {
* if (i && (i % 12) == 0)
* printf("\n ");
* printf("0x%02x, ", vec[i]);
* }
* printf("},\n");
* }
*
* int main(void)
* {
* uint8_t key[BLAKE2S_KEYBYTES];
* uint8_t buf[BLAKE2S_TESTVEC_COUNT];
* uint8_t hash[BLAKE2S_OUTBYTES];
* int i, j;
*
* key[0] = key[1] = 1;
* for (i = 2; i < BLAKE2S_KEYBYTES; ++i)
* key[i] = key[i - 2] + key[i - 1];
*
* for (i = 0; i < BLAKE2S_TESTVEC_COUNT; ++i)
* buf[i] = (uint8_t)i;
*
* printf("static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {\n");
*
* for (i = 0; i < BLAKE2S_TESTVEC_COUNT; ++i) {
* int outlen = 1 + i % BLAKE2S_OUTBYTES;
* int keylen = (13 * i) % (BLAKE2S_KEYBYTES + 1);
*
* blake2s(hash, buf, key + BLAKE2S_KEYBYTES - keylen, outlen, i,
* keylen);
* print_vec(hash, outlen);
* }
* printf("};\n\n");
*
* printf("static const u8 blake2s_hmac_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {\n");
*
* HMAC(EVP_blake2s256(), key, sizeof(key), buf, sizeof(buf), hash, NULL);
* print_vec(hash, BLAKE2S_OUTBYTES);
*
* HMAC(EVP_blake2s256(), buf, sizeof(buf), key, sizeof(key), hash, NULL);
* print_vec(hash, BLAKE2S_OUTBYTES);
*
* printf("};\n");
*
* return 0;
*}
*/
static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {
{ 0xa1, },
{ 0x7c, 0x89, },
{ 0x74, 0x0e, 0xd4, },
{ 0x47, 0x0c, 0x21, 0x15, },
{ 0x18, 0xd6, 0x9c, 0xa6, 0xc4, },
{ 0x13, 0x5d, 0x16, 0x63, 0x2e, 0xf9, },
{ 0x2c, 0xb5, 0x04, 0xb7, 0x99, 0xe2, 0x73, },
{ 0x9a, 0x0f, 0xd2, 0x39, 0xd6, 0x68, 0x1b, 0x92, },
{ 0xc8, 0xde, 0x7a, 0xea, 0x2f, 0xf4, 0xd2, 0xe3, 0x2b, },
{ 0x5b, 0xf9, 0x43, 0x52, 0x0c, 0x12, 0xba, 0xb5, 0x93, 0x9f, },
{ 0xc6, 0x2c, 0x4e, 0x80, 0xfc, 0x32, 0x5b, 0x33, 0xb8, 0xb8, 0x0a, },
{ 0xa7, 0x5c, 0xfd, 0x3a, 0xcc, 0xbf, 0x90, 0xca, 0xb7, 0x97, 0xde, 0xd8, },
{ 0x66, 0xca, 0x3c, 0xc4, 0x19, 0xef, 0x92, 0x66, 0x3f, 0x21, 0x8f, 0xda,
0xb7, },
{ 0xba, 0xe5, 0xbb, 0x30, 0x25, 0x94, 0x6d, 0xc3, 0x89, 0x09, 0xc4, 0x25,
0x52, 0x3e, },
{ 0xa2, 0xef, 0x0e, 0x52, 0x0b, 0x5f, 0xa2, 0x01, 0x6d, 0x0a, 0x25, 0xbc,
0x57, 0xe2, 0x27, },
{ 0x4f, 0xe0, 0xf9, 0x52, 0x12, 0xda, 0x84, 0xb7, 0xab, 0xae, 0xb0, 0xa6,
0x47, 0x2a, 0xc7, 0xf5, },
{ 0x56, 0xe7, 0xa8, 0x1c, 0x4c, 0xca, 0xed, 0x90, 0x31, 0xec, 0x87, 0x43,
0xe7, 0x72, 0x08, 0xec, 0xbe, },
{ 0x7e, 0xdf, 0x80, 0x1c, 0x93, 0x33, 0xfd, 0x53, 0x44, 0xba, 0xfd, 0x96,
0xe1, 0xbb, 0xb5, 0x65, 0xa5, 0x00, },
{ 0xec, 0x6b, 0xed, 0xf7, 0x7b, 0x62, 0x1d, 0x7d, 0xf4, 0x82, 0xf3, 0x1e,
0x18, 0xff, 0x2b, 0xc4, 0x06, 0x20, 0x2a, },
{ 0x74, 0x98, 0xd7, 0x68, 0x63, 0xed, 0x87, 0xe4, 0x5d, 0x8d, 0x9e, 0x1d,
0xfd, 0x2a, 0xbb, 0x86, 0xac, 0xe9, 0x2a, 0x89, },
{ 0x89, 0xc3, 0x88, 0xce, 0x2b, 0x33, 0x1e, 0x10, 0xd1, 0x37, 0x20, 0x86,
0x28, 0x43, 0x70, 0xd9, 0xfb, 0x96, 0xd9, 0xb5, 0xd3, },
{ 0xcb, 0x56, 0x74, 0x41, 0x8d, 0x80, 0x01, 0x9a, 0x6b, 0x38, 0xe1, 0x41,
0xad, 0x9c, 0x62, 0x74, 0xce, 0x35, 0xd5, 0x6c, 0x89, 0x6e, },
{ 0x79, 0xaf, 0x94, 0x59, 0x99, 0x26, 0xe1, 0xc9, 0x34, 0xfe, 0x7c, 0x22,
0xf7, 0x43, 0xd7, 0x65, 0xd4, 0x48, 0x18, 0xac, 0x3d, 0xfd, 0x93, },
{ 0x85, 0x0d, 0xff, 0xb8, 0x3e, 0x87, 0x41, 0xb0, 0x95, 0xd3, 0x3d, 0x00,
0x47, 0x55, 0x9e, 0xd2, 0x69, 0xea, 0xbf, 0xe9, 0x7a, 0x2d, 0x61, 0x45, },
{ 0x03, 0xe0, 0x85, 0xec, 0x54, 0xb5, 0x16, 0x53, 0xa8, 0xc4, 0x71, 0xe9,
0x6a, 0xe7, 0xcb, 0xc4, 0x15, 0x02, 0xfc, 0x34, 0xa4, 0xa4, 0x28, 0x13,
0xd1, },
{ 0xe3, 0x34, 0x4b, 0xe1, 0xd0, 0x4b, 0x55, 0x61, 0x8f, 0xc0, 0x24, 0x05,
0xe6, 0xe0, 0x3d, 0x70, 0x24, 0x4d, 0xda, 0xb8, 0x91, 0x05, 0x29, 0x07,
0x01, 0x3e, },
{ 0x61, 0xff, 0x01, 0x72, 0xb1, 0x4d, 0xf6, 0xfe, 0xd1, 0xd1, 0x08, 0x74,
0xe6, 0x91, 0x44, 0xeb, 0x61, 0xda, 0x40, 0xaf, 0xfc, 0x8c, 0x91, 0x6b,
0xec, 0x13, 0xed, },
{ 0xd4, 0x40, 0xd2, 0xa0, 0x7f, 0xc1, 0x58, 0x0c, 0x85, 0xa0, 0x86, 0xc7,
0x86, 0xb9, 0x61, 0xc9, 0xea, 0x19, 0x86, 0x1f, 0xab, 0x07, 0xce, 0x37,
0x72, 0x67, 0x09, 0xfc, },
{ 0x9e, 0xf8, 0x18, 0x67, 0x93, 0x10, 0x9b, 0x39, 0x75, 0xe8, 0x8b, 0x38,
0x82, 0x7d, 0xb8, 0xb7, 0xa5, 0xaf, 0xe6, 0x6a, 0x22, 0x5e, 0x1f, 0x9c,
0x95, 0x29, 0x19, 0xf2, 0x4b, },
{ 0xc8, 0x62, 0x25, 0xf5, 0x98, 0xc9, 0xea, 0xe5, 0x29, 0x3a, 0xd3, 0x22,
0xeb, 0xeb, 0x07, 0x7c, 0x15, 0x07, 0xee, 0x15, 0x61, 0xbb, 0x05, 0x30,
0x99, 0x7f, 0x11, 0xf6, 0x0a, 0x1d, },
{ 0x68, 0x70, 0xf7, 0x90, 0xa1, 0x8b, 0x1f, 0x0f, 0xbb, 0xce, 0xd2, 0x0e,
0x33, 0x1f, 0x7f, 0xa9, 0x78, 0xa8, 0xa6, 0x81, 0x66, 0xab, 0x8d, 0xcd,
0x58, 0x55, 0x3a, 0x0b, 0x7a, 0xdb, 0xb5, },
{ 0xdd, 0x35, 0xd2, 0xb4, 0xf6, 0xc7, 0xea, 0xab, 0x64, 0x24, 0x4e, 0xfe,
0xe5, 0x3d, 0x4e, 0x95, 0x8b, 0x6d, 0x6c, 0xbc, 0xb0, 0xf8, 0x88, 0x61,
0x09, 0xb7, 0x78, 0xa3, 0x31, 0xfe, 0xd9, 0x2f, },
{ 0x0a, },
{ 0x6e, 0xd4, },
{ 0x64, 0xe9, 0xd1, },
{ 0x30, 0xdd, 0x71, 0xef, },
{ 0x11, 0xb5, 0x0c, 0x87, 0xc9, },
{ 0x06, 0x1c, 0x6d, 0x04, 0x82, 0xd0, },
{ 0x5c, 0x42, 0x0b, 0xee, 0xc5, 0x9c, 0xb2, },
{ 0xe8, 0x29, 0xd6, 0xb4, 0x5d, 0xf7, 0x2b, 0x93, },
{ 0x18, 0xca, 0x27, 0x72, 0x43, 0x39, 0x16, 0xbc, 0x6a, },
{ 0x39, 0x8f, 0xfd, 0x64, 0xf5, 0x57, 0x23, 0xb0, 0x45, 0xf8, },
{ 0xbb, 0x3a, 0x78, 0x6b, 0x02, 0x1d, 0x0b, 0x16, 0xe3, 0xb2, 0x9a, },
{ 0xb8, 0xb4, 0x0b, 0xe5, 0xd4, 0x1d, 0x0d, 0x85, 0x49, 0x91, 0x35, 0xfa, },
{ 0x6d, 0x48, 0x2a, 0x0c, 0x42, 0x08, 0xbd, 0xa9, 0x78, 0x6f, 0x18, 0xaf,
0xe2, },
{ 0x10, 0x45, 0xd4, 0x58, 0x88, 0xec, 0x4e, 0x1e, 0xf6, 0x14, 0x92, 0x64,
0x7e, 0xb0, },
{ 0x8b, 0x0b, 0x95, 0xee, 0x92, 0xc6, 0x3b, 0x91, 0xf1, 0x1e, 0xeb, 0x51,
0x98, 0x0a, 0x8d, },
{ 0xa3, 0x50, 0x4d, 0xa5, 0x1d, 0x03, 0x68, 0xe9, 0x57, 0x78, 0xd6, 0x04,
0xf1, 0xc3, 0x94, 0xd8, },
{ 0xb8, 0x66, 0x6e, 0xdd, 0x46, 0x15, 0xae, 0x3d, 0x83, 0x7e, 0xcf, 0xe7,
0x2c, 0xe8, 0x8f, 0xc7, 0x34, },
{ 0x2e, 0xc0, 0x1f, 0x29, 0xea, 0xf6, 0xb9, 0xe2, 0xc2, 0x93, 0xeb, 0x41,
0x0d, 0xf0, 0x0a, 0x13, 0x0e, 0xa2, },
{ 0x71, 0xb8, 0x33, 0xa9, 0x1b, 0xac, 0xf1, 0xb5, 0x42, 0x8f, 0x5e, 0x81,
0x34, 0x43, 0xb7, 0xa4, 0x18, 0x5c, 0x47, },
{ 0xda, 0x45, 0xb8, 0x2e, 0x82, 0x1e, 0xc0, 0x59, 0x77, 0x9d, 0xfa, 0xb4,
0x1c, 0x5e, 0xa0, 0x2b, 0x33, 0x96, 0x5a, 0x58, },
{ 0xe3, 0x09, 0x05, 0xa9, 0xeb, 0x48, 0x13, 0xad, 0x71, 0x88, 0x81, 0x9a,
0x3e, 0x2c, 0xe1, 0x23, 0x99, 0x13, 0x35, 0x9f, 0xb5, },
{ 0xb7, 0x86, 0x2d, 0x16, 0xe1, 0x04, 0x00, 0x47, 0x47, 0x61, 0x31, 0xfb,
0x14, 0xac, 0xd8, 0xe9, 0xe3, 0x49, 0xbd, 0xf7, 0x9c, 0x3f, },
{ 0x7f, 0xd9, 0x95, 0xa8, 0xa7, 0xa0, 0xcc, 0xba, 0xef, 0xb1, 0x0a, 0xa9,
0x21, 0x62, 0x08, 0x0f, 0x1b, 0xff, 0x7b, 0x9d, 0xae, 0xb2, 0x95, },
{ 0x85, 0x99, 0xea, 0x33, 0xe0, 0x56, 0xff, 0x13, 0xc6, 0x61, 0x8c, 0xf9,
0x57, 0x05, 0x03, 0x11, 0xf9, 0xfb, 0x3a, 0xf7, 0xce, 0xbb, 0x52, 0x30, },
{ 0xb2, 0x72, 0x9c, 0xf8, 0x77, 0x4e, 0x8f, 0x6b, 0x01, 0x6c, 0xff, 0x4e,
0x4f, 0x02, 0xd2, 0xbc, 0xeb, 0x51, 0x28, 0x99, 0x50, 0xab, 0xc4, 0x42,
0xe3, },
{ 0x8b, 0x0a, 0xb5, 0x90, 0x8f, 0xf5, 0x7b, 0xdd, 0xba, 0x47, 0x37, 0xc9,
0x2a, 0xd5, 0x4b, 0x25, 0x08, 0x8b, 0x02, 0x17, 0xa7, 0x9e, 0x6b, 0x6e,
0xe3, 0x90, },
{ 0x90, 0xdd, 0xf7, 0x75, 0xa7, 0xa3, 0x99, 0x5e, 0x5b, 0x7d, 0x75, 0xc3,
0x39, 0x6b, 0xa0, 0xe2, 0x44, 0x53, 0xb1, 0x9e, 0xc8, 0xf1, 0x77, 0x10,
0x58, 0x06, 0x9a, },
{ 0x99, 0x52, 0xf0, 0x49, 0xa8, 0x8c, 0xec, 0xa6, 0x97, 0x32, 0x13, 0xb5,
0xf7, 0xa3, 0x8e, 0xfb, 0x4b, 0x59, 0x31, 0x3d, 0x01, 0x59, 0x98, 0x5d,
0x53, 0x03, 0x1a, 0x39, },
{ 0x9f, 0xe0, 0xc2, 0xe5, 0x5d, 0x93, 0xd6, 0x9b, 0x47, 0x8f, 0x9b, 0xe0,
0x26, 0x35, 0x84, 0x20, 0x1d, 0xc5, 0x53, 0x10, 0x0f, 0x22, 0xb9, 0xb5,
0xd4, 0x36, 0xb1, 0xac, 0x73, },
{ 0x30, 0x32, 0x20, 0x3b, 0x10, 0x28, 0xec, 0x1f, 0x4f, 0x9b, 0x47, 0x59,
0xeb, 0x7b, 0xee, 0x45, 0xfb, 0x0c, 0x49, 0xd8, 0x3d, 0x69, 0xbd, 0x90,
0x2c, 0xf0, 0x9e, 0x8d, 0xbf, 0xd5, },
{ 0x2a, 0x37, 0x73, 0x7f, 0xf9, 0x96, 0x19, 0xaa, 0x25, 0xd8, 0x13, 0x28,
0x01, 0x29, 0x89, 0xdf, 0x6e, 0x0c, 0x9b, 0x43, 0x44, 0x51, 0xe9, 0x75,
0x26, 0x0c, 0xb7, 0x87, 0x66, 0x0b, 0x5f, },
{ 0x23, 0xdf, 0x96, 0x68, 0x91, 0x86, 0xd0, 0x93, 0x55, 0x33, 0x24, 0xf6,
0xba, 0x08, 0x75, 0x5b, 0x59, 0x11, 0x69, 0xb8, 0xb9, 0xe5, 0x2c, 0x77,
0x02, 0xf6, 0x47, 0xee, 0x81, 0xdd, 0xb9, 0x06, },
{ 0x9d, },
{ 0x9d, 0x7d, },
{ 0xfd, 0xc3, 0xda, },
{ 0xe8, 0x82, 0xcd, 0x21, },
{ 0xc3, 0x1d, 0x42, 0x4c, 0x74, },
{ 0xe9, 0xda, 0xf1, 0xa2, 0xe5, 0x7c, },
{ 0x52, 0xb8, 0x6f, 0x81, 0x5c, 0x3a, 0x4c, },
{ 0x5b, 0x39, 0x26, 0xfc, 0x92, 0x5e, 0xe0, 0x49, },
{ 0x59, 0xe4, 0x7c, 0x93, 0x1c, 0xf9, 0x28, 0x93, 0xde, },
{ 0xde, 0xdf, 0xb2, 0x43, 0x61, 0x0b, 0x86, 0x16, 0x4c, 0x2e, },
{ 0x14, 0x8f, 0x75, 0x51, 0xaf, 0xb9, 0xee, 0x51, 0x5a, 0xae, 0x23, },
{ 0x43, 0x5f, 0x50, 0xd5, 0x70, 0xb0, 0x5b, 0x87, 0xf5, 0xd9, 0xb3, 0x6d, },
{ 0x66, 0x0a, 0x64, 0x93, 0x79, 0x71, 0x94, 0x40, 0xb7, 0x68, 0x2d, 0xd3,
0x63, },
{ 0x15, 0x00, 0xc4, 0x0c, 0x7d, 0x1b, 0x10, 0xa9, 0x73, 0x1b, 0x90, 0x6f,
0xe6, 0xa9, },
{ 0x34, 0x75, 0xf3, 0x86, 0x8f, 0x56, 0xcf, 0x2a, 0x0a, 0xf2, 0x62, 0x0a,
0xf6, 0x0e, 0x20, },
{ 0xb1, 0xde, 0xc9, 0xf5, 0xdb, 0xf3, 0x2f, 0x4c, 0xd6, 0x41, 0x7d, 0x39,
0x18, 0x3e, 0xc7, 0xc3, },
{ 0xc5, 0x89, 0xb2, 0xf8, 0xb8, 0xc0, 0xa3, 0xb9, 0x3b, 0x10, 0x6d, 0x7c,
0x92, 0xfc, 0x7f, 0x34, 0x41, },
{ 0xc4, 0xd8, 0xef, 0xba, 0xef, 0xd2, 0xaa, 0xc5, 0x6c, 0x8e, 0x3e, 0xbb,
0x12, 0xfc, 0x0f, 0x72, 0xbf, 0x0f, },
{ 0xdd, 0x91, 0xd1, 0x15, 0x9e, 0x7d, 0xf8, 0xc1, 0xb9, 0x14, 0x63, 0x96,
0xb5, 0xcb, 0x83, 0x1d, 0x35, 0x1c, 0xec, },
{ 0xa9, 0xf8, 0x52, 0xc9, 0x67, 0x76, 0x2b, 0xad, 0xfb, 0xd8, 0x3a, 0xa6,
0x74, 0x02, 0xae, 0xb8, 0x25, 0x2c, 0x63, 0x49, },
{ 0x77, 0x1f, 0x66, 0x70, 0xfd, 0x50, 0x29, 0xaa, 0xeb, 0xdc, 0xee, 0xba,
0x75, 0x98, 0xdc, 0x93, 0x12, 0x3f, 0xdc, 0x7c, 0x38, },
{ 0xe2, 0xe1, 0x89, 0x5c, 0x37, 0x38, 0x6a, 0xa3, 0x40, 0xac, 0x3f, 0xb0,
0xca, 0xfc, 0xa7, 0xf3, 0xea, 0xf9, 0x0f, 0x5d, 0x8e, 0x39, },
{ 0x0f, 0x67, 0xc8, 0x38, 0x01, 0xb1, 0xb7, 0xb8, 0xa2, 0xe7, 0x0a, 0x6d,
0xd2, 0x63, 0x69, 0x9e, 0xcc, 0xf0, 0xf2, 0xbe, 0x9b, 0x98, 0xdd, },
{ 0x13, 0xe1, 0x36, 0x30, 0xfe, 0xc6, 0x01, 0x8a, 0xa1, 0x63, 0x96, 0x59,
0xc2, 0xa9, 0x68, 0x3f, 0x58, 0xd4, 0x19, 0x0c, 0x40, 0xf3, 0xde, 0x02, },
{ 0xa3, 0x9e, 0xce, 0xda, 0x42, 0xee, 0x8c, 0x6c, 0x5a, 0x7d, 0xdc, 0x89,
0x02, 0x77, 0xdd, 0xe7, 0x95, 0xbb, 0xff, 0x0d, 0xa4, 0xb5, 0x38, 0x1e,
0xaf, },
{ 0x9a, 0xf6, 0xb5, 0x9a, 0x4f, 0xa9, 0x4f, 0x2c, 0x35, 0x3c, 0x24, 0xdc,
0x97, 0x6f, 0xd9, 0xa1, 0x7d, 0x1a, 0x85, 0x0b, 0xf5, 0xda, 0x2e, 0xe7,
0xb1, 0x1d, },
{ 0x84, 0x1e, 0x8e, 0x3d, 0x45, 0xa5, 0xf2, 0x27, 0xf3, 0x31, 0xfe, 0xb9,
0xfb, 0xc5, 0x45, 0x99, 0x99, 0xdd, 0x93, 0x43, 0x02, 0xee, 0x58, 0xaf,
0xee, 0x6a, 0xbe, },
{ 0x07, 0x2f, 0xc0, 0xa2, 0x04, 0xc4, 0xab, 0x7c, 0x26, 0xbb, 0xa8, 0xd8,
0xe3, 0x1c, 0x75, 0x15, 0x64, 0x5d, 0x02, 0x6a, 0xf0, 0x86, 0xe9, 0xcd,
0x5c, 0xef, 0xa3, 0x25, },
{ 0x2f, 0x3b, 0x1f, 0xb5, 0x91, 0x8f, 0x86, 0xe0, 0xdc, 0x31, 0x48, 0xb6,
0xa1, 0x8c, 0xfd, 0x75, 0xbb, 0x7d, 0x3d, 0xc1, 0xf0, 0x10, 0x9a, 0xd8,
0x4b, 0x0e, 0xe3, 0x94, 0x9f, },
{ 0x29, 0xbb, 0x8f, 0x6c, 0xd1, 0xf2, 0xb6, 0xaf, 0xe5, 0xe3, 0x2d, 0xdc,
0x6f, 0xa4, 0x53, 0x88, 0xd8, 0xcf, 0x4d, 0x45, 0x42, 0x62, 0xdb, 0xdf,
0xf8, 0x45, 0xc2, 0x13, 0xec, 0x35, },
{ 0x06, 0x3c, 0xe3, 0x2c, 0x15, 0xc6, 0x43, 0x03, 0x81, 0xfb, 0x08, 0x76,
0x33, 0xcb, 0x02, 0xc1, 0xba, 0x33, 0xe5, 0xe0, 0xd1, 0x92, 0xa8, 0x46,
0x28, 0x3f, 0x3e, 0x9d, 0x2c, 0x44, 0x54, },
{ 0xea, 0xbb, 0x96, 0xf8, 0xd1, 0x8b, 0x04, 0x11, 0x40, 0x78, 0x42, 0x02,
0x19, 0xd1, 0xbc, 0x65, 0x92, 0xd3, 0xc3, 0xd6, 0xd9, 0x19, 0xe7, 0xc3,
0x40, 0x97, 0xbd, 0xd4, 0xed, 0xfa, 0x5e, 0x28, },
{ 0x02, },
{ 0x52, 0xa8, },
{ 0x38, 0x25, 0x0d, },
{ 0xe3, 0x04, 0xd4, 0x92, },
{ 0x97, 0xdb, 0xf7, 0x81, 0xca, },
{ 0x8a, 0x56, 0x9d, 0x62, 0x56, 0xcc, },
{ 0xa1, 0x8e, 0x3c, 0x72, 0x8f, 0x63, 0x03, },
{ 0xf7, 0xf3, 0x39, 0x09, 0x0a, 0xa1, 0xbb, 0x23, },
{ 0x6b, 0x03, 0xc0, 0xe9, 0xd9, 0x83, 0x05, 0x22, 0x01, },
{ 0x1b, 0x4b, 0xf5, 0xd6, 0x4f, 0x05, 0x75, 0x91, 0x4c, 0x7f, },
{ 0x4c, 0x8c, 0x25, 0x20, 0x21, 0xcb, 0xc2, 0x4b, 0x3a, 0x5b, 0x8d, },
{ 0x56, 0xe2, 0x77, 0xa0, 0xb6, 0x9f, 0x81, 0xec, 0x83, 0x75, 0xc4, 0xf9, },
{ 0x71, 0x70, 0x0f, 0xad, 0x4d, 0x35, 0x81, 0x9d, 0x88, 0x69, 0xf9, 0xaa,
0xd3, },
{ 0x50, 0x6e, 0x86, 0x6e, 0x43, 0xc0, 0xc2, 0x44, 0xc2, 0xe2, 0xa0, 0x1c,
0xb7, 0x9a, },
{ 0xe4, 0x7e, 0x72, 0xc6, 0x12, 0x8e, 0x7c, 0xfc, 0xbd, 0xe2, 0x08, 0x31,
0x3d, 0x47, 0x3d, },
{ 0x08, 0x97, 0x5b, 0x80, 0xae, 0xc4, 0x1d, 0x50, 0x77, 0xdf, 0x1f, 0xd0,
0x24, 0xf0, 0x17, 0xc0, },
{ 0x01, 0xb6, 0x29, 0xf4, 0xaf, 0x78, 0x5f, 0xb6, 0x91, 0xdd, 0x76, 0x76,
0xd2, 0xfd, 0x0c, 0x47, 0x40, },
{ 0xa1, 0xd8, 0x09, 0x97, 0x7a, 0xa6, 0xc8, 0x94, 0xf6, 0x91, 0x7b, 0xae,
0x2b, 0x9f, 0x0d, 0x83, 0x48, 0xf7, },
{ 0x12, 0xd5, 0x53, 0x7d, 0x9a, 0xb0, 0xbe, 0xd9, 0xed, 0xe9, 0x9e, 0xee,
0x61, 0x5b, 0x42, 0xf2, 0xc0, 0x73, 0xc0, },
{ 0xd5, 0x77, 0xd6, 0x5c, 0x6e, 0xa5, 0x69, 0x2b, 0x3b, 0x8c, 0xd6, 0x7d,
0x1d, 0xbe, 0x2c, 0xa1, 0x02, 0x21, 0xcd, 0x29, },
{ 0xa4, 0x98, 0x80, 0xca, 0x22, 0xcf, 0x6a, 0xab, 0x5e, 0x40, 0x0d, 0x61,
0x08, 0x21, 0xef, 0xc0, 0x6c, 0x52, 0xb4, 0xb0, 0x53, },
{ 0xbf, 0xaf, 0x8f, 0x3b, 0x7a, 0x97, 0x33, 0xe5, 0xca, 0x07, 0x37, 0xfd,
0x15, 0xdf, 0xce, 0x26, 0x2a, 0xb1, 0xa7, 0x0b, 0xb3, 0xac, },
{ 0x16, 0x22, 0xe1, 0xbc, 0x99, 0x4e, 0x01, 0xf0, 0xfa, 0xff, 0x8f, 0xa5,
0x0c, 0x61, 0xb0, 0xad, 0xcc, 0xb1, 0xe1, 0x21, 0x46, 0xfa, 0x2e, },
{ 0x11, 0x5b, 0x0b, 0x2b, 0xe6, 0x14, 0xc1, 0xd5, 0x4d, 0x71, 0x5e, 0x17,
0xea, 0x23, 0xdd, 0x6c, 0xbd, 0x1d, 0xbe, 0x12, 0x1b, 0xee, 0x4c, 0x1a, },
{ 0x40, 0x88, 0x22, 0xf3, 0x20, 0x6c, 0xed, 0xe1, 0x36, 0x34, 0x62, 0x2c,
0x98, 0x83, 0x52, 0xe2, 0x25, 0xee, 0xe9, 0xf5, 0xe1, 0x17, 0xf0, 0x5c,
0xae, },
{ 0xc3, 0x76, 0x37, 0xde, 0x95, 0x8c, 0xca, 0x2b, 0x0c, 0x23, 0xe7, 0xb5,
0x38, 0x70, 0x61, 0xcc, 0xff, 0xd3, 0x95, 0x7b, 0xf3, 0xff, 0x1f, 0x9d,
0x59, 0x00, },
{ 0x0c, 0x19, 0x52, 0x05, 0x22, 0x53, 0xcb, 0x48, 0xd7, 0x10, 0x0e, 0x7e,
0x14, 0x69, 0xb5, 0xa2, 0x92, 0x43, 0xa3, 0x9e, 0x4b, 0x8f, 0x51, 0x2c,
0x5a, 0x2c, 0x3b, },
{ 0xe1, 0x9d, 0x70, 0x70, 0x28, 0xec, 0x86, 0x40, 0x55, 0x33, 0x56, 0xda,
0x88, 0xca, 0xee, 0xc8, 0x6a, 0x20, 0xb1, 0xe5, 0x3d, 0x57, 0xf8, 0x3c,
0x10, 0x07, 0x2a, 0xc4, },
{ 0x0b, 0xae, 0xf1, 0xc4, 0x79, 0xee, 0x1b, 0x3d, 0x27, 0x35, 0x8d, 0x14,
0xd6, 0xae, 0x4e, 0x3c, 0xe9, 0x53, 0x50, 0xb5, 0xcc, 0x0c, 0xf7, 0xdf,
0xee, 0xa1, 0x74, 0xd6, 0x71, },
{ 0xe6, 0xa4, 0xf4, 0x99, 0x98, 0xb9, 0x80, 0xea, 0x96, 0x7f, 0x4f, 0x33,
0xcf, 0x74, 0x25, 0x6f, 0x17, 0x6c, 0xbf, 0xf5, 0x5c, 0x38, 0xd0, 0xff,
0x96, 0xcb, 0x13, 0xf9, 0xdf, 0xfd, },
{ 0xbe, 0x92, 0xeb, 0xba, 0x44, 0x2c, 0x24, 0x74, 0xd4, 0x03, 0x27, 0x3c,
0x5d, 0x5b, 0x03, 0x30, 0x87, 0x63, 0x69, 0xe0, 0xb8, 0x94, 0xf4, 0x44,
0x7e, 0xad, 0xcd, 0x20, 0x12, 0x16, 0x79, },
{ 0x30, 0xf1, 0xc4, 0x8e, 0x05, 0x90, 0x2a, 0x97, 0x63, 0x94, 0x46, 0xff,
0xce, 0xd8, 0x67, 0xa7, 0xac, 0x33, 0x8c, 0x95, 0xb7, 0xcd, 0xa3, 0x23,
0x98, 0x9d, 0x76, 0x6c, 0x9d, 0xa8, 0xd6, 0x8a, },
{ 0xbe, },
{ 0x17, 0x6c, },
{ 0x1a, 0x42, 0x4f, },
{ 0xba, 0xaf, 0xb7, 0x65, },
{ 0xc2, 0x63, 0x43, 0x6a, 0xea, },
{ 0xe4, 0x4d, 0xad, 0xf2, 0x0b, 0x02, },
{ 0x04, 0xc7, 0xc4, 0x7f, 0xa9, 0x2b, 0xce, },
{ 0x66, 0xf6, 0x67, 0xcb, 0x03, 0x53, 0xc8, 0xf1, },
{ 0x56, 0xa3, 0x60, 0x78, 0xc9, 0x5f, 0x70, 0x1b, 0x5e, },
{ 0x99, 0xff, 0x81, 0x7c, 0x13, 0x3c, 0x29, 0x79, 0x4b, 0x65, },
{ 0x51, 0x10, 0x50, 0x93, 0x01, 0x93, 0xb7, 0x01, 0xc9, 0x18, 0xb7, },
{ 0x8e, 0x3c, 0x42, 0x1e, 0x5e, 0x7d, 0xc1, 0x50, 0x70, 0x1f, 0x00, 0x98, },
{ 0x5f, 0xd9, 0x9b, 0xc8, 0xd7, 0xb2, 0x72, 0x62, 0x1a, 0x1e, 0xba, 0x92,
0xe9, },
{ 0x70, 0x2b, 0xba, 0xfe, 0xad, 0x5d, 0x96, 0x3f, 0x27, 0xc2, 0x41, 0x6d,
0xc4, 0xb3, },
{ 0xae, 0xe0, 0xd5, 0xd4, 0xc7, 0xae, 0x15, 0x5e, 0xdc, 0xdd, 0x33, 0x60,
0xd7, 0xd3, 0x5e, },
{ 0x79, 0x8e, 0xbc, 0x9e, 0x20, 0xb9, 0x19, 0x4b, 0x63, 0x80, 0xf3, 0x16,
0xaf, 0x39, 0xbd, 0x92, },
{ 0xc2, 0x0e, 0x85, 0xa0, 0x0b, 0x9a, 0xb0, 0xec, 0xde, 0x38, 0xd3, 0x10,
0xd9, 0xa7, 0x66, 0x27, 0xcf, },
{ 0x0e, 0x3b, 0x75, 0x80, 0x67, 0x14, 0x0c, 0x02, 0x90, 0xd6, 0xb3, 0x02,
0x81, 0xf6, 0xa6, 0x87, 0xce, 0x58, },
{ 0x79, 0xb5, 0xe9, 0x5d, 0x52, 0x4d, 0xf7, 0x59, 0xf4, 0x2e, 0x27, 0xdd,
0xb3, 0xed, 0x57, 0x5b, 0x82, 0xea, 0x6f, },
{ 0xa2, 0x97, 0xf5, 0x80, 0x02, 0x3d, 0xde, 0xa3, 0xf9, 0xf6, 0xab, 0xe3,
0x57, 0x63, 0x7b, 0x9b, 0x10, 0x42, 0x6f, 0xf2, },
{ 0x12, 0x7a, 0xfc, 0xb7, 0x67, 0x06, 0x0c, 0x78, 0x1a, 0xfe, 0x88, 0x4f,
0xc6, 0xac, 0x52, 0x96, 0x64, 0x28, 0x97, 0x84, 0x06, },
{ 0xc5, 0x04, 0x44, 0x6b, 0xb2, 0xa5, 0xa4, 0x66, 0xe1, 0x76, 0xa2, 0x51,
0xf9, 0x59, 0x69, 0x97, 0x56, 0x0b, 0xbf, 0x50, 0xb3, 0x34, },
{ 0x21, 0x32, 0x6b, 0x42, 0xb5, 0xed, 0x71, 0x8d, 0xf7, 0x5a, 0x35, 0xe3,
0x90, 0xe2, 0xee, 0xaa, 0x89, 0xf6, 0xc9, 0x9c, 0x4d, 0x73, 0xf4, },
{ 0x4c, 0xa6, 0x09, 0xf4, 0x48, 0xe7, 0x46, 0xbc, 0x49, 0xfc, 0xe5, 0xda,
0xd1, 0x87, 0x13, 0x17, 0x4c, 0x59, 0x71, 0x26, 0x5b, 0x2c, 0x42, 0xb7, },
{ 0x13, 0x63, 0xf3, 0x40, 0x02, 0xe5, 0xa3, 0x3a, 0x5e, 0x8e, 0xf8, 0xb6,
0x8a, 0x49, 0x60, 0x76, 0x34, 0x72, 0x94, 0x73, 0xf6, 0xd9, 0x21, 0x6a,
0x26, },
{ 0xdf, 0x75, 0x16, 0x10, 0x1b, 0x5e, 0x81, 0xc3, 0xc8, 0xde, 0x34, 0x24,
0xb0, 0x98, 0xeb, 0x1b, 0x8f, 0xa1, 0x9b, 0x05, 0xee, 0xa5, 0xe9, 0x35,
0xf4, 0x1d, },
{ 0xcd, 0x21, 0x93, 0x6e, 0x5b, 0xa0, 0x26, 0x2b, 0x21, 0x0e, 0xa0, 0xb9,
0x1c, 0xb5, 0xbb, 0xb8, 0xf8, 0x1e, 0xff, 0x5c, 0xa8, 0xf9, 0x39, 0x46,
0x4e, 0x29, 0x26, },
{ 0x73, 0x7f, 0x0e, 0x3b, 0x0b, 0x5c, 0xf9, 0x60, 0xaa, 0x88, 0xa1, 0x09,
0xb1, 0x5d, 0x38, 0x7b, 0x86, 0x8f, 0x13, 0x7a, 0x8d, 0x72, 0x7a, 0x98,
0x1a, 0x5b, 0xff, 0xc9, },
{ 0xd3, 0x3c, 0x61, 0x71, 0x44, 0x7e, 0x31, 0x74, 0x98, 0x9d, 0x9a, 0xd2,
0x27, 0xf3, 0x46, 0x43, 0x42, 0x51, 0xd0, 0x5f, 0xe9, 0x1c, 0x5c, 0x69,
0xbf, 0xf6, 0xbe, 0x3c, 0x40, },
{ 0x31, 0x99, 0x31, 0x9f, 0xaa, 0x43, 0x2e, 0x77, 0x3e, 0x74, 0x26, 0x31,
0x5e, 0x61, 0xf1, 0x87, 0xe2, 0xeb, 0x9b, 0xcd, 0xd0, 0x3a, 0xee, 0x20,
0x7e, 0x10, 0x0a, 0x0b, 0x7e, 0xfa, },
{ 0xa4, 0x27, 0x80, 0x67, 0x81, 0x2a, 0xa7, 0x62, 0xf7, 0x6e, 0xda, 0xd4,
0x5c, 0x39, 0x74, 0xad, 0x7e, 0xbe, 0xad, 0xa5, 0x84, 0x7f, 0xa9, 0x30,
0x5d, 0xdb, 0xe2, 0x05, 0x43, 0xf7, 0x1b, },
{ 0x0b, 0x37, 0xd8, 0x02, 0xe1, 0x83, 0xd6, 0x80, 0xf2, 0x35, 0xc2, 0xb0,
0x37, 0xef, 0xef, 0x5e, 0x43, 0x93, 0xf0, 0x49, 0x45, 0x0a, 0xef, 0xb5,
0x76, 0x70, 0x12, 0x44, 0xc4, 0xdb, 0xf5, 0x7a, },
{ 0x1f, },
{ 0x82, 0x60, },
{ 0xcc, 0xe3, 0x08, },
{ 0x56, 0x17, 0xe4, 0x59, },
{ 0xe2, 0xd7, 0x9e, 0xc4, 0x4c, },
{ 0xb2, 0xad, 0xd3, 0x78, 0x58, 0x5a, },
{ 0xce, 0x43, 0xb4, 0x02, 0x96, 0xab, 0x3c, },
{ 0xe6, 0x05, 0x1a, 0x73, 0x22, 0x32, 0xbb, 0x77, },
{ 0x23, 0xe7, 0xda, 0xfe, 0x2c, 0xef, 0x8c, 0x22, 0xec, },
{ 0xe9, 0x8e, 0x55, 0x38, 0xd1, 0xd7, 0x35, 0x23, 0x98, 0xc7, },
{ 0xb5, 0x81, 0x1a, 0xe5, 0xb5, 0xa5, 0xd9, 0x4d, 0xca, 0x41, 0xe7, },
{ 0x41, 0x16, 0x16, 0x95, 0x8d, 0x9e, 0x0c, 0xea, 0x8c, 0x71, 0x9a, 0xc1, },
{ 0x7c, 0x33, 0xc0, 0xa4, 0x00, 0x62, 0xea, 0x60, 0x67, 0xe4, 0x20, 0xbc,
0x5b, },
{ 0xdb, 0xb1, 0xdc, 0xfd, 0x08, 0xc0, 0xde, 0x82, 0xd1, 0xde, 0x38, 0xc0,
0x90, 0x48, },
{ 0x37, 0x18, 0x2e, 0x0d, 0x61, 0xaa, 0x61, 0xd7, 0x86, 0x20, 0x16, 0x60,
0x04, 0xd9, 0xd5, },
{ 0xb0, 0xcf, 0x2c, 0x4c, 0x5e, 0x5b, 0x4f, 0x2a, 0x23, 0x25, 0x58, 0x47,
0xe5, 0x31, 0x06, 0x70, },
{ 0x91, 0xa0, 0xa3, 0x86, 0x4e, 0xe0, 0x72, 0x38, 0x06, 0x67, 0x59, 0x5c,
0x70, 0x25, 0xdb, 0x33, 0x27, },
{ 0x44, 0x58, 0x66, 0xb8, 0x58, 0xc7, 0x13, 0xed, 0x4c, 0xc0, 0xf4, 0x9a,
0x1e, 0x67, 0x75, 0x33, 0xb6, 0xb8, },
{ 0x7f, 0x98, 0x4a, 0x8e, 0x50, 0xa2, 0x5c, 0xcd, 0x59, 0xde, 0x72, 0xb3,
0x9d, 0xc3, 0x09, 0x8a, 0xab, 0x56, 0xf1, },
{ 0x80, 0x96, 0x49, 0x1a, 0x59, 0xa2, 0xc5, 0xd5, 0xa7, 0x20, 0x8a, 0xb7,
0x27, 0x62, 0x84, 0x43, 0xc6, 0xe1, 0x1b, 0x5d, },
{ 0x6b, 0xb7, 0x2b, 0x26, 0x62, 0x14, 0x70, 0x19, 0x3d, 0x4d, 0xac, 0xac,
0x63, 0x58, 0x5e, 0x94, 0xb5, 0xb7, 0xe8, 0xe8, 0xa2, },
{ 0x20, 0xa8, 0xc0, 0xfd, 0x63, 0x3d, 0x6e, 0x98, 0xcf, 0x0c, 0x49, 0x98,
0xe4, 0x5a, 0xfe, 0x8c, 0xaa, 0x70, 0x82, 0x1c, 0x7b, 0x74, },
{ 0xc8, 0xe8, 0xdd, 0xdf, 0x69, 0x30, 0x01, 0xc2, 0x0f, 0x7e, 0x2f, 0x11,
0xcc, 0x3e, 0x17, 0xa5, 0x69, 0x40, 0x3f, 0x0e, 0x79, 0x7f, 0xcf, },
{ 0xdb, 0x61, 0xc0, 0xe2, 0x2e, 0x49, 0x07, 0x31, 0x1d, 0x91, 0x42, 0x8a,
0xfc, 0x5e, 0xd3, 0xf8, 0x56, 0x1f, 0x2b, 0x73, 0xfd, 0x9f, 0xb2, 0x8e, },
{ 0x0c, 0x89, 0x55, 0x0c, 0x1f, 0x59, 0x2c, 0x9d, 0x1b, 0x29, 0x1d, 0x41,
0x1d, 0xe6, 0x47, 0x8f, 0x8c, 0x2b, 0xea, 0x8f, 0xf0, 0xff, 0x21, 0x70,
0x88, },
{ 0x12, 0x18, 0x95, 0xa6, 0x59, 0xb1, 0x31, 0x24, 0x45, 0x67, 0x55, 0xa4,
0x1a, 0x2d, 0x48, 0x67, 0x1b, 0x43, 0x88, 0x2d, 0x8e, 0xa0, 0x70, 0xb3,
0xc6, 0xbb, },
{ 0xe7, 0xb1, 0x1d, 0xb2, 0x76, 0x4d, 0x68, 0x68, 0x68, 0x23, 0x02, 0x55,
0x3a, 0xe2, 0xe5, 0xd5, 0x4b, 0x43, 0xf9, 0x34, 0x77, 0x5c, 0xa1, 0xf5,
0x55, 0xfd, 0x4f, },
{ 0x8c, 0x87, 0x5a, 0x08, 0x3a, 0x73, 0xad, 0x61, 0xe1, 0xe7, 0x99, 0x7e,
0xf0, 0x5d, 0xe9, 0x5d, 0x16, 0x43, 0x80, 0x2f, 0xd0, 0x66, 0x34, 0xe2,
0x42, 0x64, 0x3b, 0x1a, },
{ 0x39, 0xc1, 0x99, 0xcf, 0x22, 0xbf, 0x16, 0x8f, 0x9f, 0x80, 0x7f, 0x95,
0x0a, 0x05, 0x67, 0x27, 0xe7, 0x15, 0xdf, 0x9d, 0xb2, 0xfe, 0x1c, 0xb5,
0x1d, 0x60, 0x8f, 0x8a, 0x1d, },
{ 0x9b, 0x6e, 0x08, 0x09, 0x06, 0x73, 0xab, 0x68, 0x02, 0x62, 0x1a, 0xe4,
0xd4, 0xdf, 0xc7, 0x02, 0x4c, 0x6a, 0x5f, 0xfd, 0x23, 0xac, 0xae, 0x6d,
0x43, 0xa4, 0x7a, 0x50, 0x60, 0x3c, },
{ 0x1d, 0xb4, 0xc6, 0xe1, 0xb1, 0x4b, 0xe3, 0xf2, 0xe2, 0x1a, 0x73, 0x1b,
0xa0, 0x92, 0xa7, 0xf5, 0xff, 0x8f, 0x8b, 0x5d, 0xdf, 0xa8, 0x04, 0xb3,
0xb0, 0xf7, 0xcc, 0x12, 0xfa, 0x35, 0x46, },
{ 0x49, 0x45, 0x97, 0x11, 0x0f, 0x1c, 0x60, 0x8e, 0xe8, 0x47, 0x30, 0xcf,
0x60, 0xa8, 0x71, 0xc5, 0x1b, 0xe9, 0x39, 0x4d, 0x49, 0xb6, 0x12, 0x1f,
0x24, 0xab, 0x37, 0xff, 0x83, 0xc2, 0xe1, 0x3a, },
{ 0x60, },
{ 0x24, 0x26, },
{ 0x47, 0xeb, 0xc9, },
{ 0x4a, 0xd0, 0xbc, 0xf0, },
{ 0x8e, 0x2b, 0xc9, 0x85, 0x3c, },
{ 0xa2, 0x07, 0x15, 0xb8, 0x12, 0x74, },
{ 0x0f, 0xdb, 0x5b, 0x33, 0x69, 0xfe, 0x4b, },
{ 0xa2, 0x86, 0x54, 0xf4, 0xfd, 0xb2, 0xd4, 0xe6, },
{ 0xbb, 0x84, 0x78, 0x49, 0x27, 0x8e, 0x61, 0xda, 0x60, },
{ 0x04, 0xc3, 0xcd, 0xaa, 0x8f, 0xa7, 0x03, 0xc9, 0xf9, 0xb6, },
{ 0xf8, 0x27, 0x1d, 0x61, 0xdc, 0x21, 0x42, 0xdd, 0xad, 0x92, 0x40, },
{ 0x12, 0x87, 0xdf, 0xc2, 0x41, 0x45, 0x5a, 0x36, 0x48, 0x5b, 0x51, 0x2b, },
{ 0xbb, 0x37, 0x5d, 0x1f, 0xf1, 0x68, 0x7a, 0xc4, 0xa5, 0xd2, 0xa4, 0x91,
0x8d, },
{ 0x5b, 0x27, 0xd1, 0x04, 0x54, 0x52, 0x9f, 0xa3, 0x47, 0x86, 0x33, 0x33,
0xbf, 0xa0, },
{ 0xcf, 0x04, 0xea, 0xf8, 0x03, 0x2a, 0x43, 0xff, 0xa6, 0x68, 0x21, 0x4c,
0xd5, 0x4b, 0xed, },
{ 0xaf, 0xb8, 0xbc, 0x63, 0x0f, 0x18, 0x4d, 0xe2, 0x7a, 0xdd, 0x46, 0x44,
0xc8, 0x24, 0x0a, 0xb7, },
{ 0x3e, 0xdc, 0x36, 0xe4, 0x89, 0xb1, 0xfa, 0xc6, 0x40, 0x93, 0x2e, 0x75,
0xb2, 0x15, 0xd1, 0xb1, 0x10, },
{ 0x6c, 0xd8, 0x20, 0x3b, 0x82, 0x79, 0xf9, 0xc8, 0xbc, 0x9d, 0xe0, 0x35,
0xbe, 0x1b, 0x49, 0x1a, 0xbc, 0x3a, },
{ 0x78, 0x65, 0x2c, 0xbe, 0x35, 0x67, 0xdc, 0x78, 0xd4, 0x41, 0xf6, 0xc9,
0xde, 0xde, 0x1f, 0x18, 0x13, 0x31, 0x11, },
{ 0x8a, 0x7f, 0xb1, 0x33, 0x8f, 0x0c, 0x3c, 0x0a, 0x06, 0x61, 0xf0, 0x47,
0x29, 0x1b, 0x29, 0xbc, 0x1c, 0x47, 0xef, 0x7a, },
{ 0x65, 0x91, 0xf1, 0xe6, 0xb3, 0x96, 0xd3, 0x8c, 0xc2, 0x4a, 0x59, 0x35,
0x72, 0x8e, 0x0b, 0x9a, 0x87, 0xca, 0x34, 0x7b, 0x63, },
{ 0x5f, 0x08, 0x87, 0x80, 0x56, 0x25, 0x89, 0x77, 0x61, 0x8c, 0x64, 0xa1,
0x59, 0x6d, 0x59, 0x62, 0xe8, 0x4a, 0xc8, 0x58, 0x99, 0xd1, },
{ 0x23, 0x87, 0x1d, 0xed, 0x6f, 0xf2, 0x91, 0x90, 0xe2, 0xfe, 0x43, 0x21,
0xaf, 0x97, 0xc6, 0xbc, 0xd7, 0x15, 0xc7, 0x2d, 0x08, 0x77, 0x91, },
{ 0x90, 0x47, 0x9a, 0x9e, 0x3a, 0xdf, 0xf3, 0xc9, 0x4c, 0x1e, 0xa7, 0xd4,
0x6a, 0x32, 0x90, 0xfe, 0xb7, 0xb6, 0x7b, 0xfa, 0x96, 0x61, 0xfb, 0xa4, },
{ 0xb1, 0x67, 0x60, 0x45, 0xb0, 0x96, 0xc5, 0x15, 0x9f, 0x4d, 0x26, 0xd7,
0x9d, 0xf1, 0xf5, 0x6d, 0x21, 0x00, 0x94, 0x31, 0x64, 0x94, 0xd3, 0xa7,
0xd3, },
{ 0x02, 0x3e, 0xaf, 0xf3, 0x79, 0x73, 0xa5, 0xf5, 0xcc, 0x7a, 0x7f, 0xfb,
0x79, 0x2b, 0x85, 0x8c, 0x88, 0x72, 0x06, 0xbe, 0xfe, 0xaf, 0xc1, 0x16,
0xa6, 0xd6, },
{ 0x2a, 0xb0, 0x1a, 0xe5, 0xaa, 0x6e, 0xb3, 0xae, 0x53, 0x85, 0x33, 0x80,
0x75, 0xae, 0x30, 0xe6, 0xb8, 0x72, 0x42, 0xf6, 0x25, 0x4f, 0x38, 0x88,
0x55, 0xd1, 0xa9, },
{ 0x90, 0xd8, 0x0c, 0xc0, 0x93, 0x4b, 0x4f, 0x9e, 0x65, 0x6c, 0xa1, 0x54,
0xa6, 0xf6, 0x6e, 0xca, 0xd2, 0xbb, 0x7e, 0x6a, 0x1c, 0xd3, 0xce, 0x46,
0xef, 0xb0, 0x00, 0x8d, },
{ 0xed, 0x9c, 0x49, 0xcd, 0xc2, 0xde, 0x38, 0x0e, 0xe9, 0x98, 0x6c, 0xc8,
0x90, 0x9e, 0x3c, 0xd4, 0xd3, 0xeb, 0x88, 0x32, 0xc7, 0x28, 0xe3, 0x94,
0x1c, 0x9f, 0x8b, 0xf3, 0xcb, },
{ 0xac, 0xe7, 0x92, 0x16, 0xb4, 0x14, 0xa0, 0xe4, 0x04, 0x79, 0xa2, 0xf4,
0x31, 0xe6, 0x0c, 0x26, 0xdc, 0xbf, 0x2f, 0x69, 0x1b, 0x55, 0x94, 0x67,
0xda, 0x0c, 0xd7, 0x32, 0x1f, 0xef, },
{ 0x68, 0x63, 0x85, 0x57, 0x95, 0x9e, 0x42, 0x27, 0x41, 0x43, 0x42, 0x02,
0xa5, 0x78, 0xa7, 0xc6, 0x43, 0xc1, 0x6a, 0xba, 0x70, 0x80, 0xcd, 0x04,
0xb6, 0x78, 0x76, 0x29, 0xf3, 0xe8, 0xa0, },
{ 0xe6, 0xac, 0x8d, 0x9d, 0xf0, 0xc0, 0xf7, 0xf7, 0xe3, 0x3e, 0x4e, 0x28,
0x0f, 0x59, 0xb2, 0x67, 0x9e, 0x84, 0x34, 0x42, 0x96, 0x30, 0x2b, 0xca,
0x49, 0xb6, 0xc5, 0x9a, 0x84, 0x59, 0xa7, 0x81, },
{ 0x7e, },
{ 0x1e, 0x21, },
{ 0x26, 0xd3, 0xdd, },
{ 0x2c, 0xd4, 0xb3, 0x3d, },
{ 0x86, 0x7b, 0x76, 0x3c, 0xf0, },
{ 0x12, 0xc3, 0x70, 0x1d, 0x55, 0x18, },
{ 0x96, 0xc2, 0xbd, 0x61, 0x55, 0xf4, 0x24, },
{ 0x20, 0x51, 0xf7, 0x86, 0x58, 0x8f, 0x07, 0x2a, },
{ 0x93, 0x15, 0xa8, 0x1d, 0xda, 0x97, 0xee, 0x0e, 0x6c, },
{ 0x39, 0x93, 0xdf, 0xd5, 0x0e, 0xca, 0xdc, 0x7a, 0x92, 0xce, },
{ 0x60, 0xd5, 0xfd, 0xf5, 0x1b, 0x26, 0x82, 0x26, 0x73, 0x02, 0xbc, },
{ 0x98, 0xf2, 0x34, 0xe1, 0xf5, 0xfb, 0x00, 0xac, 0x10, 0x4a, 0x38, 0x9f, },
{ 0xda, 0x3a, 0x92, 0x8a, 0xd0, 0xcd, 0x12, 0xcd, 0x15, 0xbb, 0xab, 0x77,
0x66, },
{ 0xa2, 0x92, 0x1a, 0xe5, 0xca, 0x0c, 0x30, 0x75, 0xeb, 0xaf, 0x00, 0x31,
0x55, 0x66, },
{ 0x06, 0xea, 0xfd, 0x3e, 0x86, 0x38, 0x62, 0x4e, 0xa9, 0x12, 0xa4, 0x12,
0x43, 0xbf, 0xa1, },
{ 0xe4, 0x71, 0x7b, 0x94, 0xdb, 0xa0, 0xd2, 0xff, 0x9b, 0xeb, 0xad, 0x8e,
0x95, 0x8a, 0xc5, 0xed, },
{ 0x25, 0x5a, 0x77, 0x71, 0x41, 0x0e, 0x7a, 0xe9, 0xed, 0x0c, 0x10, 0xef,
0xf6, 0x2b, 0x3a, 0xba, 0x60, },
{ 0xee, 0xe2, 0xa3, 0x67, 0x64, 0x1d, 0xc6, 0x04, 0xc4, 0xe1, 0x68, 0xd2,
0x6e, 0xd2, 0x91, 0x75, 0x53, 0x07, },
{ 0xe0, 0xf6, 0x4d, 0x8f, 0x68, 0xfc, 0x06, 0x7e, 0x18, 0x79, 0x7f, 0x2b,
0x6d, 0xef, 0x46, 0x7f, 0xab, 0xb2, 0xad, },
{ 0x3d, 0x35, 0x88, 0x9f, 0x2e, 0xcf, 0x96, 0x45, 0x07, 0x60, 0x71, 0x94,
0x00, 0x8d, 0xbf, 0xf4, 0xef, 0x46, 0x2e, 0x3c, },
{ 0x43, 0xcf, 0x98, 0xf7, 0x2d, 0xf4, 0x17, 0xe7, 0x8c, 0x05, 0x2d, 0x9b,
0x24, 0xfb, 0x4d, 0xea, 0x4a, 0xec, 0x01, 0x25, 0x29, },
{ 0x8e, 0x73, 0x9a, 0x78, 0x11, 0xfe, 0x48, 0xa0, 0x3b, 0x1a, 0x26, 0xdf,
0x25, 0xe9, 0x59, 0x1c, 0x70, 0x07, 0x9f, 0xdc, 0xa0, 0xa6, },
{ 0xe8, 0x47, 0x71, 0xc7, 0x3e, 0xdf, 0xb5, 0x13, 0xb9, 0x85, 0x13, 0xa8,
0x54, 0x47, 0x6e, 0x59, 0x96, 0x09, 0x13, 0x5f, 0x82, 0x16, 0x0b, },
{ 0xfb, 0xc0, 0x8c, 0x03, 0x21, 0xb3, 0xc4, 0xb5, 0x43, 0x32, 0x6c, 0xea,
0x7f, 0xa8, 0x43, 0x91, 0xe8, 0x4e, 0x3f, 0xbf, 0x45, 0x58, 0x6a, 0xa3, },
{ 0x55, 0xf8, 0xf3, 0x00, 0x76, 0x09, 0xef, 0x69, 0x5d, 0xd2, 0x8a, 0xf2,
0x65, 0xc3, 0xcb, 0x9b, 0x43, 0xfd, 0xb1, 0x7e, 0x7f, 0xa1, 0x94, 0xb0,
0xd7, },
{ 0xaa, 0x13, 0xc1, 0x51, 0x40, 0x6d, 0x8d, 0x4c, 0x0a, 0x95, 0x64, 0x7b,
0xd1, 0x96, 0xb6, 0x56, 0xb4, 0x5b, 0xcf, 0xd6, 0xd9, 0x15, 0x97, 0xdd,
0xb6, 0xef, },
{ 0xaf, 0xb7, 0x36, 0xb0, 0x04, 0xdb, 0xd7, 0x9c, 0x9a, 0x44, 0xc4, 0xf6,
0x1f, 0x12, 0x21, 0x2d, 0x59, 0x30, 0x54, 0xab, 0x27, 0x61, 0xa3, 0x57,
0xef, 0xf8, 0x53, },
{ 0x97, 0x34, 0x45, 0x3e, 0xce, 0x7c, 0x35, 0xa2, 0xda, 0x9f, 0x4b, 0x46,
0x6c, 0x11, 0x67, 0xff, 0x2f, 0x76, 0x58, 0x15, 0x71, 0xfa, 0x44, 0x89,
0x89, 0xfd, 0xf7, 0x99, },
{ 0x1f, 0xb1, 0x62, 0xeb, 0x83, 0xc5, 0x9c, 0x89, 0xf9, 0x2c, 0xd2, 0x03,
0x61, 0xbc, 0xbb, 0xa5, 0x74, 0x0e, 0x9b, 0x7e, 0x82, 0x3e, 0x70, 0x0a,
0xa9, 0x8f, 0x2b, 0x59, 0xfb, },
{ 0xf8, 0xca, 0x5e, 0x3a, 0x4f, 0x9e, 0x10, 0x69, 0x10, 0xd5, 0x4c, 0xeb,
0x1a, 0x0f, 0x3c, 0x6a, 0x98, 0xf5, 0xb0, 0x97, 0x5b, 0x37, 0x2f, 0x0d,
0xbd, 0x42, 0x4b, 0x69, 0xa1, 0x82, },
{ 0x12, 0x8c, 0x6d, 0x52, 0x08, 0xef, 0x74, 0xb2, 0xe6, 0xaa, 0xd3, 0xb0,
0x26, 0xb0, 0xd9, 0x94, 0xb6, 0x11, 0x45, 0x0e, 0x36, 0x71, 0x14, 0x2d,
0x41, 0x8c, 0x21, 0x53, 0x31, 0xe9, 0x68, },
{ 0xee, 0xea, 0x0d, 0x89, 0x47, 0x7e, 0x72, 0xd1, 0xd8, 0xce, 0x58, 0x4c,
0x94, 0x1f, 0x0d, 0x51, 0x08, 0xa3, 0xb6, 0x3d, 0xe7, 0x82, 0x46, 0x92,
0xd6, 0x98, 0x6b, 0x07, 0x10, 0x65, 0x52, 0x65, },
};
static const u8 blake2s_hmac_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {
{ 0xce, 0xe1, 0x57, 0x69, 0x82, 0xdc, 0xbf, 0x43, 0xad, 0x56, 0x4c, 0x70,
0xed, 0x68, 0x16, 0x96, 0xcf, 0xa4, 0x73, 0xe8, 0xe8, 0xfc, 0x32, 0x79,
0x08, 0x0a, 0x75, 0x82, 0xda, 0x3f, 0x05, 0x11, },
{ 0x77, 0x2f, 0x0c, 0x71, 0x41, 0xf4, 0x4b, 0x2b, 0xb3, 0xc6, 0xb6, 0xf9,
0x60, 0xde, 0xe4, 0x52, 0x38, 0x66, 0xe8, 0xbf, 0x9b, 0x96, 0xc4, 0x9f,
0x60, 0xd9, 0x24, 0x37, 0x99, 0xd6, 0xec, 0x31, },
};
bool __init blake2s_selftest(void)
{
u8 key[BLAKE2S_KEY_SIZE];
u8 buf[ARRAY_SIZE(blake2s_testvecs)];
u8 hash[BLAKE2S_HASH_SIZE];
struct blake2s_state state;
bool success = true;
int i, l;
key[0] = key[1] = 1;
for (i = 2; i < sizeof(key); ++i)
key[i] = key[i - 2] + key[i - 1];
for (i = 0; i < sizeof(buf); ++i)
buf[i] = (u8)i;
for (i = l = 0; i < ARRAY_SIZE(blake2s_testvecs); l = (l + 37) % ++i) {
int outlen = 1 + i % BLAKE2S_HASH_SIZE;
int keylen = (13 * i) % (BLAKE2S_KEY_SIZE + 1);
blake2s(hash, buf, key + BLAKE2S_KEY_SIZE - keylen, outlen, i,
keylen);
if (memcmp(hash, blake2s_testvecs[i], outlen)) {
pr_err("blake2s self-test %d: FAIL\n", i + 1);
success = false;
}
if (!keylen)
blake2s_init(&state, outlen);
else
blake2s_init_key(&state, outlen,
key + BLAKE2S_KEY_SIZE - keylen,
keylen);
blake2s_update(&state, buf, l);
blake2s_update(&state, buf + l, i - l);
blake2s_final(&state, hash);
if (memcmp(hash, blake2s_testvecs[i], outlen)) {
pr_err("blake2s init/update/final self-test %d: FAIL\n",
i + 1);
success = false;
}
}
if (success) {
blake2s256_hmac(hash, buf, key, sizeof(buf), sizeof(key));
success &= !memcmp(hash, blake2s_hmac_testvecs[0], BLAKE2S_HASH_SIZE);
blake2s256_hmac(hash, key, buf, sizeof(key), sizeof(buf));
success &= !memcmp(hash, blake2s_hmac_testvecs[1], BLAKE2S_HASH_SIZE);
if (!success)
pr_err("blake2s256_hmac self-test: FAIL\n");
}
return success;
}
lib/crypto/blake2s.c
+126
View File
@@ -0,0 +1,126 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is an implementation of the BLAKE2s hash and PRF functions.
*
* Information: https://blake2.net/
*
*/
#include <crypto/internal/blake2s.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bug.h>
#include <asm/unaligned.h>
bool blake2s_selftest(void);
void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen)
{
const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen;
if (unlikely(!inlen))
return;
if (inlen > fill) {
memcpy(state->buf + state->buflen, in, fill);
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S))
blake2s_compress_arch(state, state->buf, 1,
BLAKE2S_BLOCK_SIZE);
else
blake2s_compress_generic(state, state->buf, 1,
BLAKE2S_BLOCK_SIZE);
state->buflen = 0;
in += fill;
inlen -= fill;
}
if (inlen > BLAKE2S_BLOCK_SIZE) {
const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE);
/* Hash one less (full) block than strictly possible */
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S))
blake2s_compress_arch(state, in, nblocks - 1,
BLAKE2S_BLOCK_SIZE);
else
blake2s_compress_generic(state, in, nblocks - 1,
BLAKE2S_BLOCK_SIZE);
in += BLAKE2S_BLOCK_SIZE * (nblocks - 1);
inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1);
}
memcpy(state->buf + state->buflen, in, inlen);
state->buflen += inlen;
}
EXPORT_SYMBOL(blake2s_update);
void blake2s_final(struct blake2s_state *state, u8 *out)
{
WARN_ON(IS_ENABLED(DEBUG) && !out);
blake2s_set_lastblock(state);
memset(state->buf + state->buflen, 0,
BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */
if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_BLAKE2S))
blake2s_compress_arch(state, state->buf, 1, state->buflen);
else
blake2s_compress_generic(state, state->buf, 1, state->buflen);
cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
memcpy(out, state->h, state->outlen);
memzero_explicit(state, sizeof(*state));
}
EXPORT_SYMBOL(blake2s_final);
void blake2s256_hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen,
const size_t keylen)
{
struct blake2s_state state;
u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 };
u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32));
int i;
if (keylen > BLAKE2S_BLOCK_SIZE) {
blake2s_init(&state, BLAKE2S_HASH_SIZE);
blake2s_update(&state, key, keylen);
blake2s_final(&state, x_key);
} else
memcpy(x_key, key, keylen);
for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i)
x_key[i] ^= 0x36;
blake2s_init(&state, BLAKE2S_HASH_SIZE);
blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE);
blake2s_update(&state, in, inlen);
blake2s_final(&state, i_hash);
for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i)
x_key[i] ^= 0x5c ^ 0x36;
blake2s_init(&state, BLAKE2S_HASH_SIZE);
blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE);
blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE);
blake2s_final(&state, i_hash);
memcpy(out, i_hash, BLAKE2S_HASH_SIZE);
memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE);
memzero_explicit(i_hash, BLAKE2S_HASH_SIZE);
}
EXPORT_SYMBOL(blake2s256_hmac);
static int __init mod_init(void)
{
if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
WARN_ON(!blake2s_selftest()))
return -ENODEV;
return 0;
}
static void __exit mod_exit(void)
{
}
module_init(mod_init);
module_exit(mod_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("BLAKE2s hash function");
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
lib/crypto/chacha.c
+119
View File
@@ -0,0 +1,119 @@
/*
* The "hash function" used as the core of the ChaCha stream cipher (RFC7539)
*
* Copyright (C) 2015 Martin Willi
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/bitops.h>
#include <linux/string.h>
#include <linux/cryptohash.h>
#include <asm/unaligned.h>
#include <crypto/chacha.h>
static void chacha_permute(u32 *x, int nrounds)
{
int i;
/* whitelist the allowed round counts */
WARN_ON_ONCE(nrounds != 20 && nrounds != 12);
for (i = 0; i < nrounds; i += 2) {
x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 16);
x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 16);
x[2] += x[6]; x[14] = rol32(x[14] ^ x[2], 16);
x[3] += x[7]; x[15] = rol32(x[15] ^ x[3], 16);
x[8] += x[12]; x[4] = rol32(x[4] ^ x[8], 12);
x[9] += x[13]; x[5] = rol32(x[5] ^ x[9], 12);
x[10] += x[14]; x[6] = rol32(x[6] ^ x[10], 12);
x[11] += x[15]; x[7] = rol32(x[7] ^ x[11], 12);
x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 8);
x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 8);
x[2] += x[6]; x[14] = rol32(x[14] ^ x[2], 8);
x[3] += x[7]; x[15] = rol32(x[15] ^ x[3], 8);
x[8] += x[12]; x[4] = rol32(x[4] ^ x[8], 7);
x[9] += x[13]; x[5] = rol32(x[5] ^ x[9], 7);
x[10] += x[14]; x[6] = rol32(x[6] ^ x[10], 7);
x[11] += x[15]; x[7] = rol32(x[7] ^ x[11], 7);
x[0] += x[5]; x[15] = rol32(x[15] ^ x[0], 16);
x[1] += x[6]; x[12] = rol32(x[12] ^ x[1], 16);
x[2] += x[7]; x[13] = rol32(x[13] ^ x[2], 16);
x[3] += x[4]; x[14] = rol32(x[14] ^ x[3], 16);
x[10] += x[15]; x[5] = rol32(x[5] ^ x[10], 12);
x[11] += x[12]; x[6] = rol32(x[6] ^ x[11], 12);
x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 12);
x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 12);
x[0] += x[5]; x[15] = rol32(x[15] ^ x[0], 8);
x[1] += x[6]; x[12] = rol32(x[12] ^ x[1], 8);
x[2] += x[7]; x[13] = rol32(x[13] ^ x[2], 8);
x[3] += x[4]; x[14] = rol32(x[14] ^ x[3], 8);
x[10] += x[15]; x[5] = rol32(x[5] ^ x[10], 7);
x[11] += x[12]; x[6] = rol32(x[6] ^ x[11], 7);
x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 7);
x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 7);
}
}
/**
* chacha_block - generate one keystream block and increment block counter
* @state: input state matrix (16 32-bit words)
* @stream: output keystream block (64 bytes)
* @nrounds: number of rounds (20 or 12; 20 is recommended)
*
* This is the ChaCha core, a function from 64-byte strings to 64-byte strings.
* The caller has already converted the endianness of the input. This function
* also handles incrementing the block counter in the input matrix.
*/
void chacha_block_generic(u32 *state, u8 *stream, int nrounds)
{
u32 x[16];
int i;
memcpy(x, state, 64);
chacha_permute(x, nrounds);
for (i = 0; i < ARRAY_SIZE(x); i++)
put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]);
state[12]++;
}
EXPORT_SYMBOL(chacha_block_generic);
/**
* hchacha_block_generic - abbreviated ChaCha core, for XChaCha
* @state: input state matrix (16 32-bit words)
* @out: output (8 32-bit words)
* @nrounds: number of rounds (20 or 12; 20 is recommended)
*
* HChaCha is the ChaCha equivalent of HSalsa and is an intermediate step
* towards XChaCha (see https://cr.yp.to/snuffle/xsalsa-20081128.pdf). HChaCha
* skips the final addition of the initial state, and outputs only certain words
* of the state. It should not be used for streaming directly.
*/
void hchacha_block_generic(const u32 *state, u32 *stream, int nrounds)
{
u32 x[16];
memcpy(x, state, 64);
chacha_permute(x, nrounds);
memcpy(&stream[0], &x[0], 16);
memcpy(&stream[4], &x[12], 16);
}
EXPORT_SYMBOL(hchacha_block_generic);
lib/crypto/chacha20poly1305-selftest.c
+9082
View File
File diff suppressed because it is too large. Load diff
lib/crypto/chacha20poly1305.c
+370
View File
@@ -0,0 +1,370 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is an implementation of the ChaCha20Poly1305 AEAD construction.
*
* Information: https://tools.ietf.org/html/rfc8439
*/
#include <crypto/algapi.h>
#include <crypto/chacha20poly1305.h>
#include <crypto/chacha.h>
#include <crypto/poly1305.h>
#include <crypto/scatterwalk.h>
#include <asm/unaligned.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/module.h>
#define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32))
static void chacha_load_key(u32 *k, const u8 *in)
{
k[0] = get_unaligned_le32(in);
k[1] = get_unaligned_le32(in + 4);
k[2] = get_unaligned_le32(in + 8);
k[3] = get_unaligned_le32(in + 12);
k[4] = get_unaligned_le32(in + 16);
k[5] = get_unaligned_le32(in + 20);
k[6] = get_unaligned_le32(in + 24);
k[7] = get_unaligned_le32(in + 28);
}
static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
{
u32 k[CHACHA_KEY_WORDS];
u8 iv[CHACHA_IV_SIZE];
memset(iv, 0, 8);
memcpy(iv + 8, nonce + 16, 8);
chacha_load_key(k, key);
/* Compute the subkey given the original key and first 128 nonce bits */
chacha_init(chacha_state, k, nonce);
hchacha_block(chacha_state, k, 20);
chacha_init(chacha_state, k, iv);
memzero_explicit(k, sizeof(k));
memzero_explicit(iv, sizeof(iv));
}
static void
__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len, u32 *chacha_state)
{
const u8 *pad0 = page_address(ZERO_PAGE(0));
struct poly1305_desc_ctx poly1305_state;
union {
u8 block0[POLY1305_KEY_SIZE];
__le64 lens[2];
} b;
chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
poly1305_init(&poly1305_state, b.block0);
poly1305_update(&poly1305_state, ad, ad_len);
if (ad_len & 0xf)
poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
chacha20_crypt(chacha_state, dst, src, src_len);
poly1305_update(&poly1305_state, dst, src_len);
if (src_len & 0xf)
poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
b.lens[0] = cpu_to_le64(ad_len);
b.lens[1] = cpu_to_le64(src_len);
poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
poly1305_final(&poly1305_state, dst + src_len);
memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
memzero_explicit(&b, sizeof(b));
}
void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
u32 chacha_state[CHACHA_STATE_WORDS];
u32 k[CHACHA_KEY_WORDS];
__le64 iv[2];
chacha_load_key(k, key);
iv[0] = 0;
iv[1] = cpu_to_le64(nonce);
chacha_init(chacha_state, k, (u8 *)iv);
__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
memzero_explicit(iv, sizeof(iv));
memzero_explicit(k, sizeof(k));
}
EXPORT_SYMBOL(chacha20poly1305_encrypt);
void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len,
const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
u32 chacha_state[CHACHA_STATE_WORDS];
xchacha_init(chacha_state, key, nonce);
__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
}
EXPORT_SYMBOL(xchacha20poly1305_encrypt);
static bool
__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len, u32 *chacha_state)
{
const u8 *pad0 = page_address(ZERO_PAGE(0));
struct poly1305_desc_ctx poly1305_state;
size_t dst_len;
int ret;
union {
u8 block0[POLY1305_KEY_SIZE];
u8 mac[POLY1305_DIGEST_SIZE];
__le64 lens[2];
} b;
if (unlikely(src_len < POLY1305_DIGEST_SIZE))
return false;
chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
poly1305_init(&poly1305_state, b.block0);
poly1305_update(&poly1305_state, ad, ad_len);
if (ad_len & 0xf)
poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
dst_len = src_len - POLY1305_DIGEST_SIZE;
poly1305_update(&poly1305_state, src, dst_len);
if (dst_len & 0xf)
poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
b.lens[0] = cpu_to_le64(ad_len);
b.lens[1] = cpu_to_le64(dst_len);
poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
poly1305_final(&poly1305_state, b.mac);
ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
if (likely(!ret))
chacha20_crypt(chacha_state, dst, src, dst_len);
memzero_explicit(&b, sizeof(b));
return !ret;
}
bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
u32 chacha_state[CHACHA_STATE_WORDS];
u32 k[CHACHA_KEY_WORDS];
__le64 iv[2];
bool ret;
chacha_load_key(k, key);
iv[0] = 0;
iv[1] = cpu_to_le64(nonce);
chacha_init(chacha_state, k, (u8 *)iv);
ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
chacha_state);
memzero_explicit(chacha_state, sizeof(chacha_state));
memzero_explicit(iv, sizeof(iv));
memzero_explicit(k, sizeof(k));
return ret;
}
EXPORT_SYMBOL(chacha20poly1305_decrypt);
bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len,
const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
u32 chacha_state[CHACHA_STATE_WORDS];
xchacha_init(chacha_state, key, nonce);
return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
chacha_state);
}
EXPORT_SYMBOL(xchacha20poly1305_decrypt);
static
bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
const size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE],
int encrypt)
{
const u8 *pad0 = page_address(ZERO_PAGE(0));
struct poly1305_desc_ctx poly1305_state;
u32 chacha_state[CHACHA_STATE_WORDS];
struct sg_mapping_iter miter;
size_t partial = 0;
unsigned int flags;
bool ret = true;
int sl;
union {
struct {
u32 k[CHACHA_KEY_WORDS];
__le64 iv[2];
};
u8 block0[POLY1305_KEY_SIZE];
u8 chacha_stream[CHACHA_BLOCK_SIZE];
struct {
u8 mac[2][POLY1305_DIGEST_SIZE];
};
__le64 lens[2];
} b __aligned(16);
if (WARN_ON(src_len > INT_MAX))
return false;
chacha_load_key(b.k, key);
b.iv[0] = 0;
b.iv[1] = cpu_to_le64(nonce);
chacha_init(chacha_state, b.k, (u8 *)b.iv);
chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
poly1305_init(&poly1305_state, b.block0);
if (unlikely(ad_len)) {
poly1305_update(&poly1305_state, ad, ad_len);
if (ad_len & 0xf)
poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
}
flags = SG_MITER_TO_SG;
if (!preemptible())
flags |= SG_MITER_ATOMIC;
sg_miter_start(&miter, src, sg_nents(src), flags);
for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
u8 *addr = miter.addr;
size_t length = min_t(size_t, sl, miter.length);
if (!encrypt)
poly1305_update(&poly1305_state, addr, length);
if (unlikely(partial)) {
size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
crypto_xor(addr, b.chacha_stream + partial, l);
partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
addr += l;
length -= l;
}
if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
size_t l = length;
if (unlikely(length < sl))
l &= ~(CHACHA_BLOCK_SIZE - 1);
chacha20_crypt(chacha_state, addr, addr, l);
addr += l;
length -= l;
}
if (unlikely(length > 0)) {
chacha20_crypt(chacha_state, b.chacha_stream, pad0,
CHACHA_BLOCK_SIZE);
crypto_xor(addr, b.chacha_stream, length);
partial = length;
}
if (encrypt)
poly1305_update(&poly1305_state, miter.addr,
min_t(size_t, sl, miter.length));
}
if (src_len & 0xf)
poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
b.lens[0] = cpu_to_le64(ad_len);
b.lens[1] = cpu_to_le64(src_len);
poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
if (encrypt) {
poly1305_final(&poly1305_state,
miter.addr + miter.length + sl);
ret = true;
} else {
poly1305_final(&poly1305_state, b.mac[0]);
ret = !crypto_memneq(b.mac[0],
miter.addr + miter.length + sl,
POLY1305_DIGEST_SIZE);
}
}
sg_miter_stop(&miter);
if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
poly1305_final(&poly1305_state, b.mac[1]);
scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
sizeof(b.mac[1]), encrypt);
ret = encrypt ||
!crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
}
memzero_explicit(chacha_state, sizeof(chacha_state));
memzero_explicit(&b, sizeof(b));
return ret;
}
bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
nonce, key, 1);
}
EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
if (unlikely(src_len < POLY1305_DIGEST_SIZE))
return false;
return chacha20poly1305_crypt_sg_inplace(src,
src_len - POLY1305_DIGEST_SIZE,
ad, ad_len, nonce, key, 0);
}
EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
static int __init mod_init(void)
{
if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
WARN_ON(!chacha20poly1305_selftest()))
return -ENODEV;
return 0;
}
module_init(mod_init);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
lib/crypto/curve25519-fiat32.c
+864
View File
@@ -0,0 +1,864 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2016 The fiat-crypto Authors.
* Copyright (C) 2018-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is a machine-generated formally verified implementation of Curve25519
* ECDH from: <https://github.com/mit-plv/fiat-crypto>. Though originally
* machine generated, it has been tweaked to be suitable for use in the kernel.
* It is optimized for 32-bit machines and machines that cannot work efficiently
* with 128-bit integer types.
*/
#include <asm/unaligned.h>
#include <crypto/curve25519.h>
#include <linux/string.h>
/* fe means field element. Here the field is \Z/(2^255-19). An element t,
* entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77
* t[3]+2^102 t[4]+...+2^230 t[9].
* fe limbs are bounded by 1.125*2^26,1.125*2^25,1.125*2^26,1.125*2^25,etc.
* Multiplication and carrying produce fe from fe_loose.
*/
typedef struct fe { u32 v[10]; } fe;
/* fe_loose limbs are bounded by 3.375*2^26,3.375*2^25,3.375*2^26,3.375*2^25,etc
* Addition and subtraction produce fe_loose from (fe, fe).
*/
typedef struct fe_loose { u32 v[10]; } fe_loose;
static __always_inline void fe_frombytes_impl(u32 h[10], const u8 *s)
{
/* Ignores top bit of s. */
u32 a0 = get_unaligned_le32(s);
u32 a1 = get_unaligned_le32(s+4);
u32 a2 = get_unaligned_le32(s+8);
u32 a3 = get_unaligned_le32(s+12);
u32 a4 = get_unaligned_le32(s+16);
u32 a5 = get_unaligned_le32(s+20);
u32 a6 = get_unaligned_le32(s+24);
u32 a7 = get_unaligned_le32(s+28);
h[0] = a0&((1<<26)-1); /* 26 used, 32-26 left. 26 */
h[1] = (a0>>26) | ((a1&((1<<19)-1))<< 6); /* (32-26) + 19 = 6+19 = 25 */
h[2] = (a1>>19) | ((a2&((1<<13)-1))<<13); /* (32-19) + 13 = 13+13 = 26 */
h[3] = (a2>>13) | ((a3&((1<< 6)-1))<<19); /* (32-13) + 6 = 19+ 6 = 25 */
h[4] = (a3>> 6); /* (32- 6) = 26 */
h[5] = a4&((1<<25)-1); /* 25 */
h[6] = (a4>>25) | ((a5&((1<<19)-1))<< 7); /* (32-25) + 19 = 7+19 = 26 */
h[7] = (a5>>19) | ((a6&((1<<12)-1))<<13); /* (32-19) + 12 = 13+12 = 25 */
h[8] = (a6>>12) | ((a7&((1<< 6)-1))<<20); /* (32-12) + 6 = 20+ 6 = 26 */
h[9] = (a7>> 6)&((1<<25)-1); /* 25 */
}
static __always_inline void fe_frombytes(fe *h, const u8 *s)
{
fe_frombytes_impl(h->v, s);
}
static __always_inline u8 /*bool*/
addcarryx_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low)
{
/* This function extracts 25 bits of result and 1 bit of carry
* (26 total), so a 32-bit intermediate is sufficient.
*/
u32 x = a + b + c;
*low = x & ((1 << 25) - 1);
return (x >> 25) & 1;
}
static __always_inline u8 /*bool*/
addcarryx_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low)
{
/* This function extracts 26 bits of result and 1 bit of carry
* (27 total), so a 32-bit intermediate is sufficient.
*/
u32 x = a + b + c;
*low = x & ((1 << 26) - 1);
return (x >> 26) & 1;
}
static __always_inline u8 /*bool*/
subborrow_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low)
{
/* This function extracts 25 bits of result and 1 bit of borrow
* (26 total), so a 32-bit intermediate is sufficient.
*/
u32 x = a - b - c;
*low = x & ((1 << 25) - 1);
return x >> 31;
}
static __always_inline u8 /*bool*/
subborrow_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low)
{
/* This function extracts 26 bits of result and 1 bit of borrow
*(27 total), so a 32-bit intermediate is sufficient.
*/
u32 x = a - b - c;
*low = x & ((1 << 26) - 1);
return x >> 31;
}
static __always_inline u32 cmovznz32(u32 t, u32 z, u32 nz)
{
t = -!!t; /* all set if nonzero, 0 if 0 */
return (t&nz) | ((~t)&z);
}
static __always_inline void fe_freeze(u32 out[10], const u32 in1[10])
{
{ const u32 x17 = in1[9];
{ const u32 x18 = in1[8];
{ const u32 x16 = in1[7];
{ const u32 x14 = in1[6];
{ const u32 x12 = in1[5];
{ const u32 x10 = in1[4];
{ const u32 x8 = in1[3];
{ const u32 x6 = in1[2];
{ const u32 x4 = in1[1];
{ const u32 x2 = in1[0];
{ u32 x20; u8/*bool*/ x21 = subborrow_u26(0x0, x2, 0x3ffffed, &x20);
{ u32 x23; u8/*bool*/ x24 = subborrow_u25(x21, x4, 0x1ffffff, &x23);
{ u32 x26; u8/*bool*/ x27 = subborrow_u26(x24, x6, 0x3ffffff, &x26);
{ u32 x29; u8/*bool*/ x30 = subborrow_u25(x27, x8, 0x1ffffff, &x29);
{ u32 x32; u8/*bool*/ x33 = subborrow_u26(x30, x10, 0x3ffffff, &x32);
{ u32 x35; u8/*bool*/ x36 = subborrow_u25(x33, x12, 0x1ffffff, &x35);
{ u32 x38; u8/*bool*/ x39 = subborrow_u26(x36, x14, 0x3ffffff, &x38);
{ u32 x41; u8/*bool*/ x42 = subborrow_u25(x39, x16, 0x1ffffff, &x41);
{ u32 x44; u8/*bool*/ x45 = subborrow_u26(x42, x18, 0x3ffffff, &x44);
{ u32 x47; u8/*bool*/ x48 = subborrow_u25(x45, x17, 0x1ffffff, &x47);
{ u32 x49 = cmovznz32(x48, 0x0, 0xffffffff);
{ u32 x50 = (x49 & 0x3ffffed);
{ u32 x52; u8/*bool*/ x53 = addcarryx_u26(0x0, x20, x50, &x52);
{ u32 x54 = (x49 & 0x1ffffff);
{ u32 x56; u8/*bool*/ x57 = addcarryx_u25(x53, x23, x54, &x56);
{ u32 x58 = (x49 & 0x3ffffff);
{ u32 x60; u8/*bool*/ x61 = addcarryx_u26(x57, x26, x58, &x60);
{ u32 x62 = (x49 & 0x1ffffff);
{ u32 x64; u8/*bool*/ x65 = addcarryx_u25(x61, x29, x62, &x64);
{ u32 x66 = (x49 & 0x3ffffff);
{ u32 x68; u8/*bool*/ x69 = addcarryx_u26(x65, x32, x66, &x68);
{ u32 x70 = (x49 & 0x1ffffff);
{ u32 x72; u8/*bool*/ x73 = addcarryx_u25(x69, x35, x70, &x72);
{ u32 x74 = (x49 & 0x3ffffff);
{ u32 x76; u8/*bool*/ x77 = addcarryx_u26(x73, x38, x74, &x76);
{ u32 x78 = (x49 & 0x1ffffff);
{ u32 x80; u8/*bool*/ x81 = addcarryx_u25(x77, x41, x78, &x80);
{ u32 x82 = (x49 & 0x3ffffff);
{ u32 x84; u8/*bool*/ x85 = addcarryx_u26(x81, x44, x82, &x84);
{ u32 x86 = (x49 & 0x1ffffff);
{ u32 x88; addcarryx_u25(x85, x47, x86, &x88);
out[0] = x52;
out[1] = x56;
out[2] = x60;
out[3] = x64;
out[4] = x68;
out[5] = x72;
out[6] = x76;
out[7] = x80;
out[8] = x84;
out[9] = x88;
}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
}
static __always_inline void fe_tobytes(u8 s[32], const fe *f)
{
u32 h[10];
fe_freeze(h, f->v);
s[0] = h[0] >> 0;
s[1] = h[0] >> 8;
s[2] = h[0] >> 16;
s[3] = (h[0] >> 24) | (h[1] << 2);
s[4] = h[1] >> 6;
s[5] = h[1] >> 14;
s[6] = (h[1] >> 22) | (h[2] << 3);
s[7] = h[2] >> 5;
s[8] = h[2] >> 13;
s[9] = (h[2] >> 21) | (h[3] << 5);
s[10] = h[3] >> 3;
s[11] = h[3] >> 11;
s[12] = (h[3] >> 19) | (h[4] << 6);
s[13] = h[4] >> 2;
s[14] = h[4] >> 10;
s[15] = h[4] >> 18;
s[16] = h[5] >> 0;
s[17] = h[5] >> 8;
s[18] = h[5] >> 16;
s[19] = (h[5] >> 24) | (h[6] << 1);
s[20] = h[6] >> 7;
s[21] = h[6] >> 15;
s[22] = (h[6] >> 23) | (h[7] << 3);
s[23] = h[7] >> 5;
s[24] = h[7] >> 13;
s[25] = (h[7] >> 21) | (h[8] << 4);
s[26] = h[8] >> 4;
s[27] = h[8] >> 12;
s[28] = (h[8] >> 20) | (h[9] << 6);
s[29] = h[9] >> 2;
s[30] = h[9] >> 10;
s[31] = h[9] >> 18;
}
/* h = f */
static __always_inline void fe_copy(fe *h, const fe *f)
{
memmove(h, f, sizeof(u32) * 10);
}
static __always_inline void fe_copy_lt(fe_loose *h, const fe *f)
{
memmove(h, f, sizeof(u32) * 10);
}
/* h = 0 */
static __always_inline void fe_0(fe *h)
{
memset(h, 0, sizeof(u32) * 10);
}
/* h = 1 */
static __always_inline void fe_1(fe *h)
{
memset(h, 0, sizeof(u32) * 10);
h->v[0] = 1;
}
static noinline void fe_add_impl(u32 out[10], const u32 in1[10], const u32 in2[10])
{
{ const u32 x20 = in1[9];
{ const u32 x21 = in1[8];
{ const u32 x19 = in1[7];
{ const u32 x17 = in1[6];
{ const u32 x15 = in1[5];
{ const u32 x13 = in1[4];
{ const u32 x11 = in1[3];
{ const u32 x9 = in1[2];
{ const u32 x7 = in1[1];
{ const u32 x5 = in1[0];
{ const u32 x38 = in2[9];
{ const u32 x39 = in2[8];
{ const u32 x37 = in2[7];
{ const u32 x35 = in2[6];
{ const u32 x33 = in2[5];
{ const u32 x31 = in2[4];
{ const u32 x29 = in2[3];
{ const u32 x27 = in2[2];
{ const u32 x25 = in2[1];
{ const u32 x23 = in2[0];
out[0] = (x5 + x23);
out[1] = (x7 + x25);
out[2] = (x9 + x27);
out[3] = (x11 + x29);
out[4] = (x13 + x31);
out[5] = (x15 + x33);
out[6] = (x17 + x35);
out[7] = (x19 + x37);
out[8] = (x21 + x39);
out[9] = (x20 + x38);
}}}}}}}}}}}}}}}}}}}}
}
/* h = f + g
* Can overlap h with f or g.
*/
static __always_inline void fe_add(fe_loose *h, const fe *f, const fe *g)
{
fe_add_impl(h->v, f->v, g->v);
}
static noinline void fe_sub_impl(u32 out[10], const u32 in1[10], const u32 in2[10])
{
{ const u32 x20 = in1[9];
{ const u32 x21 = in1[8];
{ const u32 x19 = in1[7];
{ const u32 x17 = in1[6];
{ const u32 x15 = in1[5];
{ const u32 x13 = in1[4];
{ const u32 x11 = in1[3];
{ const u32 x9 = in1[2];
{ const u32 x7 = in1[1];
{ const u32 x5 = in1[0];
{ const u32 x38 = in2[9];
{ const u32 x39 = in2[8];
{ const u32 x37 = in2[7];
{ const u32 x35 = in2[6];
{ const u32 x33 = in2[5];
{ const u32 x31 = in2[4];
{ const u32 x29 = in2[3];
{ const u32 x27 = in2[2];
{ const u32 x25 = in2[1];
{ const u32 x23 = in2[0];
out[0] = ((0x7ffffda + x5) - x23);
out[1] = ((0x3fffffe + x7) - x25);
out[2] = ((0x7fffffe + x9) - x27);
out[3] = ((0x3fffffe + x11) - x29);
out[4] = ((0x7fffffe + x13) - x31);
out[5] = ((0x3fffffe + x15) - x33);
out[6] = ((0x7fffffe + x17) - x35);
out[7] = ((0x3fffffe + x19) - x37);
out[8] = ((0x7fffffe + x21) - x39);
out[9] = ((0x3fffffe + x20) - x38);
}}}}}}}}}}}}}}}}}}}}
}
/* h = f - g
* Can overlap h with f or g.
*/
static __always_inline void fe_sub(fe_loose *h, const fe *f, const fe *g)
{
fe_sub_impl(h->v, f->v, g->v);
}
static noinline void fe_mul_impl(u32 out[10], const u32 in1[10], const u32 in2[10])
{
{ const u32 x20 = in1[9];
{ const u32 x21 = in1[8];
{ const u32 x19 = in1[7];
{ const u32 x17 = in1[6];
{ const u32 x15 = in1[5];
{ const u32 x13 = in1[4];
{ const u32 x11 = in1[3];
{ const u32 x9 = in1[2];
{ const u32 x7 = in1[1];
{ const u32 x5 = in1[0];
{ const u32 x38 = in2[9];
{ const u32 x39 = in2[8];
{ const u32 x37 = in2[7];
{ const u32 x35 = in2[6];
{ const u32 x33 = in2[5];
{ const u32 x31 = in2[4];
{ const u32 x29 = in2[3];
{ const u32 x27 = in2[2];
{ const u32 x25 = in2[1];
{ const u32 x23 = in2[0];
{ u64 x40 = ((u64)x23 * x5);
{ u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5));
{ u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5));
{ u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5));
{ u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5));
{ u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5));
{ u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5));
{ u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5));
{ u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5));
{ u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5));
{ u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9));
{ u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9));
{ u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13));
{ u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13));
{ u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17));
{ u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17));
{ u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19))));
{ u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21));
{ u64 x58 = ((u64)(0x2 * x38) * x20);
{ u64 x59 = (x48 + (x58 << 0x4));
{ u64 x60 = (x59 + (x58 << 0x1));
{ u64 x61 = (x60 + x58);
{ u64 x62 = (x47 + (x57 << 0x4));
{ u64 x63 = (x62 + (x57 << 0x1));
{ u64 x64 = (x63 + x57);
{ u64 x65 = (x46 + (x56 << 0x4));
{ u64 x66 = (x65 + (x56 << 0x1));
{ u64 x67 = (x66 + x56);
{ u64 x68 = (x45 + (x55 << 0x4));
{ u64 x69 = (x68 + (x55 << 0x1));
{ u64 x70 = (x69 + x55);
{ u64 x71 = (x44 + (x54 << 0x4));
{ u64 x72 = (x71 + (x54 << 0x1));
{ u64 x73 = (x72 + x54);
{ u64 x74 = (x43 + (x53 << 0x4));
{ u64 x75 = (x74 + (x53 << 0x1));
{ u64 x76 = (x75 + x53);
{ u64 x77 = (x42 + (x52 << 0x4));
{ u64 x78 = (x77 + (x52 << 0x1));
{ u64 x79 = (x78 + x52);
{ u64 x80 = (x41 + (x51 << 0x4));
{ u64 x81 = (x80 + (x51 << 0x1));
{ u64 x82 = (x81 + x51);
{ u64 x83 = (x40 + (x50 << 0x4));
{ u64 x84 = (x83 + (x50 << 0x1));
{ u64 x85 = (x84 + x50);
{ u64 x86 = (x85 >> 0x1a);
{ u32 x87 = ((u32)x85 & 0x3ffffff);
{ u64 x88 = (x86 + x82);
{ u64 x89 = (x88 >> 0x19);
{ u32 x90 = ((u32)x88 & 0x1ffffff);
{ u64 x91 = (x89 + x79);
{ u64 x92 = (x91 >> 0x1a);
{ u32 x93 = ((u32)x91 & 0x3ffffff);
{ u64 x94 = (x92 + x76);
{ u64 x95 = (x94 >> 0x19);
{ u32 x96 = ((u32)x94 & 0x1ffffff);
{ u64 x97 = (x95 + x73);
{ u64 x98 = (x97 >> 0x1a);
{ u32 x99 = ((u32)x97 & 0x3ffffff);
{ u64 x100 = (x98 + x70);
{ u64 x101 = (x100 >> 0x19);
{ u32 x102 = ((u32)x100 & 0x1ffffff);
{ u64 x103 = (x101 + x67);
{ u64 x104 = (x103 >> 0x1a);
{ u32 x105 = ((u32)x103 & 0x3ffffff);
{ u64 x106 = (x104 + x64);
{ u64 x107 = (x106 >> 0x19);
{ u32 x108 = ((u32)x106 & 0x1ffffff);
{ u64 x109 = (x107 + x61);
{ u64 x110 = (x109 >> 0x1a);
{ u32 x111 = ((u32)x109 & 0x3ffffff);
{ u64 x112 = (x110 + x49);
{ u64 x113 = (x112 >> 0x19);
{ u32 x114 = ((u32)x112 & 0x1ffffff);
{ u64 x115 = (x87 + (0x13 * x113));
{ u32 x116 = (u32) (x115 >> 0x1a);
{ u32 x117 = ((u32)x115 & 0x3ffffff);
{ u32 x118 = (x116 + x90);
{ u32 x119 = (x118 >> 0x19);
{ u32 x120 = (x118 & 0x1ffffff);
out[0] = x117;
out[1] = x120;
out[2] = (x119 + x93);
out[3] = x96;
out[4] = x99;
out[5] = x102;
out[6] = x105;
out[7] = x108;
out[8] = x111;
out[9] = x114;
}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
}
static __always_inline void fe_mul_ttt(fe *h, const fe *f, const fe *g)
{
fe_mul_impl(h->v, f->v, g->v);
}
static __always_inline void fe_mul_tlt(fe *h, const fe_loose *f, const fe *g)
{
fe_mul_impl(h->v, f->v, g->v);
}
static __always_inline void
fe_mul_tll(fe *h, const fe_loose *f, const fe_loose *g)
{
fe_mul_impl(h->v, f->v, g->v);
}
static noinline void fe_sqr_impl(u32 out[10], const u32 in1[10])
{
{ const u32 x17 = in1[9];
{ const u32 x18 = in1[8];
{ const u32 x16 = in1[7];
{ const u32 x14 = in1[6];
{ const u32 x12 = in1[5];
{ const u32 x10 = in1[4];
{ const u32 x8 = in1[3];
{ const u32 x6 = in1[2];
{ const u32 x4 = in1[1];
{ const u32 x2 = in1[0];
{ u64 x19 = ((u64)x2 * x2);
{ u64 x20 = ((u64)(0x2 * x2) * x4);
{ u64 x21 = (0x2 * (((u64)x4 * x4) + ((u64)x2 * x6)));
{ u64 x22 = (0x2 * (((u64)x4 * x6) + ((u64)x2 * x8)));
{ u64 x23 = ((((u64)x6 * x6) + ((u64)(0x4 * x4) * x8)) + ((u64)(0x2 * x2) * x10));
{ u64 x24 = (0x2 * ((((u64)x6 * x8) + ((u64)x4 * x10)) + ((u64)x2 * x12)));
{ u64 x25 = (0x2 * (((((u64)x8 * x8) + ((u64)x6 * x10)) + ((u64)x2 * x14)) + ((u64)(0x2 * x4) * x12)));
{ u64 x26 = (0x2 * (((((u64)x8 * x10) + ((u64)x6 * x12)) + ((u64)x4 * x14)) + ((u64)x2 * x16)));
{ u64 x27 = (((u64)x10 * x10) + (0x2 * ((((u64)x6 * x14) + ((u64)x2 * x18)) + (0x2 * (((u64)x4 * x16) + ((u64)x8 * x12))))));
{ u64 x28 = (0x2 * ((((((u64)x10 * x12) + ((u64)x8 * x14)) + ((u64)x6 * x16)) + ((u64)x4 * x18)) + ((u64)x2 * x17)));
{ u64 x29 = (0x2 * (((((u64)x12 * x12) + ((u64)x10 * x14)) + ((u64)x6 * x18)) + (0x2 * (((u64)x8 * x16) + ((u64)x4 * x17)))));
{ u64 x30 = (0x2 * (((((u64)x12 * x14) + ((u64)x10 * x16)) + ((u64)x8 * x18)) + ((u64)x6 * x17)));
{ u64 x31 = (((u64)x14 * x14) + (0x2 * (((u64)x10 * x18) + (0x2 * (((u64)x12 * x16) + ((u64)x8 * x17))))));
{ u64 x32 = (0x2 * ((((u64)x14 * x16) + ((u64)x12 * x18)) + ((u64)x10 * x17)));
{ u64 x33 = (0x2 * ((((u64)x16 * x16) + ((u64)x14 * x18)) + ((u64)(0x2 * x12) * x17)));
{ u64 x34 = (0x2 * (((u64)x16 * x18) + ((u64)x14 * x17)));
{ u64 x35 = (((u64)x18 * x18) + ((u64)(0x4 * x16) * x17));
{ u64 x36 = ((u64)(0x2 * x18) * x17);
{ u64 x37 = ((u64)(0x2 * x17) * x17);
{ u64 x38 = (x27 + (x37 << 0x4));
{ u64 x39 = (x38 + (x37 << 0x1));
{ u64 x40 = (x39 + x37);
{ u64 x41 = (x26 + (x36 << 0x4));
{ u64 x42 = (x41 + (x36 << 0x1));
{ u64 x43 = (x42 + x36);
{ u64 x44 = (x25 + (x35 << 0x4));
{ u64 x45 = (x44 + (x35 << 0x1));
{ u64 x46 = (x45 + x35);
{ u64 x47 = (x24 + (x34 << 0x4));
{ u64 x48 = (x47 + (x34 << 0x1));
{ u64 x49 = (x48 + x34);
{ u64 x50 = (x23 + (x33 << 0x4));
{ u64 x51 = (x50 + (x33 << 0x1));
{ u64 x52 = (x51 + x33);
{ u64 x53 = (x22 + (x32 << 0x4));
{ u64 x54 = (x53 + (x32 << 0x1));
{ u64 x55 = (x54 + x32);
{ u64 x56 = (x21 + (x31 << 0x4));
{ u64 x57 = (x56 + (x31 << 0x1));
{ u64 x58 = (x57 + x31);
{ u64 x59 = (x20 + (x30 << 0x4));
{ u64 x60 = (x59 + (x30 << 0x1));
{ u64 x61 = (x60 + x30);
{ u64 x62 = (x19 + (x29 << 0x4));
{ u64 x63 = (x62 + (x29 << 0x1));
{ u64 x64 = (x63 + x29);
{ u64 x65 = (x64 >> 0x1a);
{ u32 x66 = ((u32)x64 & 0x3ffffff);
{ u64 x67 = (x65 + x61);
{ u64 x68 = (x67 >> 0x19);
{ u32 x69 = ((u32)x67 & 0x1ffffff);
{ u64 x70 = (x68 + x58);
{ u64 x71 = (x70 >> 0x1a);
{ u32 x72 = ((u32)x70 & 0x3ffffff);
{ u64 x73 = (x71 + x55);
{ u64 x74 = (x73 >> 0x19);
{ u32 x75 = ((u32)x73 & 0x1ffffff);
{ u64 x76 = (x74 + x52);
{ u64 x77 = (x76 >> 0x1a);
{ u32 x78 = ((u32)x76 & 0x3ffffff);
{ u64 x79 = (x77 + x49);
{ u64 x80 = (x79 >> 0x19);
{ u32 x81 = ((u32)x79 & 0x1ffffff);
{ u64 x82 = (x80 + x46);
{ u64 x83 = (x82 >> 0x1a);
{ u32 x84 = ((u32)x82 & 0x3ffffff);
{ u64 x85 = (x83 + x43);
{ u64 x86 = (x85 >> 0x19);
{ u32 x87 = ((u32)x85 & 0x1ffffff);
{ u64 x88 = (x86 + x40);
{ u64 x89 = (x88 >> 0x1a);
{ u32 x90 = ((u32)x88 & 0x3ffffff);
{ u64 x91 = (x89 + x28);
{ u64 x92 = (x91 >> 0x19);
{ u32 x93 = ((u32)x91 & 0x1ffffff);
{ u64 x94 = (x66 + (0x13 * x92));
{ u32 x95 = (u32) (x94 >> 0x1a);
{ u32 x96 = ((u32)x94 & 0x3ffffff);
{ u32 x97 = (x95 + x69);
{ u32 x98 = (x97 >> 0x19);
{ u32 x99 = (x97 & 0x1ffffff);
out[0] = x96;
out[1] = x99;
out[2] = (x98 + x72);
out[3] = x75;
out[4] = x78;
out[5] = x81;
out[6] = x84;
out[7] = x87;
out[8] = x90;
out[9] = x93;
}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
}
static __always_inline void fe_sq_tl(fe *h, const fe_loose *f)
{
fe_sqr_impl(h->v, f->v);
}
static __always_inline void fe_sq_tt(fe *h, const fe *f)
{
fe_sqr_impl(h->v, f->v);
}
static __always_inline void fe_loose_invert(fe *out, const fe_loose *z)
{
fe t0;
fe t1;
fe t2;
fe t3;
int i;
fe_sq_tl(&t0, z);
fe_sq_tt(&t1, &t0);
for (i = 1; i < 2; ++i)
fe_sq_tt(&t1, &t1);
fe_mul_tlt(&t1, z, &t1);
fe_mul_ttt(&t0, &t0, &t1);
fe_sq_tt(&t2, &t0);
fe_mul_ttt(&t1, &t1, &t2);
fe_sq_tt(&t2, &t1);
for (i = 1; i < 5; ++i)
fe_sq_tt(&t2, &t2);
fe_mul_ttt(&t1, &t2, &t1);
fe_sq_tt(&t2, &t1);
for (i = 1; i < 10; ++i)
fe_sq_tt(&t2, &t2);
fe_mul_ttt(&t2, &t2, &t1);
fe_sq_tt(&t3, &t2);
for (i = 1; i < 20; ++i)
fe_sq_tt(&t3, &t3);
fe_mul_ttt(&t2, &t3, &t2);
fe_sq_tt(&t2, &t2);
for (i = 1; i < 10; ++i)
fe_sq_tt(&t2, &t2);
fe_mul_ttt(&t1, &t2, &t1);
fe_sq_tt(&t2, &t1);
for (i = 1; i < 50; ++i)
fe_sq_tt(&t2, &t2);
fe_mul_ttt(&t2, &t2, &t1);
fe_sq_tt(&t3, &t2);
for (i = 1; i < 100; ++i)
fe_sq_tt(&t3, &t3);
fe_mul_ttt(&t2, &t3, &t2);
fe_sq_tt(&t2, &t2);
for (i = 1; i < 50; ++i)
fe_sq_tt(&t2, &t2);
fe_mul_ttt(&t1, &t2, &t1);
fe_sq_tt(&t1, &t1);
for (i = 1; i < 5; ++i)
fe_sq_tt(&t1, &t1);
fe_mul_ttt(out, &t1, &t0);
}
static __always_inline void fe_invert(fe *out, const fe *z)
{
fe_loose l;
fe_copy_lt(&l, z);
fe_loose_invert(out, &l);
}
/* Replace (f,g) with (g,f) if b == 1;
* replace (f,g) with (f,g) if b == 0.
*
* Preconditions: b in {0,1}
*/
static noinline void fe_cswap(fe *f, fe *g, unsigned int b)
{
unsigned i;
b = 0 - b;
for (i = 0; i < 10; i++) {
u32 x = f->v[i] ^ g->v[i];
x &= b;
f->v[i] ^= x;
g->v[i] ^= x;
}
}
/* NOTE: based on fiat-crypto fe_mul, edited for in2=121666, 0, 0.*/
static __always_inline void fe_mul_121666_impl(u32 out[10], const u32 in1[10])
{
{ const u32 x20 = in1[9];
{ const u32 x21 = in1[8];
{ const u32 x19 = in1[7];
{ const u32 x17 = in1[6];
{ const u32 x15 = in1[5];
{ const u32 x13 = in1[4];
{ const u32 x11 = in1[3];
{ const u32 x9 = in1[2];
{ const u32 x7 = in1[1];
{ const u32 x5 = in1[0];
{ const u32 x38 = 0;
{ const u32 x39 = 0;
{ const u32 x37 = 0;
{ const u32 x35 = 0;
{ const u32 x33 = 0;
{ const u32 x31 = 0;
{ const u32 x29 = 0;
{ const u32 x27 = 0;
{ const u32 x25 = 0;
{ const u32 x23 = 121666;
{ u64 x40 = ((u64)x23 * x5);
{ u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5));
{ u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5));
{ u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5));
{ u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5));
{ u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5));
{ u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5));
{ u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5));
{ u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5));
{ u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5));
{ u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9));
{ u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9));
{ u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13));
{ u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13));
{ u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17));
{ u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17));
{ u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19))));
{ u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21));
{ u64 x58 = ((u64)(0x2 * x38) * x20);
{ u64 x59 = (x48 + (x58 << 0x4));
{ u64 x60 = (x59 + (x58 << 0x1));
{ u64 x61 = (x60 + x58);
{ u64 x62 = (x47 + (x57 << 0x4));
{ u64 x63 = (x62 + (x57 << 0x1));
{ u64 x64 = (x63 + x57);
{ u64 x65 = (x46 + (x56 << 0x4));
{ u64 x66 = (x65 + (x56 << 0x1));
{ u64 x67 = (x66 + x56);
{ u64 x68 = (x45 + (x55 << 0x4));
{ u64 x69 = (x68 + (x55 << 0x1));
{ u64 x70 = (x69 + x55);
{ u64 x71 = (x44 + (x54 << 0x4));
{ u64 x72 = (x71 + (x54 << 0x1));
{ u64 x73 = (x72 + x54);
{ u64 x74 = (x43 + (x53 << 0x4));
{ u64 x75 = (x74 + (x53 << 0x1));
{ u64 x76 = (x75 + x53);
{ u64 x77 = (x42 + (x52 << 0x4));
{ u64 x78 = (x77 + (x52 << 0x1));
{ u64 x79 = (x78 + x52);
{ u64 x80 = (x41 + (x51 << 0x4));
{ u64 x81 = (x80 + (x51 << 0x1));
{ u64 x82 = (x81 + x51);
{ u64 x83 = (x40 + (x50 << 0x4));
{ u64 x84 = (x83 + (x50 << 0x1));
{ u64 x85 = (x84 + x50);
{ u64 x86 = (x85 >> 0x1a);
{ u32 x87 = ((u32)x85 & 0x3ffffff);
{ u64 x88 = (x86 + x82);
{ u64 x89 = (x88 >> 0x19);
{ u32 x90 = ((u32)x88 & 0x1ffffff);
{ u64 x91 = (x89 + x79);
{ u64 x92 = (x91 >> 0x1a);
{ u32 x93 = ((u32)x91 & 0x3ffffff);
{ u64 x94 = (x92 + x76);
{ u64 x95 = (x94 >> 0x19);
{ u32 x96 = ((u32)x94 & 0x1ffffff);
{ u64 x97 = (x95 + x73);
{ u64 x98 = (x97 >> 0x1a);
{ u32 x99 = ((u32)x97 & 0x3ffffff);
{ u64 x100 = (x98 + x70);
{ u64 x101 = (x100 >> 0x19);
{ u32 x102 = ((u32)x100 & 0x1ffffff);
{ u64 x103 = (x101 + x67);
{ u64 x104 = (x103 >> 0x1a);
{ u32 x105 = ((u32)x103 & 0x3ffffff);
{ u64 x106 = (x104 + x64);
{ u64 x107 = (x106 >> 0x19);
{ u32 x108 = ((u32)x106 & 0x1ffffff);
{ u64 x109 = (x107 + x61);
{ u64 x110 = (x109 >> 0x1a);
{ u32 x111 = ((u32)x109 & 0x3ffffff);
{ u64 x112 = (x110 + x49);
{ u64 x113 = (x112 >> 0x19);
{ u32 x114 = ((u32)x112 & 0x1ffffff);
{ u64 x115 = (x87 + (0x13 * x113));
{ u32 x116 = (u32) (x115 >> 0x1a);
{ u32 x117 = ((u32)x115 & 0x3ffffff);
{ u32 x118 = (x116 + x90);
{ u32 x119 = (x118 >> 0x19);
{ u32 x120 = (x118 & 0x1ffffff);
out[0] = x117;
out[1] = x120;
out[2] = (x119 + x93);
out[3] = x96;
out[4] = x99;
out[5] = x102;
out[6] = x105;
out[7] = x108;
out[8] = x111;
out[9] = x114;
}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}
}
static __always_inline void fe_mul121666(fe *h, const fe_loose *f)
{
fe_mul_121666_impl(h->v, f->v);
}
void curve25519_generic(u8 out[CURVE25519_KEY_SIZE],
const u8 scalar[CURVE25519_KEY_SIZE],
const u8 point[CURVE25519_KEY_SIZE])
{
fe x1, x2, z2, x3, z3;
fe_loose x2l, z2l, x3l;
unsigned swap = 0;
int pos;
u8 e[32];
memcpy(e, scalar, 32);
curve25519_clamp_secret(e);
/* The following implementation was transcribed to Coq and proven to
* correspond to unary scalar multiplication in affine coordinates given
* that x1 != 0 is the x coordinate of some point on the curve. It was
* also checked in Coq that doing a ladderstep with x1 = x3 = 0 gives
* z2' = z3' = 0, and z2 = z3 = 0 gives z2' = z3' = 0. The statement was
* quantified over the underlying field, so it applies to Curve25519
* itself and the quadratic twist of Curve25519. It was not proven in
* Coq that prime-field arithmetic correctly simulates extension-field
* arithmetic on prime-field values. The decoding of the byte array
* representation of e was not considered.
*
* Specification of Montgomery curves in affine coordinates:
* <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Spec/MontgomeryCurve.v#L27>
*
* Proof that these form a group that is isomorphic to a Weierstrass
* curve:
* <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/AffineProofs.v#L35>
*
* Coq transcription and correctness proof of the loop
* (where scalarbits=255):
* <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZ.v#L118>
* <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L278>
* preconditions: 0 <= e < 2^255 (not necessarily e < order),
* fe_invert(0) = 0
*/
fe_frombytes(&x1, point);
fe_1(&x2);
fe_0(&z2);
fe_copy(&x3, &x1);
fe_1(&z3);
for (pos = 254; pos >= 0; --pos) {
fe tmp0, tmp1;
fe_loose tmp0l, tmp1l;
/* loop invariant as of right before the test, for the case
* where x1 != 0:
* pos >= -1; if z2 = 0 then x2 is nonzero; if z3 = 0 then x3
* is nonzero
* let r := e >> (pos+1) in the following equalities of
* projective points:
* to_xz (r*P) === if swap then (x3, z3) else (x2, z2)
* to_xz ((r+1)*P) === if swap then (x2, z2) else (x3, z3)
* x1 is the nonzero x coordinate of the nonzero
* point (r*P-(r+1)*P)
*/
unsigned b = 1 & (e[pos / 8] >> (pos & 7));
swap ^= b;
fe_cswap(&x2, &x3, swap);
fe_cswap(&z2, &z3, swap);
swap = b;
/* Coq transcription of ladderstep formula (called from
* transcribed loop):
* <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZ.v#L89>
* <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L131>
* x1 != 0 <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L217>
* x1 = 0 <https://github.com/mit-plv/fiat-crypto/blob/2456d821825521f7e03e65882cc3521795b0320f/src/Curves/Montgomery/XZProofs.v#L147>
*/
fe_sub(&tmp0l, &x3, &z3);
fe_sub(&tmp1l, &x2, &z2);
fe_add(&x2l, &x2, &z2);
fe_add(&z2l, &x3, &z3);
fe_mul_tll(&z3, &tmp0l, &x2l);
fe_mul_tll(&z2, &z2l, &tmp1l);
fe_sq_tl(&tmp0, &tmp1l);
fe_sq_tl(&tmp1, &x2l);
fe_add(&x3l, &z3, &z2);
fe_sub(&z2l, &z3, &z2);
fe_mul_ttt(&x2, &tmp1, &tmp0);
fe_sub(&tmp1l, &tmp1, &tmp0);
fe_sq_tl(&z2, &z2l);
fe_mul121666(&z3, &tmp1l);
fe_sq_tl(&x3, &x3l);
fe_add(&tmp0l, &tmp0, &z3);
fe_mul_ttt(&z3, &x1, &z2);
fe_mul_tll(&z2, &tmp1l, &tmp0l);
}
/* here pos=-1, so r=e, so to_xz (e*P) === if swap then (x3, z3)
* else (x2, z2)
*/
fe_cswap(&x2, &x3, swap);
fe_cswap(&z2, &z3, swap);
fe_invert(&z2, &z2);
fe_mul_ttt(&x2, &x2, &z2);
fe_tobytes(out, &x2);
memzero_explicit(&x1, sizeof(x1));
memzero_explicit(&x2, sizeof(x2));
memzero_explicit(&z2, sizeof(z2));
memzero_explicit(&x3, sizeof(x3));
memzero_explicit(&z3, sizeof(z3));
memzero_explicit(&x2l, sizeof(x2l));
memzero_explicit(&z2l, sizeof(z2l));
memzero_explicit(&x3l, sizeof(x3l));
memzero_explicit(&e, sizeof(e));
}
lib/crypto/curve25519-generic.c
+24
View File
@@ -0,0 +1,24 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is an implementation of the Curve25519 ECDH algorithm, using either
* a 32-bit implementation or a 64-bit implementation with 128-bit integers,
* depending on what is supported by the target compiler.
*
* Information: https://cr.yp.to/ecdh.html
*/
#include <crypto/curve25519.h>
#include <linux/module.h>
const u8 curve25519_null_point[CURVE25519_KEY_SIZE] __aligned(32) = { 0 };
const u8 curve25519_base_point[CURVE25519_KEY_SIZE] __aligned(32) = { 9 };
EXPORT_SYMBOL(curve25519_null_point);
EXPORT_SYMBOL(curve25519_base_point);
EXPORT_SYMBOL(curve25519_generic);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Curve25519 scalar multiplication");
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
lib/crypto/curve25519-hacl64.c
+788
View File
@@ -0,0 +1,788 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2016-2017 INRIA and Microsoft Corporation.
* Copyright (C) 2018-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is a machine-generated formally verified implementation of Curve25519
* ECDH from: <https://github.com/mitls/hacl-star>. Though originally machine
* generated, it has been tweaked to be suitable for use in the kernel. It is
* optimized for 64-bit machines that can efficiently work with 128-bit
* integer types.
*/
#include <asm/unaligned.h>
#include <crypto/curve25519.h>
#include <linux/string.h>
typedef __uint128_t u128;
static __always_inline u64 u64_eq_mask(u64 a, u64 b)
{
u64 x = a ^ b;
u64 minus_x = ~x + (u64)1U;
u64 x_or_minus_x = x | minus_x;
u64 xnx = x_or_minus_x >> (u32)63U;
u64 c = xnx - (u64)1U;
return c;
}
static __always_inline u64 u64_gte_mask(u64 a, u64 b)
{
u64 x = a;
u64 y = b;
u64 x_xor_y = x ^ y;
u64 x_sub_y = x - y;
u64 x_sub_y_xor_y = x_sub_y ^ y;
u64 q = x_xor_y | x_sub_y_xor_y;
u64 x_xor_q = x ^ q;
u64 x_xor_q_ = x_xor_q >> (u32)63U;
u64 c = x_xor_q_ - (u64)1U;
return c;
}
static __always_inline void modulo_carry_top(u64 *b)
{
u64 b4 = b[4];
u64 b0 = b[0];
u64 b4_ = b4 & 0x7ffffffffffffLLU;
u64 b0_ = b0 + 19 * (b4 >> 51);
b[4] = b4_;
b[0] = b0_;
}
static __always_inline void fproduct_copy_from_wide_(u64 *output, u128 *input)
{
{
u128 xi = input[0];
output[0] = ((u64)(xi));
}
{
u128 xi = input[1];
output[1] = ((u64)(xi));
}
{
u128 xi = input[2];
output[2] = ((u64)(xi));
}
{
u128 xi = input[3];
output[3] = ((u64)(xi));
}
{
u128 xi = input[4];
output[4] = ((u64)(xi));
}
}
static __always_inline void
fproduct_sum_scalar_multiplication_(u128 *output, u64 *input, u64 s)
{
output[0] += (u128)input[0] * s;
output[1] += (u128)input[1] * s;
output[2] += (u128)input[2] * s;
output[3] += (u128)input[3] * s;
output[4] += (u128)input[4] * s;
}
static __always_inline void fproduct_carry_wide_(u128 *tmp)
{
{
u32 ctr = 0;
u128 tctr = tmp[ctr];
u128 tctrp1 = tmp[ctr + 1];
u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
u128 c = ((tctr) >> (51));
tmp[ctr] = ((u128)(r0));
tmp[ctr + 1] = ((tctrp1) + (c));
}
{
u32 ctr = 1;
u128 tctr = tmp[ctr];
u128 tctrp1 = tmp[ctr + 1];
u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
u128 c = ((tctr) >> (51));
tmp[ctr] = ((u128)(r0));
tmp[ctr + 1] = ((tctrp1) + (c));
}
{
u32 ctr = 2;
u128 tctr = tmp[ctr];
u128 tctrp1 = tmp[ctr + 1];
u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
u128 c = ((tctr) >> (51));
tmp[ctr] = ((u128)(r0));
tmp[ctr + 1] = ((tctrp1) + (c));
}
{
u32 ctr = 3;
u128 tctr = tmp[ctr];
u128 tctrp1 = tmp[ctr + 1];
u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
u128 c = ((tctr) >> (51));
tmp[ctr] = ((u128)(r0));
tmp[ctr + 1] = ((tctrp1) + (c));
}
}
static __always_inline void fmul_shift_reduce(u64 *output)
{
u64 tmp = output[4];
u64 b0;
{
u32 ctr = 5 - 0 - 1;
u64 z = output[ctr - 1];
output[ctr] = z;
}
{
u32 ctr = 5 - 1 - 1;
u64 z = output[ctr - 1];
output[ctr] = z;
}
{
u32 ctr = 5 - 2 - 1;
u64 z = output[ctr - 1];
output[ctr] = z;
}
{
u32 ctr = 5 - 3 - 1;
u64 z = output[ctr - 1];
output[ctr] = z;
}
output[0] = tmp;
b0 = output[0];
output[0] = 19 * b0;
}
static __always_inline void fmul_mul_shift_reduce_(u128 *output, u64 *input,
u64 *input21)
{
u32 i;
u64 input2i;
{
u64 input2i = input21[0];
fproduct_sum_scalar_multiplication_(output, input, input2i);
fmul_shift_reduce(input);
}
{
u64 input2i = input21[1];
fproduct_sum_scalar_multiplication_(output, input, input2i);
fmul_shift_reduce(input);
}
{
u64 input2i = input21[2];
fproduct_sum_scalar_multiplication_(output, input, input2i);
fmul_shift_reduce(input);
}
{
u64 input2i = input21[3];
fproduct_sum_scalar_multiplication_(output, input, input2i);
fmul_shift_reduce(input);
}
i = 4;
input2i = input21[i];
fproduct_sum_scalar_multiplication_(output, input, input2i);
}
static __always_inline void fmul_fmul(u64 *output, u64 *input, u64 *input21)
{
u64 tmp[5] = { input[0], input[1], input[2], input[3], input[4] };
{
u128 b4;
u128 b0;
u128 b4_;
u128 b0_;
u64 i0;
u64 i1;
u64 i0_;
u64 i1_;
u128 t[5] = { 0 };
fmul_mul_shift_reduce_(t, tmp, input21);
fproduct_carry_wide_(t);
b4 = t[4];
b0 = t[0];
b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
t[4] = b4_;
t[0] = b0_;
fproduct_copy_from_wide_(output, t);
i0 = output[0];
i1 = output[1];
i0_ = i0 & 0x7ffffffffffffLLU;
i1_ = i1 + (i0 >> 51);
output[0] = i0_;
output[1] = i1_;
}
}
static __always_inline void fsquare_fsquare__(u128 *tmp, u64 *output)
{
u64 r0 = output[0];
u64 r1 = output[1];
u64 r2 = output[2];
u64 r3 = output[3];
u64 r4 = output[4];
u64 d0 = r0 * 2;
u64 d1 = r1 * 2;
u64 d2 = r2 * 2 * 19;
u64 d419 = r4 * 19;
u64 d4 = d419 * 2;
u128 s0 = ((((((u128)(r0) * (r0))) + (((u128)(d4) * (r1))))) +
(((u128)(d2) * (r3))));
u128 s1 = ((((((u128)(d0) * (r1))) + (((u128)(d4) * (r2))))) +
(((u128)(r3 * 19) * (r3))));
u128 s2 = ((((((u128)(d0) * (r2))) + (((u128)(r1) * (r1))))) +
(((u128)(d4) * (r3))));
u128 s3 = ((((((u128)(d0) * (r3))) + (((u128)(d1) * (r2))))) +
(((u128)(r4) * (d419))));
u128 s4 = ((((((u128)(d0) * (r4))) + (((u128)(d1) * (r3))))) +
(((u128)(r2) * (r2))));
tmp[0] = s0;
tmp[1] = s1;
tmp[2] = s2;
tmp[3] = s3;
tmp[4] = s4;
}
static __always_inline void fsquare_fsquare_(u128 *tmp, u64 *output)
{
u128 b4;
u128 b0;
u128 b4_;
u128 b0_;
u64 i0;
u64 i1;
u64 i0_;
u64 i1_;
fsquare_fsquare__(tmp, output);
fproduct_carry_wide_(tmp);
b4 = tmp[4];
b0 = tmp[0];
b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
tmp[4] = b4_;
tmp[0] = b0_;
fproduct_copy_from_wide_(output, tmp);
i0 = output[0];
i1 = output[1];
i0_ = i0 & 0x7ffffffffffffLLU;
i1_ = i1 + (i0 >> 51);
output[0] = i0_;
output[1] = i1_;
}
static __always_inline void fsquare_fsquare_times_(u64 *output, u128 *tmp,
u32 count1)
{
u32 i;
fsquare_fsquare_(tmp, output);
for (i = 1; i < count1; ++i)
fsquare_fsquare_(tmp, output);
}
static __always_inline void fsquare_fsquare_times(u64 *output, u64 *input,
u32 count1)
{
u128 t[5];
memcpy(output, input, 5 * sizeof(*input));
fsquare_fsquare_times_(output, t, count1);
}
static __always_inline void fsquare_fsquare_times_inplace(u64 *output,
u32 count1)
{
u128 t[5];
fsquare_fsquare_times_(output, t, count1);
}
static __always_inline void crecip_crecip(u64 *out, u64 *z)
{
u64 buf[20] = { 0 };
u64 *a0 = buf;
u64 *t00 = buf + 5;
u64 *b0 = buf + 10;
u64 *t01;
u64 *b1;
u64 *c0;
u64 *a;
u64 *t0;
u64 *b;
u64 *c;
fsquare_fsquare_times(a0, z, 1);
fsquare_fsquare_times(t00, a0, 2);
fmul_fmul(b0, t00, z);
fmul_fmul(a0, b0, a0);
fsquare_fsquare_times(t00, a0, 1);
fmul_fmul(b0, t00, b0);
fsquare_fsquare_times(t00, b0, 5);
t01 = buf + 5;
b1 = buf + 10;
c0 = buf + 15;
fmul_fmul(b1, t01, b1);
fsquare_fsquare_times(t01, b1, 10);
fmul_fmul(c0, t01, b1);
fsquare_fsquare_times(t01, c0, 20);
fmul_fmul(t01, t01, c0);
fsquare_fsquare_times_inplace(t01, 10);
fmul_fmul(b1, t01, b1);
fsquare_fsquare_times(t01, b1, 50);
a = buf;
t0 = buf + 5;
b = buf + 10;
c = buf + 15;
fmul_fmul(c, t0, b);
fsquare_fsquare_times(t0, c, 100);
fmul_fmul(t0, t0, c);
fsquare_fsquare_times_inplace(t0, 50);
fmul_fmul(t0, t0, b);
fsquare_fsquare_times_inplace(t0, 5);
fmul_fmul(out, t0, a);
}
static __always_inline void fsum(u64 *a, u64 *b)
{
a[0] += b[0];
a[1] += b[1];
a[2] += b[2];
a[3] += b[3];
a[4] += b[4];
}
static __always_inline void fdifference(u64 *a, u64 *b)
{
u64 tmp[5] = { 0 };
u64 b0;
u64 b1;
u64 b2;
u64 b3;
u64 b4;
memcpy(tmp, b, 5 * sizeof(*b));
b0 = tmp[0];
b1 = tmp[1];
b2 = tmp[2];
b3 = tmp[3];
b4 = tmp[4];
tmp[0] = b0 + 0x3fffffffffff68LLU;
tmp[1] = b1 + 0x3ffffffffffff8LLU;
tmp[2] = b2 + 0x3ffffffffffff8LLU;
tmp[3] = b3 + 0x3ffffffffffff8LLU;
tmp[4] = b4 + 0x3ffffffffffff8LLU;
{
u64 xi = a[0];
u64 yi = tmp[0];
a[0] = yi - xi;
}
{
u64 xi = a[1];
u64 yi = tmp[1];
a[1] = yi - xi;
}
{
u64 xi = a[2];
u64 yi = tmp[2];
a[2] = yi - xi;
}
{
u64 xi = a[3];
u64 yi = tmp[3];
a[3] = yi - xi;
}
{
u64 xi = a[4];
u64 yi = tmp[4];
a[4] = yi - xi;
}
}
static __always_inline void fscalar(u64 *output, u64 *b, u64 s)
{
u128 tmp[5];
u128 b4;
u128 b0;
u128 b4_;
u128 b0_;
{
u64 xi = b[0];
tmp[0] = ((u128)(xi) * (s));
}
{
u64 xi = b[1];
tmp[1] = ((u128)(xi) * (s));
}
{
u64 xi = b[2];
tmp[2] = ((u128)(xi) * (s));
}
{
u64 xi = b[3];
tmp[3] = ((u128)(xi) * (s));
}
{
u64 xi = b[4];
tmp[4] = ((u128)(xi) * (s));
}
fproduct_carry_wide_(tmp);
b4 = tmp[4];
b0 = tmp[0];
b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
tmp[4] = b4_;
tmp[0] = b0_;
fproduct_copy_from_wide_(output, tmp);
}
static __always_inline void fmul(u64 *output, u64 *a, u64 *b)
{
fmul_fmul(output, a, b);
}
static __always_inline void crecip(u64 *output, u64 *input)
{
crecip_crecip(output, input);
}
static __always_inline void point_swap_conditional_step(u64 *a, u64 *b,
u64 swap1, u32 ctr)
{
u32 i = ctr - 1;
u64 ai = a[i];
u64 bi = b[i];
u64 x = swap1 & (ai ^ bi);
u64 ai1 = ai ^ x;
u64 bi1 = bi ^ x;
a[i] = ai1;
b[i] = bi1;
}
static __always_inline void point_swap_conditional5(u64 *a, u64 *b, u64 swap1)
{
point_swap_conditional_step(a, b, swap1, 5);
point_swap_conditional_step(a, b, swap1, 4);
point_swap_conditional_step(a, b, swap1, 3);
point_swap_conditional_step(a, b, swap1, 2);
point_swap_conditional_step(a, b, swap1, 1);
}
static __always_inline void point_swap_conditional(u64 *a, u64 *b, u64 iswap)
{
u64 swap1 = 0 - iswap;
point_swap_conditional5(a, b, swap1);
point_swap_conditional5(a + 5, b + 5, swap1);
}
static __always_inline void point_copy(u64 *output, u64 *input)
{
memcpy(output, input, 5 * sizeof(*input));
memcpy(output + 5, input + 5, 5 * sizeof(*input));
}
static __always_inline void addanddouble_fmonty(u64 *pp, u64 *ppq, u64 *p,
u64 *pq, u64 *qmqp)
{
u64 *qx = qmqp;
u64 *x2 = pp;
u64 *z2 = pp + 5;
u64 *x3 = ppq;
u64 *z3 = ppq + 5;
u64 *x = p;
u64 *z = p + 5;
u64 *xprime = pq;
u64 *zprime = pq + 5;
u64 buf[40] = { 0 };
u64 *origx = buf;
u64 *origxprime0 = buf + 5;
u64 *xxprime0;
u64 *zzprime0;
u64 *origxprime;
xxprime0 = buf + 25;
zzprime0 = buf + 30;
memcpy(origx, x, 5 * sizeof(*x));
fsum(x, z);
fdifference(z, origx);
memcpy(origxprime0, xprime, 5 * sizeof(*xprime));
fsum(xprime, zprime);
fdifference(zprime, origxprime0);
fmul(xxprime0, xprime, z);
fmul(zzprime0, x, zprime);
origxprime = buf + 5;
{
u64 *xx0;
u64 *zz0;
u64 *xxprime;
u64 *zzprime;
u64 *zzzprime;
xx0 = buf + 15;
zz0 = buf + 20;
xxprime = buf + 25;
zzprime = buf + 30;
zzzprime = buf + 35;
memcpy(origxprime, xxprime, 5 * sizeof(*xxprime));
fsum(xxprime, zzprime);
fdifference(zzprime, origxprime);
fsquare_fsquare_times(x3, xxprime, 1);
fsquare_fsquare_times(zzzprime, zzprime, 1);
fmul(z3, zzzprime, qx);
fsquare_fsquare_times(xx0, x, 1);
fsquare_fsquare_times(zz0, z, 1);
{
u64 *zzz;
u64 *xx;
u64 *zz;
u64 scalar;
zzz = buf + 10;
xx = buf + 15;
zz = buf + 20;
fmul(x2, xx, zz);
fdifference(zz, xx);
scalar = 121665;
fscalar(zzz, zz, scalar);
fsum(zzz, xx);
fmul(z2, zzz, zz);
}
}
}
static __always_inline void
ladder_smallloop_cmult_small_loop_step(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2,
u64 *q, u8 byt)
{
u64 bit0 = (u64)(byt >> 7);
u64 bit;
point_swap_conditional(nq, nqpq, bit0);
addanddouble_fmonty(nq2, nqpq2, nq, nqpq, q);
bit = (u64)(byt >> 7);
point_swap_conditional(nq2, nqpq2, bit);
}
static __always_inline void
ladder_smallloop_cmult_small_loop_double_step(u64 *nq, u64 *nqpq, u64 *nq2,
u64 *nqpq2, u64 *q, u8 byt)
{
u8 byt1;
ladder_smallloop_cmult_small_loop_step(nq, nqpq, nq2, nqpq2, q, byt);
byt1 = byt << 1;
ladder_smallloop_cmult_small_loop_step(nq2, nqpq2, nq, nqpq, q, byt1);
}
static __always_inline void
ladder_smallloop_cmult_small_loop(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2,
u64 *q, u8 byt, u32 i)
{
while (i--) {
ladder_smallloop_cmult_small_loop_double_step(nq, nqpq, nq2,
nqpq2, q, byt);
byt <<= 2;
}
}
static __always_inline void ladder_bigloop_cmult_big_loop(u8 *n1, u64 *nq,
u64 *nqpq, u64 *nq2,
u64 *nqpq2, u64 *q,
u32 i)
{
while (i--) {
u8 byte = n1[i];
ladder_smallloop_cmult_small_loop(nq, nqpq, nq2, nqpq2, q,
byte, 4);
}
}
static void ladder_cmult(u64 *result, u8 *n1, u64 *q)
{
u64 point_buf[40] = { 0 };
u64 *nq = point_buf;
u64 *nqpq = point_buf + 10;
u64 *nq2 = point_buf + 20;
u64 *nqpq2 = point_buf + 30;
point_copy(nqpq, q);
nq[0] = 1;
ladder_bigloop_cmult_big_loop(n1, nq, nqpq, nq2, nqpq2, q, 32);
point_copy(result, nq);
}
static __always_inline void format_fexpand(u64 *output, const u8 *input)
{
const u8 *x00 = input + 6;
const u8 *x01 = input + 12;
const u8 *x02 = input + 19;
const u8 *x0 = input + 24;
u64 i0, i1, i2, i3, i4, output0, output1, output2, output3, output4;
i0 = get_unaligned_le64(input);
i1 = get_unaligned_le64(x00);
i2 = get_unaligned_le64(x01);
i3 = get_unaligned_le64(x02);
i4 = get_unaligned_le64(x0);
output0 = i0 & 0x7ffffffffffffLLU;
output1 = i1 >> 3 & 0x7ffffffffffffLLU;
output2 = i2 >> 6 & 0x7ffffffffffffLLU;
output3 = i3 >> 1 & 0x7ffffffffffffLLU;
output4 = i4 >> 12 & 0x7ffffffffffffLLU;
output[0] = output0;
output[1] = output1;
output[2] = output2;
output[3] = output3;
output[4] = output4;
}
static __always_inline void format_fcontract_first_carry_pass(u64 *input)
{
u64 t0 = input[0];
u64 t1 = input[1];
u64 t2 = input[2];
u64 t3 = input[3];
u64 t4 = input[4];
u64 t1_ = t1 + (t0 >> 51);
u64 t0_ = t0 & 0x7ffffffffffffLLU;
u64 t2_ = t2 + (t1_ >> 51);
u64 t1__ = t1_ & 0x7ffffffffffffLLU;
u64 t3_ = t3 + (t2_ >> 51);
u64 t2__ = t2_ & 0x7ffffffffffffLLU;
u64 t4_ = t4 + (t3_ >> 51);
u64 t3__ = t3_ & 0x7ffffffffffffLLU;
input[0] = t0_;
input[1] = t1__;
input[2] = t2__;
input[3] = t3__;
input[4] = t4_;
}
static __always_inline void format_fcontract_first_carry_full(u64 *input)
{
format_fcontract_first_carry_pass(input);
modulo_carry_top(input);
}
static __always_inline void format_fcontract_second_carry_pass(u64 *input)
{
u64 t0 = input[0];
u64 t1 = input[1];
u64 t2 = input[2];
u64 t3 = input[3];
u64 t4 = input[4];
u64 t1_ = t1 + (t0 >> 51);
u64 t0_ = t0 & 0x7ffffffffffffLLU;
u64 t2_ = t2 + (t1_ >> 51);
u64 t1__ = t1_ & 0x7ffffffffffffLLU;
u64 t3_ = t3 + (t2_ >> 51);
u64 t2__ = t2_ & 0x7ffffffffffffLLU;
u64 t4_ = t4 + (t3_ >> 51);
u64 t3__ = t3_ & 0x7ffffffffffffLLU;
input[0] = t0_;
input[1] = t1__;
input[2] = t2__;
input[3] = t3__;
input[4] = t4_;
}
static __always_inline void format_fcontract_second_carry_full(u64 *input)
{
u64 i0;
u64 i1;
u64 i0_;
u64 i1_;
format_fcontract_second_carry_pass(input);
modulo_carry_top(input);
i0 = input[0];
i1 = input[1];
i0_ = i0 & 0x7ffffffffffffLLU;
i1_ = i1 + (i0 >> 51);
input[0] = i0_;
input[1] = i1_;
}
static __always_inline void format_fcontract_trim(u64 *input)
{
u64 a0 = input[0];
u64 a1 = input[1];
u64 a2 = input[2];
u64 a3 = input[3];
u64 a4 = input[4];
u64 mask0 = u64_gte_mask(a0, 0x7ffffffffffedLLU);
u64 mask1 = u64_eq_mask(a1, 0x7ffffffffffffLLU);
u64 mask2 = u64_eq_mask(a2, 0x7ffffffffffffLLU);
u64 mask3 = u64_eq_mask(a3, 0x7ffffffffffffLLU);
u64 mask4 = u64_eq_mask(a4, 0x7ffffffffffffLLU);
u64 mask = (((mask0 & mask1) & mask2) & mask3) & mask4;
u64 a0_ = a0 - (0x7ffffffffffedLLU & mask);
u64 a1_ = a1 - (0x7ffffffffffffLLU & mask);
u64 a2_ = a2 - (0x7ffffffffffffLLU & mask);
u64 a3_ = a3 - (0x7ffffffffffffLLU & mask);
u64 a4_ = a4 - (0x7ffffffffffffLLU & mask);
input[0] = a0_;
input[1] = a1_;
input[2] = a2_;
input[3] = a3_;
input[4] = a4_;
}
static __always_inline void format_fcontract_store(u8 *output, u64 *input)
{
u64 t0 = input[0];
u64 t1 = input[1];
u64 t2 = input[2];
u64 t3 = input[3];
u64 t4 = input[4];
u64 o0 = t1 << 51 | t0;
u64 o1 = t2 << 38 | t1 >> 13;
u64 o2 = t3 << 25 | t2 >> 26;
u64 o3 = t4 << 12 | t3 >> 39;
u8 *b0 = output;
u8 *b1 = output + 8;
u8 *b2 = output + 16;
u8 *b3 = output + 24;
put_unaligned_le64(o0, b0);
put_unaligned_le64(o1, b1);
put_unaligned_le64(o2, b2);
put_unaligned_le64(o3, b3);
}
static __always_inline void format_fcontract(u8 *output, u64 *input)
{
format_fcontract_first_carry_full(input);
format_fcontract_second_carry_full(input);
format_fcontract_trim(input);
format_fcontract_store(output, input);
}
static __always_inline void format_scalar_of_point(u8 *scalar, u64 *point)
{
u64 *x = point;
u64 *z = point + 5;
u64 buf[10] __aligned(32) = { 0 };
u64 *zmone = buf;
u64 *sc = buf + 5;
crecip(zmone, z);
fmul(sc, x, zmone);
format_fcontract(scalar, sc);
}
void curve25519_generic(u8 mypublic[CURVE25519_KEY_SIZE],
const u8 secret[CURVE25519_KEY_SIZE],
const u8 basepoint[CURVE25519_KEY_SIZE])
{
u64 buf0[10] __aligned(32) = { 0 };
u64 *x0 = buf0;
u64 *z = buf0 + 5;
u64 *q;
format_fexpand(x0, basepoint);
z[0] = 1;
q = buf0;
{
u8 e[32] __aligned(32) = { 0 };
u8 *scalar;
memcpy(e, secret, 32);
curve25519_clamp_secret(e);
scalar = e;
{
u64 buf[15] = { 0 };
u64 *nq = buf;
u64 *x = nq;
x[0] = 1;
ladder_cmult(nq, scalar, q);
format_scalar_of_point(mypublic, nq);
memzero_explicit(buf, sizeof(buf));
}
memzero_explicit(e, sizeof(e));
}
memzero_explicit(buf0, sizeof(buf0));
}
lib/crypto/curve25519-selftest.c
+1321
View File
File diff suppressed because it is too large. Load diff
lib/crypto/curve25519.c
+35
View File
@@ -0,0 +1,35 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is an implementation of the Curve25519 ECDH algorithm, using either
* a 32-bit implementation or a 64-bit implementation with 128-bit integers,
* depending on what is supported by the target compiler.
*
* Information: https://cr.yp.to/ecdh.html
*/
#include <crypto/curve25519.h>
#include <linux/module.h>
#include <linux/init.h>
bool curve25519_selftest(void);
static int __init mod_init(void)
{
if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
WARN_ON(!curve25519_selftest()))
return -ENODEV;
return 0;
}
static void __exit mod_exit(void)
{
}
module_init(mod_init);
module_exit(mod_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Curve25519 scalar multiplication");
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
lib/crypto/libchacha.c
+35
View File
@@ -0,0 +1,35 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* The ChaCha stream cipher (RFC7539)
*
* Copyright (C) 2015 Martin Willi
*/
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/module.h>
#include <crypto/algapi.h> // for crypto_xor_cpy
#include <crypto/chacha.h>
void chacha_crypt_generic(u32 *state, u8 *dst, const u8 *src,
unsigned int bytes, int nrounds)
{
/* aligned to potentially speed up crypto_xor() */
u8 stream[CHACHA_BLOCK_SIZE] __aligned(sizeof(long));
while (bytes >= CHACHA_BLOCK_SIZE) {
chacha_block_generic(state, stream, nrounds);
crypto_xor_cpy(dst, src, stream, CHACHA_BLOCK_SIZE);
bytes -= CHACHA_BLOCK_SIZE;
dst += CHACHA_BLOCK_SIZE;
src += CHACHA_BLOCK_SIZE;
}
if (bytes) {
chacha_block_generic(state, stream, nrounds);
crypto_xor_cpy(dst, src, stream, bytes);
}
}
EXPORT_SYMBOL(chacha_crypt_generic);
MODULE_LICENSE("GPL");
lib/crypto/poly1305-donna32.c
+204
View File
@@ -0,0 +1,204 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is based in part on Andrew Moon's poly1305-donna, which is in the
* public domain.
*/
#include <linux/kernel.h>
#include <asm/unaligned.h>
#include <crypto/internal/poly1305.h>
void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16])
{
/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
key->key.r[0] = (get_unaligned_le32(&raw_key[0])) & 0x3ffffff;
key->key.r[1] = (get_unaligned_le32(&raw_key[3]) >> 2) & 0x3ffff03;
key->key.r[2] = (get_unaligned_le32(&raw_key[6]) >> 4) & 0x3ffc0ff;
key->key.r[3] = (get_unaligned_le32(&raw_key[9]) >> 6) & 0x3f03fff;
key->key.r[4] = (get_unaligned_le32(&raw_key[12]) >> 8) & 0x00fffff;
/* s = 5*r */
key->precomputed_s.r[0] = key->key.r[1] * 5;
key->precomputed_s.r[1] = key->key.r[2] * 5;
key->precomputed_s.r[2] = key->key.r[3] * 5;
key->precomputed_s.r[3] = key->key.r[4] * 5;
}
EXPORT_SYMBOL(poly1305_core_setkey);
void poly1305_core_blocks(struct poly1305_state *state,
const struct poly1305_core_key *key, const void *src,
unsigned int nblocks, u32 hibit)
{
const u8 *input = src;
u32 r0, r1, r2, r3, r4;
u32 s1, s2, s3, s4;
u32 h0, h1, h2, h3, h4;
u64 d0, d1, d2, d3, d4;
u32 c;
if (!nblocks)
return;
hibit <<= 24;
r0 = key->key.r[0];
r1 = key->key.r[1];
r2 = key->key.r[2];
r3 = key->key.r[3];
r4 = key->key.r[4];
s1 = key->precomputed_s.r[0];
s2 = key->precomputed_s.r[1];
s3 = key->precomputed_s.r[2];
s4 = key->precomputed_s.r[3];
h0 = state->h[0];
h1 = state->h[1];
h2 = state->h[2];
h3 = state->h[3];
h4 = state->h[4];
do {
/* h += m[i] */
h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff;
h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff;
h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff;
h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff;
h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit;
/* h *= r */
d0 = ((u64)h0 * r0) + ((u64)h1 * s4) +
((u64)h2 * s3) + ((u64)h3 * s2) +
((u64)h4 * s1);
d1 = ((u64)h0 * r1) + ((u64)h1 * r0) +
((u64)h2 * s4) + ((u64)h3 * s3) +
((u64)h4 * s2);
d2 = ((u64)h0 * r2) + ((u64)h1 * r1) +
((u64)h2 * r0) + ((u64)h3 * s4) +
((u64)h4 * s3);
d3 = ((u64)h0 * r3) + ((u64)h1 * r2) +
((u64)h2 * r1) + ((u64)h3 * r0) +
((u64)h4 * s4);
d4 = ((u64)h0 * r4) + ((u64)h1 * r3) +
((u64)h2 * r2) + ((u64)h3 * r1) +
((u64)h4 * r0);
/* (partial) h %= p */
c = (u32)(d0 >> 26);
h0 = (u32)d0 & 0x3ffffff;
d1 += c;
c = (u32)(d1 >> 26);
h1 = (u32)d1 & 0x3ffffff;
d2 += c;
c = (u32)(d2 >> 26);
h2 = (u32)d2 & 0x3ffffff;
d3 += c;
c = (u32)(d3 >> 26);
h3 = (u32)d3 & 0x3ffffff;
d4 += c;
c = (u32)(d4 >> 26);
h4 = (u32)d4 & 0x3ffffff;
h0 += c * 5;
c = (h0 >> 26);
h0 = h0 & 0x3ffffff;
h1 += c;
input += POLY1305_BLOCK_SIZE;
} while (--nblocks);
state->h[0] = h0;
state->h[1] = h1;
state->h[2] = h2;
state->h[3] = h3;
state->h[4] = h4;
}
EXPORT_SYMBOL(poly1305_core_blocks);
void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
void *dst)
{
u8 *mac = dst;
u32 h0, h1, h2, h3, h4, c;
u32 g0, g1, g2, g3, g4;
u64 f;
u32 mask;
/* fully carry h */
h0 = state->h[0];
h1 = state->h[1];
h2 = state->h[2];
h3 = state->h[3];
h4 = state->h[4];
c = h1 >> 26;
h1 = h1 & 0x3ffffff;
h2 += c;
c = h2 >> 26;
h2 = h2 & 0x3ffffff;
h3 += c;
c = h3 >> 26;
h3 = h3 & 0x3ffffff;
h4 += c;
c = h4 >> 26;
h4 = h4 & 0x3ffffff;
h0 += c * 5;
c = h0 >> 26;
h0 = h0 & 0x3ffffff;
h1 += c;
/* compute h + -p */
g0 = h0 + 5;
c = g0 >> 26;
g0 &= 0x3ffffff;
g1 = h1 + c;
c = g1 >> 26;
g1 &= 0x3ffffff;
g2 = h2 + c;
c = g2 >> 26;
g2 &= 0x3ffffff;
g3 = h3 + c;
c = g3 >> 26;
g3 &= 0x3ffffff;
g4 = h4 + c - (1UL << 26);
/* select h if h < p, or h + -p if h >= p */
mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1;
g0 &= mask;
g1 &= mask;
g2 &= mask;
g3 &= mask;
g4 &= mask;
mask = ~mask;
h0 = (h0 & mask) | g0;
h1 = (h1 & mask) | g1;
h2 = (h2 & mask) | g2;
h3 = (h3 & mask) | g3;
h4 = (h4 & mask) | g4;
/* h = h % (2^128) */
h0 = ((h0) | (h1 << 26)) & 0xffffffff;
h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff;
h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff;
h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff;
if (likely(nonce)) {
/* mac = (h + nonce) % (2^128) */
f = (u64)h0 + nonce[0];
h0 = (u32)f;
f = (u64)h1 + nonce[1] + (f >> 32);
h1 = (u32)f;
f = (u64)h2 + nonce[2] + (f >> 32);
h2 = (u32)f;
f = (u64)h3 + nonce[3] + (f >> 32);
h3 = (u32)f;
}
put_unaligned_le32(h0, &mac[0]);
put_unaligned_le32(h1, &mac[4]);
put_unaligned_le32(h2, &mac[8]);
put_unaligned_le32(h3, &mac[12]);
}
EXPORT_SYMBOL(poly1305_core_emit);
lib/crypto/poly1305-donna64.c
+185
View File
@@ -0,0 +1,185 @@
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
*
* This is based in part on Andrew Moon's poly1305-donna, which is in the
* public domain.
*/
#include <linux/kernel.h>
#include <asm/unaligned.h>
#include <crypto/internal/poly1305.h>
typedef __uint128_t u128;
void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16])
{
u64 t0, t1;
/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
t0 = get_unaligned_le64(&raw_key[0]);
t1 = get_unaligned_le64(&raw_key[8]);
key->key.r64[0] = t0 & 0xffc0fffffffULL;
key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
/* s = 20*r */
key->precomputed_s.r64[0] = key->key.r64[1] * 20;
key->precomputed_s.r64[1] = key->key.r64[2] * 20;
}
EXPORT_SYMBOL(poly1305_core_setkey);
void poly1305_core_blocks(struct poly1305_state *state,
const struct poly1305_core_key *key, const void *src,
unsigned int nblocks, u32 hibit)
{
const u8 *input = src;
u64 hibit64;
u64 r0, r1, r2;
u64 s1, s2;
u64 h0, h1, h2;
u64 c;
u128 d0, d1, d2, d;
if (!nblocks)
return;
hibit64 = ((u64)hibit) << 40;
r0 = key->key.r64[0];
r1 = key->key.r64[1];
r2 = key->key.r64[2];
h0 = state->h64[0];
h1 = state->h64[1];
h2 = state->h64[2];
s1 = key->precomputed_s.r64[0];
s2 = key->precomputed_s.r64[1];
do {
u64 t0, t1;
/* h += m[i] */
t0 = get_unaligned_le64(&input[0]);
t1 = get_unaligned_le64(&input[8]);
h0 += t0 & 0xfffffffffffULL;
h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;
/* h *= r */
d0 = (u128)h0 * r0;
d = (u128)h1 * s2;
d0 += d;
d = (u128)h2 * s1;
d0 += d;
d1 = (u128)h0 * r1;
d = (u128)h1 * r0;
d1 += d;
d = (u128)h2 * s2;
d1 += d;
d2 = (u128)h0 * r2;
d = (u128)h1 * r1;
d2 += d;
d = (u128)h2 * r0;
d2 += d;
/* (partial) h %= p */
c = (u64)(d0 >> 44);
h0 = (u64)d0 & 0xfffffffffffULL;
d1 += c;
c = (u64)(d1 >> 44);
h1 = (u64)d1 & 0xfffffffffffULL;
d2 += c;
c = (u64)(d2 >> 42);
h2 = (u64)d2 & 0x3ffffffffffULL;
h0 += c * 5;
c = h0 >> 44;
h0 = h0 & 0xfffffffffffULL;
h1 += c;
input += POLY1305_BLOCK_SIZE;
} while (--nblocks);
state->h64[0] = h0;
state->h64[1] = h1;
state->h64[2] = h2;
}
EXPORT_SYMBOL(poly1305_core_blocks);
void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
void *dst)
{
u8 *mac = dst;
u64 h0, h1, h2, c;
u64 g0, g1, g2;
u64 t0, t1;
/* fully carry h */
h0 = state->h64[0];
h1 = state->h64[1];
h2 = state->h64[2];
c = h1 >> 44;
h1 &= 0xfffffffffffULL;
h2 += c;
c = h2 >> 42;
h2 &= 0x3ffffffffffULL;
h0 += c * 5;
c = h0 >> 44;
h0 &= 0xfffffffffffULL;
h1 += c;
c = h1 >> 44;
h1 &= 0xfffffffffffULL;
h2 += c;
c = h2 >> 42;
h2 &= 0x3ffffffffffULL;
h0 += c * 5;
c = h0 >> 44;
h0 &= 0xfffffffffffULL;
h1 += c;
/* compute h + -p */
g0 = h0 + 5;
c = g0 >> 44;
g0 &= 0xfffffffffffULL;
g1 = h1 + c;
c = g1 >> 44;
g1 &= 0xfffffffffffULL;
g2 = h2 + c - (1ULL << 42);
/* select h if h < p, or h + -p if h >= p */
c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
g0 &= c;
g1 &= c;
g2 &= c;
c = ~c;
h0 = (h0 & c) | g0;
h1 = (h1 & c) | g1;
h2 = (h2 & c) | g2;
if (likely(nonce)) {
/* h = (h + nonce) */
t0 = ((u64)nonce[1] << 32) | nonce[0];
t1 = ((u64)nonce[3] << 32) | nonce[2];
h0 += t0 & 0xfffffffffffULL;
c = h0 >> 44;
h0 &= 0xfffffffffffULL;
h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
c = h1 >> 44;
h1 &= 0xfffffffffffULL;
h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
h2 &= 0x3ffffffffffULL;
}
/* mac = h % (2^128) */
h0 = h0 | (h1 << 44);
h1 = (h1 >> 20) | (h2 << 24);
put_unaligned_le64(h0, &mac[0]);
put_unaligned_le64(h1, &mac[8]);
}
EXPORT_SYMBOL(poly1305_core_emit);
lib/crypto/poly1305.c
+77
View File
@@ -0,0 +1,77 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Poly1305 authenticator algorithm, RFC7539
*
* Copyright (C) 2015 Martin Willi
*
* Based on public domain code by Andrew Moon and Daniel J. Bernstein.
*/
#include <crypto/internal/poly1305.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/unaligned.h>
void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key)
{
poly1305_core_setkey(&desc->core_r, key);
desc->s[0] = get_unaligned_le32(key + 16);
desc->s[1] = get_unaligned_le32(key + 20);
desc->s[2] = get_unaligned_le32(key + 24);
desc->s[3] = get_unaligned_le32(key + 28);
poly1305_core_init(&desc->h);
desc->buflen = 0;
desc->sset = true;
desc->rset = 2;
}
EXPORT_SYMBOL_GPL(poly1305_init_generic);
void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src,
unsigned int nbytes)
{
unsigned int bytes;
if (unlikely(desc->buflen)) {
bytes = min(nbytes, POLY1305_BLOCK_SIZE - desc->buflen);
memcpy(desc->buf + desc->buflen, src, bytes);
src += bytes;
nbytes -= bytes;
desc->buflen += bytes;
if (desc->buflen == POLY1305_BLOCK_SIZE) {
poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf,
1, 1);
desc->buflen = 0;
}
}
if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
poly1305_core_blocks(&desc->h, &desc->core_r, src,
nbytes / POLY1305_BLOCK_SIZE, 1);
src += nbytes - (nbytes % POLY1305_BLOCK_SIZE);
nbytes %= POLY1305_BLOCK_SIZE;
}
if (unlikely(nbytes)) {
desc->buflen = nbytes;
memcpy(desc->buf, src, nbytes);
}
}
EXPORT_SYMBOL_GPL(poly1305_update_generic);
void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst)
{
if (unlikely(desc->buflen)) {
desc->buf[desc->buflen++] = 1;
memset(desc->buf + desc->buflen, 0,
POLY1305_BLOCK_SIZE - desc->buflen);
poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, 1, 0);
}
poly1305_core_emit(&desc->h, desc->s, dst);
*desc = (struct poly1305_desc_ctx){};
}
EXPORT_SYMBOL_GPL(poly1305_final_generic);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
lib/ctype.c
+38
View File
@@ -0,0 +1,38 @@
// SPDX-License-Identifier: GPL-2.0
/*
* linux/lib/ctype.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
#include <linux/ctype.h>
#include <linux/compiler.h>
#include <linux/export.h>
const unsigned char _ctype[] = {
_C,_C,_C,_C,_C,_C,_C,_C, /* 0-7 */
_C,_C|_S,_C|_S,_C|_S,_C|_S,_C|_S,_C,_C, /* 8-15 */
_C,_C,_C,_C,_C,_C,_C,_C, /* 16-23 */
_C,_C,_C,_C,_C,_C,_C,_C, /* 24-31 */
_S|_SP,_P,_P,_P,_P,_P,_P,_P, /* 32-39 */
_P,_P,_P,_P,_P,_P,_P,_P, /* 40-47 */
_D,_D,_D,_D,_D,_D,_D,_D, /* 48-55 */
_D,_D,_P,_P,_P,_P,_P,_P, /* 56-63 */
_P,_U|_X,_U|_X,_U|_X,_U|_X,_U|_X,_U|_X,_U, /* 64-71 */
_U,_U,_U,_U,_U,_U,_U,_U, /* 72-79 */
_U,_U,_U,_U,_U,_U,_U,_U, /* 80-87 */
_U,_U,_U,_P,_P,_P,_P,_P, /* 88-95 */
_P,_L|_X,_L|_X,_L|_X,_L|_X,_L|_X,_L|_X,_L, /* 96-103 */
_L,_L,_L,_L,_L,_L,_L,_L, /* 104-111 */
_L,_L,_L,_L,_L,_L,_L,_L, /* 112-119 */
_L,_L,_L,_P,_P,_P,_P,_C, /* 120-127 */
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 128-143 */
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 144-159 */
_S|_SP,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P, /* 160-175 */
_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P, /* 176-191 */
_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U, /* 192-207 */
_U,_U,_U,_U,_U,_U,_U,_P,_U,_U,_U,_U,_U,_U,_U,_L, /* 208-223 */
_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L, /* 224-239 */
_L,_L,_L,_L,_L,_L,_L,_P,_L,_L,_L,_L,_L,_L,_L,_L}; /* 240-255 */
EXPORT_SYMBOL(_ctype);
lib/debug_info.c
+28
View File
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: GPL-2.0
/*
* This file exists solely to ensure debug information for some core
* data structures is included in the final image even for
* CONFIG_DEBUG_INFO_REDUCED. Please do not add actual code. However,
* adding appropriate #includes is fine.
*/
#include <stdarg.h>
#include <linux/cred.h>
#include <linux/crypto.h>
#include <linux/dcache.h>
#include <linux/device.h>
#include <linux/fs.h>
#include <linux/fscache-cache.h>
#include <linux/io.h>
#include <linux/kallsyms.h>
#include <linux/kernel.h>
#include <linux/kobject.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/net.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <net/addrconf.h>
#include <net/sock.h>
#include <net/tcp.h>
lib/debug_locks.c
+56
View File
@@ -0,0 +1,56 @@
/*
* lib/debug_locks.c
*
* Generic place for common debugging facilities for various locks:
* spinlocks, rwlocks, mutexes and rwsems.
*
* Started by Ingo Molnar:
*
* Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
*/
#include <linux/rwsem.h>
#include <linux/mutex.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/debug_locks.h>
#ifdef CONFIG_LOCKDEP
#include <linux/sched/clock.h>
unsigned long long debug_locks_off_ts;
#endif
/*
* We want to turn all lock-debugging facilities on/off at once,
* via a global flag. The reason is that once a single bug has been
* detected and reported, there might be cascade of followup bugs
* that would just muddy the log. So we report the first one and
* shut up after that.
*/
int debug_locks = 1;
EXPORT_SYMBOL_GPL(debug_locks);
/*
* The locking-testsuite uses <debug_locks_silent> to get a
* 'silent failure': nothing is printed to the console when
* a locking bug is detected.
*/
int debug_locks_silent;
EXPORT_SYMBOL_GPL(debug_locks_silent);
/*
* Generic 'turn off all lock debugging' function:
*/
int debug_locks_off(void)
{
if (debug_locks && __debug_locks_off()) {
#ifdef CONFIG_LOCKDEP
debug_locks_off_ts = sched_clock();
#endif
if (!debug_locks_silent) {
console_verbose();
return 1;
}
}
return 0;
}
EXPORT_SYMBOL_GPL(debug_locks_off);
lib/debugobjects.c
+1201
View File
File diff suppressed because it is too large. Load diff
lib/dec_and_lock.c
+51
View File
@@ -0,0 +1,51 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/atomic.h>
/*
* This is an implementation of the notion of "decrement a
* reference count, and return locked if it decremented to zero".
*
* NOTE NOTE NOTE! This is _not_ equivalent to
*
* if (atomic_dec_and_test(&atomic)) {
* spin_lock(&lock);
* return 1;
* }
* return 0;
*
* because the spin-lock and the decrement must be
* "atomic".
*/
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
{
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
if (atomic_add_unless(atomic, -1, 1))
return 0;
/* Otherwise do it the slow way */
spin_lock(lock);
if (atomic_dec_and_test(atomic))
return 1;
spin_unlock(lock);
return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_lock);
int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
unsigned long *flags)
{
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
if (atomic_add_unless(atomic, -1, 1))
return 0;
/* Otherwise do it the slow way */
spin_lock_irqsave(lock, *flags);
if (atomic_dec_and_test(atomic))
return 1;
spin_unlock_irqrestore(lock, *flags);
return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_lock_irqsave);
lib/decompress.c
+79
View File
@@ -0,0 +1,79 @@
// SPDX-License-Identifier: GPL-2.0
/*
* decompress.c
*
* Detect the decompression method based on magic number
*/
#include <linux/decompress/generic.h>
#include <linux/decompress/bunzip2.h>
#include <linux/decompress/unlzma.h>
#include <linux/decompress/unxz.h>
#include <linux/decompress/inflate.h>
#include <linux/decompress/unlzo.h>
#include <linux/decompress/unlz4.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/printk.h>
#ifndef CONFIG_DECOMPRESS_GZIP
# define gunzip NULL
#endif
#ifndef CONFIG_DECOMPRESS_BZIP2
# define bunzip2 NULL
#endif
#ifndef CONFIG_DECOMPRESS_LZMA
# define unlzma NULL
#endif
#ifndef CONFIG_DECOMPRESS_XZ
# define unxz NULL
#endif
#ifndef CONFIG_DECOMPRESS_LZO
# define unlzo NULL
#endif
#ifndef CONFIG_DECOMPRESS_LZ4
# define unlz4 NULL
#endif
struct compress_format {
unsigned char magic[2];
const char *name;
decompress_fn decompressor;
};
static const struct compress_format compressed_formats[] __initconst = {
{ {0x1f, 0x8b}, "gzip", gunzip },
{ {0x1f, 0x9e}, "gzip", gunzip },
{ {0x42, 0x5a}, "bzip2", bunzip2 },
{ {0x5d, 0x00}, "lzma", unlzma },
{ {0xfd, 0x37}, "xz", unxz },
{ {0x89, 0x4c}, "lzo", unlzo },
{ {0x02, 0x21}, "lz4", unlz4 },
{ {0, 0}, NULL, NULL }
};
decompress_fn __init decompress_method(const unsigned char *inbuf, long len,
const char **name)
{
const struct compress_format *cf;
if (len < 2) {
if (name)
*name = NULL;
return NULL; /* Need at least this much... */
}
pr_debug("Compressed data magic: %#.2x %#.2x\n", inbuf[0], inbuf[1]);
for (cf = compressed_formats; cf->name; cf++) {
if (!memcmp(inbuf, cf->magic, 2))
break;
}
if (name)
*name = cf->name;
return cf->decompressor;
}
lib/decompress_bunzip2.c
+756
View File
@@ -0,0 +1,756 @@
/* Small bzip2 deflate implementation, by Rob Landley (rob@landley.net).
Based on bzip2 decompression code by Julian R Seward (jseward@acm.org),
which also acknowledges contributions by Mike Burrows, David Wheeler,
Peter Fenwick, Alistair Moffat, Radford Neal, Ian H. Witten,
Robert Sedgewick, and Jon L. Bentley.
This code is licensed under the LGPLv2:
LGPL (http://www.gnu.org/copyleft/lgpl.html
*/
/*
Size and speed optimizations by Manuel Novoa III (mjn3@codepoet.org).
More efficient reading of Huffman codes, a streamlined read_bunzip()
function, and various other tweaks. In (limited) tests, approximately
20% faster than bzcat on x86 and about 10% faster on arm.
Note that about 2/3 of the time is spent in read_unzip() reversing
the Burrows-Wheeler transformation. Much of that time is delay
resulting from cache misses.
I would ask that anyone benefiting from this work, especially those
using it in commercial products, consider making a donation to my local
non-profit hospice organization in the name of the woman I loved, who
passed away Feb. 12, 2003.
In memory of Toni W. Hagan
Hospice of Acadiana, Inc.
2600 Johnston St., Suite 200
Lafayette, LA 70503-3240
Phone (337) 232-1234 or 1-800-738-2226
Fax (337) 232-1297
http://www.hospiceacadiana.com/
Manuel
*/
/*
Made it fit for running in Linux Kernel by Alain Knaff (alain@knaff.lu)
*/
#ifdef STATIC
#define PREBOOT
#else
#include <linux/decompress/bunzip2.h>
#endif /* STATIC */
#include <linux/decompress/mm.h>
#include <linux/crc32poly.h>
#ifndef INT_MAX
#define INT_MAX 0x7fffffff
#endif
/* Constants for Huffman coding */
#define MAX_GROUPS 6
#define GROUP_SIZE 50 /* 64 would have been more efficient */
#define MAX_HUFCODE_BITS 20 /* Longest Huffman code allowed */
#define MAX_SYMBOLS 258 /* 256 literals + RUNA + RUNB */
#define SYMBOL_RUNA 0
#define SYMBOL_RUNB 1
/* Status return values */
#define RETVAL_OK 0
#define RETVAL_LAST_BLOCK (-1)
#define RETVAL_NOT_BZIP_DATA (-2)
#define RETVAL_UNEXPECTED_INPUT_EOF (-3)
#define RETVAL_UNEXPECTED_OUTPUT_EOF (-4)
#define RETVAL_DATA_ERROR (-5)
#define RETVAL_OUT_OF_MEMORY (-6)
#define RETVAL_OBSOLETE_INPUT (-7)
/* Other housekeeping constants */
#define BZIP2_IOBUF_SIZE 4096
/* This is what we know about each Huffman coding group */
struct group_data {
/* We have an extra slot at the end of limit[] for a sentinal value. */
int limit[MAX_HUFCODE_BITS+1];
int base[MAX_HUFCODE_BITS];
int permute[MAX_SYMBOLS];
int minLen, maxLen;
};
/* Structure holding all the housekeeping data, including IO buffers and
memory that persists between calls to bunzip */
struct bunzip_data {
/* State for interrupting output loop */
int writeCopies, writePos, writeRunCountdown, writeCount, writeCurrent;
/* I/O tracking data (file handles, buffers, positions, etc.) */
long (*fill)(void*, unsigned long);
long inbufCount, inbufPos /*, outbufPos*/;
unsigned char *inbuf /*,*outbuf*/;
unsigned int inbufBitCount, inbufBits;
/* The CRC values stored in the block header and calculated from the
data */
unsigned int crc32Table[256], headerCRC, totalCRC, writeCRC;
/* Intermediate buffer and its size (in bytes) */
unsigned int *dbuf, dbufSize;
/* These things are a bit too big to go on the stack */
unsigned char selectors[32768]; /* nSelectors = 15 bits */
struct group_data groups[MAX_GROUPS]; /* Huffman coding tables */
int io_error; /* non-zero if we have IO error */
int byteCount[256];
unsigned char symToByte[256], mtfSymbol[256];
};
/* Return the next nnn bits of input. All reads from the compressed input
are done through this function. All reads are big endian */
static unsigned int INIT get_bits(struct bunzip_data *bd, char bits_wanted)
{
unsigned int bits = 0;
/* If we need to get more data from the byte buffer, do so.
(Loop getting one byte at a time to enforce endianness and avoid
unaligned access.) */
while (bd->inbufBitCount < bits_wanted) {
/* If we need to read more data from file into byte buffer, do
so */
if (bd->inbufPos == bd->inbufCount) {
if (bd->io_error)
return 0;
bd->inbufCount = bd->fill(bd->inbuf, BZIP2_IOBUF_SIZE);
if (bd->inbufCount <= 0) {
bd->io_error = RETVAL_UNEXPECTED_INPUT_EOF;
return 0;
}
bd->inbufPos = 0;
}
/* Avoid 32-bit overflow (dump bit buffer to top of output) */
if (bd->inbufBitCount >= 24) {
bits = bd->inbufBits&((1 << bd->inbufBitCount)-1);
bits_wanted -= bd->inbufBitCount;
bits <<= bits_wanted;
bd->inbufBitCount = 0;
}
/* Grab next 8 bits of input from buffer. */
bd->inbufBits = (bd->inbufBits << 8)|bd->inbuf[bd->inbufPos++];
bd->inbufBitCount += 8;
}
/* Calculate result */
bd->inbufBitCount -= bits_wanted;
bits |= (bd->inbufBits >> bd->inbufBitCount)&((1 << bits_wanted)-1);
return bits;
}
/* Unpacks the next block and sets up for the inverse burrows-wheeler step. */
static int INIT get_next_block(struct bunzip_data *bd)
{
struct group_data *hufGroup = NULL;
int *base = NULL;
int *limit = NULL;
int dbufCount, nextSym, dbufSize, groupCount, selector,
i, j, k, t, runPos, symCount, symTotal, nSelectors, *byteCount;
unsigned char uc, *symToByte, *mtfSymbol, *selectors;
unsigned int *dbuf, origPtr;
dbuf = bd->dbuf;
dbufSize = bd->dbufSize;
selectors = bd->selectors;
byteCount = bd->byteCount;
symToByte = bd->symToByte;
mtfSymbol = bd->mtfSymbol;
/* Read in header signature and CRC, then validate signature.
(last block signature means CRC is for whole file, return now) */
i = get_bits(bd, 24);
j = get_bits(bd, 24);
bd->headerCRC = get_bits(bd, 32);
if ((i == 0x177245) && (j == 0x385090))
return RETVAL_LAST_BLOCK;
if ((i != 0x314159) || (j != 0x265359))
return RETVAL_NOT_BZIP_DATA;
/* We can add support for blockRandomised if anybody complains.
There was some code for this in busybox 1.0.0-pre3, but nobody ever
noticed that it didn't actually work. */
if (get_bits(bd, 1))
return RETVAL_OBSOLETE_INPUT;
origPtr = get_bits(bd, 24);
if (origPtr >= dbufSize)
return RETVAL_DATA_ERROR;
/* mapping table: if some byte values are never used (encoding things
like ascii text), the compression code removes the gaps to have fewer
symbols to deal with, and writes a sparse bitfield indicating which
values were present. We make a translation table to convert the
symbols back to the corresponding bytes. */
t = get_bits(bd, 16);
symTotal = 0;
for (i = 0; i < 16; i++) {
if (t&(1 << (15-i))) {
k = get_bits(bd, 16);
for (j = 0; j < 16; j++)
if (k&(1 << (15-j)))
symToByte[symTotal++] = (16*i)+j;
}
}
/* How many different Huffman coding groups does this block use? */
groupCount = get_bits(bd, 3);
if (groupCount < 2 || groupCount > MAX_GROUPS)
return RETVAL_DATA_ERROR;
/* nSelectors: Every GROUP_SIZE many symbols we select a new
Huffman coding group. Read in the group selector list,
which is stored as MTF encoded bit runs. (MTF = Move To
Front, as each value is used it's moved to the start of the
list.) */
nSelectors = get_bits(bd, 15);
if (!nSelectors)
return RETVAL_DATA_ERROR;
for (i = 0; i < groupCount; i++)
mtfSymbol[i] = i;
for (i = 0; i < nSelectors; i++) {
/* Get next value */
for (j = 0; get_bits(bd, 1); j++)
if (j >= groupCount)
return RETVAL_DATA_ERROR;
/* Decode MTF to get the next selector */
uc = mtfSymbol[j];
for (; j; j--)
mtfSymbol[j] = mtfSymbol[j-1];
mtfSymbol[0] = selectors[i] = uc;
}
/* Read the Huffman coding tables for each group, which code
for symTotal literal symbols, plus two run symbols (RUNA,
RUNB) */
symCount = symTotal+2;
for (j = 0; j < groupCount; j++) {
unsigned char length[MAX_SYMBOLS], temp[MAX_HUFCODE_BITS+1];
int minLen, maxLen, pp;
/* Read Huffman code lengths for each symbol. They're
stored in a way similar to mtf; record a starting
value for the first symbol, and an offset from the
previous value for everys symbol after that.
(Subtracting 1 before the loop and then adding it
back at the end is an optimization that makes the
test inside the loop simpler: symbol length 0
becomes negative, so an unsigned inequality catches
it.) */
t = get_bits(bd, 5)-1;
for (i = 0; i < symCount; i++) {
for (;;) {
if (((unsigned)t) > (MAX_HUFCODE_BITS-1))
return RETVAL_DATA_ERROR;
/* If first bit is 0, stop. Else
second bit indicates whether to
increment or decrement the value.
Optimization: grab 2 bits and unget
the second if the first was 0. */
k = get_bits(bd, 2);
if (k < 2) {
bd->inbufBitCount++;
break;
}
/* Add one if second bit 1, else
* subtract 1. Avoids if/else */
t += (((k+1)&2)-1);
}
/* Correct for the initial -1, to get the
* final symbol length */
length[i] = t+1;
}
/* Find largest and smallest lengths in this group */
minLen = maxLen = length[0];
for (i = 1; i < symCount; i++) {
if (length[i] > maxLen)
maxLen = length[i];
else if (length[i] < minLen)
minLen = length[i];
}
/* Calculate permute[], base[], and limit[] tables from
* length[].
*
* permute[] is the lookup table for converting
* Huffman coded symbols into decoded symbols. base[]
* is the amount to subtract from the value of a
* Huffman symbol of a given length when using
* permute[].
*
* limit[] indicates the largest numerical value a
* symbol with a given number of bits can have. This
* is how the Huffman codes can vary in length: each
* code with a value > limit[length] needs another
* bit.
*/
hufGroup = bd->groups+j;
hufGroup->minLen = minLen;
hufGroup->maxLen = maxLen;
/* Note that minLen can't be smaller than 1, so we
adjust the base and limit array pointers so we're
not always wasting the first entry. We do this
again when using them (during symbol decoding).*/
base = hufGroup->base-1;
limit = hufGroup->limit-1;
/* Calculate permute[]. Concurrently, initialize
* temp[] and limit[]. */
pp = 0;
for (i = minLen; i <= maxLen; i++) {
temp[i] = limit[i] = 0;
for (t = 0; t < symCount; t++)
if (length[t] == i)
hufGroup->permute[pp++] = t;
}
/* Count symbols coded for at each bit length */
for (i = 0; i < symCount; i++)
temp[length[i]]++;
/* Calculate limit[] (the largest symbol-coding value
*at each bit length, which is (previous limit <<
*1)+symbols at this level), and base[] (number of
*symbols to ignore at each bit length, which is limit
*minus the cumulative count of symbols coded for
*already). */
pp = t = 0;
for (i = minLen; i < maxLen; i++) {
pp += temp[i];
/* We read the largest possible symbol size
and then unget bits after determining how
many we need, and those extra bits could be
set to anything. (They're noise from
future symbols.) At each level we're
really only interested in the first few
bits, so here we set all the trailing
to-be-ignored bits to 1 so they don't
affect the value > limit[length]
comparison. */
limit[i] = (pp << (maxLen - i)) - 1;
pp <<= 1;
base[i+1] = pp-(t += temp[i]);
}
limit[maxLen+1] = INT_MAX; /* Sentinal value for
* reading next sym. */
limit[maxLen] = pp+temp[maxLen]-1;
base[minLen] = 0;
}
/* We've finished reading and digesting the block header. Now
read this block's Huffman coded symbols from the file and
undo the Huffman coding and run length encoding, saving the
result into dbuf[dbufCount++] = uc */
/* Initialize symbol occurrence counters and symbol Move To
* Front table */
for (i = 0; i < 256; i++) {
byteCount[i] = 0;
mtfSymbol[i] = (unsigned char)i;
}
/* Loop through compressed symbols. */
runPos = dbufCount = symCount = selector = 0;
for (;;) {
/* Determine which Huffman coding group to use. */
if (!(symCount--)) {
symCount = GROUP_SIZE-1;
if (selector >= nSelectors)
return RETVAL_DATA_ERROR;
hufGroup = bd->groups+selectors[selector++];
base = hufGroup->base-1;
limit = hufGroup->limit-1;
}
/* Read next Huffman-coded symbol. */
/* Note: It is far cheaper to read maxLen bits and
back up than it is to read minLen bits and then an
additional bit at a time, testing as we go.
Because there is a trailing last block (with file
CRC), there is no danger of the overread causing an
unexpected EOF for a valid compressed file. As a
further optimization, we do the read inline
(falling back to a call to get_bits if the buffer
runs dry). The following (up to got_huff_bits:) is
equivalent to j = get_bits(bd, hufGroup->maxLen);
*/
while (bd->inbufBitCount < hufGroup->maxLen) {
if (bd->inbufPos == bd->inbufCount) {
j = get_bits(bd, hufGroup->maxLen);
goto got_huff_bits;
}
bd->inbufBits =
(bd->inbufBits << 8)|bd->inbuf[bd->inbufPos++];
bd->inbufBitCount += 8;
};
bd->inbufBitCount -= hufGroup->maxLen;
j = (bd->inbufBits >> bd->inbufBitCount)&
((1 << hufGroup->maxLen)-1);
got_huff_bits:
/* Figure how how many bits are in next symbol and
* unget extras */
i = hufGroup->minLen;
while (j > limit[i])
++i;
bd->inbufBitCount += (hufGroup->maxLen - i);
/* Huffman decode value to get nextSym (with bounds checking) */
if ((i > hufGroup->maxLen)
|| (((unsigned)(j = (j>>(hufGroup->maxLen-i))-base[i]))
>= MAX_SYMBOLS))
return RETVAL_DATA_ERROR;
nextSym = hufGroup->permute[j];
/* We have now decoded the symbol, which indicates
either a new literal byte, or a repeated run of the
most recent literal byte. First, check if nextSym
indicates a repeated run, and if so loop collecting
how many times to repeat the last literal. */
if (((unsigned)nextSym) <= SYMBOL_RUNB) { /* RUNA or RUNB */
/* If this is the start of a new run, zero out
* counter */
if (!runPos) {
runPos = 1;
t = 0;
}
/* Neat trick that saves 1 symbol: instead of
or-ing 0 or 1 at each bit position, add 1
or 2 instead. For example, 1011 is 1 << 0
+ 1 << 1 + 2 << 2. 1010 is 2 << 0 + 2 << 1
+ 1 << 2. You can make any bit pattern
that way using 1 less symbol than the basic
or 0/1 method (except all bits 0, which
would use no symbols, but a run of length 0
doesn't mean anything in this context).
Thus space is saved. */
t += (runPos << nextSym);
/* +runPos if RUNA; +2*runPos if RUNB */
runPos <<= 1;
continue;
}
/* When we hit the first non-run symbol after a run,
we now know how many times to repeat the last
literal, so append that many copies to our buffer
of decoded symbols (dbuf) now. (The last literal
used is the one at the head of the mtfSymbol
array.) */
if (runPos) {
runPos = 0;
if (dbufCount+t >= dbufSize)
return RETVAL_DATA_ERROR;
uc = symToByte[mtfSymbol[0]];
byteCount[uc] += t;
while (t--)
dbuf[dbufCount++] = uc;
}
/* Is this the terminating symbol? */
if (nextSym > symTotal)
break;
/* At this point, nextSym indicates a new literal
character. Subtract one to get the position in the
MTF array at which this literal is currently to be
found. (Note that the result can't be -1 or 0,
because 0 and 1 are RUNA and RUNB. But another
instance of the first symbol in the mtf array,
position 0, would have been handled as part of a
run above. Therefore 1 unused mtf position minus 2
non-literal nextSym values equals -1.) */
if (dbufCount >= dbufSize)
return RETVAL_DATA_ERROR;
i = nextSym - 1;
uc = mtfSymbol[i];
/* Adjust the MTF array. Since we typically expect to
*move only a small number of symbols, and are bound
*by 256 in any case, using memmove here would
*typically be bigger and slower due to function call
*overhead and other assorted setup costs. */
do {
mtfSymbol[i] = mtfSymbol[i-1];
} while (--i);
mtfSymbol[0] = uc;
uc = symToByte[uc];
/* We have our literal byte. Save it into dbuf. */
byteCount[uc]++;
dbuf[dbufCount++] = (unsigned int)uc;
}
/* At this point, we've read all the Huffman-coded symbols
(and repeated runs) for this block from the input stream,
and decoded them into the intermediate buffer. There are
dbufCount many decoded bytes in dbuf[]. Now undo the
Burrows-Wheeler transform on dbuf. See
http://dogma.net/markn/articles/bwt/bwt.htm
*/
/* Turn byteCount into cumulative occurrence counts of 0 to n-1. */
j = 0;
for (i = 0; i < 256; i++) {
k = j+byteCount[i];
byteCount[i] = j;
j = k;
}
/* Figure out what order dbuf would be in if we sorted it. */
for (i = 0; i < dbufCount; i++) {
uc = (unsigned char)(dbuf[i] & 0xff);
dbuf[byteCount[uc]] |= (i << 8);
byteCount[uc]++;
}
/* Decode first byte by hand to initialize "previous" byte.
Note that it doesn't get output, and if the first three
characters are identical it doesn't qualify as a run (hence
writeRunCountdown = 5). */
if (dbufCount) {
if (origPtr >= dbufCount)
return RETVAL_DATA_ERROR;
bd->writePos = dbuf[origPtr];
bd->writeCurrent = (unsigned char)(bd->writePos&0xff);
bd->writePos >>= 8;
bd->writeRunCountdown = 5;
}
bd->writeCount = dbufCount;
return RETVAL_OK;
}
/* Undo burrows-wheeler transform on intermediate buffer to produce output.
If start_bunzip was initialized with out_fd =-1, then up to len bytes of
data are written to outbuf. Return value is number of bytes written or
error (all errors are negative numbers). If out_fd!=-1, outbuf and len
are ignored, data is written to out_fd and return is RETVAL_OK or error.
*/
static int INIT read_bunzip(struct bunzip_data *bd, char *outbuf, int len)
{
const unsigned int *dbuf;
int pos, xcurrent, previous, gotcount;
/* If last read was short due to end of file, return last block now */
if (bd->writeCount < 0)
return bd->writeCount;
gotcount = 0;
dbuf = bd->dbuf;
pos = bd->writePos;
xcurrent = bd->writeCurrent;
/* We will always have pending decoded data to write into the output
buffer unless this is the very first call (in which case we haven't
Huffman-decoded a block into the intermediate buffer yet). */
if (bd->writeCopies) {
/* Inside the loop, writeCopies means extra copies (beyond 1) */
--bd->writeCopies;
/* Loop outputting bytes */
for (;;) {
/* If the output buffer is full, snapshot
* state and return */
if (gotcount >= len) {
bd->writePos = pos;
bd->writeCurrent = xcurrent;
bd->writeCopies++;
return len;
}
/* Write next byte into output buffer, updating CRC */
outbuf[gotcount++] = xcurrent;
bd->writeCRC = (((bd->writeCRC) << 8)
^bd->crc32Table[((bd->writeCRC) >> 24)
^xcurrent]);
/* Loop now if we're outputting multiple
* copies of this byte */
if (bd->writeCopies) {
--bd->writeCopies;
continue;
}
decode_next_byte:
if (!bd->writeCount--)
break;
/* Follow sequence vector to undo
* Burrows-Wheeler transform */
previous = xcurrent;
pos = dbuf[pos];
xcurrent = pos&0xff;
pos >>= 8;
/* After 3 consecutive copies of the same
byte, the 4th is a repeat count. We count
down from 4 instead *of counting up because
testing for non-zero is faster */
if (--bd->writeRunCountdown) {
if (xcurrent != previous)
bd->writeRunCountdown = 4;
} else {
/* We have a repeated run, this byte
* indicates the count */
bd->writeCopies = xcurrent;
xcurrent = previous;
bd->writeRunCountdown = 5;
/* Sometimes there are just 3 bytes
* (run length 0) */
if (!bd->writeCopies)
goto decode_next_byte;
/* Subtract the 1 copy we'd output
* anyway to get extras */
--bd->writeCopies;
}
}
/* Decompression of this block completed successfully */
bd->writeCRC = ~bd->writeCRC;
bd->totalCRC = ((bd->totalCRC << 1) |
(bd->totalCRC >> 31)) ^ bd->writeCRC;
/* If this block had a CRC error, force file level CRC error. */
if (bd->writeCRC != bd->headerCRC) {
bd->totalCRC = bd->headerCRC+1;
return RETVAL_LAST_BLOCK;
}
}
/* Refill the intermediate buffer by Huffman-decoding next
* block of input */
/* (previous is just a convenient unused temp variable here) */
previous = get_next_block(bd);
if (previous) {
bd->writeCount = previous;
return (previous != RETVAL_LAST_BLOCK) ? previous : gotcount;
}
bd->writeCRC = 0xffffffffUL;
pos = bd->writePos;
xcurrent = bd->writeCurrent;
goto decode_next_byte;
}
static long INIT nofill(void *buf, unsigned long len)
{
return -1;
}
/* Allocate the structure, read file header. If in_fd ==-1, inbuf must contain
a complete bunzip file (len bytes long). If in_fd!=-1, inbuf and len are
ignored, and data is read from file handle into temporary buffer. */
static int INIT start_bunzip(struct bunzip_data **bdp, void *inbuf, long len,
long (*fill)(void*, unsigned long))
{
struct bunzip_data *bd;
unsigned int i, j, c;
const unsigned int BZh0 =
(((unsigned int)'B') << 24)+(((unsigned int)'Z') << 16)
+(((unsigned int)'h') << 8)+(unsigned int)'0';
/* Figure out how much data to allocate */
i = sizeof(struct bunzip_data);
/* Allocate bunzip_data. Most fields initialize to zero. */
bd = *bdp = malloc(i);
if (!bd)
return RETVAL_OUT_OF_MEMORY;
memset(bd, 0, sizeof(struct bunzip_data));
/* Setup input buffer */
bd->inbuf = inbuf;
bd->inbufCount = len;
if (fill != NULL)
bd->fill = fill;
else
bd->fill = nofill;
/* Init the CRC32 table (big endian) */
for (i = 0; i < 256; i++) {
c = i << 24;
for (j = 8; j; j--)
c = c&0x80000000 ? (c << 1)^(CRC32_POLY_BE) : (c << 1);
bd->crc32Table[i] = c;
}
/* Ensure that file starts with "BZh['1'-'9']." */
i = get_bits(bd, 32);
if (((unsigned int)(i-BZh0-1)) >= 9)
return RETVAL_NOT_BZIP_DATA;
/* Fourth byte (ascii '1'-'9'), indicates block size in units of 100k of
uncompressed data. Allocate intermediate buffer for block. */
bd->dbufSize = 100000*(i-BZh0);
bd->dbuf = large_malloc(bd->dbufSize * sizeof(int));
if (!bd->dbuf)
return RETVAL_OUT_OF_MEMORY;
return RETVAL_OK;
}
/* Example usage: decompress src_fd to dst_fd. (Stops at end of bzip2 data,
not end of file.) */
STATIC int INIT bunzip2(unsigned char *buf, long len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *outbuf,
long *pos,
void(*error)(char *x))
{
struct bunzip_data *bd;
int i = -1;
unsigned char *inbuf;
if (flush)
outbuf = malloc(BZIP2_IOBUF_SIZE);
if (!outbuf) {
error("Could not allocate output buffer");
return RETVAL_OUT_OF_MEMORY;
}
if (buf)
inbuf = buf;
else
inbuf = malloc(BZIP2_IOBUF_SIZE);
if (!inbuf) {
error("Could not allocate input buffer");
i = RETVAL_OUT_OF_MEMORY;
goto exit_0;
}
i = start_bunzip(&bd, inbuf, len, fill);
if (!i) {
for (;;) {
i = read_bunzip(bd, outbuf, BZIP2_IOBUF_SIZE);
if (i <= 0)
break;
if (!flush)
outbuf += i;
else
if (i != flush(outbuf, i)) {
i = RETVAL_UNEXPECTED_OUTPUT_EOF;
break;
}
}
}
/* Check CRC and release memory */
if (i == RETVAL_LAST_BLOCK) {
if (bd->headerCRC != bd->totalCRC)
error("Data integrity error when decompressing.");
else
i = RETVAL_OK;
} else if (i == RETVAL_UNEXPECTED_OUTPUT_EOF) {
error("Compressed file ends unexpectedly");
}
if (!bd)
goto exit_1;
if (bd->dbuf)
large_free(bd->dbuf);
if (pos)
*pos = bd->inbufPos;
free(bd);
exit_1:
if (!buf)
free(inbuf);
exit_0:
if (flush)
free(outbuf);
return i;
}
#ifdef PREBOOT
STATIC int INIT __decompress(unsigned char *buf, long len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *outbuf, long olen,
long *pos,
void (*error)(char *x))
{
return bunzip2(buf, len - 4, fill, flush, outbuf, pos, error);
}
#endif
lib/decompress_inflate.c
+206
View File
@@ -0,0 +1,206 @@
// SPDX-License-Identifier: GPL-2.0
#ifdef STATIC
#define PREBOOT
/* Pre-boot environment: included */
/* prevent inclusion of _LINUX_KERNEL_H in pre-boot environment: lots
* errors about console_printk etc... on ARM */
#define _LINUX_KERNEL_H
#include "zlib_inflate/inftrees.c"
#include "zlib_inflate/inffast.c"
#include "zlib_inflate/inflate.c"
#else /* STATIC */
/* initramfs et al: linked */
#include <linux/zutil.h>
#include "zlib_inflate/inftrees.h"
#include "zlib_inflate/inffast.h"
#include "zlib_inflate/inflate.h"
#include "zlib_inflate/infutil.h"
#include <linux/decompress/inflate.h>
#endif /* STATIC */
#include <linux/decompress/mm.h>
#define GZIP_IOBUF_SIZE (16*1024)
static long INIT nofill(void *buffer, unsigned long len)
{
return -1;
}
/* Included from initramfs et al code */
STATIC int INIT __gunzip(unsigned char *buf, long len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *out_buf, long out_len,
long *pos,
void(*error)(char *x)) {
u8 *zbuf;
struct z_stream_s *strm;
int rc;
rc = -1;
if (flush) {
out_len = 0x8000; /* 32 K */
out_buf = malloc(out_len);
} else {
if (!out_len)
out_len = ((size_t)~0) - (size_t)out_buf; /* no limit */
}
if (!out_buf) {
error("Out of memory while allocating output buffer");
goto gunzip_nomem1;
}
if (buf)
zbuf = buf;
else {
zbuf = malloc(GZIP_IOBUF_SIZE);
len = 0;
}
if (!zbuf) {
error("Out of memory while allocating input buffer");
goto gunzip_nomem2;
}
strm = malloc(sizeof(*strm));
if (strm == NULL) {
error("Out of memory while allocating z_stream");
goto gunzip_nomem3;
}
strm->workspace = malloc(flush ? zlib_inflate_workspacesize() :
sizeof(struct inflate_state));
if (strm->workspace == NULL) {
error("Out of memory while allocating workspace");
goto gunzip_nomem4;
}
if (!fill)
fill = nofill;
if (len == 0)
len = fill(zbuf, GZIP_IOBUF_SIZE);
/* verify the gzip header */
if (len < 10 ||
zbuf[0] != 0x1f || zbuf[1] != 0x8b || zbuf[2] != 0x08) {
if (pos)
*pos = 0;
error("Not a gzip file");
goto gunzip_5;
}
/* skip over gzip header (1f,8b,08... 10 bytes total +
* possible asciz filename)
*/
strm->next_in = zbuf + 10;
strm->avail_in = len - 10;
/* skip over asciz filename */
if (zbuf[3] & 0x8) {
do {
/*
* If the filename doesn't fit into the buffer,
* the file is very probably corrupt. Don't try
* to read more data.
*/
if (strm->avail_in == 0) {
error("header error");
goto gunzip_5;
}
--strm->avail_in;
} while (*strm->next_in++);
}
strm->next_out = out_buf;
strm->avail_out = out_len;
rc = zlib_inflateInit2(strm, -MAX_WBITS);
if (!flush) {
WS(strm)->inflate_state.wsize = 0;
WS(strm)->inflate_state.window = NULL;
}
while (rc == Z_OK) {
if (strm->avail_in == 0) {
/* TODO: handle case where both pos and fill are set */
len = fill(zbuf, GZIP_IOBUF_SIZE);
if (len < 0) {
rc = -1;
error("read error");
break;
}
strm->next_in = zbuf;
strm->avail_in = len;
}
rc = zlib_inflate(strm, 0);
/* Write any data generated */
if (flush && strm->next_out > out_buf) {
long l = strm->next_out - out_buf;
if (l != flush(out_buf, l)) {
rc = -1;
error("write error");
break;
}
strm->next_out = out_buf;
strm->avail_out = out_len;
}
/* after Z_FINISH, only Z_STREAM_END is "we unpacked it all" */
if (rc == Z_STREAM_END) {
rc = 0;
break;
} else if (rc != Z_OK) {
error("uncompression error");
rc = -1;
}
}
zlib_inflateEnd(strm);
if (pos)
/* add + 8 to skip over trailer */
*pos = strm->next_in - zbuf+8;
gunzip_5:
free(strm->workspace);
gunzip_nomem4:
free(strm);
gunzip_nomem3:
if (!buf)
free(zbuf);
gunzip_nomem2:
if (flush)
free(out_buf);
gunzip_nomem1:
return rc; /* returns Z_OK (0) if successful */
}
#ifndef PREBOOT
STATIC int INIT gunzip(unsigned char *buf, long len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *out_buf,
long *pos,
void (*error)(char *x))
{
return __gunzip(buf, len, fill, flush, out_buf, 0, pos, error);
}
#else
STATIC int INIT __decompress(unsigned char *buf, long len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *out_buf, long out_len,
long *pos,
void (*error)(char *x))
{
return __gunzip(buf, len, fill, flush, out_buf, out_len, pos, error);
}
#endif
lib/decompress_unlz4.c
+212
View File
@@ -0,0 +1,212 @@
/*
* Wrapper for decompressing LZ4-compressed kernel, initramfs, and initrd
*
* Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik.lee@lge.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifdef STATIC
#define PREBOOT
#include "lz4/lz4_decompress.c"
#else
#include <linux/decompress/unlz4.h>
#endif
#include <linux/types.h>
#include <linux/lz4.h>
#include <linux/decompress/mm.h>
#include <linux/compiler.h>
#include <asm/unaligned.h>
/*
* Note: Uncompressed chunk size is used in the compressor side
* (userspace side for compression).
* It is hardcoded because there is not proper way to extract it
* from the binary stream which is generated by the preliminary
* version of LZ4 tool so far.
*/
#define LZ4_DEFAULT_UNCOMPRESSED_CHUNK_SIZE (8 << 20)
#define ARCHIVE_MAGICNUMBER 0x184C2102
STATIC inline int INIT unlz4(u8 *input, long in_len,
long (*fill)(void *, unsigned long),
long (*flush)(void *, unsigned long),
u8 *output, long *posp,
void (*error) (char *x))
{
int ret = -1;
size_t chunksize = 0;
size_t uncomp_chunksize = LZ4_DEFAULT_UNCOMPRESSED_CHUNK_SIZE;
u8 *inp;
u8 *inp_start;
u8 *outp;
long size = in_len;
#ifdef PREBOOT
size_t out_len = get_unaligned_le32(input + in_len);
#endif
size_t dest_len;
if (output) {
outp = output;
} else if (!flush) {
error("NULL output pointer and no flush function provided");
goto exit_0;
} else {
outp = large_malloc(uncomp_chunksize);
if (!outp) {
error("Could not allocate output buffer");
goto exit_0;
}
}
if (input && fill) {
error("Both input pointer and fill function provided,");
goto exit_1;
} else if (input) {
inp = input;
} else if (!fill) {
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
}
}
inp_start = inp;
if (posp)
*posp = 0;
if (fill) {
size = fill(inp, 4);
if (size < 4) {
error("data corrupted");
goto exit_2;
}
}
chunksize = get_unaligned_le32(inp);
if (chunksize == ARCHIVE_MAGICNUMBER) {
if (!fill) {
inp += 4;
size -= 4;
}
} else {
error("invalid header");
goto exit_2;
}
if (posp)
*posp += 4;
for (;;) {
if (fill) {
size = fill(inp, 4);
if (size == 0)
break;
if (size < 4) {
error("data corrupted");
goto exit_2;
}
}
chunksize = get_unaligned_le32(inp);
if (chunksize == ARCHIVE_MAGICNUMBER) {
if (!fill) {
inp += 4;
size -= 4;
}
if (posp)
*posp += 4;
continue;
}
if (posp)
*posp += 4;
if (!fill) {
inp += 4;
size -= 4;
} else {
if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
size = fill(inp, chunksize);
if (size < chunksize) {
error("data corrupted");
goto exit_2;
}
}
#ifdef PREBOOT
if (out_len >= uncomp_chunksize) {
dest_len = uncomp_chunksize;
out_len -= dest_len;
} else
dest_len = out_len;
ret = LZ4_decompress_fast(inp, outp, dest_len);
chunksize = ret;
#else
dest_len = uncomp_chunksize;
ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
dest_len = ret;
#endif
if (ret < 0) {
error("Decoding failed");
goto exit_2;
}
ret = -1;
if (flush && flush(outp, dest_len) != dest_len)
goto exit_2;
if (output)
outp += dest_len;
if (posp)
*posp += chunksize;
if (!fill) {
size -= chunksize;
if (size == 0)
break;
else if (size < 0) {
error("data corrupted");
goto exit_2;
}
inp += chunksize;
}
}
ret = 0;
exit_2:
if (!input)
large_free(inp_start);
exit_1:
if (!output)
large_free(outp);
exit_0:
return ret;
}
#ifdef PREBOOT
STATIC int INIT __decompress(unsigned char *buf, long in_len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *output, long out_len,
long *posp,
void (*error)(char *x)
)
{
return unlz4(buf, in_len - 4, fill, flush, output, posp, error);
}
#endif
lib/decompress_unlzma.c
+679
View File
@@ -0,0 +1,679 @@
/* Lzma decompressor for Linux kernel. Shamelessly snarfed
*from busybox 1.1.1
*
*Linux kernel adaptation
*Copyright (C) 2006 Alain < alain@knaff.lu >
*
*Based on small lzma deflate implementation/Small range coder
*implementation for lzma.
*Copyright (C) 2006 Aurelien Jacobs < aurel@gnuage.org >
*
*Based on LzmaDecode.c from the LZMA SDK 4.22 (http://www.7-zip.org/)
*Copyright (C) 1999-2005 Igor Pavlov
*
*Copyrights of the parts, see headers below.
*
*
*This program is free software; you can redistribute it and/or
*modify it under the terms of the GNU Lesser General Public
*License as published by the Free Software Foundation; either
*version 2.1 of the License, or (at your option) any later version.
*
*This program is distributed in the hope that it will be useful,
*but WITHOUT ANY WARRANTY; without even the implied warranty of
*MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
*Lesser General Public License for more details.
*
*You should have received a copy of the GNU Lesser General Public
*License along with this library; if not, write to the Free Software
*Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifdef STATIC
#define PREBOOT
#else
#include <linux/decompress/unlzma.h>
#endif /* STATIC */
#include <linux/decompress/mm.h>
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
static long long INIT read_int(unsigned char *ptr, int size)
{
int i;
long long ret = 0;
for (i = 0; i < size; i++)
ret = (ret << 8) | ptr[size-i-1];
return ret;
}
#define ENDIAN_CONVERT(x) \
x = (typeof(x))read_int((unsigned char *)&x, sizeof(x))
/* Small range coder implementation for lzma.
*Copyright (C) 2006 Aurelien Jacobs < aurel@gnuage.org >
*
*Based on LzmaDecode.c from the LZMA SDK 4.22 (http://www.7-zip.org/)
*Copyright (c) 1999-2005 Igor Pavlov
*/
#include <linux/compiler.h>
#define LZMA_IOBUF_SIZE 0x10000
struct rc {
long (*fill)(void*, unsigned long);
uint8_t *ptr;
uint8_t *buffer;
uint8_t *buffer_end;
long buffer_size;
uint32_t code;
uint32_t range;
uint32_t bound;
void (*error)(char *);
};
#define RC_TOP_BITS 24
#define RC_MOVE_BITS 5
#define RC_MODEL_TOTAL_BITS 11
static long INIT nofill(void *buffer, unsigned long len)
{
return -1;
}
/* Called twice: once at startup and once in rc_normalize() */
static void INIT rc_read(struct rc *rc)
{
rc->buffer_size = rc->fill((char *)rc->buffer, LZMA_IOBUF_SIZE);
if (rc->buffer_size <= 0)
rc->error("unexpected EOF");
rc->ptr = rc->buffer;
rc->buffer_end = rc->buffer + rc->buffer_size;
}
/* Called once */
static inline void INIT rc_init(struct rc *rc,
long (*fill)(void*, unsigned long),
char *buffer, long buffer_size)
{
if (fill)
rc->fill = fill;
else
rc->fill = nofill;
rc->buffer = (uint8_t *)buffer;
rc->buffer_size = buffer_size;
rc->buffer_end = rc->buffer + rc->buffer_size;
rc->ptr = rc->buffer;
rc->code = 0;
rc->range = 0xFFFFFFFF;
}
static inline void INIT rc_init_code(struct rc *rc)
{
int i;
for (i = 0; i < 5; i++) {
if (rc->ptr >= rc->buffer_end)
rc_read(rc);
rc->code = (rc->code << 8) | *rc->ptr++;
}
}
/* Called twice, but one callsite is in inline'd rc_is_bit_0_helper() */
static void INIT rc_do_normalize(struct rc *rc)
{
if (rc->ptr >= rc->buffer_end)
rc_read(rc);
rc->range <<= 8;
rc->code = (rc->code << 8) | *rc->ptr++;
}
static inline void INIT rc_normalize(struct rc *rc)
{
if (rc->range < (1 << RC_TOP_BITS))
rc_do_normalize(rc);
}
/* Called 9 times */
/* Why rc_is_bit_0_helper exists?
*Because we want to always expose (rc->code < rc->bound) to optimizer
*/
static inline uint32_t INIT rc_is_bit_0_helper(struct rc *rc, uint16_t *p)
{
rc_normalize(rc);
rc->bound = *p * (rc->range >> RC_MODEL_TOTAL_BITS);
return rc->bound;
}
static inline int INIT rc_is_bit_0(struct rc *rc, uint16_t *p)
{
uint32_t t = rc_is_bit_0_helper(rc, p);
return rc->code < t;
}
/* Called ~10 times, but very small, thus inlined */
static inline void INIT rc_update_bit_0(struct rc *rc, uint16_t *p)
{
rc->range = rc->bound;
*p += ((1 << RC_MODEL_TOTAL_BITS) - *p) >> RC_MOVE_BITS;
}
static inline void INIT rc_update_bit_1(struct rc *rc, uint16_t *p)
{
rc->range -= rc->bound;
rc->code -= rc->bound;
*p -= *p >> RC_MOVE_BITS;
}
/* Called 4 times in unlzma loop */
static int INIT rc_get_bit(struct rc *rc, uint16_t *p, int *symbol)
{
if (rc_is_bit_0(rc, p)) {
rc_update_bit_0(rc, p);
*symbol *= 2;
return 0;
} else {
rc_update_bit_1(rc, p);
*symbol = *symbol * 2 + 1;
return 1;
}
}
/* Called once */
static inline int INIT rc_direct_bit(struct rc *rc)
{
rc_normalize(rc);
rc->range >>= 1;
if (rc->code >= rc->range) {
rc->code -= rc->range;
return 1;
}
return 0;
}
/* Called twice */
static inline void INIT
rc_bit_tree_decode(struct rc *rc, uint16_t *p, int num_levels, int *symbol)
{
int i = num_levels;
*symbol = 1;
while (i--)
rc_get_bit(rc, p + *symbol, symbol);
*symbol -= 1 << num_levels;
}
/*
* Small lzma deflate implementation.
* Copyright (C) 2006 Aurelien Jacobs < aurel@gnuage.org >
*
* Based on LzmaDecode.c from the LZMA SDK 4.22 (http://www.7-zip.org/)
* Copyright (C) 1999-2005 Igor Pavlov
*/
struct lzma_header {
uint8_t pos;
uint32_t dict_size;
uint64_t dst_size;
} __attribute__ ((packed)) ;
#define LZMA_BASE_SIZE 1846
#define LZMA_LIT_SIZE 768
#define LZMA_NUM_POS_BITS_MAX 4
#define LZMA_LEN_NUM_LOW_BITS 3
#define LZMA_LEN_NUM_MID_BITS 3
#define LZMA_LEN_NUM_HIGH_BITS 8
#define LZMA_LEN_CHOICE 0
#define LZMA_LEN_CHOICE_2 (LZMA_LEN_CHOICE + 1)
#define LZMA_LEN_LOW (LZMA_LEN_CHOICE_2 + 1)
#define LZMA_LEN_MID (LZMA_LEN_LOW \
+ (1 << (LZMA_NUM_POS_BITS_MAX + LZMA_LEN_NUM_LOW_BITS)))
#define LZMA_LEN_HIGH (LZMA_LEN_MID \
+(1 << (LZMA_NUM_POS_BITS_MAX + LZMA_LEN_NUM_MID_BITS)))
#define LZMA_NUM_LEN_PROBS (LZMA_LEN_HIGH + (1 << LZMA_LEN_NUM_HIGH_BITS))
#define LZMA_NUM_STATES 12
#define LZMA_NUM_LIT_STATES 7
#define LZMA_START_POS_MODEL_INDEX 4
#define LZMA_END_POS_MODEL_INDEX 14
#define LZMA_NUM_FULL_DISTANCES (1 << (LZMA_END_POS_MODEL_INDEX >> 1))
#define LZMA_NUM_POS_SLOT_BITS 6
#define LZMA_NUM_LEN_TO_POS_STATES 4
#define LZMA_NUM_ALIGN_BITS 4
#define LZMA_MATCH_MIN_LEN 2
#define LZMA_IS_MATCH 0
#define LZMA_IS_REP (LZMA_IS_MATCH + (LZMA_NUM_STATES << LZMA_NUM_POS_BITS_MAX))
#define LZMA_IS_REP_G0 (LZMA_IS_REP + LZMA_NUM_STATES)
#define LZMA_IS_REP_G1 (LZMA_IS_REP_G0 + LZMA_NUM_STATES)
#define LZMA_IS_REP_G2 (LZMA_IS_REP_G1 + LZMA_NUM_STATES)
#define LZMA_IS_REP_0_LONG (LZMA_IS_REP_G2 + LZMA_NUM_STATES)
#define LZMA_POS_SLOT (LZMA_IS_REP_0_LONG \
+ (LZMA_NUM_STATES << LZMA_NUM_POS_BITS_MAX))
#define LZMA_SPEC_POS (LZMA_POS_SLOT \
+(LZMA_NUM_LEN_TO_POS_STATES << LZMA_NUM_POS_SLOT_BITS))
#define LZMA_ALIGN (LZMA_SPEC_POS \
+ LZMA_NUM_FULL_DISTANCES - LZMA_END_POS_MODEL_INDEX)
#define LZMA_LEN_CODER (LZMA_ALIGN + (1 << LZMA_NUM_ALIGN_BITS))
#define LZMA_REP_LEN_CODER (LZMA_LEN_CODER + LZMA_NUM_LEN_PROBS)
#define LZMA_LITERAL (LZMA_REP_LEN_CODER + LZMA_NUM_LEN_PROBS)
struct writer {
uint8_t *buffer;
uint8_t previous_byte;
size_t buffer_pos;
int bufsize;
size_t global_pos;
long (*flush)(void*, unsigned long);
struct lzma_header *header;
};
struct cstate {
int state;
uint32_t rep0, rep1, rep2, rep3;
};
static inline size_t INIT get_pos(struct writer *wr)
{
return
wr->global_pos + wr->buffer_pos;
}
static inline uint8_t INIT peek_old_byte(struct writer *wr,
uint32_t offs)
{
if (!wr->flush) {
int32_t pos;
while (offs > wr->header->dict_size)
offs -= wr->header->dict_size;
pos = wr->buffer_pos - offs;
return wr->buffer[pos];
} else {
uint32_t pos = wr->buffer_pos - offs;
while (pos >= wr->header->dict_size)
pos += wr->header->dict_size;
return wr->buffer[pos];
}
}
static inline int INIT write_byte(struct writer *wr, uint8_t byte)
{
wr->buffer[wr->buffer_pos++] = wr->previous_byte = byte;
if (wr->flush && wr->buffer_pos == wr->header->dict_size) {
wr->buffer_pos = 0;
wr->global_pos += wr->header->dict_size;
if (wr->flush((char *)wr->buffer, wr->header->dict_size)
!= wr->header->dict_size)
return -1;
}
return 0;
}
static inline int INIT copy_byte(struct writer *wr, uint32_t offs)
{
return write_byte(wr, peek_old_byte(wr, offs));
}
static inline int INIT copy_bytes(struct writer *wr,
uint32_t rep0, int len)
{
do {
if (copy_byte(wr, rep0))
return -1;
len--;
} while (len != 0 && wr->buffer_pos < wr->header->dst_size);
return len;
}
static inline int INIT process_bit0(struct writer *wr, struct rc *rc,
struct cstate *cst, uint16_t *p,
int pos_state, uint16_t *prob,
int lc, uint32_t literal_pos_mask) {
int mi = 1;
rc_update_bit_0(rc, prob);
prob = (p + LZMA_LITERAL +
(LZMA_LIT_SIZE
* (((get_pos(wr) & literal_pos_mask) << lc)
+ (wr->previous_byte >> (8 - lc))))
);
if (cst->state >= LZMA_NUM_LIT_STATES) {
int match_byte = peek_old_byte(wr, cst->rep0);
do {
int bit;
uint16_t *prob_lit;
match_byte <<= 1;
bit = match_byte & 0x100;
prob_lit = prob + 0x100 + bit + mi;
if (rc_get_bit(rc, prob_lit, &mi)) {
if (!bit)
break;
} else {
if (bit)
break;
}
} while (mi < 0x100);
}
while (mi < 0x100) {
uint16_t *prob_lit = prob + mi;
rc_get_bit(rc, prob_lit, &mi);
}
if (cst->state < 4)
cst->state = 0;
else if (cst->state < 10)
cst->state -= 3;
else
cst->state -= 6;
return write_byte(wr, mi);
}
static inline int INIT process_bit1(struct writer *wr, struct rc *rc,
struct cstate *cst, uint16_t *p,
int pos_state, uint16_t *prob) {
int offset;
uint16_t *prob_len;
int num_bits;
int len;
rc_update_bit_1(rc, prob);
prob = p + LZMA_IS_REP + cst->state;
if (rc_is_bit_0(rc, prob)) {
rc_update_bit_0(rc, prob);
cst->rep3 = cst->rep2;
cst->rep2 = cst->rep1;
cst->rep1 = cst->rep0;
cst->state = cst->state < LZMA_NUM_LIT_STATES ? 0 : 3;
prob = p + LZMA_LEN_CODER;
} else {
rc_update_bit_1(rc, prob);
prob = p + LZMA_IS_REP_G0 + cst->state;
if (rc_is_bit_0(rc, prob)) {
rc_update_bit_0(rc, prob);
prob = (p + LZMA_IS_REP_0_LONG
+ (cst->state <<
LZMA_NUM_POS_BITS_MAX) +
pos_state);
if (rc_is_bit_0(rc, prob)) {
rc_update_bit_0(rc, prob);
cst->state = cst->state < LZMA_NUM_LIT_STATES ?
9 : 11;
return copy_byte(wr, cst->rep0);
} else {
rc_update_bit_1(rc, prob);
}
} else {
uint32_t distance;
rc_update_bit_1(rc, prob);
prob = p + LZMA_IS_REP_G1 + cst->state;
if (rc_is_bit_0(rc, prob)) {
rc_update_bit_0(rc, prob);
distance = cst->rep1;
} else {
rc_update_bit_1(rc, prob);
prob = p + LZMA_IS_REP_G2 + cst->state;
if (rc_is_bit_0(rc, prob)) {
rc_update_bit_0(rc, prob);
distance = cst->rep2;
} else {
rc_update_bit_1(rc, prob);
distance = cst->rep3;
cst->rep3 = cst->rep2;
}
cst->rep2 = cst->rep1;
}
cst->rep1 = cst->rep0;
cst->rep0 = distance;
}
cst->state = cst->state < LZMA_NUM_LIT_STATES ? 8 : 11;
prob = p + LZMA_REP_LEN_CODER;
}
prob_len = prob + LZMA_LEN_CHOICE;
if (rc_is_bit_0(rc, prob_len)) {
rc_update_bit_0(rc, prob_len);
prob_len = (prob + LZMA_LEN_LOW
+ (pos_state <<
LZMA_LEN_NUM_LOW_BITS));
offset = 0;
num_bits = LZMA_LEN_NUM_LOW_BITS;
} else {
rc_update_bit_1(rc, prob_len);
prob_len = prob + LZMA_LEN_CHOICE_2;
if (rc_is_bit_0(rc, prob_len)) {
rc_update_bit_0(rc, prob_len);
prob_len = (prob + LZMA_LEN_MID
+ (pos_state <<
LZMA_LEN_NUM_MID_BITS));
offset = 1 << LZMA_LEN_NUM_LOW_BITS;
num_bits = LZMA_LEN_NUM_MID_BITS;
} else {
rc_update_bit_1(rc, prob_len);
prob_len = prob + LZMA_LEN_HIGH;
offset = ((1 << LZMA_LEN_NUM_LOW_BITS)
+ (1 << LZMA_LEN_NUM_MID_BITS));
num_bits = LZMA_LEN_NUM_HIGH_BITS;
}
}
rc_bit_tree_decode(rc, prob_len, num_bits, &len);
len += offset;
if (cst->state < 4) {
int pos_slot;
cst->state += LZMA_NUM_LIT_STATES;
prob =
p + LZMA_POS_SLOT +
((len <
LZMA_NUM_LEN_TO_POS_STATES ? len :
LZMA_NUM_LEN_TO_POS_STATES - 1)
<< LZMA_NUM_POS_SLOT_BITS);
rc_bit_tree_decode(rc, prob,
LZMA_NUM_POS_SLOT_BITS,
&pos_slot);
if (pos_slot >= LZMA_START_POS_MODEL_INDEX) {
int i, mi;
num_bits = (pos_slot >> 1) - 1;
cst->rep0 = 2 | (pos_slot & 1);
if (pos_slot < LZMA_END_POS_MODEL_INDEX) {
cst->rep0 <<= num_bits;
prob = p + LZMA_SPEC_POS +
cst->rep0 - pos_slot - 1;
} else {
num_bits -= LZMA_NUM_ALIGN_BITS;
while (num_bits--)
cst->rep0 = (cst->rep0 << 1) |
rc_direct_bit(rc);
prob = p + LZMA_ALIGN;
cst->rep0 <<= LZMA_NUM_ALIGN_BITS;
num_bits = LZMA_NUM_ALIGN_BITS;
}
i = 1;
mi = 1;
while (num_bits--) {
if (rc_get_bit(rc, prob + mi, &mi))
cst->rep0 |= i;
i <<= 1;
}
} else
cst->rep0 = pos_slot;
if (++(cst->rep0) == 0)
return 0;
if (cst->rep0 > wr->header->dict_size
|| cst->rep0 > get_pos(wr))
return -1;
}
len += LZMA_MATCH_MIN_LEN;
return copy_bytes(wr, cst->rep0, len);
}
STATIC inline int INIT unlzma(unsigned char *buf, long in_len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *output,
long *posp,
void(*error)(char *x)
)
{
struct lzma_header header;
int lc, pb, lp;
uint32_t pos_state_mask;
uint32_t literal_pos_mask;
uint16_t *p;
int num_probs;
struct rc rc;
int i, mi;
struct writer wr;
struct cstate cst;
unsigned char *inbuf;
int ret = -1;
rc.error = error;
if (buf)
inbuf = buf;
else
inbuf = malloc(LZMA_IOBUF_SIZE);
if (!inbuf) {
error("Could not allocate input buffer");
goto exit_0;
}
cst.state = 0;
cst.rep0 = cst.rep1 = cst.rep2 = cst.rep3 = 1;
wr.header = &header;
wr.flush = flush;
wr.global_pos = 0;
wr.previous_byte = 0;
wr.buffer_pos = 0;
rc_init(&rc, fill, inbuf, in_len);
for (i = 0; i < sizeof(header); i++) {
if (rc.ptr >= rc.buffer_end)
rc_read(&rc);
((unsigned char *)&header)[i] = *rc.ptr++;
}
if (header.pos >= (9 * 5 * 5)) {
error("bad header");
goto exit_1;
}
mi = 0;
lc = header.pos;
while (lc >= 9) {
mi++;
lc -= 9;
}
pb = 0;
lp = mi;
while (lp >= 5) {
pb++;
lp -= 5;
}
pos_state_mask = (1 << pb) - 1;
literal_pos_mask = (1 << lp) - 1;
ENDIAN_CONVERT(header.dict_size);
ENDIAN_CONVERT(header.dst_size);
if (header.dict_size == 0)
header.dict_size = 1;
if (output)
wr.buffer = output;
else {
wr.bufsize = MIN(header.dst_size, header.dict_size);
wr.buffer = large_malloc(wr.bufsize);
}
if (wr.buffer == NULL)
goto exit_1;
num_probs = LZMA_BASE_SIZE + (LZMA_LIT_SIZE << (lc + lp));
p = (uint16_t *) large_malloc(num_probs * sizeof(*p));
if (p == NULL)
goto exit_2;
num_probs = LZMA_LITERAL + (LZMA_LIT_SIZE << (lc + lp));
for (i = 0; i < num_probs; i++)
p[i] = (1 << RC_MODEL_TOTAL_BITS) >> 1;
rc_init_code(&rc);
while (get_pos(&wr) < header.dst_size) {
int pos_state = get_pos(&wr) & pos_state_mask;
uint16_t *prob = p + LZMA_IS_MATCH +
(cst.state << LZMA_NUM_POS_BITS_MAX) + pos_state;
if (rc_is_bit_0(&rc, prob)) {
if (process_bit0(&wr, &rc, &cst, p, pos_state, prob,
lc, literal_pos_mask)) {
error("LZMA data is corrupt");
goto exit_3;
}
} else {
if (process_bit1(&wr, &rc, &cst, p, pos_state, prob)) {
error("LZMA data is corrupt");
goto exit_3;
}
if (cst.rep0 == 0)
break;
}
if (rc.buffer_size <= 0)
goto exit_3;
}
if (posp)
*posp = rc.ptr-rc.buffer;
if (!wr.flush || wr.flush(wr.buffer, wr.buffer_pos) == wr.buffer_pos)
ret = 0;
exit_3:
large_free(p);
exit_2:
if (!output)
large_free(wr.buffer);
exit_1:
if (!buf)
free(inbuf);
exit_0:
return ret;
}
#ifdef PREBOOT
STATIC int INIT __decompress(unsigned char *buf, long in_len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *output, long out_len,
long *posp,
void (*error)(char *x))
{
return unlzma(buf, in_len - 4, fill, flush, output, posp, error);
}
#endif
lib/decompress_unlzo.c
+301
View File
@@ -0,0 +1,301 @@
/*
* LZO decompressor for the Linux kernel. Code borrowed from the lzo
* implementation by Markus Franz Xaver Johannes Oberhumer.
*
* Linux kernel adaptation:
* Copyright (C) 2009
* Albin Tonnerre, Free Electrons <albin.tonnerre@free-electrons.com>
*
* Original code:
* Copyright (C) 1996-2005 Markus Franz Xaver Johannes Oberhumer
* All Rights Reserved.
*
* lzop and the LZO library are free software; you can redistribute them
* and/or modify them under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; see the file COPYING.
* If not, write to the Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* Markus F.X.J. Oberhumer
* <markus@oberhumer.com>
* http://www.oberhumer.com/opensource/lzop/
*/
#ifdef STATIC
#define PREBOOT
#include "lzo/lzo1x_decompress_safe.c"
#else
#include <linux/decompress/unlzo.h>
#endif
#include <linux/types.h>
#include <linux/lzo.h>
#include <linux/decompress/mm.h>
#include <linux/compiler.h>
#include <asm/unaligned.h>
static const unsigned char lzop_magic[] = {
0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a };
#define LZO_BLOCK_SIZE (256*1024l)
#define HEADER_HAS_FILTER 0x00000800L
#define HEADER_SIZE_MIN (9 + 7 + 4 + 8 + 1 + 4)
#define HEADER_SIZE_MAX (9 + 7 + 1 + 8 + 8 + 4 + 1 + 255 + 4)
STATIC inline long INIT parse_header(u8 *input, long *skip, long in_len)
{
int l;
u8 *parse = input;
u8 *end = input + in_len;
u8 level = 0;
u16 version;
/*
* Check that there's enough input to possibly have a valid header.
* Then it is possible to parse several fields until the minimum
* size may have been used.
*/
if (in_len < HEADER_SIZE_MIN)
return 0;
/* read magic: 9 first bits */
for (l = 0; l < 9; l++) {
if (*parse++ != lzop_magic[l])
return 0;
}
/* get version (2bytes), skip library version (2),
* 'need to be extracted' version (2) and
* method (1) */
version = get_unaligned_be16(parse);
parse += 7;
if (version >= 0x0940)
level = *parse++;
if (get_unaligned_be32(parse) & HEADER_HAS_FILTER)
parse += 8; /* flags + filter info */
else
parse += 4; /* flags */
/*
* At least mode, mtime_low, filename length, and checksum must
* be left to be parsed. If also mtime_high is present, it's OK
* because the next input buffer check is after reading the
* filename length.
*/
if (end - parse < 8 + 1 + 4)
return 0;
/* skip mode and mtime_low */
parse += 8;
if (version >= 0x0940)
parse += 4; /* skip mtime_high */
l = *parse++;
/* don't care about the file name, and skip checksum */
if (end - parse < l + 4)
return 0;
parse += l + 4;
*skip = parse - input;
return 1;
}
STATIC int INIT unlzo(u8 *input, long in_len,
long (*fill)(void *, unsigned long),
long (*flush)(void *, unsigned long),
u8 *output, long *posp,
void (*error) (char *x))
{
u8 r = 0;
long skip = 0;
u32 src_len, dst_len;
size_t tmp;
u8 *in_buf, *in_buf_save, *out_buf;
int ret = -1;
if (output) {
out_buf = output;
} else if (!flush) {
error("NULL output pointer and no flush function provided");
goto exit;
} else {
out_buf = malloc(LZO_BLOCK_SIZE);
if (!out_buf) {
error("Could not allocate output buffer");
goto exit;
}
}
if (input && fill) {
error("Both input pointer and fill function provided, don't know what to do");
goto exit_1;
} else if (input) {
in_buf = input;
} else if (!fill) {
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
in_buf = malloc(lzo1x_worst_compress(LZO_BLOCK_SIZE));
if (!in_buf) {
error("Could not allocate input buffer");
goto exit_1;
}
}
in_buf_save = in_buf;
if (posp)
*posp = 0;
if (fill) {
/*
* Start from in_buf + HEADER_SIZE_MAX to make it possible
* to use memcpy() to copy the unused data to the beginning
* of the buffer. This way memmove() isn't needed which
* is missing from pre-boot environments of most archs.
*/
in_buf += HEADER_SIZE_MAX;
in_len = fill(in_buf, HEADER_SIZE_MAX);
}
if (!parse_header(in_buf, &skip, in_len)) {
error("invalid header");
goto exit_2;
}
in_buf += skip;
in_len -= skip;
if (fill) {
/* Move the unused data to the beginning of the buffer. */
memcpy(in_buf_save, in_buf, in_len);
in_buf = in_buf_save;
}
if (posp)
*posp = skip;
for (;;) {
/* read uncompressed block size */
if (fill && in_len < 4) {
skip = fill(in_buf + in_len, 4 - in_len);
if (skip > 0)
in_len += skip;
}
if (in_len < 4) {
error("file corrupted");
goto exit_2;
}
dst_len = get_unaligned_be32(in_buf);
in_buf += 4;
in_len -= 4;
/* exit if last block */
if (dst_len == 0) {
if (posp)
*posp += 4;
break;
}
if (dst_len > LZO_BLOCK_SIZE) {
error("dest len longer than block size");
goto exit_2;
}
/* read compressed block size, and skip block checksum info */
if (fill && in_len < 8) {
skip = fill(in_buf + in_len, 8 - in_len);
if (skip > 0)
in_len += skip;
}
if (in_len < 8) {
error("file corrupted");
goto exit_2;
}
src_len = get_unaligned_be32(in_buf);
in_buf += 8;
in_len -= 8;
if (src_len <= 0 || src_len > dst_len) {
error("file corrupted");
goto exit_2;
}
/* decompress */
if (fill && in_len < src_len) {
skip = fill(in_buf + in_len, src_len - in_len);
if (skip > 0)
in_len += skip;
}
if (in_len < src_len) {
error("file corrupted");
goto exit_2;
}
tmp = dst_len;
/* When the input data is not compressed at all,
* lzo1x_decompress_safe will fail, so call memcpy()
* instead */
if (unlikely(dst_len == src_len))
memcpy(out_buf, in_buf, src_len);
else {
r = lzo1x_decompress_safe((u8 *) in_buf, src_len,
out_buf, &tmp);
if (r != LZO_E_OK || dst_len != tmp) {
error("Compressed data violation");
goto exit_2;
}
}
if (flush && flush(out_buf, dst_len) != dst_len)
goto exit_2;
if (output)
out_buf += dst_len;
if (posp)
*posp += src_len + 12;
in_buf += src_len;
in_len -= src_len;
if (fill) {
/*
* If there happens to still be unused data left in
* in_buf, move it to the beginning of the buffer.
* Use a loop to avoid memmove() dependency.
*/
if (in_len > 0)
for (skip = 0; skip < in_len; ++skip)
in_buf_save[skip] = in_buf[skip];
in_buf = in_buf_save;
}
}
ret = 0;
exit_2:
if (!input)
free(in_buf_save);
exit_1:
if (!output)
free(out_buf);
exit:
return ret;
}
#ifdef PREBOOT
STATIC int INIT __decompress(unsigned char *buf, long len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *out_buf, long olen,
long *pos,
void (*error)(char *x))
{
return unlzo(buf, len, fill, flush, out_buf, pos, error);
}
#endif
lib/decompress_unxz.c
+407
View File
@@ -0,0 +1,407 @@
/*
* Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd
*
* Author: Lasse Collin <lasse.collin@tukaani.org>
*
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
/*
* Important notes about in-place decompression
*
* At least on x86, the kernel is decompressed in place: the compressed data
* is placed to the end of the output buffer, and the decompressor overwrites
* most of the compressed data. There must be enough safety margin to
* guarantee that the write position is always behind the read position.
*
* The safety margin for XZ with LZMA2 or BCJ+LZMA2 is calculated below.
* Note that the margin with XZ is bigger than with Deflate (gzip)!
*
* The worst case for in-place decompression is that the beginning of
* the file is compressed extremely well, and the rest of the file is
* uncompressible. Thus, we must look for worst-case expansion when the
* compressor is encoding uncompressible data.
*
* The structure of the .xz file in case of a compresed kernel is as follows.
* Sizes (as bytes) of the fields are in parenthesis.
*
* Stream Header (12)
* Block Header:
* Block Header (8-12)
* Compressed Data (N)
* Block Padding (0-3)
* CRC32 (4)
* Index (8-20)
* Stream Footer (12)
*
* Normally there is exactly one Block, but let's assume that there are
* 2-4 Blocks just in case. Because Stream Header and also Block Header
* of the first Block don't make the decompressor produce any uncompressed
* data, we can ignore them from our calculations. Block Headers of possible
* additional Blocks have to be taken into account still. With these
* assumptions, it is safe to assume that the total header overhead is
* less than 128 bytes.
*
* Compressed Data contains LZMA2 or BCJ+LZMA2 encoded data. Since BCJ
* doesn't change the size of the data, it is enough to calculate the
* safety margin for LZMA2.
*
* LZMA2 stores the data in chunks. Each chunk has a header whose size is
* a maximum of 6 bytes, but to get round 2^n numbers, let's assume that
* the maximum chunk header size is 8 bytes. After the chunk header, there
* may be up to 64 KiB of actual payload in the chunk. Often the payload is
* quite a bit smaller though; to be safe, let's assume that an average
* chunk has only 32 KiB of payload.
*
* The maximum uncompressed size of the payload is 2 MiB. The minimum
* uncompressed size of the payload is in practice never less than the
* payload size itself. The LZMA2 format would allow uncompressed size
* to be less than the payload size, but no sane compressor creates such
* files. LZMA2 supports storing uncompressible data in uncompressed form,
* so there's never a need to create payloads whose uncompressed size is
* smaller than the compressed size.
*
* The assumption, that the uncompressed size of the payload is never
* smaller than the payload itself, is valid only when talking about
* the payload as a whole. It is possible that the payload has parts where
* the decompressor consumes more input than it produces output. Calculating
* the worst case for this would be tricky. Instead of trying to do that,
* let's simply make sure that the decompressor never overwrites any bytes
* of the payload which it is currently reading.
*
* Now we have enough information to calculate the safety margin. We need
* - 128 bytes for the .xz file format headers;
* - 8 bytes per every 32 KiB of uncompressed size (one LZMA2 chunk header
* per chunk, each chunk having average payload size of 32 KiB); and
* - 64 KiB (biggest possible LZMA2 chunk payload size) to make sure that
* the decompressor never overwrites anything from the LZMA2 chunk
* payload it is currently reading.
*
* We get the following formula:
*
* safety_margin = 128 + uncompressed_size * 8 / 32768 + 65536
* = 128 + (uncompressed_size >> 12) + 65536
*
* For comparison, according to arch/x86/boot/compressed/misc.c, the
* equivalent formula for Deflate is this:
*
* safety_margin = 18 + (uncompressed_size >> 12) + 32768
*
* Thus, when updating Deflate-only in-place kernel decompressor to
* support XZ, the fixed overhead has to be increased from 18+32768 bytes
* to 128+65536 bytes.
*/
/*
* STATIC is defined to "static" if we are being built for kernel
* decompression (pre-boot code). <linux/decompress/mm.h> will define
* STATIC to empty if it wasn't already defined. Since we will need to
* know later if we are being used for kernel decompression, we define
* XZ_PREBOOT here.
*/
#ifdef STATIC
# define XZ_PREBOOT
#endif
#ifdef __KERNEL__
# include <linux/decompress/mm.h>
#endif
#define XZ_EXTERN STATIC
#ifndef XZ_PREBOOT
# include <linux/slab.h>
# include <linux/xz.h>
#else
/*
* Use the internal CRC32 code instead of kernel's CRC32 module, which
* is not available in early phase of booting.
*/
#define XZ_INTERNAL_CRC32 1
/*
* For boot time use, we enable only the BCJ filter of the current
* architecture or none if no BCJ filter is available for the architecture.
*/
#ifdef CONFIG_X86
# define XZ_DEC_X86
#endif
#ifdef CONFIG_PPC
# define XZ_DEC_POWERPC
#endif
#ifdef CONFIG_ARM
# define XZ_DEC_ARM
#endif
#ifdef CONFIG_IA64
# define XZ_DEC_IA64
#endif
#ifdef CONFIG_SPARC
# define XZ_DEC_SPARC
#endif
/*
* This will get the basic headers so that memeq() and others
* can be defined.
*/
#include "xz/xz_private.h"
/*
* Replace the normal allocation functions with the versions from
* <linux/decompress/mm.h>. vfree() needs to support vfree(NULL)
* when XZ_DYNALLOC is used, but the pre-boot free() doesn't support it.
* Workaround it here because the other decompressors don't need it.
*/
#undef kmalloc
#undef kfree
#undef vmalloc
#undef vfree
#define kmalloc(size, flags) malloc(size)
#define kfree(ptr) free(ptr)
#define vmalloc(size) malloc(size)
#define vfree(ptr) do { if (ptr != NULL) free(ptr); } while (0)
/*
* FIXME: Not all basic memory functions are provided in architecture-specific
* files (yet). We define our own versions here for now, but this should be
* only a temporary solution.
*
* memeq and memzero are not used much and any remotely sane implementation
* is fast enough. memcpy/memmove speed matters in multi-call mode, but
* the kernel image is decompressed in single-call mode, in which only
* memcpy speed can matter and only if there is a lot of uncompressible data
* (LZMA2 stores uncompressible chunks in uncompressed form). Thus, the
* functions below should just be kept small; it's probably not worth
* optimizing for speed.
*/
#ifndef memeq
static bool memeq(const void *a, const void *b, size_t size)
{
const uint8_t *x = a;
const uint8_t *y = b;
size_t i;
for (i = 0; i < size; ++i)
if (x[i] != y[i])
return false;
return true;
}
#endif
#ifndef memzero
static void memzero(void *buf, size_t size)
{
uint8_t *b = buf;
uint8_t *e = b + size;
while (b != e)
*b++ = '\0';
}
#endif
#ifndef memmove
/* Not static to avoid a conflict with the prototype in the Linux headers. */
void *memmove(void *dest, const void *src, size_t size)
{
uint8_t *d = dest;
const uint8_t *s = src;
size_t i;
if (d < s) {
for (i = 0; i < size; ++i)
d[i] = s[i];
} else if (d > s) {
i = size;
while (i-- > 0)
d[i] = s[i];
}
return dest;
}
#endif
/*
* Since we need memmove anyway, would use it as memcpy too.
* Commented out for now to avoid breaking things.
*/
/*
#ifndef memcpy
# define memcpy memmove
#endif
*/
#include "xz/xz_crc32.c"
#include "xz/xz_dec_stream.c"
#include "xz/xz_dec_lzma2.c"
#include "xz/xz_dec_bcj.c"
#endif /* XZ_PREBOOT */
/* Size of the input and output buffers in multi-call mode */
#define XZ_IOBUF_SIZE 4096
/*
* This function implements the API defined in <linux/decompress/generic.h>.
*
* This wrapper will automatically choose single-call or multi-call mode
* of the native XZ decoder API. The single-call mode can be used only when
* both input and output buffers are available as a single chunk, i.e. when
* fill() and flush() won't be used.
*/
STATIC int INIT unxz(unsigned char *in, long in_size,
long (*fill)(void *dest, unsigned long size),
long (*flush)(void *src, unsigned long size),
unsigned char *out, long *in_used,
void (*error)(char *x))
{
struct xz_buf b;
struct xz_dec *s;
enum xz_ret ret;
bool must_free_in = false;
#if XZ_INTERNAL_CRC32
xz_crc32_init();
#endif
if (in_used != NULL)
*in_used = 0;
if (fill == NULL && flush == NULL)
s = xz_dec_init(XZ_SINGLE, 0);
else
s = xz_dec_init(XZ_DYNALLOC, (uint32_t)-1);
if (s == NULL)
goto error_alloc_state;
if (flush == NULL) {
b.out = out;
b.out_size = (size_t)-1;
} else {
b.out_size = XZ_IOBUF_SIZE;
b.out = malloc(XZ_IOBUF_SIZE);
if (b.out == NULL)
goto error_alloc_out;
}
if (in == NULL) {
must_free_in = true;
in = malloc(XZ_IOBUF_SIZE);
if (in == NULL)
goto error_alloc_in;
}
b.in = in;
b.in_pos = 0;
b.in_size = in_size;
b.out_pos = 0;
if (fill == NULL && flush == NULL) {
ret = xz_dec_run(s, &b);
} else {
do {
if (b.in_pos == b.in_size && fill != NULL) {
if (in_used != NULL)
*in_used += b.in_pos;
b.in_pos = 0;
in_size = fill(in, XZ_IOBUF_SIZE);
if (in_size < 0) {
/*
* This isn't an optimal error code
* but it probably isn't worth making
* a new one either.
*/
ret = XZ_BUF_ERROR;
break;
}
b.in_size = in_size;
}
ret = xz_dec_run(s, &b);
if (flush != NULL && (b.out_pos == b.out_size
|| (ret != XZ_OK && b.out_pos > 0))) {
/*
* Setting ret here may hide an error
* returned by xz_dec_run(), but probably
* it's not too bad.
*/
if (flush(b.out, b.out_pos) != (long)b.out_pos)
ret = XZ_BUF_ERROR;
b.out_pos = 0;
}
} while (ret == XZ_OK);
if (must_free_in)
free(in);
if (flush != NULL)
free(b.out);
}
if (in_used != NULL)
*in_used += b.in_pos;
xz_dec_end(s);
switch (ret) {
case XZ_STREAM_END:
return 0;
case XZ_MEM_ERROR:
/* This can occur only in multi-call mode. */
error("XZ decompressor ran out of memory");
break;
case XZ_FORMAT_ERROR:
error("Input is not in the XZ format (wrong magic bytes)");
break;
case XZ_OPTIONS_ERROR:
error("Input was encoded with settings that are not "
"supported by this XZ decoder");
break;
case XZ_DATA_ERROR:
case XZ_BUF_ERROR:
error("XZ-compressed data is corrupt");
break;
default:
error("Bug in the XZ decompressor");
break;
}
return -1;
error_alloc_in:
if (flush != NULL)
free(b.out);
error_alloc_out:
xz_dec_end(s);
error_alloc_state:
error("XZ decompressor ran out of memory");
return -1;
}
/*
* This macro is used by architecture-specific files to decompress
* the kernel image.
*/
#ifdef XZ_PREBOOT
STATIC int INIT __decompress(unsigned char *buf, long len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
unsigned char *out_buf, long olen,
long *pos,
void (*error)(char *x))
{
return unxz(buf, len, fill, flush, out_buf, pos, error);
}
#endif
lib/devres.c
+485
View File
@@ -0,0 +1,485 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/err.h>
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/gfp.h>
#include <linux/export.h>
#include <linux/of_address.h>
enum devm_ioremap_type {
DEVM_IOREMAP = 0,
DEVM_IOREMAP_NC,
DEVM_IOREMAP_UC,
DEVM_IOREMAP_WC,
};
void devm_ioremap_release(struct device *dev, void *res)
{
iounmap(*(void __iomem **)res);
}
static int devm_ioremap_match(struct device *dev, void *res, void *match_data)
{
return *(void **)res == match_data;
}
static void __iomem *__devm_ioremap(struct device *dev, resource_size_t offset,
resource_size_t size,
enum devm_ioremap_type type)
{
void __iomem **ptr, *addr = NULL;
ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL);
if (!ptr)
return NULL;
switch (type) {
case DEVM_IOREMAP:
addr = ioremap(offset, size);
break;
case DEVM_IOREMAP_NC:
addr = ioremap_nocache(offset, size);
break;
case DEVM_IOREMAP_UC:
addr = ioremap_uc(offset, size);
break;
case DEVM_IOREMAP_WC:
addr = ioremap_wc(offset, size);
break;
}
if (addr) {
*ptr = addr;
devres_add(dev, ptr);
} else
devres_free(ptr);
return addr;
}
/**
* devm_ioremap - Managed ioremap()
* @dev: Generic device to remap IO address for
* @offset: Resource address to map
* @size: Size of map
*
* Managed ioremap(). Map is automatically unmapped on driver detach.
*/
void __iomem *devm_ioremap(struct device *dev, resource_size_t offset,
resource_size_t size)
{
return __devm_ioremap(dev, offset, size, DEVM_IOREMAP);
}
EXPORT_SYMBOL(devm_ioremap);
/**
* devm_ioremap_uc - Managed ioremap_uc()
* @dev: Generic device to remap IO address for
* @offset: Resource address to map
* @size: Size of map
*
* Managed ioremap_uc(). Map is automatically unmapped on driver detach.
*/
void __iomem *devm_ioremap_uc(struct device *dev, resource_size_t offset,
resource_size_t size)
{
return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_UC);
}
EXPORT_SYMBOL_GPL(devm_ioremap_uc);
/**
* devm_ioremap_nocache - Managed ioremap_nocache()
* @dev: Generic device to remap IO address for
* @offset: Resource address to map
* @size: Size of map
*
* Managed ioremap_nocache(). Map is automatically unmapped on driver
* detach.
*/
void __iomem *devm_ioremap_nocache(struct device *dev, resource_size_t offset,
resource_size_t size)
{
return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_NC);
}
EXPORT_SYMBOL(devm_ioremap_nocache);
/**
* devm_ioremap_wc - Managed ioremap_wc()
* @dev: Generic device to remap IO address for
* @offset: Resource address to map
* @size: Size of map
*
* Managed ioremap_wc(). Map is automatically unmapped on driver detach.
*/
void __iomem *devm_ioremap_wc(struct device *dev, resource_size_t offset,
resource_size_t size)
{
return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_WC);
}
EXPORT_SYMBOL(devm_ioremap_wc);
/**
* devm_iounmap - Managed iounmap()
* @dev: Generic device to unmap for
* @addr: Address to unmap
*
* Managed iounmap(). @addr must have been mapped using devm_ioremap*().
*/
void devm_iounmap(struct device *dev, void __iomem *addr)
{
WARN_ON(devres_destroy(dev, devm_ioremap_release, devm_ioremap_match,
(__force void *)addr));
iounmap(addr);
}
EXPORT_SYMBOL(devm_iounmap);
/**
* devm_ioremap_resource() - check, request region, and ioremap resource
* @dev: generic device to handle the resource for
* @res: resource to be handled
*
* Checks that a resource is a valid memory region, requests the memory
* region and ioremaps it. All operations are managed and will be undone
* on driver detach.
*
* Returns a pointer to the remapped memory or an ERR_PTR() encoded error code
* on failure. Usage example:
*
* res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
* base = devm_ioremap_resource(&pdev->dev, res);
* if (IS_ERR(base))
* return PTR_ERR(base);
*/
void __iomem *devm_ioremap_resource(struct device *dev,
const struct resource *res)
{
resource_size_t size;
const char *name;
void __iomem *dest_ptr;
BUG_ON(!dev);
if (!res || resource_type(res) != IORESOURCE_MEM) {
dev_err(dev, "invalid resource\n");
return IOMEM_ERR_PTR(-EINVAL);
}
size = resource_size(res);
name = res->name ?: dev_name(dev);
if (!devm_request_mem_region(dev, res->start, size, name)) {
dev_err(dev, "can't request region for resource %pR\n", res);
return IOMEM_ERR_PTR(-EBUSY);
}
dest_ptr = devm_ioremap(dev, res->start, size);
if (!dest_ptr) {
dev_err(dev, "ioremap failed for resource %pR\n", res);
devm_release_mem_region(dev, res->start, size);
dest_ptr = IOMEM_ERR_PTR(-ENOMEM);
}
return dest_ptr;
}
EXPORT_SYMBOL(devm_ioremap_resource);
/*
* devm_of_iomap - Requests a resource and maps the memory mapped IO
* for a given device_node managed by a given device
*
* Checks that a resource is a valid memory region, requests the memory
* region and ioremaps it. All operations are managed and will be undone
* on driver detach of the device.
*
* This is to be used when a device requests/maps resources described
* by other device tree nodes (children or otherwise).
*
* @dev: The device "managing" the resource
* @node: The device-tree node where the resource resides
* @index: index of the MMIO range in the "reg" property
* @size: Returns the size of the resource (pass NULL if not needed)
* Returns a pointer to the requested and mapped memory or an ERR_PTR() encoded
* error code on failure. Usage example:
*
* base = devm_of_iomap(&pdev->dev, node, 0, NULL);
* if (IS_ERR(base))
* return PTR_ERR(base);
*/
void __iomem *devm_of_iomap(struct device *dev, struct device_node *node, int index,
resource_size_t *size)
{
struct resource res;
if (of_address_to_resource(node, index, &res))
return IOMEM_ERR_PTR(-EINVAL);
if (size)
*size = resource_size(&res);
return devm_ioremap_resource(dev, &res);
}
EXPORT_SYMBOL(devm_of_iomap);
#ifdef CONFIG_HAS_IOPORT_MAP
/*
* Generic iomap devres
*/
static void devm_ioport_map_release(struct device *dev, void *res)
{
ioport_unmap(*(void __iomem **)res);
}
static int devm_ioport_map_match(struct device *dev, void *res,
void *match_data)
{
return *(void **)res == match_data;
}
/**
* devm_ioport_map - Managed ioport_map()
* @dev: Generic device to map ioport for
* @port: Port to map
* @nr: Number of ports to map
*
* Managed ioport_map(). Map is automatically unmapped on driver
* detach.
*/
void __iomem *devm_ioport_map(struct device *dev, unsigned long port,
unsigned int nr)
{
void __iomem **ptr, *addr;
ptr = devres_alloc(devm_ioport_map_release, sizeof(*ptr), GFP_KERNEL);
if (!ptr)
return NULL;
addr = ioport_map(port, nr);
if (addr) {
*ptr = addr;
devres_add(dev, ptr);
} else
devres_free(ptr);
return addr;
}
EXPORT_SYMBOL(devm_ioport_map);
/**
* devm_ioport_unmap - Managed ioport_unmap()
* @dev: Generic device to unmap for
* @addr: Address to unmap
*
* Managed ioport_unmap(). @addr must have been mapped using
* devm_ioport_map().
*/
void devm_ioport_unmap(struct device *dev, void __iomem *addr)
{
ioport_unmap(addr);
WARN_ON(devres_destroy(dev, devm_ioport_map_release,
devm_ioport_map_match, (__force void *)addr));
}
EXPORT_SYMBOL(devm_ioport_unmap);
#endif /* CONFIG_HAS_IOPORT_MAP */
#ifdef CONFIG_PCI
/*
* PCI iomap devres
*/
#define PCIM_IOMAP_MAX PCI_ROM_RESOURCE
struct pcim_iomap_devres {
void __iomem *table[PCIM_IOMAP_MAX];
};
static void pcim_iomap_release(struct device *gendev, void *res)
{
struct pci_dev *dev = to_pci_dev(gendev);
struct pcim_iomap_devres *this = res;
int i;
for (i = 0; i < PCIM_IOMAP_MAX; i++)
if (this->table[i])
pci_iounmap(dev, this->table[i]);
}
/**
* pcim_iomap_table - access iomap allocation table
* @pdev: PCI device to access iomap table for
*
* Access iomap allocation table for @dev. If iomap table doesn't
* exist and @pdev is managed, it will be allocated. All iomaps
* recorded in the iomap table are automatically unmapped on driver
* detach.
*
* This function might sleep when the table is first allocated but can
* be safely called without context and guaranteed to succed once
* allocated.
*/
void __iomem * const *pcim_iomap_table(struct pci_dev *pdev)
{
struct pcim_iomap_devres *dr, *new_dr;
dr = devres_find(&pdev->dev, pcim_iomap_release, NULL, NULL);
if (dr)
return dr->table;
new_dr = devres_alloc(pcim_iomap_release, sizeof(*new_dr), GFP_KERNEL);
if (!new_dr)
return NULL;
dr = devres_get(&pdev->dev, new_dr, NULL, NULL);
return dr->table;
}
EXPORT_SYMBOL(pcim_iomap_table);
/**
* pcim_iomap - Managed pcim_iomap()
* @pdev: PCI device to iomap for
* @bar: BAR to iomap
* @maxlen: Maximum length of iomap
*
* Managed pci_iomap(). Map is automatically unmapped on driver
* detach.
*/
void __iomem *pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen)
{
void __iomem **tbl;
BUG_ON(bar >= PCIM_IOMAP_MAX);
tbl = (void __iomem **)pcim_iomap_table(pdev);
if (!tbl || tbl[bar]) /* duplicate mappings not allowed */
return NULL;
tbl[bar] = pci_iomap(pdev, bar, maxlen);
return tbl[bar];
}
EXPORT_SYMBOL(pcim_iomap);
/**
* pcim_iounmap - Managed pci_iounmap()
* @pdev: PCI device to iounmap for
* @addr: Address to unmap
*
* Managed pci_iounmap(). @addr must have been mapped using pcim_iomap().
*/
void pcim_iounmap(struct pci_dev *pdev, void __iomem *addr)
{
void __iomem **tbl;
int i;
pci_iounmap(pdev, addr);
tbl = (void __iomem **)pcim_iomap_table(pdev);
BUG_ON(!tbl);
for (i = 0; i < PCIM_IOMAP_MAX; i++)
if (tbl[i] == addr) {
tbl[i] = NULL;
return;
}
WARN_ON(1);
}
EXPORT_SYMBOL(pcim_iounmap);
/**
* pcim_iomap_regions - Request and iomap PCI BARs
* @pdev: PCI device to map IO resources for
* @mask: Mask of BARs to request and iomap
* @name: Name used when requesting regions
*
* Request and iomap regions specified by @mask.
*/
int pcim_iomap_regions(struct pci_dev *pdev, int mask, const char *name)
{
void __iomem * const *iomap;
int i, rc;
iomap = pcim_iomap_table(pdev);
if (!iomap)
return -ENOMEM;
for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
unsigned long len;
if (!(mask & (1 << i)))
continue;
rc = -EINVAL;
len = pci_resource_len(pdev, i);
if (!len)
goto err_inval;
rc = pci_request_region(pdev, i, name);
if (rc)
goto err_inval;
rc = -ENOMEM;
if (!pcim_iomap(pdev, i, 0))
goto err_region;
}
return 0;
err_region:
pci_release_region(pdev, i);
err_inval:
while (--i >= 0) {
if (!(mask & (1 << i)))
continue;
pcim_iounmap(pdev, iomap[i]);
pci_release_region(pdev, i);
}
return rc;
}
EXPORT_SYMBOL(pcim_iomap_regions);
/**
* pcim_iomap_regions_request_all - Request all BARs and iomap specified ones
* @pdev: PCI device to map IO resources for
* @mask: Mask of BARs to iomap
* @name: Name used when requesting regions
*
* Request all PCI BARs and iomap regions specified by @mask.
*/
int pcim_iomap_regions_request_all(struct pci_dev *pdev, int mask,
const char *name)
{
int request_mask = ((1 << 6) - 1) & ~mask;
int rc;
rc = pci_request_selected_regions(pdev, request_mask, name);
if (rc)
return rc;
rc = pcim_iomap_regions(pdev, mask, name);
if (rc)
pci_release_selected_regions(pdev, request_mask);
return rc;
}
EXPORT_SYMBOL(pcim_iomap_regions_request_all);
/**
* pcim_iounmap_regions - Unmap and release PCI BARs
* @pdev: PCI device to map IO resources for
* @mask: Mask of BARs to unmap and release
*
* Unmap and release regions specified by @mask.
*/
void pcim_iounmap_regions(struct pci_dev *pdev, int mask)
{
void __iomem * const *iomap;
int i;
iomap = pcim_iomap_table(pdev);
if (!iomap)
return;
for (i = 0; i < PCIM_IOMAP_MAX; i++) {
if (!(mask & (1 << i)))
continue;
pcim_iounmap(pdev, iomap[i]);
pci_release_region(pdev, i);
}
}
EXPORT_SYMBOL(pcim_iounmap_regions);
#endif /* CONFIG_PCI */
lib/digsig.c
+283
View File
@@ -0,0 +1,283 @@
/*
* Copyright (C) 2011 Nokia Corporation
* Copyright (C) 2011 Intel Corporation
*
* Author:
* Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
* <dmitry.kasatkin@intel.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 2 of the License.
*
* File: sign.c
* implements signature (RSA) verification
* pkcs decoding is based on LibTomCrypt code
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/err.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/key.h>
#include <linux/crypto.h>
#include <crypto/hash.h>
#include <crypto/sha.h>
#include <keys/user-type.h>
#include <linux/mpi.h>
#include <linux/digsig.h>
static struct crypto_shash *shash;
static const char *pkcs_1_v1_5_decode_emsa(const unsigned char *msg,
unsigned long msglen,
unsigned long modulus_bitlen,
unsigned long *outlen)
{
unsigned long modulus_len, ps_len, i;
modulus_len = (modulus_bitlen >> 3) + (modulus_bitlen & 7 ? 1 : 0);
/* test message size */
if ((msglen > modulus_len) || (modulus_len < 11))
return NULL;
/* separate encoded message */
if (msg[0] != 0x00 || msg[1] != 0x01)
return NULL;
for (i = 2; i < modulus_len - 1; i++)
if (msg[i] != 0xFF)
break;
/* separator check */
if (msg[i] != 0)
/* There was no octet with hexadecimal value 0x00
to separate ps from m. */
return NULL;
ps_len = i - 2;
*outlen = (msglen - (2 + ps_len + 1));
return msg + 2 + ps_len + 1;
}
/*
* RSA Signature verification with public key
*/
static int digsig_verify_rsa(struct key *key,
const char *sig, int siglen,
const char *h, int hlen)
{
int err = -EINVAL;
unsigned long len;
unsigned long mlen, mblen;
unsigned nret, l;
int head, i;
unsigned char *out1 = NULL;
const char *m;
MPI in = NULL, res = NULL, pkey[2];
uint8_t *p, *datap;
const uint8_t *endp;
const struct user_key_payload *ukp;
struct pubkey_hdr *pkh;
down_read(&key->sem);
ukp = user_key_payload_locked(key);
if (!ukp) {
/* key was revoked before we acquired its semaphore */
err = -EKEYREVOKED;
goto err1;
}
if (ukp->datalen < sizeof(*pkh))
goto err1;
pkh = (struct pubkey_hdr *)ukp->data;
if (pkh->version != 1)
goto err1;
if (pkh->algo != PUBKEY_ALGO_RSA)
goto err1;
if (pkh->nmpi != 2)
goto err1;
datap = pkh->mpi;
endp = ukp->data + ukp->datalen;
for (i = 0; i < pkh->nmpi; i++) {
unsigned int remaining = endp - datap;
pkey[i] = mpi_read_from_buffer(datap, &remaining);
if (IS_ERR(pkey[i])) {
err = PTR_ERR(pkey[i]);
goto err;
}
datap += remaining;
}
mblen = mpi_get_nbits(pkey[0]);
mlen = DIV_ROUND_UP(mblen, 8);
if (mlen == 0) {
err = -EINVAL;
goto err;
}
err = -ENOMEM;
out1 = kzalloc(mlen, GFP_KERNEL);
if (!out1)
goto err;
nret = siglen;
in = mpi_read_from_buffer(sig, &nret);
if (IS_ERR(in)) {
err = PTR_ERR(in);
goto err;
}
res = mpi_alloc(mpi_get_nlimbs(in) * 2);
if (!res)
goto err;
err = mpi_powm(res, in, pkey[1], pkey[0]);
if (err)
goto err;
if (mpi_get_nlimbs(res) * BYTES_PER_MPI_LIMB > mlen) {
err = -EINVAL;
goto err;
}
p = mpi_get_buffer(res, &l, NULL);
if (!p) {
err = -EINVAL;
goto err;
}
len = mlen;
head = len - l;
memset(out1, 0, head);
memcpy(out1 + head, p, l);
kfree(p);
m = pkcs_1_v1_5_decode_emsa(out1, len, mblen, &len);
if (!m || len != hlen || memcmp(m, h, hlen))
err = -EINVAL;
err:
mpi_free(in);
mpi_free(res);
kfree(out1);
while (--i >= 0)
mpi_free(pkey[i]);
err1:
up_read(&key->sem);
return err;
}
/**
* digsig_verify() - digital signature verification with public key
* @keyring: keyring to search key in
* @sig: digital signature
* @siglen: length of the signature
* @data: data
* @datalen: length of the data
*
* Returns 0 on success, -EINVAL otherwise
*
* Verifies data integrity against digital signature.
* Currently only RSA is supported.
* Normally hash of the content is used as a data for this function.
*
*/
int digsig_verify(struct key *keyring, const char *sig, int siglen,
const char *data, int datalen)
{
int err = -ENOMEM;
struct signature_hdr *sh = (struct signature_hdr *)sig;
struct shash_desc *desc = NULL;
unsigned char hash[SHA1_DIGEST_SIZE];
struct key *key;
char name[20];
if (siglen < sizeof(*sh) + 2)
return -EINVAL;
if (sh->algo != PUBKEY_ALGO_RSA)
return -ENOTSUPP;
sprintf(name, "%llX", __be64_to_cpup((uint64_t *)sh->keyid));
if (keyring) {
/* search in specific keyring */
key_ref_t kref;
kref = keyring_search(make_key_ref(keyring, 1UL),
&key_type_user, name);
if (IS_ERR(kref))
key = ERR_CAST(kref);
else
key = key_ref_to_ptr(kref);
} else {
key = request_key(&key_type_user, name, NULL);
}
if (IS_ERR(key)) {
pr_err("key not found, id: %s\n", name);
return PTR_ERR(key);
}
desc = kzalloc(sizeof(*desc) + crypto_shash_descsize(shash),
GFP_KERNEL);
if (!desc)
goto err;
desc->tfm = shash;
desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
crypto_shash_init(desc);
crypto_shash_update(desc, data, datalen);
crypto_shash_update(desc, sig, sizeof(*sh));
crypto_shash_final(desc, hash);
kfree(desc);
/* pass signature mpis address */
err = digsig_verify_rsa(key, sig + sizeof(*sh), siglen - sizeof(*sh),
hash, sizeof(hash));
err:
key_put(key);
return err ? -EINVAL : 0;
}
EXPORT_SYMBOL_GPL(digsig_verify);
static int __init digsig_init(void)
{
shash = crypto_alloc_shash("sha1", 0, 0);
if (IS_ERR(shash)) {
pr_err("shash allocation failed\n");
return PTR_ERR(shash);
}
return 0;
}
static void __exit digsig_cleanup(void)
{
crypto_free_shash(shash);
}
module_init(digsig_init);
module_exit(digsig_cleanup);
MODULE_LICENSE("GPL");
lib/div64.c
+192
View File
@@ -0,0 +1,192 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2003 Bernardo Innocenti <bernie@develer.com>
*
* Based on former do_div() implementation from asm-parisc/div64.h:
* Copyright (C) 1999 Hewlett-Packard Co
* Copyright (C) 1999 David Mosberger-Tang <davidm@hpl.hp.com>
*
*
* Generic C version of 64bit/32bit division and modulo, with
* 64bit result and 32bit remainder.
*
* The fast case for (n>>32 == 0) is handled inline by do_div().
*
* Code generated for this function might be very inefficient
* for some CPUs. __div64_32() can be overridden by linking arch-specific
* assembly versions such as arch/ppc/lib/div64.S and arch/sh/lib/div64.S
* or by defining a preprocessor macro in arch/include/asm/div64.h.
*/
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/math64.h>
/* Not needed on 64bit architectures */
#if BITS_PER_LONG == 32
#ifndef __div64_32
uint32_t __attribute__((weak)) __div64_32(uint64_t *n, uint32_t base)
{
uint64_t rem = *n;
uint64_t b = base;
uint64_t res, d = 1;
uint32_t high = rem >> 32;
/* Reduce the thing a bit first */
res = 0;
if (high >= base) {
high /= base;
res = (uint64_t) high << 32;
rem -= (uint64_t) (high*base) << 32;
}
while ((int64_t)b > 0 && b < rem) {
b = b+b;
d = d+d;
}
do {
if (rem >= b) {
rem -= b;
res += d;
}
b >>= 1;
d >>= 1;
} while (d);
*n = res;
return rem;
}
EXPORT_SYMBOL(__div64_32);
#endif
/**
* div_s64_rem - signed 64bit divide with 64bit divisor and remainder
* @dividend: 64bit dividend
* @divisor: 64bit divisor
* @remainder: 64bit remainder
*/
#ifndef div_s64_rem
s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
{
u64 quotient;
if (dividend < 0) {
quotient = div_u64_rem(-dividend, abs(divisor), (u32 *)remainder);
*remainder = -*remainder;
if (divisor > 0)
quotient = -quotient;
} else {
quotient = div_u64_rem(dividend, abs(divisor), (u32 *)remainder);
if (divisor < 0)
quotient = -quotient;
}
return quotient;
}
EXPORT_SYMBOL(div_s64_rem);
#endif
/**
* div64_u64_rem - unsigned 64bit divide with 64bit divisor and remainder
* @dividend: 64bit dividend
* @divisor: 64bit divisor
* @remainder: 64bit remainder
*
* This implementation is a comparable to algorithm used by div64_u64.
* But this operation, which includes math for calculating the remainder,
* is kept distinct to avoid slowing down the div64_u64 operation on 32bit
* systems.
*/
#ifndef div64_u64_rem
u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
{
u32 high = divisor >> 32;
u64 quot;
if (high == 0) {
u32 rem32;
quot = div_u64_rem(dividend, divisor, &rem32);
*remainder = rem32;
} else {
int n = fls(high);
quot = div_u64(dividend >> n, divisor >> n);
if (quot != 0)
quot--;
*remainder = dividend - quot * divisor;
if (*remainder >= divisor) {
quot++;
*remainder -= divisor;
}
}
return quot;
}
EXPORT_SYMBOL(div64_u64_rem);
#endif
/**
* div64_u64 - unsigned 64bit divide with 64bit divisor
* @dividend: 64bit dividend
* @divisor: 64bit divisor
*
* This implementation is a modified version of the algorithm proposed
* by the book 'Hacker's Delight'. The original source and full proof
* can be found here and is available for use without restriction.
*
* 'http://www.hackersdelight.org/hdcodetxt/divDouble.c.txt'
*/
#ifndef div64_u64
u64 div64_u64(u64 dividend, u64 divisor)
{
u32 high = divisor >> 32;
u64 quot;
if (high == 0) {
quot = div_u64(dividend, divisor);
} else {
int n = fls(high);
quot = div_u64(dividend >> n, divisor >> n);
if (quot != 0)
quot--;
if ((dividend - quot * divisor) >= divisor)
quot++;
}
return quot;
}
EXPORT_SYMBOL(div64_u64);
#endif
/**
* div64_s64 - signed 64bit divide with 64bit divisor
* @dividend: 64bit dividend
* @divisor: 64bit divisor
*/
#ifndef div64_s64
s64 div64_s64(s64 dividend, s64 divisor)
{
s64 quot, t;
quot = div64_u64(abs(dividend), abs(divisor));
t = (dividend ^ divisor) >> 63;
return (quot ^ t) - t;
}
EXPORT_SYMBOL(div64_s64);
#endif
#endif /* BITS_PER_LONG == 32 */
/*
* Iterative div/mod for use when dividend is not expected to be much
* bigger than divisor.
*/
u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder)
{
return __iter_div_u64_rem(dividend, divisor, remainder);
}
EXPORT_SYMBOL(iter_div_u64_rem);
lib/dump_stack.c
+131
View File
@@ -0,0 +1,131 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Provide a default dump_stack() function for architectures
* which don't implement their own.
*/
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/smp.h>
#include <linux/atomic.h>
#include <linux/kexec.h>
#include <linux/utsname.h>
static char dump_stack_arch_desc_str[128];
/**
* dump_stack_set_arch_desc - set arch-specific str to show with task dumps
* @fmt: printf-style format string
* @...: arguments for the format string
*
* The configured string will be printed right after utsname during task
* dumps. Usually used to add arch-specific system identifiers. If an
* arch wants to make use of such an ID string, it should initialize this
* as soon as possible during boot.
*/
void __init dump_stack_set_arch_desc(const char *fmt, ...)
{
va_list args;
va_start(args, fmt);
vsnprintf(dump_stack_arch_desc_str, sizeof(dump_stack_arch_desc_str),
fmt, args);
va_end(args);
}
/**
* dump_stack_print_info - print generic debug info for dump_stack()
* @log_lvl: log level
*
* Arch-specific dump_stack() implementations can use this function to
* print out the same debug information as the generic dump_stack().
*/
void dump_stack_print_info(const char *log_lvl)
{
printk("%sCPU: %d PID: %d Comm: %.20s %s%s %s %.*s\n",
log_lvl, raw_smp_processor_id(), current->pid, current->comm,
kexec_crash_loaded() ? "Kdump: loaded " : "",
print_tainted(),
init_utsname()->release,
(int)strcspn(init_utsname()->version, " "),
init_utsname()->version);
if (dump_stack_arch_desc_str[0] != '\0')
printk("%sHardware name: %s\n",
log_lvl, dump_stack_arch_desc_str);
print_worker_info(log_lvl, current);
}
/**
* show_regs_print_info - print generic debug info for show_regs()
* @log_lvl: log level
*
* show_regs() implementations can use this function to print out generic
* debug information.
*/
void show_regs_print_info(const char *log_lvl)
{
dump_stack_print_info(log_lvl);
}
static void __dump_stack(void)
{
dump_stack_print_info(KERN_DEFAULT);
show_stack(NULL, NULL);
}
/**
* dump_stack - dump the current task information and its stack trace
*
* Architectures can override this implementation by implementing its own.
*/
#ifdef CONFIG_SMP
static atomic_t dump_lock = ATOMIC_INIT(-1);
asmlinkage __visible void dump_stack(void)
{
unsigned long flags;
int was_locked;
int old;
int cpu;
/*
* Permit this cpu to perform nested stack dumps while serialising
* against other CPUs
*/
retry:
local_irq_save(flags);
cpu = smp_processor_id();
old = atomic_cmpxchg(&dump_lock, -1, cpu);
if (old == -1) {
was_locked = 0;
} else if (old == cpu) {
was_locked = 1;
} else {
local_irq_restore(flags);
/*
* Wait for the lock to release before jumping to
* atomic_cmpxchg() in order to mitigate the thundering herd
* problem.
*/
do { cpu_relax(); } while (atomic_read(&dump_lock) != -1);
goto retry;
}
__dump_stack();
if (!was_locked)
atomic_set(&dump_lock, -1);
local_irq_restore(flags);
}
#else
asmlinkage __visible void dump_stack(void)
{
__dump_stack();
}
#endif
EXPORT_SYMBOL(dump_stack);
lib/dynamic_debug.c
+1194
View File
File diff suppressed because it is too large. Load diff
lib/dynamic_queue_limits.c
+138
View File
@@ -0,0 +1,138 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Dynamic byte queue limits. See include/linux/dynamic_queue_limits.h
*
* Copyright (c) 2011, Tom Herbert <therbert@google.com>
*/
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/jiffies.h>
#include <linux/dynamic_queue_limits.h>
#include <linux/compiler.h>
#include <linux/export.h>
#define POSDIFF(A, B) ((int)((A) - (B)) > 0 ? (A) - (B) : 0)
#define AFTER_EQ(A, B) ((int)((A) - (B)) >= 0)
/* Records completed count and recalculates the queue limit */
void dql_completed(struct dql *dql, unsigned int count)
{
unsigned int inprogress, prev_inprogress, limit;
unsigned int ovlimit, completed, num_queued;
bool all_prev_completed;
num_queued = READ_ONCE(dql->num_queued);
/* Can't complete more than what's in queue */
BUG_ON(count > num_queued - dql->num_completed);
completed = dql->num_completed + count;
limit = dql->limit;
ovlimit = POSDIFF(num_queued - dql->num_completed, limit);
inprogress = num_queued - completed;
prev_inprogress = dql->prev_num_queued - dql->num_completed;
all_prev_completed = AFTER_EQ(completed, dql->prev_num_queued);
if ((ovlimit && !inprogress) ||
(dql->prev_ovlimit && all_prev_completed)) {
/*
* Queue considered starved if:
* - The queue was over-limit in the last interval,
* and there is no more data in the queue.
* OR
* - The queue was over-limit in the previous interval and
* when enqueuing it was possible that all queued data
* had been consumed. This covers the case when queue
* may have becomes starved between completion processing
* running and next time enqueue was scheduled.
*
* When queue is starved increase the limit by the amount
* of bytes both sent and completed in the last interval,
* plus any previous over-limit.
*/
limit += POSDIFF(completed, dql->prev_num_queued) +
dql->prev_ovlimit;
dql->slack_start_time = jiffies;
dql->lowest_slack = UINT_MAX;
} else if (inprogress && prev_inprogress && !all_prev_completed) {
/*
* Queue was not starved, check if the limit can be decreased.
* A decrease is only considered if the queue has been busy in
* the whole interval (the check above).
*
* If there is slack, the amount of execess data queued above
* the the amount needed to prevent starvation, the queue limit
* can be decreased. To avoid hysteresis we consider the
* minimum amount of slack found over several iterations of the
* completion routine.
*/
unsigned int slack, slack_last_objs;
/*
* Slack is the maximum of
* - The queue limit plus previous over-limit minus twice
* the number of objects completed. Note that two times
* number of completed bytes is a basis for an upper bound
* of the limit.
* - Portion of objects in the last queuing operation that
* was not part of non-zero previous over-limit. That is
* "round down" by non-overlimit portion of the last
* queueing operation.
*/
slack = POSDIFF(limit + dql->prev_ovlimit,
2 * (completed - dql->num_completed));
slack_last_objs = dql->prev_ovlimit ?
POSDIFF(dql->prev_last_obj_cnt, dql->prev_ovlimit) : 0;
slack = max(slack, slack_last_objs);
if (slack < dql->lowest_slack)
dql->lowest_slack = slack;
if (time_after(jiffies,
dql->slack_start_time + dql->slack_hold_time)) {
limit = POSDIFF(limit, dql->lowest_slack);
dql->slack_start_time = jiffies;
dql->lowest_slack = UINT_MAX;
}
}
/* Enforce bounds on limit */
limit = clamp(limit, dql->min_limit, dql->max_limit);
if (limit != dql->limit) {
dql->limit = limit;
ovlimit = 0;
}
dql->adj_limit = limit + completed;
dql->prev_ovlimit = ovlimit;
dql->prev_last_obj_cnt = dql->last_obj_cnt;
dql->num_completed = completed;
dql->prev_num_queued = num_queued;
}
EXPORT_SYMBOL(dql_completed);
void dql_reset(struct dql *dql)
{
/* Reset all dynamic values */
dql->limit = 0;
dql->num_queued = 0;
dql->num_completed = 0;
dql->last_obj_cnt = 0;
dql->prev_num_queued = 0;
dql->prev_last_obj_cnt = 0;
dql->prev_ovlimit = 0;
dql->lowest_slack = UINT_MAX;
dql->slack_start_time = jiffies;
}
EXPORT_SYMBOL(dql_reset);
void dql_init(struct dql *dql, unsigned int hold_time)
{
dql->max_limit = DQL_MAX_LIMIT;
dql->min_limit = 0;
dql->slack_hold_time = hold_time;
dql_reset(dql);
}
EXPORT_SYMBOL(dql_init);
lib/earlycpio.c
+149
View File
@@ -0,0 +1,149 @@
/* ----------------------------------------------------------------------- *
*
* Copyright 2012 Intel Corporation; author H. Peter Anvin
*
* This file is part of the Linux kernel, and is made available
* under the terms of the GNU General Public License version 2, as
* published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* ----------------------------------------------------------------------- */
/*
* earlycpio.c
*
* Find a specific cpio member; must precede any compressed content.
* This is used to locate data items in the initramfs used by the
* kernel itself during early boot (before the main initramfs is
* decompressed.) It is the responsibility of the initramfs creator
* to ensure that these items are uncompressed at the head of the
* blob. Depending on the boot loader or package tool that may be a
* separate file or part of the same file.
*/
#include <linux/earlycpio.h>
#include <linux/kernel.h>
#include <linux/string.h>
enum cpio_fields {
C_MAGIC,
C_INO,
C_MODE,
C_UID,
C_GID,
C_NLINK,
C_MTIME,
C_FILESIZE,
C_MAJ,
C_MIN,
C_RMAJ,
C_RMIN,
C_NAMESIZE,
C_CHKSUM,
C_NFIELDS
};
/**
* cpio_data find_cpio_data - Search for files in an uncompressed cpio
* @path: The directory to search for, including a slash at the end
* @data: Pointer to the the cpio archive or a header inside
* @len: Remaining length of the cpio based on data pointer
* @nextoff: When a matching file is found, this is the offset from the
* beginning of the cpio to the beginning of the next file, not the
* matching file itself. It can be used to iterate through the cpio
* to find all files inside of a directory path.
*
* @return: struct cpio_data containing the address, length and
* filename (with the directory path cut off) of the found file.
* If you search for a filename and not for files in a directory,
* pass the absolute path of the filename in the cpio and make sure
* the match returned an empty filename string.
*/
struct cpio_data find_cpio_data(const char *path, void *data,
size_t len, long *nextoff)
{
const size_t cpio_header_len = 8*C_NFIELDS - 2;
struct cpio_data cd = { NULL, 0, "" };
const char *p, *dptr, *nptr;
unsigned int ch[C_NFIELDS], *chp, v;
unsigned char c, x;
size_t mypathsize = strlen(path);
int i, j;
p = data;
while (len > cpio_header_len) {
if (!*p) {
/* All cpio headers need to be 4-byte aligned */
p += 4;
len -= 4;
continue;
}
j = 6; /* The magic field is only 6 characters */
chp = ch;
for (i = C_NFIELDS; i; i--) {
v = 0;
while (j--) {
v <<= 4;
c = *p++;
x = c - '0';
if (x < 10) {
v += x;
continue;
}
x = (c | 0x20) - 'a';
if (x < 6) {
v += x + 10;
continue;
}
goto quit; /* Invalid hexadecimal */
}
*chp++ = v;
j = 8; /* All other fields are 8 characters */
}
if ((ch[C_MAGIC] - 0x070701) > 1)
goto quit; /* Invalid magic */
len -= cpio_header_len;
dptr = PTR_ALIGN(p + ch[C_NAMESIZE], 4);
nptr = PTR_ALIGN(dptr + ch[C_FILESIZE], 4);
if (nptr > p + len || dptr < p || nptr < dptr)
goto quit; /* Buffer overrun */
if ((ch[C_MODE] & 0170000) == 0100000 &&
ch[C_NAMESIZE] >= mypathsize &&
!memcmp(p, path, mypathsize)) {
if (nextoff)
*nextoff = (long)nptr - (long)data;
if (ch[C_NAMESIZE] - mypathsize >= MAX_CPIO_FILE_NAME) {
pr_warn(
"File %s exceeding MAX_CPIO_FILE_NAME [%d]\n",
p, MAX_CPIO_FILE_NAME);
}
strlcpy(cd.name, p + mypathsize, MAX_CPIO_FILE_NAME);
cd.data = (void *)dptr;
cd.size = ch[C_FILESIZE];
return cd; /* Found it! */
}
len -= (nptr - p);
p = nptr;
}
quit:
return cd;
}
lib/error-inject.c
+242
View File
@@ -0,0 +1,242 @@
// SPDX-License-Identifier: GPL-2.0
// error-inject.c: Function-level error injection table
#include <linux/error-injection.h>
#include <linux/debugfs.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/slab.h>
/* Whitelist of symbols that can be overridden for error injection. */
static LIST_HEAD(error_injection_list);
static DEFINE_MUTEX(ei_mutex);
struct ei_entry {
struct list_head list;
unsigned long start_addr;
unsigned long end_addr;
int etype;
void *priv;
};
bool within_error_injection_list(unsigned long addr)
{
struct ei_entry *ent;
bool ret = false;
mutex_lock(&ei_mutex);
list_for_each_entry(ent, &error_injection_list, list) {
if (addr >= ent->start_addr && addr < ent->end_addr) {
ret = true;
break;
}
}
mutex_unlock(&ei_mutex);
return ret;
}
int get_injectable_error_type(unsigned long addr)
{
struct ei_entry *ent;
list_for_each_entry(ent, &error_injection_list, list) {
if (addr >= ent->start_addr && addr < ent->end_addr)
return ent->etype;
}
return EI_ETYPE_NONE;
}
/*
* Lookup and populate the error_injection_list.
*
* For safety reasons we only allow certain functions to be overridden with
* bpf_error_injection, so we need to populate the list of the symbols that have
* been marked as safe for overriding.
*/
static void populate_error_injection_list(struct error_injection_entry *start,
struct error_injection_entry *end,
void *priv)
{
struct error_injection_entry *iter;
struct ei_entry *ent;
unsigned long entry, offset = 0, size = 0;
mutex_lock(&ei_mutex);
for (iter = start; iter < end; iter++) {
entry = arch_deref_entry_point((void *)iter->addr);
if (!kernel_text_address(entry) ||
!kallsyms_lookup_size_offset(entry, &size, &offset)) {
pr_err("Failed to find error inject entry at %p\n",
(void *)entry);
continue;
}
ent = kmalloc(sizeof(*ent), GFP_KERNEL);
if (!ent)
break;
ent->start_addr = entry;
ent->end_addr = entry + size;
ent->etype = iter->etype;
ent->priv = priv;
INIT_LIST_HEAD(&ent->list);
list_add_tail(&ent->list, &error_injection_list);
}
mutex_unlock(&ei_mutex);
}
/* Markers of the _error_inject_whitelist section */
extern struct error_injection_entry __start_error_injection_whitelist[];
extern struct error_injection_entry __stop_error_injection_whitelist[];
static void __init populate_kernel_ei_list(void)
{
populate_error_injection_list(__start_error_injection_whitelist,
__stop_error_injection_whitelist,
NULL);
}
#ifdef CONFIG_MODULES
static void module_load_ei_list(struct module *mod)
{
if (!mod->num_ei_funcs)
return;
populate_error_injection_list(mod->ei_funcs,
mod->ei_funcs + mod->num_ei_funcs, mod);
}
static void module_unload_ei_list(struct module *mod)
{
struct ei_entry *ent, *n;
if (!mod->num_ei_funcs)
return;
mutex_lock(&ei_mutex);
list_for_each_entry_safe(ent, n, &error_injection_list, list) {
if (ent->priv == mod) {
list_del_init(&ent->list);
kfree(ent);
}
}
mutex_unlock(&ei_mutex);
}
/* Module notifier call back, checking error injection table on the module */
static int ei_module_callback(struct notifier_block *nb,
unsigned long val, void *data)
{
struct module *mod = data;
if (val == MODULE_STATE_COMING)
module_load_ei_list(mod);
else if (val == MODULE_STATE_GOING)
module_unload_ei_list(mod);
return NOTIFY_DONE;
}
static struct notifier_block ei_module_nb = {
.notifier_call = ei_module_callback,
.priority = 0
};
static __init int module_ei_init(void)
{
return register_module_notifier(&ei_module_nb);
}
#else /* !CONFIG_MODULES */
#define module_ei_init() (0)
#endif
/*
* error_injection/whitelist -- shows which functions can be overridden for
* error injection.
*/
static void *ei_seq_start(struct seq_file *m, loff_t *pos)
{
mutex_lock(&ei_mutex);
return seq_list_start(&error_injection_list, *pos);
}
static void ei_seq_stop(struct seq_file *m, void *v)
{
mutex_unlock(&ei_mutex);
}
static void *ei_seq_next(struct seq_file *m, void *v, loff_t *pos)
{
return seq_list_next(v, &error_injection_list, pos);
}
static const char *error_type_string(int etype)
{
switch (etype) {
case EI_ETYPE_NULL:
return "NULL";
case EI_ETYPE_ERRNO:
return "ERRNO";
case EI_ETYPE_ERRNO_NULL:
return "ERRNO_NULL";
default:
return "(unknown)";
}
}
static int ei_seq_show(struct seq_file *m, void *v)
{
struct ei_entry *ent = list_entry(v, struct ei_entry, list);
seq_printf(m, "%pf\t%s\n", (void *)ent->start_addr,
error_type_string(ent->etype));
return 0;
}
static const struct seq_operations ei_seq_ops = {
.start = ei_seq_start,
.next = ei_seq_next,
.stop = ei_seq_stop,
.show = ei_seq_show,
};
static int ei_open(struct inode *inode, struct file *filp)
{
return seq_open(filp, &ei_seq_ops);
}
static const struct file_operations debugfs_ei_ops = {
.open = ei_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release,
};
static int __init ei_debugfs_init(void)
{
struct dentry *dir, *file;
dir = debugfs_create_dir("error_injection", NULL);
if (!dir)
return -ENOMEM;
file = debugfs_create_file("list", 0444, dir, NULL, &debugfs_ei_ops);
if (!file) {
debugfs_remove(dir);
return -ENOMEM;
}
return 0;
}
static int __init init_error_injection(void)
{
populate_kernel_ei_list();
if (!module_ei_init())
ei_debugfs_init();
return 0;
}
late_initcall(init_error_injection);
lib/errseq.c
+206
View File
@@ -0,0 +1,206 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/err.h>
#include <linux/bug.h>
#include <linux/atomic.h>
#include <linux/errseq.h>
/*
* An errseq_t is a way of recording errors in one place, and allowing any
* number of "subscribers" to tell whether it has changed since a previous
* point where it was sampled.
*
* It's implemented as an unsigned 32-bit value. The low order bits are
* designated to hold an error code (between 0 and -MAX_ERRNO). The upper bits
* are used as a counter. This is done with atomics instead of locking so that
* these functions can be called from any context.
*
* The general idea is for consumers to sample an errseq_t value. That value
* can later be used to tell whether any new errors have occurred since that
* sampling was done.
*
* Note that there is a risk of collisions if new errors are being recorded
* frequently, since we have so few bits to use as a counter.
*
* To mitigate this, one bit is used as a flag to tell whether the value has
* been sampled since a new value was recorded. That allows us to avoid bumping
* the counter if no one has sampled it since the last time an error was
* recorded.
*
* A new errseq_t should always be zeroed out. A errseq_t value of all zeroes
* is the special (but common) case where there has never been an error. An all
* zero value thus serves as the "epoch" if one wishes to know whether there
* has ever been an error set since it was first initialized.
*/
/* The low bits are designated for error code (max of MAX_ERRNO) */
#define ERRSEQ_SHIFT ilog2(MAX_ERRNO + 1)
/* This bit is used as a flag to indicate whether the value has been seen */
#define ERRSEQ_SEEN (1 << ERRSEQ_SHIFT)
/* The lowest bit of the counter */
#define ERRSEQ_CTR_INC (1 << (ERRSEQ_SHIFT + 1))
/**
* errseq_set - set a errseq_t for later reporting
* @eseq: errseq_t field that should be set
* @err: error to set (must be between -1 and -MAX_ERRNO)
*
* This function sets the error in @eseq, and increments the sequence counter
* if the last sequence was sampled at some point in the past.
*
* Any error set will always overwrite an existing error.
*
* Return: The previous value, primarily for debugging purposes. The
* return value should not be used as a previously sampled value in later
* calls as it will not have the SEEN flag set.
*/
errseq_t errseq_set(errseq_t *eseq, int err)
{
errseq_t cur, old;
/* MAX_ERRNO must be able to serve as a mask */
BUILD_BUG_ON_NOT_POWER_OF_2(MAX_ERRNO + 1);
/*
* Ensure the error code actually fits where we want it to go. If it
* doesn't then just throw a warning and don't record anything. We
* also don't accept zero here as that would effectively clear a
* previous error.
*/
old = READ_ONCE(*eseq);
if (WARN(unlikely(err == 0 || (unsigned int)-err > MAX_ERRNO),
"err = %d\n", err))
return old;
for (;;) {
errseq_t new;
/* Clear out error bits and set new error */
new = (old & ~(MAX_ERRNO|ERRSEQ_SEEN)) | -err;
/* Only increment if someone has looked at it */
if (old & ERRSEQ_SEEN)
new += ERRSEQ_CTR_INC;
/* If there would be no change, then call it done */
if (new == old) {
cur = new;
break;
}
/* Try to swap the new value into place */
cur = cmpxchg(eseq, old, new);
/*
* Call it success if we did the swap or someone else beat us
* to it for the same value.
*/
if (likely(cur == old || cur == new))
break;
/* Raced with an update, try again */
old = cur;
}
return cur;
}
EXPORT_SYMBOL(errseq_set);
/**
* errseq_sample() - Grab current errseq_t value.
* @eseq: Pointer to errseq_t to be sampled.
*
* This function allows callers to initialise their errseq_t variable.
* If the error has been "seen", new callers will not see an old error.
* If there is an unseen error in @eseq, the caller of this function will
* see it the next time it checks for an error.
*
* Context: Any context.
* Return: The current errseq value.
*/
errseq_t errseq_sample(errseq_t *eseq)
{
errseq_t old = READ_ONCE(*eseq);
/* If nobody has seen this error yet, then we can be the first. */
if (!(old & ERRSEQ_SEEN))
old = 0;
return old;
}
EXPORT_SYMBOL(errseq_sample);
/**
* errseq_check() - Has an error occurred since a particular sample point?
* @eseq: Pointer to errseq_t value to be checked.
* @since: Previously-sampled errseq_t from which to check.
*
* Grab the value that eseq points to, and see if it has changed @since
* the given value was sampled. The @since value is not advanced, so there
* is no need to mark the value as seen.
*
* Return: The latest error set in the errseq_t or 0 if it hasn't changed.
*/
int errseq_check(errseq_t *eseq, errseq_t since)
{
errseq_t cur = READ_ONCE(*eseq);
if (likely(cur == since))
return 0;
return -(cur & MAX_ERRNO);
}
EXPORT_SYMBOL(errseq_check);
/**
* errseq_check_and_advance() - Check an errseq_t and advance to current value.
* @eseq: Pointer to value being checked and reported.
* @since: Pointer to previously-sampled errseq_t to check against and advance.
*
* Grab the eseq value, and see whether it matches the value that @since
* points to. If it does, then just return 0.
*
* If it doesn't, then the value has changed. Set the "seen" flag, and try to
* swap it into place as the new eseq value. Then, set that value as the new
* "since" value, and return whatever the error portion is set to.
*
* Note that no locking is provided here for concurrent updates to the "since"
* value. The caller must provide that if necessary. Because of this, callers
* may want to do a lockless errseq_check before taking the lock and calling
* this.
*
* Return: Negative errno if one has been stored, or 0 if no new error has
* occurred.
*/
int errseq_check_and_advance(errseq_t *eseq, errseq_t *since)
{
int err = 0;
errseq_t old, new;
/*
* Most callers will want to use the inline wrapper to check this,
* so that the common case of no error is handled without needing
* to take the lock that protects the "since" value.
*/
old = READ_ONCE(*eseq);
if (old != *since) {
/*
* Set the flag and try to swap it into place if it has
* changed.
*
* We don't care about the outcome of the swap here. If the
* swap doesn't occur, then it has either been updated by a
* writer who is altering the value in some way (updating
* counter or resetting the error), or another reader who is
* just setting the "seen" flag. Either outcome is OK, and we
* can advance "since" and return an error based on what we
* have.
*/
new = old | ERRSEQ_SEEN;
if (new != old)
cmpxchg(eseq, old, new);
*since = new;
err = -(new & MAX_ERRNO);
}
return err;
}
EXPORT_SYMBOL(errseq_check_and_advance);
lib/extable.c
+126
View File
@@ -0,0 +1,126 @@
/*
* Derived from arch/ppc/mm/extable.c and arch/i386/mm/extable.c.
*
* Copyright (C) 2004 Paul Mackerras, IBM Corp.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#include <linux/bsearch.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/sort.h>
#include <linux/uaccess.h>
#ifndef ARCH_HAS_RELATIVE_EXTABLE
#define ex_to_insn(x) ((x)->insn)
#else
static inline unsigned long ex_to_insn(const struct exception_table_entry *x)
{
return (unsigned long)&x->insn + x->insn;
}
#endif
#ifndef ARCH_HAS_SORT_EXTABLE
#ifndef ARCH_HAS_RELATIVE_EXTABLE
#define swap_ex NULL
#else
static void swap_ex(void *a, void *b, int size)
{
struct exception_table_entry *x = a, *y = b, tmp;
int delta = b - a;
tmp = *x;
x->insn = y->insn + delta;
y->insn = tmp.insn - delta;
#ifdef swap_ex_entry_fixup
swap_ex_entry_fixup(x, y, tmp, delta);
#else
x->fixup = y->fixup + delta;
y->fixup = tmp.fixup - delta;
#endif
}
#endif /* ARCH_HAS_RELATIVE_EXTABLE */
/*
* The exception table needs to be sorted so that the binary
* search that we use to find entries in it works properly.
* This is used both for the kernel exception table and for
* the exception tables of modules that get loaded.
*/
static int cmp_ex_sort(const void *a, const void *b)
{
const struct exception_table_entry *x = a, *y = b;
/* avoid overflow */
if (ex_to_insn(x) > ex_to_insn(y))
return 1;
if (ex_to_insn(x) < ex_to_insn(y))
return -1;
return 0;
}
void sort_extable(struct exception_table_entry *start,
struct exception_table_entry *finish)
{
sort(start, finish - start, sizeof(struct exception_table_entry),
cmp_ex_sort, swap_ex);
}
#ifdef CONFIG_MODULES
/*
* If the exception table is sorted, any referring to the module init
* will be at the beginning or the end.
*/
void trim_init_extable(struct module *m)
{
/*trim the beginning*/
while (m->num_exentries &&
within_module_init(ex_to_insn(&m->extable[0]), m)) {
m->extable++;
m->num_exentries--;
}
/*trim the end*/
while (m->num_exentries &&
within_module_init(ex_to_insn(&m->extable[m->num_exentries - 1]),
m))
m->num_exentries--;
}
#endif /* CONFIG_MODULES */
#endif /* !ARCH_HAS_SORT_EXTABLE */
#ifndef ARCH_HAS_SEARCH_EXTABLE
static int cmp_ex_search(const void *key, const void *elt)
{
const struct exception_table_entry *_elt = elt;
unsigned long _key = *(unsigned long *)key;
/* avoid overflow */
if (_key > ex_to_insn(_elt))
return 1;
if (_key < ex_to_insn(_elt))
return -1;
return 0;
}
/*
* Search one exception table for an entry corresponding to the
* given instruction address, and return the address of the entry,
* or NULL if none is found.
* We use a binary search, and thus we assume that the table is
* already sorted.
*/
const struct exception_table_entry *
search_extable(const struct exception_table_entry *base,
const size_t num,
unsigned long value)
{
return bsearch(&value, base, num,
sizeof(struct exception_table_entry), cmp_ex_search);
}
#endif
lib/fault-inject.c
+258
View File
@@ -0,0 +1,258 @@
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/random.h>
#include <linux/sched.h>
#include <linux/stat.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/stacktrace.h>
#include <linux/fault-inject.h>
/*
* setup_fault_attr() is a helper function for various __setup handlers, so it
* returns 0 on error, because that is what __setup handlers do.
*/
int setup_fault_attr(struct fault_attr *attr, char *str)
{
unsigned long probability;
unsigned long interval;
int times;
int space;
/* "<interval>,<probability>,<space>,<times>" */
if (sscanf(str, "%lu,%lu,%d,%d",
&interval, &probability, &space, &times) < 4) {
printk(KERN_WARNING
"FAULT_INJECTION: failed to parse arguments\n");
return 0;
}
attr->probability = probability;
attr->interval = interval;
atomic_set(&attr->times, times);
atomic_set(&attr->space, space);
return 1;
}
EXPORT_SYMBOL_GPL(setup_fault_attr);
static void fail_dump(struct fault_attr *attr)
{
if (attr->verbose > 0 && __ratelimit(&attr->ratelimit_state)) {
printk(KERN_NOTICE "FAULT_INJECTION: forcing a failure.\n"
"name %pd, interval %lu, probability %lu, "
"space %d, times %d\n", attr->dname,
attr->interval, attr->probability,
atomic_read(&attr->space),
atomic_read(&attr->times));
if (attr->verbose > 1)
dump_stack();
}
}
#define atomic_dec_not_zero(v) atomic_add_unless((v), -1, 0)
static bool fail_task(struct fault_attr *attr, struct task_struct *task)
{
return in_task() && task->make_it_fail;
}
#define MAX_STACK_TRACE_DEPTH 32
#ifdef CONFIG_FAULT_INJECTION_STACKTRACE_FILTER
static bool fail_stacktrace(struct fault_attr *attr)
{
struct stack_trace trace;
int depth = attr->stacktrace_depth;
unsigned long entries[MAX_STACK_TRACE_DEPTH];
int n;
bool found = (attr->require_start == 0 && attr->require_end == ULONG_MAX);
if (depth == 0)
return found;
trace.nr_entries = 0;
trace.entries = entries;
trace.max_entries = depth;
trace.skip = 1;
save_stack_trace(&trace);
for (n = 0; n < trace.nr_entries; n++) {
if (attr->reject_start <= entries[n] &&
entries[n] < attr->reject_end)
return false;
if (attr->require_start <= entries[n] &&
entries[n] < attr->require_end)
found = true;
}
return found;
}
#else
static inline bool fail_stacktrace(struct fault_attr *attr)
{
return true;
}
#endif /* CONFIG_FAULT_INJECTION_STACKTRACE_FILTER */
/*
* This code is stolen from failmalloc-1.0
* http://www.nongnu.org/failmalloc/
*/
bool should_fail(struct fault_attr *attr, ssize_t size)
{
if (in_task()) {
unsigned int fail_nth = READ_ONCE(current->fail_nth);
if (fail_nth) {
if (!WRITE_ONCE(current->fail_nth, fail_nth - 1))
goto fail;
return false;
}
}
/* No need to check any other properties if the probability is 0 */
if (attr->probability == 0)
return false;
if (attr->task_filter && !fail_task(attr, current))
return false;
if (atomic_read(&attr->times) == 0)
return false;
if (atomic_read(&attr->space) > size) {
atomic_sub(size, &attr->space);
return false;
}
if (attr->interval > 1) {
attr->count++;
if (attr->count % attr->interval)
return false;
}
if (attr->probability <= prandom_u32() % 100)
return false;
if (!fail_stacktrace(attr))
return false;
fail:
fail_dump(attr);
if (atomic_read(&attr->times) != -1)
atomic_dec_not_zero(&attr->times);
return true;
}
EXPORT_SYMBOL_GPL(should_fail);
#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
static int debugfs_ul_set(void *data, u64 val)
{
*(unsigned long *)data = val;
return 0;
}
static int debugfs_ul_get(void *data, u64 *val)
{
*val = *(unsigned long *)data;
return 0;
}
DEFINE_SIMPLE_ATTRIBUTE(fops_ul, debugfs_ul_get, debugfs_ul_set, "%llu\n");
static struct dentry *debugfs_create_ul(const char *name, umode_t mode,
struct dentry *parent, unsigned long *value)
{
return debugfs_create_file(name, mode, parent, value, &fops_ul);
}
#ifdef CONFIG_FAULT_INJECTION_STACKTRACE_FILTER
static int debugfs_stacktrace_depth_set(void *data, u64 val)
{
*(unsigned long *)data =
min_t(unsigned long, val, MAX_STACK_TRACE_DEPTH);
return 0;
}
DEFINE_SIMPLE_ATTRIBUTE(fops_stacktrace_depth, debugfs_ul_get,
debugfs_stacktrace_depth_set, "%llu\n");
static struct dentry *debugfs_create_stacktrace_depth(
const char *name, umode_t mode,
struct dentry *parent, unsigned long *value)
{
return debugfs_create_file(name, mode, parent, value,
&fops_stacktrace_depth);
}
#endif /* CONFIG_FAULT_INJECTION_STACKTRACE_FILTER */
struct dentry *fault_create_debugfs_attr(const char *name,
struct dentry *parent, struct fault_attr *attr)
{
umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
struct dentry *dir;
dir = debugfs_create_dir(name, parent);
if (!dir)
return ERR_PTR(-ENOMEM);
if (!debugfs_create_ul("probability", mode, dir, &attr->probability))
goto fail;
if (!debugfs_create_ul("interval", mode, dir, &attr->interval))
goto fail;
if (!debugfs_create_atomic_t("times", mode, dir, &attr->times))
goto fail;
if (!debugfs_create_atomic_t("space", mode, dir, &attr->space))
goto fail;
if (!debugfs_create_ul("verbose", mode, dir, &attr->verbose))
goto fail;
if (!debugfs_create_u32("verbose_ratelimit_interval_ms", mode, dir,
&attr->ratelimit_state.interval))
goto fail;
if (!debugfs_create_u32("verbose_ratelimit_burst", mode, dir,
&attr->ratelimit_state.burst))
goto fail;
if (!debugfs_create_bool("task-filter", mode, dir, &attr->task_filter))
goto fail;
#ifdef CONFIG_FAULT_INJECTION_STACKTRACE_FILTER
if (!debugfs_create_stacktrace_depth("stacktrace-depth", mode, dir,
&attr->stacktrace_depth))
goto fail;
if (!debugfs_create_ul("require-start", mode, dir,
&attr->require_start))
goto fail;
if (!debugfs_create_ul("require-end", mode, dir, &attr->require_end))
goto fail;
if (!debugfs_create_ul("reject-start", mode, dir, &attr->reject_start))
goto fail;
if (!debugfs_create_ul("reject-end", mode, dir, &attr->reject_end))
goto fail;
#endif /* CONFIG_FAULT_INJECTION_STACKTRACE_FILTER */
attr->dname = dget(dir);
return dir;
fail:
debugfs_remove_recursive(dir);
return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL_GPL(fault_create_debugfs_attr);
#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
lib/fdt.c
+2
View File
@@ -0,0 +1,2 @@
#include <linux/libfdt_env.h>
#include "../scripts/dtc/libfdt/fdt.c"
lib/fdt_empty_tree.c
+2
View File
@@ -0,0 +1,2 @@
#include <linux/libfdt_env.h>
#include "../scripts/dtc/libfdt/fdt_empty_tree.c"
lib/fdt_ro.c
+2
View File
@@ -0,0 +1,2 @@
#include <linux/libfdt_env.h>
#include "../scripts/dtc/libfdt/fdt_ro.c"
lib/fdt_rw.c
+2
View File
@@ -0,0 +1,2 @@
#include <linux/libfdt_env.h>
#include "../scripts/dtc/libfdt/fdt_rw.c"
lib/fdt_strerror.c
+2
View File
@@ -0,0 +1,2 @@
#include <linux/libfdt_env.h>
#include "../scripts/dtc/libfdt/fdt_strerror.c"
lib/fdt_sw.c
+2
View File
@@ -0,0 +1,2 @@
#include <linux/libfdt_env.h>
#include "../scripts/dtc/libfdt/fdt_sw.c"
lib/fdt_wip.c
+2
View File
@@ -0,0 +1,2 @@
#include <linux/libfdt_env.h>
#include "../scripts/dtc/libfdt/fdt_wip.c"
lib/find_bit.c
+208
View File
@@ -0,0 +1,208 @@
/* bit search implementation
*
* Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
* Written by David Howells (dhowells@redhat.com)
*
* Copyright (C) 2008 IBM Corporation
* 'find_last_bit' is written by Rusty Russell <rusty@rustcorp.com.au>
* (Inspired by David Howell's find_next_bit implementation)
*
* Rewritten by Yury Norov <yury.norov@gmail.com> to decrease
* size and improve performance, 2015.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#include <linux/bitops.h>
#include <linux/bitmap.h>
#include <linux/export.h>
#include <linux/kernel.h>
#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \
!defined(find_next_and_bit)
/*
* This is a common helper function for find_next_bit, find_next_zero_bit, and
* find_next_and_bit. The differences are:
* - The "invert" argument, which is XORed with each fetched word before
* searching it for one bits.
* - The optional "addr2", which is anded with "addr1" if present.
*/
static inline unsigned long _find_next_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long nbits,
unsigned long start, unsigned long invert)
{
unsigned long tmp;
if (unlikely(start >= nbits))
return nbits;
tmp = addr1[start / BITS_PER_LONG];
if (addr2)
tmp &= addr2[start / BITS_PER_LONG];
tmp ^= invert;
/* Handle 1st word. */
tmp &= BITMAP_FIRST_WORD_MASK(start);
start = round_down(start, BITS_PER_LONG);
while (!tmp) {
start += BITS_PER_LONG;
if (start >= nbits)
return nbits;
tmp = addr1[start / BITS_PER_LONG];
if (addr2)
tmp &= addr2[start / BITS_PER_LONG];
tmp ^= invert;
}
return min(start + __ffs(tmp), nbits);
}
#endif
#ifndef find_next_bit
/*
* Find the next set bit in a memory region.
*/
unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
unsigned long offset)
{
return _find_next_bit(addr, NULL, size, offset, 0UL);
}
EXPORT_SYMBOL(find_next_bit);
#endif
#ifndef find_next_zero_bit
unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
unsigned long offset)
{
return _find_next_bit(addr, NULL, size, offset, ~0UL);
}
EXPORT_SYMBOL(find_next_zero_bit);
#endif
#if !defined(find_next_and_bit)
unsigned long find_next_and_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long size,
unsigned long offset)
{
return _find_next_bit(addr1, addr2, size, offset, 0UL);
}
EXPORT_SYMBOL(find_next_and_bit);
#endif
#ifndef find_first_bit
/*
* Find the first set bit in a memory region.
*/
unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
{
unsigned long idx;
for (idx = 0; idx * BITS_PER_LONG < size; idx++) {
if (addr[idx])
return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size);
}
return size;
}
EXPORT_SYMBOL(find_first_bit);
#endif
#ifndef find_first_zero_bit
/*
* Find the first cleared bit in a memory region.
*/
unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
{
unsigned long idx;
for (idx = 0; idx * BITS_PER_LONG < size; idx++) {
if (addr[idx] != ~0UL)
return min(idx * BITS_PER_LONG + ffz(addr[idx]), size);
}
return size;
}
EXPORT_SYMBOL(find_first_zero_bit);
#endif
#ifndef find_last_bit
unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
{
if (size) {
unsigned long val = BITMAP_LAST_WORD_MASK(size);
unsigned long idx = (size-1) / BITS_PER_LONG;
do {
val &= addr[idx];
if (val)
return idx * BITS_PER_LONG + __fls(val);
val = ~0ul;
} while (idx--);
}
return size;
}
EXPORT_SYMBOL(find_last_bit);
#endif
#ifdef __BIG_ENDIAN
#if !defined(find_next_bit_le) || !defined(find_next_zero_bit_le)
static inline unsigned long _find_next_bit_le(const unsigned long *addr1,
const unsigned long *addr2, unsigned long nbits,
unsigned long start, unsigned long invert)
{
unsigned long tmp;
if (unlikely(start >= nbits))
return nbits;
tmp = addr1[start / BITS_PER_LONG];
if (addr2)
tmp &= addr2[start / BITS_PER_LONG];
tmp ^= invert;
/* Handle 1st word. */
tmp &= swab(BITMAP_FIRST_WORD_MASK(start));
start = round_down(start, BITS_PER_LONG);
while (!tmp) {
start += BITS_PER_LONG;
if (start >= nbits)
return nbits;
tmp = addr1[start / BITS_PER_LONG];
if (addr2)
tmp &= addr2[start / BITS_PER_LONG];
tmp ^= invert;
}
return min(start + __ffs(swab(tmp)), nbits);
}
#endif
#ifndef find_next_zero_bit_le
unsigned long find_next_zero_bit_le(const void *addr, unsigned
long size, unsigned long offset)
{
return _find_next_bit_le(addr, NULL, size, offset, ~0UL);
}
EXPORT_SYMBOL(find_next_zero_bit_le);
#endif
#ifndef find_next_bit_le
unsigned long find_next_bit_le(const void *addr, unsigned
long size, unsigned long offset)
{
return _find_next_bit_le(addr, NULL, size, offset, 0UL);
}
EXPORT_SYMBOL(find_next_bit_le);
#endif
#endif /* __BIG_ENDIAN */
lib/find_bit_benchmark.c
+167
View File
@@ -0,0 +1,167 @@
/*
* Test for find_*_bit functions.
*
* Copyright (c) 2017 Cavium.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of version 2 of the GNU General Public
* License as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
/*
* find_bit functions are widely used in kernel, so the successful boot
* is good enough test for correctness.
*
* This test is focused on performance of traversing bitmaps. Two typical
* scenarios are reproduced:
* - randomly filled bitmap with approximately equal number of set and
* cleared bits;
* - sparse bitmap with few set bits at random positions.
*/
#include <linux/bitops.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/printk.h>
#include <linux/random.h>
#define BITMAP_LEN (4096UL * 8 * 10)
#define SPARSE 500
static DECLARE_BITMAP(bitmap, BITMAP_LEN) __initdata;
static DECLARE_BITMAP(bitmap2, BITMAP_LEN) __initdata;
/*
* This is Schlemiel the Painter's algorithm. It should be called after
* all other tests for the same bitmap because it sets all bits of bitmap to 1.
*/
static int __init test_find_first_bit(void *bitmap, unsigned long len)
{
unsigned long i, cnt;
ktime_t time;
time = ktime_get();
for (cnt = i = 0; i < len; cnt++) {
i = find_first_bit(bitmap, len);
__clear_bit(i, bitmap);
}
time = ktime_get() - time;
pr_err("find_first_bit: %18llu ns, %6ld iterations\n", time, cnt);
return 0;
}
static int __init test_find_next_bit(const void *bitmap, unsigned long len)
{
unsigned long i, cnt;
ktime_t time;
time = ktime_get();
for (cnt = i = 0; i < BITMAP_LEN; cnt++)
i = find_next_bit(bitmap, BITMAP_LEN, i) + 1;
time = ktime_get() - time;
pr_err("find_next_bit: %18llu ns, %6ld iterations\n", time, cnt);
return 0;
}
static int __init test_find_next_zero_bit(const void *bitmap, unsigned long len)
{
unsigned long i, cnt;
ktime_t time;
time = ktime_get();
for (cnt = i = 0; i < BITMAP_LEN; cnt++)
i = find_next_zero_bit(bitmap, len, i) + 1;
time = ktime_get() - time;
pr_err("find_next_zero_bit: %18llu ns, %6ld iterations\n", time, cnt);
return 0;
}
static int __init test_find_last_bit(const void *bitmap, unsigned long len)
{
unsigned long l, cnt = 0;
ktime_t time;
time = ktime_get();
do {
cnt++;
l = find_last_bit(bitmap, len);
if (l >= len)
break;
len = l;
} while (len);
time = ktime_get() - time;
pr_err("find_last_bit: %18llu ns, %6ld iterations\n", time, cnt);
return 0;
}
static int __init test_find_next_and_bit(const void *bitmap,
const void *bitmap2, unsigned long len)
{
unsigned long i, cnt;
cycles_t cycles;
cycles = get_cycles();
for (cnt = i = 0; i < BITMAP_LEN; cnt++)
i = find_next_and_bit(bitmap, bitmap2, BITMAP_LEN, i+1);
cycles = get_cycles() - cycles;
pr_err("find_next_and_bit:\t\t%llu cycles, %ld iterations\n",
(u64)cycles, cnt);
return 0;
}
static int __init find_bit_test(void)
{
unsigned long nbits = BITMAP_LEN / SPARSE;
pr_err("\nStart testing find_bit() with random-filled bitmap\n");
get_random_bytes(bitmap, sizeof(bitmap));
get_random_bytes(bitmap2, sizeof(bitmap2));
test_find_next_bit(bitmap, BITMAP_LEN);
test_find_next_zero_bit(bitmap, BITMAP_LEN);
test_find_last_bit(bitmap, BITMAP_LEN);
/*
* test_find_first_bit() may take some time, so
* traverse only part of bitmap to avoid soft lockup.
*/
test_find_first_bit(bitmap, BITMAP_LEN / 10);
test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN);
pr_err("\nStart testing find_bit() with sparse bitmap\n");
bitmap_zero(bitmap, BITMAP_LEN);
bitmap_zero(bitmap2, BITMAP_LEN);
while (nbits--) {
__set_bit(prandom_u32() % BITMAP_LEN, bitmap);
__set_bit(prandom_u32() % BITMAP_LEN, bitmap2);
}
test_find_next_bit(bitmap, BITMAP_LEN);
test_find_next_zero_bit(bitmap, BITMAP_LEN);
test_find_last_bit(bitmap, BITMAP_LEN);
test_find_first_bit(bitmap, BITMAP_LEN);
test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN);
/*
* Everything is OK. Return error just to let user run benchmark
* again without annoying rmmod.
*/
return -EINVAL;
}
module_init(find_bit_test);
MODULE_LICENSE("GPL");
lib/flex_array.c
+398
View File
@@ -0,0 +1,398 @@
/*
* Flexible array managed in PAGE_SIZE parts
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* Copyright IBM Corporation, 2009
*
* Author: Dave Hansen <dave@linux.vnet.ibm.com>
*/
#include <linux/flex_array.h>
#include <linux/slab.h>
#include <linux/stddef.h>
#include <linux/export.h>
#include <linux/reciprocal_div.h>
struct flex_array_part {
char elements[FLEX_ARRAY_PART_SIZE];
};
/*
* If a user requests an allocation which is small
* enough, we may simply use the space in the
* flex_array->parts[] array to store the user
* data.
*/
static inline int elements_fit_in_base(struct flex_array *fa)
{
int data_size = fa->element_size * fa->total_nr_elements;
if (data_size <= FLEX_ARRAY_BASE_BYTES_LEFT)
return 1;
return 0;
}
/**
* flex_array_alloc - allocate a new flexible array
* @element_size: the size of individual elements in the array
* @total: total number of elements that this should hold
* @flags: page allocation flags to use for base array
*
* Note: all locking must be provided by the caller.
*
* @total is used to size internal structures. If the user ever
* accesses any array indexes >=@total, it will produce errors.
*
* The maximum number of elements is defined as: the number of
* elements that can be stored in a page times the number of
* page pointers that we can fit in the base structure or (using
* integer math):
*
* (PAGE_SIZE/element_size) * (PAGE_SIZE-8)/sizeof(void *)
*
* Here's a table showing example capacities. Note that the maximum
* index that the get/put() functions is just nr_objects-1. This
* basically means that you get 4MB of storage on 32-bit and 2MB on
* 64-bit.
*
*
* Element size | Objects | Objects |
* PAGE_SIZE=4k | 32-bit | 64-bit |
* ---------------------------------|
* 1 bytes | 4177920 | 2088960 |
* 2 bytes | 2088960 | 1044480 |
* 3 bytes | 1392300 | 696150 |
* 4 bytes | 1044480 | 522240 |
* 32 bytes | 130560 | 65408 |
* 33 bytes | 126480 | 63240 |
* 2048 bytes | 2040 | 1020 |
* 2049 bytes | 1020 | 510 |
* void * | 1044480 | 261120 |
*
* Since 64-bit pointers are twice the size, we lose half the
* capacity in the base structure. Also note that no effort is made
* to efficiently pack objects across page boundaries.
*/
struct flex_array *flex_array_alloc(int element_size, unsigned int total,
gfp_t flags)
{
struct flex_array *ret;
int elems_per_part = 0;
int max_size = 0;
struct reciprocal_value reciprocal_elems = { 0 };
if (element_size) {
elems_per_part = FLEX_ARRAY_ELEMENTS_PER_PART(element_size);
reciprocal_elems = reciprocal_value(elems_per_part);
max_size = FLEX_ARRAY_NR_BASE_PTRS * elems_per_part;
}
/* max_size will end up 0 if element_size > PAGE_SIZE */
if (total > max_size)
return NULL;
ret = kzalloc(sizeof(struct flex_array), flags);
if (!ret)
return NULL;
ret->element_size = element_size;
ret->total_nr_elements = total;
ret->elems_per_part = elems_per_part;
ret->reciprocal_elems = reciprocal_elems;
if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO))
memset(&ret->parts[0], FLEX_ARRAY_FREE,
FLEX_ARRAY_BASE_BYTES_LEFT);
return ret;
}
EXPORT_SYMBOL(flex_array_alloc);
static int fa_element_to_part_nr(struct flex_array *fa,
unsigned int element_nr)
{
/*
* if element_size == 0 we don't get here, so we never touch
* the zeroed fa->reciprocal_elems, which would yield invalid
* results
*/
return reciprocal_divide(element_nr, fa->reciprocal_elems);
}
/**
* flex_array_free_parts - just free the second-level pages
* @fa: the flex array from which to free parts
*
* This is to be used in cases where the base 'struct flex_array'
* has been statically allocated and should not be free.
*/
void flex_array_free_parts(struct flex_array *fa)
{
int part_nr;
if (elements_fit_in_base(fa))
return;
for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++)
kfree(fa->parts[part_nr]);
}
EXPORT_SYMBOL(flex_array_free_parts);
void flex_array_free(struct flex_array *fa)
{
flex_array_free_parts(fa);
kfree(fa);
}
EXPORT_SYMBOL(flex_array_free);
static unsigned int index_inside_part(struct flex_array *fa,
unsigned int element_nr,
unsigned int part_nr)
{
unsigned int part_offset;
part_offset = element_nr - part_nr * fa->elems_per_part;
return part_offset * fa->element_size;
}
static struct flex_array_part *
__fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags)
{
struct flex_array_part *part = fa->parts[part_nr];
if (!part) {
part = kmalloc(sizeof(struct flex_array_part), flags);
if (!part)
return NULL;
if (!(flags & __GFP_ZERO))
memset(part, FLEX_ARRAY_FREE,
sizeof(struct flex_array_part));
fa->parts[part_nr] = part;
}
return part;
}
/**
* flex_array_put - copy data into the array at @element_nr
* @fa: the flex array to copy data into
* @element_nr: index of the position in which to insert
* the new element.
* @src: address of data to copy into the array
* @flags: page allocation flags to use for array expansion
*
*
* Note that this *copies* the contents of @src into
* the array. If you are trying to store an array of
* pointers, make sure to pass in &ptr instead of ptr.
* You may instead wish to use the flex_array_put_ptr()
* helper function.
*
* Locking must be provided by the caller.
*/
int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src,
gfp_t flags)
{
int part_nr = 0;
struct flex_array_part *part;
void *dst;
if (element_nr >= fa->total_nr_elements)
return -ENOSPC;
if (!fa->element_size)
return 0;
if (elements_fit_in_base(fa))
part = (struct flex_array_part *)&fa->parts[0];
else {
part_nr = fa_element_to_part_nr(fa, element_nr);
part = __fa_get_part(fa, part_nr, flags);
if (!part)
return -ENOMEM;
}
dst = &part->elements[index_inside_part(fa, element_nr, part_nr)];
memcpy(dst, src, fa->element_size);
return 0;
}
EXPORT_SYMBOL(flex_array_put);
/**
* flex_array_clear - clear element in array at @element_nr
* @fa: the flex array of the element.
* @element_nr: index of the position to clear.
*
* Locking must be provided by the caller.
*/
int flex_array_clear(struct flex_array *fa, unsigned int element_nr)
{
int part_nr = 0;
struct flex_array_part *part;
void *dst;
if (element_nr >= fa->total_nr_elements)
return -ENOSPC;
if (!fa->element_size)
return 0;
if (elements_fit_in_base(fa))
part = (struct flex_array_part *)&fa->parts[0];
else {
part_nr = fa_element_to_part_nr(fa, element_nr);
part = fa->parts[part_nr];
if (!part)
return -EINVAL;
}
dst = &part->elements[index_inside_part(fa, element_nr, part_nr)];
memset(dst, FLEX_ARRAY_FREE, fa->element_size);
return 0;
}
EXPORT_SYMBOL(flex_array_clear);
/**
* flex_array_prealloc - guarantee that array space exists
* @fa: the flex array for which to preallocate parts
* @start: index of first array element for which space is allocated
* @nr_elements: number of elements for which space is allocated
* @flags: page allocation flags
*
* This will guarantee that no future calls to flex_array_put()
* will allocate memory. It can be used if you are expecting to
* be holding a lock or in some atomic context while writing
* data into the array.
*
* Locking must be provided by the caller.
*/
int flex_array_prealloc(struct flex_array *fa, unsigned int start,
unsigned int nr_elements, gfp_t flags)
{
int start_part;
int end_part;
int part_nr;
unsigned int end;
struct flex_array_part *part;
if (!start && !nr_elements)
return 0;
if (start >= fa->total_nr_elements)
return -ENOSPC;
if (!nr_elements)
return 0;
end = start + nr_elements - 1;
if (end >= fa->total_nr_elements)
return -ENOSPC;
if (!fa->element_size)
return 0;
if (elements_fit_in_base(fa))
return 0;
start_part = fa_element_to_part_nr(fa, start);
end_part = fa_element_to_part_nr(fa, end);
for (part_nr = start_part; part_nr <= end_part; part_nr++) {
part = __fa_get_part(fa, part_nr, flags);
if (!part)
return -ENOMEM;
}
return 0;
}
EXPORT_SYMBOL(flex_array_prealloc);
/**
* flex_array_get - pull data back out of the array
* @fa: the flex array from which to extract data
* @element_nr: index of the element to fetch from the array
*
* Returns a pointer to the data at index @element_nr. Note
* that this is a copy of the data that was passed in. If you
* are using this to store pointers, you'll get back &ptr. You
* may instead wish to use the flex_array_get_ptr helper.
*
* Locking must be provided by the caller.
*/
void *flex_array_get(struct flex_array *fa, unsigned int element_nr)
{
int part_nr = 0;
struct flex_array_part *part;
if (!fa->element_size)
return NULL;
if (element_nr >= fa->total_nr_elements)
return NULL;
if (elements_fit_in_base(fa))
part = (struct flex_array_part *)&fa->parts[0];
else {
part_nr = fa_element_to_part_nr(fa, element_nr);
part = fa->parts[part_nr];
if (!part)
return NULL;
}
return &part->elements[index_inside_part(fa, element_nr, part_nr)];
}
EXPORT_SYMBOL(flex_array_get);
/**
* flex_array_get_ptr - pull a ptr back out of the array
* @fa: the flex array from which to extract data
* @element_nr: index of the element to fetch from the array
*
* Returns the pointer placed in the flex array at element_nr using
* flex_array_put_ptr(). This function should not be called if the
* element in question was not set using the _put_ptr() helper.
*/
void *flex_array_get_ptr(struct flex_array *fa, unsigned int element_nr)
{
void **tmp;
tmp = flex_array_get(fa, element_nr);
if (!tmp)
return NULL;
return *tmp;
}
EXPORT_SYMBOL(flex_array_get_ptr);
static int part_is_free(struct flex_array_part *part)
{
int i;
for (i = 0; i < sizeof(struct flex_array_part); i++)
if (part->elements[i] != FLEX_ARRAY_FREE)
return 0;
return 1;
}
/**
* flex_array_shrink - free unused second-level pages
* @fa: the flex array to shrink
*
* Frees all second-level pages that consist solely of unused
* elements. Returns the number of pages freed.
*
* Locking must be provided by the caller.
*/
int flex_array_shrink(struct flex_array *fa)
{
struct flex_array_part *part;
int part_nr;
int ret = 0;
if (!fa->total_nr_elements || !fa->element_size)
return 0;
if (elements_fit_in_base(fa))
return ret;
for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) {
part = fa->parts[part_nr];
if (!part)
continue;
if (part_is_free(part)) {
fa->parts[part_nr] = NULL;
kfree(part);
ret++;
}
}
return ret;
}
EXPORT_SYMBOL(flex_array_shrink);
Loaded 100 of 80519 files, more files were not shown because too many files have changed in this diff. Show more