Line data Source code
1 : /* bufhelp.h - Some buffer manipulation helpers
2 : * Copyright (C) 2012-2017 Jussi Kivilinna <jussi.kivilinna@iki.fi>
3 : *
4 : * This file is part of Libgcrypt.
5 : *
6 : * Libgcrypt is free software; you can redistribute it and/or modify
7 : * it under the terms of the GNU Lesser General Public License as
8 : * published by the Free Software Foundation; either version 2.1 of
9 : * the License, or (at your option) any later version.
10 : *
11 : * Libgcrypt is distributed in the hope that it will be useful,
12 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : * GNU Lesser General Public License for more details.
15 : *
16 : * You should have received a copy of the GNU Lesser General Public
17 : * License along with this program; if not, see <http://www.gnu.org/licenses/>.
18 : */
19 : #ifndef GCRYPT_BUFHELP_H
20 : #define GCRYPT_BUFHELP_H
21 :
22 :
23 : #include "g10lib.h"
24 : #include "bithelp.h"
25 :
26 :
27 : #undef BUFHELP_UNALIGNED_ACCESS
28 : #if defined(HAVE_GCC_ATTRIBUTE_PACKED) && \
29 : defined(HAVE_GCC_ATTRIBUTE_ALIGNED) && \
30 : defined(HAVE_GCC_ATTRIBUTE_MAY_ALIAS)
31 : /* Compiler is supports attributes needed for automatically issuing unaligned
32 : memory access instructions.
33 : */
34 : # define BUFHELP_UNALIGNED_ACCESS 1
35 : #endif
36 :
37 :
38 : #undef BUFHELP_FAST_UNALIGNED_ACCESS
39 : #if defined(BUFHELP_UNALIGNED_ACCESS) && \
40 : (defined(__i386__) || defined(__x86_64__) || \
41 : (defined(__arm__) && defined(__ARM_FEATURE_UNALIGNED)) || \
42 : defined(__aarch64__))
43 : /* These architectures are able of unaligned memory accesses and can
44 : handle those fast.
45 : */
46 : # define BUFHELP_FAST_UNALIGNED_ACCESS 1
47 : #endif
48 :
49 :
50 : #ifdef BUFHELP_FAST_UNALIGNED_ACCESS
51 : /* Define type with one-byte alignment on architectures with fast unaligned
52 : memory accesses.
53 : */
54 : typedef struct bufhelp_int_s
55 : {
56 : uintptr_t a;
57 : } __attribute__((packed, aligned(1), may_alias)) bufhelp_int_t;
58 : #else
59 : /* Define type with default alignment for other architectures (unaligned
60 : accessed handled in per byte loops).
61 : */
62 : #ifdef HAVE_GCC_ATTRIBUTE_MAY_ALIAS
63 : typedef struct bufhelp_int_s
64 : {
65 : uintptr_t a;
66 : } __attribute__((may_alias)) bufhelp_int_t;
67 : #else
68 : typedef struct bufhelp_int_s
69 : {
70 : uintptr_t a;
71 : } bufhelp_int_t;
72 : #endif
73 : #endif
74 :
75 :
76 : /* Optimized function for small buffer copying */
77 : static inline void
78 17848000 : buf_cpy(void *_dst, const void *_src, size_t len)
79 : {
80 : #if __GNUC__ >= 4 && (defined(__x86_64__) || defined(__i386__))
81 : /* For AMD64 and i386, memcpy is faster. */
82 17848000 : memcpy(_dst, _src, len);
83 : #else
84 : byte *dst = _dst;
85 : const byte *src = _src;
86 : bufhelp_int_t *ldst;
87 : const bufhelp_int_t *lsrc;
88 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
89 : const unsigned int longmask = sizeof(bufhelp_int_t) - 1;
90 :
91 : /* Skip fast processing if buffers are unaligned. */
92 : if (UNLIKELY(((uintptr_t)dst | (uintptr_t)src) & longmask))
93 : goto do_bytes;
94 : #endif
95 :
96 : ldst = (bufhelp_int_t *)(void *)dst;
97 : lsrc = (const bufhelp_int_t *)(const void *)src;
98 :
99 : for (; len >= sizeof(bufhelp_int_t); len -= sizeof(bufhelp_int_t))
100 : (ldst++)->a = (lsrc++)->a;
101 :
102 : dst = (byte *)ldst;
103 : src = (const byte *)lsrc;
104 :
105 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
106 : do_bytes:
107 : #endif
108 : /* Handle tail. */
109 : for (; len; len--)
110 : *dst++ = *src++;
111 : #endif /*__GNUC__ >= 4 && (__x86_64__ || __i386__)*/
112 17848000 : }
113 :
114 :
115 : /* Optimized function for buffer xoring */
116 : static inline void
117 45255156 : buf_xor(void *_dst, const void *_src1, const void *_src2, size_t len)
118 : {
119 45255156 : byte *dst = _dst;
120 45255156 : const byte *src1 = _src1;
121 45255156 : const byte *src2 = _src2;
122 : bufhelp_int_t *ldst;
123 : const bufhelp_int_t *lsrc1, *lsrc2;
124 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
125 : const unsigned int longmask = sizeof(bufhelp_int_t) - 1;
126 :
127 : /* Skip fast processing if buffers are unaligned. */
128 : if (UNLIKELY(((uintptr_t)dst | (uintptr_t)src1 | (uintptr_t)src2) & longmask))
129 : goto do_bytes;
130 : #endif
131 :
132 45255156 : ldst = (bufhelp_int_t *)(void *)dst;
133 45255156 : lsrc1 = (const bufhelp_int_t *)(const void *)src1;
134 45255156 : lsrc2 = (const bufhelp_int_t *)(const void *)src2;
135 :
136 133375314 : for (; len >= sizeof(bufhelp_int_t); len -= sizeof(bufhelp_int_t))
137 88120158 : (ldst++)->a = (lsrc1++)->a ^ (lsrc2++)->a;
138 :
139 45255156 : dst = (byte *)ldst;
140 45255156 : src1 = (const byte *)lsrc1;
141 45255156 : src2 = (const byte *)lsrc2;
142 :
143 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
144 : do_bytes:
145 : #endif
146 : /* Handle tail. */
147 45891928 : for (; len; len--)
148 636772 : *dst++ = *src1++ ^ *src2++;
149 45255156 : }
150 :
151 :
152 : /* Optimized function for in-place buffer xoring. */
153 : static inline void
154 44051998 : buf_xor_1(void *_dst, const void *_src, size_t len)
155 : {
156 44051998 : byte *dst = _dst;
157 44051998 : const byte *src = _src;
158 : bufhelp_int_t *ldst;
159 : const bufhelp_int_t *lsrc;
160 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
161 : const unsigned int longmask = sizeof(bufhelp_int_t) - 1;
162 :
163 : /* Skip fast processing if buffers are unaligned. */
164 : if (UNLIKELY(((uintptr_t)dst | (uintptr_t)src) & longmask))
165 : goto do_bytes;
166 : #endif
167 :
168 44051998 : ldst = (bufhelp_int_t *)(void *)dst;
169 44051998 : lsrc = (const bufhelp_int_t *)(const void *)src;
170 :
171 132155994 : for (; len >= sizeof(bufhelp_int_t); len -= sizeof(bufhelp_int_t))
172 88103996 : (ldst++)->a ^= (lsrc++)->a;
173 :
174 44051998 : dst = (byte *)ldst;
175 44051998 : src = (const byte *)lsrc;
176 :
177 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
178 : do_bytes:
179 : #endif
180 : /* Handle tail. */
181 44051998 : for (; len; len--)
182 0 : *dst++ ^= *src++;
183 44051998 : }
184 :
185 :
186 : /* Optimized function for buffer xoring with two destination buffers. Used
187 : mainly by CFB mode encryption. */
188 : static inline void
189 5817765 : buf_xor_2dst(void *_dst1, void *_dst2, const void *_src, size_t len)
190 : {
191 5817765 : byte *dst1 = _dst1;
192 5817765 : byte *dst2 = _dst2;
193 5817765 : const byte *src = _src;
194 : bufhelp_int_t *ldst1, *ldst2;
195 : const bufhelp_int_t *lsrc;
196 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
197 : const unsigned int longmask = sizeof(bufhelp_int_t) - 1;
198 :
199 : /* Skip fast processing if buffers are unaligned. */
200 : if (UNLIKELY(((uintptr_t)src | (uintptr_t)dst1 | (uintptr_t)dst2) & longmask))
201 : goto do_bytes;
202 : #endif
203 :
204 5817765 : ldst1 = (bufhelp_int_t *)(void *)dst1;
205 5817765 : ldst2 = (bufhelp_int_t *)(void *)dst2;
206 5817765 : lsrc = (const bufhelp_int_t *)(const void *)src;
207 :
208 14164867 : for (; len >= sizeof(bufhelp_int_t); len -= sizeof(bufhelp_int_t))
209 8347102 : (ldst1++)->a = ((ldst2++)->a ^= (lsrc++)->a);
210 :
211 5817765 : dst1 = (byte *)ldst1;
212 5817765 : dst2 = (byte *)ldst2;
213 5817765 : src = (const byte *)lsrc;
214 :
215 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
216 : do_bytes:
217 : #endif
218 : /* Handle tail. */
219 5817765 : for (; len; len--)
220 0 : *dst1++ = (*dst2++ ^= *src++);
221 5817765 : }
222 :
223 :
224 : /* Optimized function for combined buffer xoring and copying. Used by mainly
225 : CBC mode decryption. */
226 : static inline void
227 1995844 : buf_xor_n_copy_2(void *_dst_xor, const void *_src_xor, void *_srcdst_cpy,
228 : const void *_src_cpy, size_t len)
229 : {
230 1995844 : byte *dst_xor = _dst_xor;
231 1995844 : byte *srcdst_cpy = _srcdst_cpy;
232 1995844 : const byte *src_xor = _src_xor;
233 1995844 : const byte *src_cpy = _src_cpy;
234 : byte temp;
235 : bufhelp_int_t *ldst_xor, *lsrcdst_cpy;
236 : const bufhelp_int_t *lsrc_cpy, *lsrc_xor;
237 : uintptr_t ltemp;
238 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
239 : const unsigned int longmask = sizeof(bufhelp_int_t) - 1;
240 :
241 : /* Skip fast processing if buffers are unaligned. */
242 : if (UNLIKELY(((uintptr_t)src_cpy | (uintptr_t)src_xor | (uintptr_t)dst_xor |
243 : (uintptr_t)srcdst_cpy) & longmask))
244 : goto do_bytes;
245 : #endif
246 :
247 1995844 : ldst_xor = (bufhelp_int_t *)(void *)dst_xor;
248 1995844 : lsrc_xor = (const bufhelp_int_t *)(void *)src_xor;
249 1995844 : lsrcdst_cpy = (bufhelp_int_t *)(void *)srcdst_cpy;
250 1995844 : lsrc_cpy = (const bufhelp_int_t *)(const void *)src_cpy;
251 :
252 4351872 : for (; len >= sizeof(bufhelp_int_t); len -= sizeof(bufhelp_int_t))
253 : {
254 2356028 : ltemp = (lsrc_cpy++)->a;
255 2356028 : (ldst_xor++)->a = (lsrcdst_cpy)->a ^ (lsrc_xor++)->a;
256 2356028 : (lsrcdst_cpy++)->a = ltemp;
257 : }
258 :
259 1995844 : dst_xor = (byte *)ldst_xor;
260 1995844 : src_xor = (const byte *)lsrc_xor;
261 1995844 : srcdst_cpy = (byte *)lsrcdst_cpy;
262 1995844 : src_cpy = (const byte *)lsrc_cpy;
263 :
264 : #ifndef BUFHELP_FAST_UNALIGNED_ACCESS
265 : do_bytes:
266 : #endif
267 : /* Handle tail. */
268 1995844 : for (; len; len--)
269 : {
270 0 : temp = *src_cpy++;
271 0 : *dst_xor++ = *srcdst_cpy ^ *src_xor++;
272 0 : *srcdst_cpy++ = temp;
273 : }
274 1995844 : }
275 :
276 :
277 : /* Optimized function for combined buffer xoring and copying. Used by mainly
278 : CFB mode decryption. */
279 : static inline void
280 1863715 : buf_xor_n_copy(void *_dst_xor, void *_srcdst_cpy, const void *_src, size_t len)
281 : {
282 1863715 : buf_xor_n_copy_2(_dst_xor, _src, _srcdst_cpy, _src, len);
283 1863715 : }
284 :
285 :
286 : /* Constant-time compare of two buffers. Returns 1 if buffers are equal,
287 : and 0 if buffers differ. */
288 : static inline int
289 44670 : buf_eq_const(const void *_a, const void *_b, size_t len)
290 : {
291 44670 : const byte *a = _a;
292 44670 : const byte *b = _b;
293 : size_t diff, i;
294 :
295 : /* Constant-time compare. */
296 572030 : for (i = 0, diff = 0; i < len; i++)
297 527360 : diff -= !!(a[i] - b[i]);
298 :
299 44670 : return !diff;
300 : }
301 :
302 :
303 : #ifndef BUFHELP_UNALIGNED_ACCESS
304 :
305 : /* Functions for loading and storing unaligned u32 values of different
306 : endianness. */
307 : static inline u32 buf_get_be32(const void *_buf)
308 : {
309 : const byte *in = _buf;
310 : return ((u32)in[0] << 24) | ((u32)in[1] << 16) | \
311 : ((u32)in[2] << 8) | (u32)in[3];
312 : }
313 :
314 : static inline u32 buf_get_le32(const void *_buf)
315 : {
316 : const byte *in = _buf;
317 : return ((u32)in[3] << 24) | ((u32)in[2] << 16) | \
318 : ((u32)in[1] << 8) | (u32)in[0];
319 : }
320 :
321 : static inline void buf_put_be32(void *_buf, u32 val)
322 : {
323 : byte *out = _buf;
324 : out[0] = val >> 24;
325 : out[1] = val >> 16;
326 : out[2] = val >> 8;
327 : out[3] = val;
328 : }
329 :
330 : static inline void buf_put_le32(void *_buf, u32 val)
331 : {
332 : byte *out = _buf;
333 : out[3] = val >> 24;
334 : out[2] = val >> 16;
335 : out[1] = val >> 8;
336 : out[0] = val;
337 : }
338 :
339 :
340 : /* Functions for loading and storing unaligned u64 values of different
341 : endianness. */
342 : static inline u64 buf_get_be64(const void *_buf)
343 : {
344 : const byte *in = _buf;
345 : return ((u64)in[0] << 56) | ((u64)in[1] << 48) | \
346 : ((u64)in[2] << 40) | ((u64)in[3] << 32) | \
347 : ((u64)in[4] << 24) | ((u64)in[5] << 16) | \
348 : ((u64)in[6] << 8) | (u64)in[7];
349 : }
350 :
351 : static inline u64 buf_get_le64(const void *_buf)
352 : {
353 : const byte *in = _buf;
354 : return ((u64)in[7] << 56) | ((u64)in[6] << 48) | \
355 : ((u64)in[5] << 40) | ((u64)in[4] << 32) | \
356 : ((u64)in[3] << 24) | ((u64)in[2] << 16) | \
357 : ((u64)in[1] << 8) | (u64)in[0];
358 : }
359 :
360 : static inline void buf_put_be64(void *_buf, u64 val)
361 : {
362 : byte *out = _buf;
363 : out[0] = val >> 56;
364 : out[1] = val >> 48;
365 : out[2] = val >> 40;
366 : out[3] = val >> 32;
367 : out[4] = val >> 24;
368 : out[5] = val >> 16;
369 : out[6] = val >> 8;
370 : out[7] = val;
371 : }
372 :
373 : static inline void buf_put_le64(void *_buf, u64 val)
374 : {
375 : byte *out = _buf;
376 : out[7] = val >> 56;
377 : out[6] = val >> 48;
378 : out[5] = val >> 40;
379 : out[4] = val >> 32;
380 : out[3] = val >> 24;
381 : out[2] = val >> 16;
382 : out[1] = val >> 8;
383 : out[0] = val;
384 : }
385 :
386 : #else /*BUFHELP_UNALIGNED_ACCESS*/
387 :
388 : typedef struct bufhelp_u32_s
389 : {
390 : u32 a;
391 : } __attribute__((packed, aligned(1), may_alias)) bufhelp_u32_t;
392 :
393 : /* Functions for loading and storing unaligned u32 values of different
394 : endianness. */
395 96865912 : static inline u32 buf_get_be32(const void *_buf)
396 : {
397 96865912 : return be_bswap32(((const bufhelp_u32_t *)_buf)->a);
398 : }
399 :
400 65130436 : static inline u32 buf_get_le32(const void *_buf)
401 : {
402 65130436 : return le_bswap32(((const bufhelp_u32_t *)_buf)->a);
403 : }
404 :
405 95477453 : static inline void buf_put_be32(void *_buf, u32 val)
406 : {
407 95477453 : bufhelp_u32_t *out = _buf;
408 95477453 : out->a = be_bswap32(val);
409 95477453 : }
410 :
411 34586482 : static inline void buf_put_le32(void *_buf, u32 val)
412 : {
413 34586482 : bufhelp_u32_t *out = _buf;
414 34586482 : out->a = le_bswap32(val);
415 34586482 : }
416 :
417 :
418 : typedef struct bufhelp_u64_s
419 : {
420 : u64 a;
421 : } __attribute__((packed, aligned(1), may_alias)) bufhelp_u64_t;
422 :
423 : /* Functions for loading and storing unaligned u64 values of different
424 : endianness. */
425 2904328 : static inline u64 buf_get_be64(const void *_buf)
426 : {
427 2904328 : return be_bswap64(((const bufhelp_u64_t *)_buf)->a);
428 : }
429 :
430 30008774 : static inline u64 buf_get_le64(const void *_buf)
431 : {
432 30008774 : return le_bswap64(((const bufhelp_u64_t *)_buf)->a);
433 : }
434 :
435 3018277 : static inline void buf_put_be64(void *_buf, u64 val)
436 : {
437 3018277 : bufhelp_u64_t *out = _buf;
438 3018277 : out->a = be_bswap64(val);
439 3018277 : }
440 :
441 7465188 : static inline void buf_put_le64(void *_buf, u64 val)
442 : {
443 7465188 : bufhelp_u64_t *out = _buf;
444 7465188 : out->a = le_bswap64(val);
445 7465188 : }
446 :
447 :
448 : #endif /*BUFHELP_UNALIGNED_ACCESS*/
449 :
450 : #endif /*GCRYPT_BUFHELP_H*/
|
