Update slow-hash.c

1. Added AES-NI support for modern processors.

1 files changed, 223 insertions, 117 deletions

diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c
index 747cdec65..35438dcab 100644
--- a/src/crypto/slow-hash.c
+++ b/src/crypto/slow-hash.c
@@ -11,145 +11,251 @@
 #include "hash-ops.h"
 #include "oaes_lib.h"
 
-static void (*const extra_hashes[4])(const void *, size_t, char *) = {
-  hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein
-};
+#include <emmintrin.h>
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#include <intrin.h>
+#define STATIC
+#define INLINE __inline
+#if !defined(RDATA_ALIGN16)
+#define RDATA_ALIGN16 __declspec(align(16))
+#endif
+#else
+#include <wmmintrin.h>
+#define STATIC static
+#define INLINE inline
+#if !defined(RDATA_ALIGN16)
+#define RDATA_ALIGN16 __attribute__ ((aligned(16)))
+#endif
+#endif
 
-#define MEMORY         (1 << 21) /* 2 MiB */
+#define MEMORY         (1 << 21) // 2MB scratchpad
 #define ITER           (1 << 20)
 #define AES_BLOCK_SIZE  16
-#define AES_KEY_SIZE    32 /*16*/
+#define AES_KEY_SIZE    32
 #define INIT_SIZE_BLK   8
 #define INIT_SIZE_BYTE (INIT_SIZE_BLK * AES_BLOCK_SIZE)
 
+#define U64(x) ((uint64_t *) (x))
+#define R128(x) ((__m128i *) (x))
+
 extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey);
 extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey);
 
-static size_t e2i(const uint8_t* a, size_t count) { return (*((uint64_t*)a) / AES_BLOCK_SIZE) & (count - 1); }
+#pragma pack(push, 1)
+union cn_slow_hash_state
+{
+    union hash_state hs;
+    struct
+    {
+        uint8_t k[64];
+        uint8_t init[INIT_SIZE_BYTE];
+    };
+};
+#pragma pack(pop)
+
+#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
+#define cpuid(info,x)    __cpuidex(info,x,0)
+#else
+void cpuid(int CPUInfo[4], int InfoType)
+{
+    __asm__ __volatile__
+    (
+        "cpuid":
+        "=a" (CPUInfo[0]),
+        "=b" (CPUInfo[1]),
+        "=c" (CPUInfo[2]),
+        "=d" (CPUInfo[3]) :
+        "a" (InfoType), "c" (0)
+    );
+}
+#endif
 
-static void mul(const uint8_t* a, const uint8_t* b, uint8_t* res) {
-  uint64_t a0, b0;
-  uint64_t hi, lo;
+STATIC INLINE void mul(const uint8_t *a, const uint8_t *b, uint8_t *res)
+{
+    uint64_t a0, b0;
+    uint64_t hi, lo;
 
-  a0 = SWAP64LE(((uint64_t*)a)[0]);
-  b0 = SWAP64LE(((uint64_t*)b)[0]);
-  lo = mul128(a0, b0, &hi);
-  ((uint64_t*)res)[0] = SWAP64LE(hi);
-  ((uint64_t*)res)[1] = SWAP64LE(lo);
+    a0 = U64(a)[0];
+    b0 = U64(b)[0];
+    lo = mul128(a0, b0, &hi);
+    U64(res)[0] = hi;
+    U64(res)[1] = lo;
 }
 
-static void sum_half_blocks(uint8_t* a, const uint8_t* b) {
-  uint64_t a0, a1, b0, b1;
-
-  a0 = SWAP64LE(((uint64_t*)a)[0]);
-  a1 = SWAP64LE(((uint64_t*)a)[1]);
-  b0 = SWAP64LE(((uint64_t*)b)[0]);
-  b1 = SWAP64LE(((uint64_t*)b)[1]);
-  a0 += b0;
-  a1 += b1;
-  ((uint64_t*)a)[0] = SWAP64LE(a0);
-  ((uint64_t*)a)[1] = SWAP64LE(a1);
+STATIC INLINE void sum_half_blocks(uint8_t *a, const uint8_t *b)
+{
+    uint64_t a0, a1, b0, b1;
+    a0 = U64(a)[0];
+    a1 = U64(a)[1];
+    b0 = U64(b)[0];
+    b1 = U64(b)[1];
+    a0 += b0;
+    a1 += b1;
+    U64(a)[0] = a0;
+    U64(a)[1] = a1;
 }
-#define U64(x) ((uint64_t *) (x))
 
-static void copy_block(uint8_t* dst, const uint8_t* src) {
-  memcpy(dst, src, AES_BLOCK_SIZE);
+STATIC INLINE void swap_blocks(uint8_t *a, uint8_t *b)
+{
+    uint64_t t[2];
+    U64(t)[0] = U64(a)[0];
+    U64(t)[1] = U64(a)[1];
+    U64(a)[0] = U64(b)[0];
+    U64(a)[1] = U64(b)[1];
+    U64(b)[0] = U64(t)[0];
+    U64(b)[1] = U64(t)[1];
 }
 
-static void swap_blocks(uint8_t *a, uint8_t *b){
-	uint64_t t[2];
-	U64(t)[0] = U64(a)[0];
-	U64(t)[1] = U64(a)[1];
-	U64(a)[0] = U64(b)[0];
-	U64(a)[1] = U64(b)[1];
-	U64(b)[0] = U64(t)[0];
-	U64(b)[1] = U64(t)[1];
+STATIC INLINE void xor_blocks(uint8_t *a, const uint8_t *b)
+{
+    U64(a)[0] ^= U64(b)[0];
+    U64(a)[1] ^= U64(b)[1];
 }
 
-static void xor_blocks(uint8_t* a, const uint8_t* b) {
-  size_t i;
-  for (i = 0; i < AES_BLOCK_SIZE; i++) {
-    a[i] ^= b[i];
-  }
+STATIC INLINE int check_aes_hw(void)
+{
+    int cpuid_results[4];
+    static int supported = -1;
+
+    if(supported >= 0)
+        return supported;
+
+    cpuid(cpuid_results,1);
+    return supported = cpuid_results[2] & (1 << 25);
 }
 
-#pragma pack(push, 1)
-union cn_slow_hash_state {
-  union hash_state hs;
-  struct {
-    uint8_t k[64];
-    uint8_t init[INIT_SIZE_BYTE];
-  };
-};
-#pragma pack(pop)
+STATIC INLINE void aesni_pseudo_round(const uint8_t *in, uint8_t *out,
+                                      const uint8_t *expandedKey)
+{
+    __m128i *k = R128(expandedKey);
+    __m128i d;
+
+    d = _mm_loadu_si128(R128(in));
+    d = _mm_aesenc_si128(d, *R128(&k[0]));
+    d = _mm_aesenc_si128(d, *R128(&k[1]));
+    d = _mm_aesenc_si128(d, *R128(&k[2]));
+    d = _mm_aesenc_si128(d, *R128(&k[3]));
+    d = _mm_aesenc_si128(d, *R128(&k[4]));
+    d = _mm_aesenc_si128(d, *R128(&k[5]));
+    d = _mm_aesenc_si128(d, *R128(&k[6]));
+    d = _mm_aesenc_si128(d, *R128(&k[7]));
+    d = _mm_aesenc_si128(d, *R128(&k[8]));
+    d = _mm_aesenc_si128(d, *R128(&k[9]));
+    _mm_storeu_si128((R128(out)), d);
+}
+
+void cn_slow_hash(const void *data, size_t length, char *hash)
+{
+    uint8_t long_state[MEMORY];
+    uint8_t text[INIT_SIZE_BYTE];
+    uint8_t a[AES_BLOCK_SIZE];
+    uint8_t b[AES_BLOCK_SIZE];
+    uint8_t d[AES_BLOCK_SIZE];
+    uint8_t aes_key[AES_KEY_SIZE];
+    RDATA_ALIGN16 uint8_t expandedKey[256];
+
+    union cn_slow_hash_state state;
 
-void cn_slow_hash(const void *data, size_t length, char *hash) {
-  uint8_t long_state[MEMORY];
-  union cn_slow_hash_state state;
-  uint8_t text[INIT_SIZE_BYTE];
-  uint8_t a[AES_BLOCK_SIZE];
-  uint8_t b[AES_BLOCK_SIZE];
-  uint8_t c[AES_BLOCK_SIZE];
-  uint8_t d[AES_BLOCK_SIZE];
-  size_t i, j;
-  uint8_t aes_key[AES_KEY_SIZE];
-  oaes_ctx *aes_ctx;
-
-  hash_process(&state.hs, data, length);
-  memcpy(text, state.init, INIT_SIZE_BYTE);
-  memcpy(aes_key, state.hs.b, AES_KEY_SIZE);
-  aes_ctx = (oaes_ctx *) oaes_alloc();
-  
-  oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE);
-  for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) {
-    for (j = 0; j < INIT_SIZE_BLK; j++) {    
-		  aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data);
+    size_t i, j;
+    uint8_t *p = NULL;
+    oaes_ctx *aes_ctx;
+
+    int useAes = check_aes_hw();
+    static void (*const extra_hashes[4])(const void *, size_t, char *) =
+    {
+        hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein
+    };
+
+    hash_process(&state.hs, data, length);
+    memcpy(text, state.init, INIT_SIZE_BYTE);
+
+    aes_ctx = (oaes_ctx *) oaes_alloc();
+    oaes_key_import_data(aes_ctx, state.hs.b, AES_KEY_SIZE);
+
+    // use aligned data
+    memcpy(expandedKey, aes_ctx->key->exp_data, aes_ctx->key->exp_data_len);
+
+    if(useAes)
+    {
+        for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++)
+        {
+            for(j = 0; j < INIT_SIZE_BLK; j++)
+                aesni_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], expandedKey);
+            memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE);
+        }
+    }
+    else
+    {
+        for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++)
+        {
+            for(j = 0; j < INIT_SIZE_BLK; j++)
+                aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], expandedKey);
+
+            memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE);
+        }
+    }
+
+    U64(a)[0] = U64(&state.k[0])[0] ^ U64(&state.k[32])[0];
+    U64(a)[1] = U64(&state.k[0])[1] ^ U64(&state.k[32])[1];
+    U64(b)[0] = U64(&state.k[16])[0] ^ U64(&state.k[48])[0];
+    U64(b)[1] = U64(&state.k[16])[1] ^ U64(&state.k[48])[1];
+
+    for(i = 0; i < ITER / 2; i++)
+    {
+				#define TOTALBLOCKS (MEMORY / AES_BLOCK_SIZE)
+				#define state_index(x) (((*((uint64_t *)x) >> 4) & (TOTALBLOCKS - 1)) << 4)
+
+        // Iteration 1
+        p = &long_state[state_index(a)];
+
+        if(useAes)
+            _mm_storeu_si128(R128(p), _mm_aesenc_si128(_mm_loadu_si128(R128(p)), _mm_loadu_si128(R128(a))));
+        else
+            aesb_single_round(p, p, a);
+
+        xor_blocks(b, p);
+        swap_blocks(b, p);
+        swap_blocks(a, b);
+
+        // Iteration 2
+        p = &long_state[state_index(a)];
+
+        mul(a, p, d);
+        sum_half_blocks(b, d);
+        swap_blocks(b, p);
+        xor_blocks(b, p);
+        swap_blocks(a, b);
+    }
+
+    memcpy(text, state.init, INIT_SIZE_BYTE);
+    oaes_key_import_data(aes_ctx, &state.hs.b[32], AES_KEY_SIZE);
+    memcpy(expandedKey, aes_ctx->key->exp_data, aes_ctx->key->exp_data_len);
+    if(useAes)
+    {
+        for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++)
+        {
+            for(j = 0; j < INIT_SIZE_BLK; j++)
+            {
+                xor_blocks(&text[j * AES_BLOCK_SIZE], &long_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]);
+                aesni_pseudo_round(&text[j * AES_BLOCK_SIZE], &text[j * AES_BLOCK_SIZE], expandedKey);
+            }
+        }
     }
-    memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE);
-  }
-
-  for (i = 0; i < 16; i++) {
-    a[i] = state.k[     i] ^ state.k[32 + i];
-    b[i] = state.k[16 + i] ^ state.k[48 + i];
-  }
-
-  for (i = 0; i < ITER / 2; i++) {
-    /* Dependency chain: address -> read value ------+
-     * written value <-+ hard function (AES or MUL) <+
-     * next address  <-+
-     */
-    /* Iteration 1 */
-    j = e2i(a, MEMORY / AES_BLOCK_SIZE);
-    copy_block(c, &long_state[j * AES_BLOCK_SIZE]);
-	  aesb_single_round(c, c, a);
-    xor_blocks(b, c);
-    swap_blocks(b, c);
-    copy_block(&long_state[j * AES_BLOCK_SIZE], c);
-    assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE));
-    swap_blocks(a, b);
-    /* Iteration 2 */
-    j = e2i(a, MEMORY / AES_BLOCK_SIZE);
-    copy_block(c, &long_state[j * AES_BLOCK_SIZE]);
-    mul(a, c, d);
-    sum_half_blocks(b, d);
-    swap_blocks(b, c);
-    xor_blocks(b, c);
-    copy_block(&long_state[j * AES_BLOCK_SIZE], c);
-    assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE));
-    swap_blocks(a, b);
-  }
-
-  memcpy(text, state.init, INIT_SIZE_BYTE);
-  oaes_key_import_data(aes_ctx, &state.hs.b[32], AES_KEY_SIZE);
-  for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) {
-    for (j = 0; j < INIT_SIZE_BLK; j++) {
-      xor_blocks(&text[j * AES_BLOCK_SIZE], &long_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]);
-	    aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data);
+    else
+    {
+        for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++)
+        {
+            for(j = 0; j < INIT_SIZE_BLK; j++)
+            {
+                xor_blocks(&text[j * AES_BLOCK_SIZE], &long_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]);
+                aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], expandedKey);
+            }
+        }
     }
-  }
-  memcpy(state.init, text, INIT_SIZE_BYTE);
-  hash_permutation(&state.hs);
-  /*memcpy(hash, &state, 32);*/
-  extra_hashes[state.hs.b[0] & 3](&state, 200, hash);
-  oaes_free((OAES_CTX **) &aes_ctx);
+
+    oaes_free((OAES_CTX **) &aes_ctx);
+    memcpy(state.init, text, INIT_SIZE_BYTE);
+    hash_permutation(&state.hs);
+    extra_hashes[state.hs.b[0] & 3](&state, 200, hash);
 }