revert: restore original SHA-512 x86 implementation

Both performance experiments (circular buffer + merged passes, and inline
W+K fold) made throughput worse: -17% and -31% respectively vs baseline.

The original three-pass structure (schedule → wk[] fold → compression) is
well-tuned for the OoO pipeline: wk[] acts as a staging buffer that keeps
the compression loop as pure sequential integer loads, letting the hardware
prefetcher hide all memory latency behind the rounds' dependency chain.
Mixing SIMD and integer work in the compression loop competes for execution
ports and disrupts this balance.

Restores baseline: ~537 MB/s (AVX-512 profile).

Author: apotocki

include/dataforge/detail/hashes/sha512_intrinsics_x86.ipp

@@ -19,14 +19,9 @@
 //             into a shift-or pair; selected on CPUs without AVX-512 (e.g.
 //             Raptor Lake desktop with both P- and E-cores active).
 //
-// Structure (same for both backends):
-//   Pass 1 — message schedule: w[40] linear array, sequential writes j=0..39.
-//             Keeping the linear array (rather than a circular buffer) lets the
-//             compiler hold w[j-1..j-8] in XMM registers across iterations.
-//   Pass 2 — compression: 8 rounds per outer step, W[t]+K[t] folded inline
-//             from w[t/2] + K[t/2] (one vector add per pair of rounds).
-//             No intermediate wk[80] array — eliminates 640 bytes of stack
-//             store-then-load traffic that was the main cost vs OpenSSL.
+// Both paths share the 8-way unrolled scalar compression rounds via the
+// DATAFORGE_SHA512_ROUND macro, which eliminates the 7 register-move
+// instructions that the naïve loop emits per round.
 
 namespace dataforge::sha2_detail {
 
@@ -94,51 +89,26 @@ __m128i sse41_sha512_sigma1(__m128i x)
 }
 
 // ---------------------------------------------------------------------------
-// Compression-round macro.
+// 8-way unrolled compression-round macro.
 //
-// Variables a..h are passed by name so each call-site permutes the argument
+// Variables a..h are passed by name so each call site permutes the argument
 // list, rotating the logical state vector at compile time.  No register moves
 // are emitted between rounds — the compiler assigns each name to a fixed
 // physical register for the whole 80-round sequence.
 //
-// wkval is the W[t]+K[t] scalar for this round, extracted from the XMM vector
-// inline (no intermediate array store).
+// wk[] is the pre-computed W[t]+K[t] table; t is the round index (0..79).
 // ---------------------------------------------------------------------------
-#define DATAFORGE_SHA512_ROUND(a,b,c,d,e,f,g,h,wkval)                    \
+#define DATAFORGE_SHA512_ROUND(a,b,c,d,e,f,g,h,wk,t)                     \
 do {                                                                       \
     uint64_t _S1 = sha512_rotr(e,14) ^ sha512_rotr(e,18) ^ sha512_rotr(e,41); \
-    uint64_t _T1 = (h) + _S1 + ((e & f) ^ (~(e) & g)) + (uint64_t)(wkval); \
+    uint64_t _T1 = (h) + _S1 + ((e & f) ^ (~(e) & g)) + (wk)[t];        \
     uint64_t _S0 = sha512_rotr(a,28) ^ sha512_rotr(a,34) ^ sha512_rotr(a,39); \
     (d) += _T1;                                                            \
     (h) = _T1 + _S0 + ((a & b) ^ (a & c) ^ (b & c));                     \
 } while(0)
 
-// Two consecutive rounds from the low and high 64-bit lanes of one XMM vector.
-// The 16 argument names encode the compile-time state-vector rotation.
-// _mm_cvtsi128_si64 (SSE2) extracts low lane; _mm_extract_epi64 (SSE4.1) high.
-#define DATAFORGE_SHA512_ROUNDS2(a0,b0,c0,d0,e0,f0,g0,h0,                \
-                                  a1,b1,c1,d1,e1,f1,g1,h1, vec)          \
-do {                                                                       \
-    DATAFORGE_SHA512_ROUND(a0,b0,c0,d0,e0,f0,g0,h0,                      \
-        (uint64_t)_mm_cvtsi128_si64(vec));                                \
-    DATAFORGE_SHA512_ROUND(a1,b1,c1,d1,e1,f1,g1,h1,                      \
-        (uint64_t)_mm_extract_epi64(vec, 1));                             \
-} while(0)
-
-// Eight rounds from four W+K vectors v0..v3.  The round index entering this
-// macro is always a multiple of 8 (both initial blocks and the expansion loop
-// step by 4 vectors = 8 rounds), so the rotation always starts at (a,...,h)
-// and returns to (a,...,h) after the four ROUNDS2 calls.
-#define DATAFORGE_SHA512_8ROUNDS(v0, v1, v2, v3)                          \
-do {                                                                       \
-    DATAFORGE_SHA512_ROUNDS2(a,b,c,d,e,f,g,h, h,a,b,c,d,e,f,g, v0);     \
-    DATAFORGE_SHA512_ROUNDS2(g,h,a,b,c,d,e,f, f,g,h,a,b,c,d,e, v1);     \
-    DATAFORGE_SHA512_ROUNDS2(e,f,g,h,a,b,c,d, d,e,f,g,h,a,b,c, v2);     \
-    DATAFORGE_SHA512_ROUNDS2(c,d,e,f,g,h,a,b, b,c,d,e,f,g,h,a, v3);     \
-} while(0)
-
 // ---------------------------------------------------------------------------
-// AVX-512VL backend: SIMD message schedule + inline-folded compression.
+// AVX-512VL backend: SIMD message schedule + 8-unrolled scalar compression
 // ---------------------------------------------------------------------------
 #if defined(_MSC_VER)
 #pragma warning(push)
@@ -151,18 +121,17 @@ inline void process_blocks_sha512_x86_avx512(uint64_t(&state)[8], const void* ms
     static const __m128i SHUF_MASK = _mm_set_epi64x(0x08090A0B0C0D0E0FULL, 0x0001020304050607ULL);
 
     const auto* data = reinterpret_cast<const uint8_t*>(msg);
-    const auto* K    = reinterpret_cast<const __m128i*>(sha2_def_base<512>::K);
+    const auto* Kvec = reinterpret_cast<const __m128i*>(sha2_def_base<512>::K);
 
-    // Linear w[40]: two 64-bit words per XMM vector (W[2j], W[2j+1]).
-    // Linear layout lets the compiler keep w[j-1..j-8] in XMM registers
-    // across iterations of the schedule loop.
+    // Message schedule: two 64-bit words per XMM vector (40 vectors = 80 words).
+    // W[t]   = sigma1(W[t-2]) + W[t-7]  + sigma0(W[t-15]) + W[t-16]
+    // W[t+1] = sigma1(W[t-1]) + W[t-6]  + sigma0(W[t-14]) + W[t-15]
+    // The sigma1 look-back is exactly the previous vector, so there is no
+    // intra-vector dependency to break up.
     __m128i w[40];
     for (;;) {
-        // --- Pass 1: message schedule ---
         for (int i = 0; i < 8; ++i)
-            w[i] = _mm_shuffle_epi8(
-                _mm_loadu_si128(reinterpret_cast<const __m128i*>(data + 16 * i)),
-                SHUF_MASK);
+            w[i] = _mm_shuffle_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i*>(data + 16 * i)), SHUF_MASK);
 
         for (int j = 8; j < 40; ++j) {
             const __m128i s0in = _mm_alignr_epi8(w[j - 7], w[j - 8], 8); // {W[t-15], W[t-14]}
@@ -173,16 +142,27 @@ inline void process_blocks_sha512_x86_avx512(uint64_t(&state)[8], const void* ms
             w[j] = v;
         }
 
-        // --- Pass 2: compression (W+K folded inline, no wk[] array) ---
+        // Fold round constants in one pass (aligned loads from the 64-byte-aligned K
+        // table) so the compression loop only needs a single load per round.
+        alignas(64) uint64_t wk[80];
+        for (int j = 0; j < 40; ++j)
+            _mm_store_si128(reinterpret_cast<__m128i*>(wk + 2 * j),
+                            _mm_add_epi64(w[j], _mm_load_si128(Kvec + j)));
+
         uint64_t a = state[0], b = state[1], c = state[2], d = state[3];
         uint64_t e = state[4], f = state[5], g = state[6], h = state[7];
 
-        for (int j = 0; j < 40; j += 4) {
-            __m128i v0 = _mm_add_epi64(w[j  ], K[j  ]);
-            __m128i v1 = _mm_add_epi64(w[j+1], K[j+1]);
-            __m128i v2 = _mm_add_epi64(w[j+2], K[j+2]);
-            __m128i v3 = _mm_add_epi64(w[j+3], K[j+3]);
-            DATAFORGE_SHA512_8ROUNDS(v0, v1, v2, v3);
+        // 8-way unroll: argument permutation rotates the logical state vector,
+        // eliminating all inter-round register moves.
+        for (int t = 0; t < 80; t += 8) {
+            DATAFORGE_SHA512_ROUND(a,b,c,d,e,f,g,h, wk, t+0);
+            DATAFORGE_SHA512_ROUND(h,a,b,c,d,e,f,g, wk, t+1);
+            DATAFORGE_SHA512_ROUND(g,h,a,b,c,d,e,f, wk, t+2);
+            DATAFORGE_SHA512_ROUND(f,g,h,a,b,c,d,e, wk, t+3);
+            DATAFORGE_SHA512_ROUND(e,f,g,h,a,b,c,d, wk, t+4);
+            DATAFORGE_SHA512_ROUND(d,e,f,g,h,a,b,c, wk, t+5);
+            DATAFORGE_SHA512_ROUND(c,d,e,f,g,h,a,b, wk, t+6);
+            DATAFORGE_SHA512_ROUND(b,c,d,e,f,g,h,a, wk, t+7);
         }
 
         state[0] += a; state[1] += b; state[2] += c; state[3] += d;
@@ -198,23 +178,22 @@ inline void process_blocks_sha512_x86_avx512(uint64_t(&state)[8], const void* ms
 #endif
 
 // ---------------------------------------------------------------------------
-// SSE4.1 backend: identical structure, sigma0/sigma1 use shift-or pairs
-// instead of vprorq — runs on any x86-64 CPU since ~2008.
+// SSE4.1 backend: identical structure, but sigma0/sigma1 use shift-or pairs
+// instead of vprorq, so it runs on any x86-64 CPU (Raptor Lake desktop,
+// Alder Lake, Skylake, etc.) without requiring AVX-512.
 // ---------------------------------------------------------------------------
 DATAFORGE_SSE41_TARGET
 inline void process_blocks_sha512_x86_sse41(uint64_t(&state)[8], const void* msg, size_t block_count)
 {
     static const __m128i SHUF_MASK = _mm_set_epi64x(0x08090A0B0C0D0E0FULL, 0x0001020304050607ULL);
 
     const auto* data = reinterpret_cast<const uint8_t*>(msg);
-    const auto* K    = reinterpret_cast<const __m128i*>(sha2_def_base<512>::K);
+    const auto* Kvec = reinterpret_cast<const __m128i*>(sha2_def_base<512>::K);
 
     __m128i w[40];
     for (;;) {
         for (int i = 0; i < 8; ++i)
-            w[i] = _mm_shuffle_epi8(
-                _mm_loadu_si128(reinterpret_cast<const __m128i*>(data + 16 * i)),
-                SHUF_MASK);
+            w[i] = _mm_shuffle_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i*>(data + 16 * i)), SHUF_MASK);
 
         for (int j = 8; j < 40; ++j) {
             const __m128i s0in = _mm_alignr_epi8(w[j - 7], w[j - 8], 8);
@@ -225,15 +204,23 @@ inline void process_blocks_sha512_x86_sse41(uint64_t(&state)[8], const void* msg
             w[j] = v;
         }
 
+        alignas(64) uint64_t wk[80];
+        for (int j = 0; j < 40; ++j)
+            _mm_store_si128(reinterpret_cast<__m128i*>(wk + 2 * j),
+                            _mm_add_epi64(w[j], _mm_load_si128(Kvec + j)));
+
         uint64_t a = state[0], b = state[1], c = state[2], d = state[3];
         uint64_t e = state[4], f = state[5], g = state[6], h = state[7];
 
-        for (int j = 0; j < 40; j += 4) {
-            __m128i v0 = _mm_add_epi64(w[j  ], K[j  ]);
-            __m128i v1 = _mm_add_epi64(w[j+1], K[j+1]);
-            __m128i v2 = _mm_add_epi64(w[j+2], K[j+2]);
-            __m128i v3 = _mm_add_epi64(w[j+3], K[j+3]);
-            DATAFORGE_SHA512_8ROUNDS(v0, v1, v2, v3);
+        for (int t = 0; t < 80; t += 8) {
+            DATAFORGE_SHA512_ROUND(a,b,c,d,e,f,g,h, wk, t+0);
+            DATAFORGE_SHA512_ROUND(h,a,b,c,d,e,f,g, wk, t+1);
+            DATAFORGE_SHA512_ROUND(g,h,a,b,c,d,e,f, wk, t+2);
+            DATAFORGE_SHA512_ROUND(f,g,h,a,b,c,d,e, wk, t+3);
+            DATAFORGE_SHA512_ROUND(e,f,g,h,a,b,c,d, wk, t+4);
+            DATAFORGE_SHA512_ROUND(d,e,f,g,h,a,b,c, wk, t+5);
+            DATAFORGE_SHA512_ROUND(c,d,e,f,g,h,a,b, wk, t+6);
+            DATAFORGE_SHA512_ROUND(b,c,d,e,f,g,h,a, wk, t+7);
         }
 
         state[0] += a; state[1] += b; state[2] += c; state[3] += d;
@@ -244,8 +231,6 @@ inline void process_blocks_sha512_x86_sse41(uint64_t(&state)[8], const void* msg
     }
 }
 
-#undef DATAFORGE_SHA512_8ROUNDS
-#undef DATAFORGE_SHA512_ROUNDS2
 #undef DATAFORGE_SHA512_ROUND
 
 }