3ds GenHashConsoleUnique bruteforcer for Gen 6 initial seed offset finding. Heavily based on https://github.com/zoogie/seedminer/

console_unique.cl

// sha256_12 taken from https://github.com/zoogie/bfCL/blob/master/cl/sha256_16.cl
// which was modified from https://github.com/ARMmbed/mbedtls/blob/development/library/sha256.c

typedef unsigned int uint32_t;

typedef unsigned char u8;
typedef unsigned short u16;
typedef unsigned int u32;
typedef unsigned long u64;

#ifndef GET_UINT32_BE
#define GET_UINT32_BE(n,b,i)                            \
{                                                       \
    (n) = ( (uint32_t) (b)[(i)    ] << 24 )             \
        | ( (uint32_t) (b)[(i) + 1] << 16 )             \
        | ( (uint32_t) (b)[(i) + 2] <<  8 )             \
        | ( (uint32_t) (b)[(i) + 3]       );            \
}
#endif

#ifndef PUT_UINT32_BE
#define PUT_UINT32_BE(n,b,i)                            \
{                                                       \
    (b)[(i)    ] = (unsigned char) ( (n) >> 24 );       \
    (b)[(i) + 1] = (unsigned char) ( (n) >> 16 );       \
    (b)[(i) + 2] = (unsigned char) ( (n) >>  8 );       \
    (b)[(i) + 3] = (unsigned char) ( (n)       );       \
}
#endif

__constant const uint32_t K[] =
{
    0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5,
    0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
    0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3,
    0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
    0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC,
    0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
    0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7,
    0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
    0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13,
    0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
    0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3,
    0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
    0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5,
    0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
    0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208,
    0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
};

#define  SHR(x,n) ((x & 0xFFFFFFFF) >> n)
#define ROTR(x,n) (SHR(x,n) | (x << (32 - n)))

#define S0(x) (ROTR(x, 7) ^ ROTR(x,18) ^  SHR(x, 3))
#define S1(x) (ROTR(x,17) ^ ROTR(x,19) ^  SHR(x,10))

#define S2(x) (ROTR(x, 2) ^ ROTR(x,13) ^ ROTR(x,22))
#define S3(x) (ROTR(x, 6) ^ ROTR(x,11) ^ ROTR(x,25))

#define F0(x,y,z) ((x & y) | (z & (x | y)))
#define F1(x,y,z) (z ^ (x & (y ^ z)))

#define R(t)                                    \
(                                               \
    W[t] = S1(W[t -  2]) + W[t -  7] +          \
           S0(W[t - 15]) + W[t - 16]            \
)

#define P(a,b,c,d,e,f,g,h,x,K)                  \
{                                               \
    temp1 = h + S3(e) + F1(e,f,g) + K + x;      \
    temp2 = S2(a) + F0(a,b,c);                  \
    d += temp1; h = temp1 + temp2;              \
}

void sha256_12(unsigned char *io)
{
    uint32_t temp1, temp2, W[64];
    uint32_t A[8] = {
        0x6A09E667,
        0xBB67AE85,
        0x3C6EF372,
        0xA54FF53A,
        0x510E527F,
        0x9B05688C,
        0x1F83D9AB,
        0x5BE0CD19
    };
    unsigned int i;

    GET_UINT32_BE(W[0], io, 0);
    GET_UINT32_BE(W[1], io, 4);
    GET_UINT32_BE(W[2], io, 8);

    W[3] = 0x80000000u;
    W[4] = 0;

    W[5] = 0; W[6] = 0; W[7] = 0;
    W[8] = 0; W[9] = 0; W[10] = 0; W[11] = 0;
    W[12] = 0; W[13] = 0; W[14] = 0;

    W[15] = 0x60u;

#pragma unroll 16
    for (i = 0; i < 16; i += 8)
    {
        P(A[0], A[1], A[2], A[3], A[4], A[5], A[6], A[7], W[i + 0], K[i + 0]);
        P(A[7], A[0], A[1], A[2], A[3], A[4], A[5], A[6], W[i + 1], K[i + 1]);
        P(A[6], A[7], A[0], A[1], A[2], A[3], A[4], A[5], W[i + 2], K[i + 2]);
        P(A[5], A[6], A[7], A[0], A[1], A[2], A[3], A[4], W[i + 3], K[i + 3]);
        P(A[4], A[5], A[6], A[7], A[0], A[1], A[2], A[3], W[i + 4], K[i + 4]);
        P(A[3], A[4], A[5], A[6], A[7], A[0], A[1], A[2], W[i + 5], K[i + 5]);
        P(A[2], A[3], A[4], A[5], A[6], A[7], A[0], A[1], W[i + 6], K[i + 6]);
        P(A[1], A[2], A[3], A[4], A[5], A[6], A[7], A[0], W[i + 7], K[i + 7]);
    }

#pragma unroll 48
    for (i = 16; i < 64; i += 8)
    {
        P(A[0], A[1], A[2], A[3], A[4], A[5], A[6], A[7], R(i + 0), K[i + 0]);
        P(A[7], A[0], A[1], A[2], A[3], A[4], A[5], A[6], R(i + 1), K[i + 1]);
        P(A[6], A[7], A[0], A[1], A[2], A[3], A[4], A[5], R(i + 2), K[i + 2]);
        P(A[5], A[6], A[7], A[0], A[1], A[2], A[3], A[4], R(i + 3), K[i + 3]);
        P(A[4], A[5], A[6], A[7], A[0], A[1], A[2], A[3], R(i + 4), K[i + 4]);
        P(A[3], A[4], A[5], A[6], A[7], A[0], A[1], A[2], R(i + 5), K[i + 5]);
        P(A[2], A[3], A[4], A[5], A[6], A[7], A[0], A[1], R(i + 6), K[i + 6]);
        P(A[1], A[2], A[3], A[4], A[5], A[6], A[7], A[0], R(i + 7), K[i + 7]);
    }

    A[6] += 0x1F83D9AB;
    A[7] += 0x5BE0CD19;

    PUT_UINT32_BE(A[6], io, 0);
    PUT_UINT32_BE(A[7], io, 4);
}

#ifndef DS_TYPE
#define DS_TYPE 0
#endif
#ifndef TARGET_LOW
#define TARGET_LOW 0
#endif
#ifndef TARGET_HIGH
#define TARGET_HIGH 0
#endif

__kernel void find_unique(const u32 start, __global u32 *out)
{
    if (*out) {
        return;
    }
    u32 gid = get_global_id(0);

    u32 io[3];
    io[0] = start + (gid >> 16);
    io[1] = ((u32)DS_TYPE) | (gid << 16);
    io[2] = 0;

    sha256_12((u8*)io);

    if (io[0] == TARGET_LOW && io[1] == TARGET_HIGH){
        *out = gid;
    }
}

main.py

import struct
import sys
import hashlib
import pyopencl as cl
import numpy as np
from Crypto.Cipher import AES
from tqdm import tqdm

ds_type = int(input("3ds type (0 = old, 2 = new): "))
assert ds_type in (0, 2)

LFCS_RANGE = (0, 0x05000000 if ds_type == 2 else 0x0B000000)
LFCS_HALF_RANGE = (LFCS_RANGE[1] - LFCS_RANGE[0]) >> 1

KEY = (0x59FC817E6446EA6190347B20E9BDCE52).to_bytes(16, 'big')
with open("input.bin", "rb") as f:
    enc = f.read()
assert len(enc) == 0x70
nonce = enc[:8] + b"\x00" * 4
cipher = AES.new(KEY, AES.MODE_CCM, nonce)
dec = cipher.decrypt(enc[8:0x60])
nonce = nonce[:8]
final = dec[:12] + nonce + dec[12:]
# GenHashConsoleUnique(0)
low, high = struct.unpack("<II", final[4:4 + 8])

context = cl.create_some_context()
queue = cl.CommandQueue(context)
with open("console_unique.cl", "r", encoding="utf-8") as f:
    shader = f.read()
program = cl.Program(context, shader).build(
    [
        f"-D DS_TYPE={ds_type}",
        f"-D TARGET_LOW={low}",
        f"-D TARGET_HIGH={high}",
    ]
)

host_result = np.zeros(1, np.uint32)
device_result = cl.Buffer(context, cl.mem_flags.READ_WRITE, host_result.nbytes)
cl.enqueue_copy(queue, device_result, host_result)

kernel = program.find_unique

# must divide ranges cleanly
CHUNK_SIZE = 0x1000

for offset in tqdm(range(0, LFCS_HALF_RANGE, CHUNK_SIZE)):
    for sign in (1, -1):
        if sign == -1:
            offset += CHUNK_SIZE
        start = LFCS_RANGE[0] + LFCS_HALF_RANGE + sign * offset
        kernel(
            queue,
            (CHUNK_SIZE << 16,),
            None,
            np.uint32(start),
            device_result
        )
        cl.enqueue_copy(queue, host_result, device_result)
        if host_result[0]:
            lfcs = start | int(host_result[0] >> 16)
            rand = int(host_result[0]) & 0xFFFF
            salt = 0x55d
            m = hashlib.sha256()
            m.update(
                (lfcs).to_bytes(4, 'little')
                + ((rand << 16) | ds_type).to_bytes(4, 'little')
                + (salt).to_bytes(4, 'little')
            )
            low, high = struct.unpack("<" + "I"* 8, m.digest())[-2:]
            # full_hash = (high << 32) | low
            seed_variable = low ^ high
            print(f"\nResult found! {seed_variable=:X}")
            sys.exit(1)