advance_refill.cpp

    // advance
    for (int i = 0; i < num_streams; ++i)
    {
        std::string desc = formatf("advance %d", i);
        bool is_reverse_stream = (i % 3) == 1;
        if (EARLY_CLZ != 2)
        {
            if (EARLY_CLZ == 0)
                bb->append(CLZ(bits[i], bits[i]).set_comment(desc)); // figure out how many bits we consumed
            if (is_reverse_stream)
                bb->append(SUB(inptr[i], inptr[i], LSR(bits[i], 3)).set_comment(desc));
            else
                bb->append(ADD(inptr[i], inptr[i], LSR(bits[i], 3)).set_comment(desc));
        }
        else
        {
            bb->append(UBFX(temp[i], bits[i], 3, 3).set_comment(desc));
            if (is_reverse_stream)
                bb->append(SUB(inptr[i], inptr[i], temp[i]).set_comment(desc));
            else
                bb->append(ADD(inptr[i], inptr[i], temp[i]).set_comment(desc));
        }
    }

    // refill
    for (int i = 0; i < num_streams; ++i)
    {
        std::string desc = formatf("refill %d", i);
        bool is_reverse = (i % 3) == 1;

        // load
        bb->append(AND(bits[i], bits[i], 7).set_comment(desc + " leftover bits"));
        bb->append(LDR(temp[i], Addr::base(inptr[i])).set_comment(desc + " load"));
        // byte-reverse middle streams
        if (is_reverse)
            bb->append(REV(temp[i], temp[i]).set_comment(desc + " LE->BE"));
        bb->append(ORR(temp[i], temp[i], 1ull<<63).set_comment(desc + " mark"));
        bb->append(LSR(bits[i], temp[i], bits[i]).set_comment(desc + " consume leftover"));

        // Prefetch if requested
        if (PREFETCH_DIST)
        {
            // We use 8-byte loads that access bytes [ptr,ptr+7]. So make forward
            // prefetch distances relative to ptr+7 and backwards prefetch distances
            // relative to ptr. That way, 0==no prefetch (=only prefetching bytes we
            // already accessed) naturally.
            int dist = is_reverse ? -PREFETCH_DIST : PREFETCH_DIST + 7;
            bb->append(PRFUM(PRF::PLD | PRF::L1 | PRF::STRM, Addr::base(inptr[i], dist)).set_comment(desc + " prefetch"));
        }
    }