Rabbit Stream Cipher

Rabbit.java

package org.csdgn.crypt;

import java.util.Arrays;

/**
 * Tested against the actual RFC.
 * 
 * @author Chase (Robert Maupin)
 * @see {@link http://tools.ietf.org/rfc/rfc4503.txt}
 */
public class Rabbit {
	private static final int KEYSTREAM_LENGTH = 16;
	private static final int[] A = new int[] { 0x4D34D34D, 0xD34D34D3, 0x34D34D34, 0x4D34D34D, 0xD34D34D3, 0x34D34D34, 0x4D34D34D,
			0xD34D34D3 };

	private static final int rotl(final int value, final int shift) {
		return value << shift | value >>> 32 - shift;
	}

	private final int[] X = new int[8];
	private final int[] C = new int[8];
	private byte b;
	private int keyindex = 0;
	private byte[] keystream = null;

	public Rabbit() {
		b = 0;
	}

	/**
	 * Should be fed an array with a length that is a multiple of 16 for proper
	 * key sequencing.
	 */
	public byte[] crypt(final byte[] message) {
		int index = 0;
		while(index < message.length) {
			if(keystream == null || keyindex == KEYSTREAM_LENGTH) {
				keystream = keyStream();
				keyindex = 0;
			}
			for(; keyindex < KEYSTREAM_LENGTH && index < message.length; ++keyindex)
				message[index++] ^= keystream[keyindex];
		}
		return message;
	}

	/**
	 * returns 16 bytes
	 */
	private byte[] keyStream() {
		nextState();
		final byte[] s = new byte[16];
		/* unroll */
		int x = X[6] ^ X[3] >>> 16 ^ X[1] << 16;
		s[0] = (byte) (x >>> 24);
		s[1] = (byte) (x >> 16);
		s[2] = (byte) (x >> 8);
		s[3] = (byte) x;
		x = X[4] ^ X[1] >>> 16 ^ X[7] << 16;
		s[4] = (byte) (x >>> 24);
		s[5] = (byte) (x >> 16);
		s[6] = (byte) (x >> 8);
		s[7] = (byte) x;
		x = X[2] ^ X[7] >>> 16 ^ X[5] << 16;
		s[8] = (byte) (x >>> 24);
		s[9] = (byte) (x >> 16);
		s[10] = (byte) (x >> 8);
		s[11] = (byte) x;
		x = X[0] ^ X[5] >>> 16 ^ X[3] << 16;
		s[12] = (byte) (x >>> 24);
		s[13] = (byte) (x >> 16);
		s[14] = (byte) (x >> 8);
		s[15] = (byte) x;
		return s;
	}

	private void nextState() {
		/* counter update */
		for(int j = 0; j < 8; ++j) {
			final long t = (C[j] & 0xFFFFFFFFL) + (A[j] & 0xFFFFFFFFL) + b;
			b = (byte) (t >>> 32);
			C[j] = (int) (t & 0xFFFFFFFF);
		}
		/* next state function */
		final int G[] = new int[8];
		for(int j = 0; j < 8; ++j) {
			// TODO: reduce this to use 32 bits only
			long t = X[j] + C[j] & 0xFFFFFFFFL;
			G[j] = (int) ((t *= t) ^ t >>> 32);
		}
		/* unroll */
		X[0] = G[0] + rotl(G[7], 16) + rotl(G[6], 16);
		X[1] = G[1] + rotl(G[0], 8) + G[7];
		X[2] = G[2] + rotl(G[1], 16) + rotl(G[0], 16);
		X[3] = G[3] + rotl(G[2], 8) + G[1];
		X[4] = G[4] + rotl(G[3], 16) + rotl(G[2], 16);
		X[5] = G[5] + rotl(G[4], 8) + G[3];
		X[6] = G[6] + rotl(G[5], 16) + rotl(G[4], 16);
		X[7] = G[7] + rotl(G[6], 8) + G[5];
	}

	/**
	 * Clears all internal data. You must set the key again to use this cypher.
	 */
	public void reset() {
		b = 0;
		keyindex = 0;
		keystream = null;
		Arrays.fill(X, 0);
		Arrays.fill(C, 0);
	}

	/**
	 * @param IV
	 *            An array of 8 bytes
	 */
	public void setupIV(final byte[] IV) {
		short[] sIV = new short[IV.length>>1];
		for(int i=0;i<sIV.length;++i) {
			sIV[i] = (short)((IV[i << 1] << 8) | IV[(2 << 1) + 1]);
		}
		setupIV(sIV);
	}
	
	/**
	 * @param iv
	 *            array of 4 short values
	 */
	public void setupIV(final short[] iv) {
		/* unroll */
		C[0] ^= iv[1] << 16 | iv[0] & 0xFFFF;
		C[1] ^= iv[3] << 16 | iv[1] & 0xFFFF;
		C[2] ^= iv[3] << 16 | iv[2] & 0xFFFF;
		C[3] ^= iv[2] << 16 | iv[0] & 0xFFFF;
		C[4] ^= iv[1] << 16 | iv[0] & 0xFFFF;
		C[5] ^= iv[3] << 16 | iv[1] & 0xFFFF;
		C[6] ^= iv[3] << 16 | iv[2] & 0xFFFF;
		C[7] ^= iv[2] << 16 | iv[0] & 0xFFFF;
		
		nextState();
		nextState();
		nextState();
		nextState();
	}

	/**
	 * @param key
	 *            An array of 16 bytes
	 */
	public void setupKey(final byte[] key) {
		short[] sKey = new short[key.length>>1];
		for(int i=0;i<sKey.length;++i) {
			sKey[i] = (short)((key[i << 1] << 8) | key[(2 << 1) + 1]);
		}
		setupKey(sKey);
	}

	/**
	 * @param key
	 *            An array of 8 short values
	 */
	public void setupKey(final short[] key) {
		/* unroll */
		X[0] = key[1] << 16 | key[0] & 0xFFFF;
		X[1] = key[6] << 16 | key[5] & 0xFFFF;
		X[2] = key[3] << 16 | key[2] & 0xFFFF;
		X[3] = key[0] << 16 | key[7] & 0xFFFF;
		X[4] = key[5] << 16 | key[4] & 0xFFFF;
		X[5] = key[2] << 16 | key[1] & 0xFFFF;
		X[6] = key[7] << 16 | key[6] & 0xFFFF;
		X[7] = key[4] << 16 | key[3] & 0xFFFF;
		/* unroll */
		C[0] = key[4] << 16 | key[5] & 0xFFFF;
		C[1] = key[1] << 16 | key[2] & 0xFFFF;
		C[2] = key[6] << 16 | key[7] & 0xFFFF;
		C[3] = key[3] << 16 | key[4] & 0xFFFF;
		C[4] = key[0] << 16 | key[1] & 0xFFFF;
		C[5] = key[5] << 16 | key[6] & 0xFFFF;
		C[6] = key[2] << 16 | key[3] & 0xFFFF;
		C[7] = key[7] << 16 | key[0] & 0xFFFF;
		nextState();
		nextState();
		nextState();
		nextState();
		/* unroll */
		C[0] ^= X[4];
		C[1] ^= X[5];
		C[2] ^= X[6];
		C[3] ^= X[7];
		C[4] ^= X[0];
		C[5] ^= X[1];
		C[6] ^= X[2];
		C[7] ^= X[3];
	}
}

RabbitTest.java

package org.csdgn.crypt;

import java.util.Arrays;

public class RabbitTest {
  public static void main(String[] args) {
		RabbitTest test = new RabbitTest();
		test.test(new Rabbit());
	}

	private byte[] convertData(String ... data) {
		byte[] array = new byte[data.length*16];
		int i = 0;
		for(String tdata : data) {
			for(String value : tdata.split(" ")) {
				array[i++] = (byte) (Integer.parseInt(value, 16) & 0xFF);
			}
		}
		return array;
	}
	private byte[] convertIVData(String data) {
		byte[] array = new byte[8];
		int i = 0;
		for(String value : data.split(" ")) {
			array[i++] = (byte) (Integer.parseInt(value, 16) & 0xFF);
		}
		return array;
	}
	
	private void test(Rabbit rabbit, byte[] key, byte[] iv, byte[] data) {
		rabbit.reset();
		rabbit.setupKey(key);
		if(iv != null)
			rabbit.setupIV(iv);
		byte[] crypt = rabbit.crypt(data.clone());
		rabbit.reset();
		rabbit.setupKey(key);
		if(iv != null)
			rabbit.setupIV(iv);
		rabbit.crypt(crypt);
		if(Arrays.equals(data, crypt)) {
			System.out.println("SUCCESS");
		} else {
			System.out.println("FAILED");
		}
	}

	public void test(Rabbit rabbit) {
		/* Appendix A: Test Vectors */
		/* A.1. Testing without IV Setup */
		System.out.println("Appendix A: Test Vectors");
		System.out.println();
		System.out.println("A.1. Testing without IV Setup");
		
		//TEST 1
		System.out.print("\tTEST 1... ");
		test(rabbit,
				convertData("00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"),
				null,
				convertData(
						"B1 57 54 F0 36 A5 D6 EC F5 6B 45 26 1C 4A F7 02",
						"88 E8 D8 15 C5 9C 0C 39 7B 69 6C 47 89 C6 8A A7",
						"F4 16 A1 C3 70 0C D4 51 DA 68 D1 88 16 73 D6 96"));
		
		System.out.print("\tTEST 2... ");
		test(rabbit,
				convertData("91 28 13 29 2E 3D 36 FE 3B FC 62 F1 DC 51 C3 AC"),
				null,
				convertData(
						"3D 2D F3 C8 3E F6 27 A1 E9 7F C3 84 87 E2 51 9C",
						"F5 76 CD 61 F4 40 5B 88 96 BF 53 AA 85 54 FC 19",
						"E5 54 74 73 FB DB 43 50 8A E5 3B 20 20 4D 4C 5E"));
		
		System.out.print("\tTEST 3... ");
		test(rabbit,
				convertData("83 95 74 15 87 E0 C7 33 E9 E9 AB 01 C0 9B 00 43"),
				null,
				convertData(
						"0C B1 0D CD A0 41 CD AC 32 EB 5C FD 02 D0 60 9B",
						"95 FC 9F CA 0F 17 01 5A 7B 70 92 11 4C FF 3E AD",
						"96 49 E5 DE 8B FC 7F 3F 92 41 47 AD 3A 94 74 28"));
		
		System.out.println();
		System.out.println("A.2. Testing with IV Setup");
		
		System.out.print("\tTEST 1... ");
		test(rabbit,
				convertData("00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"),
				convertIVData("00 00 00 00 00 00 00 00"),
				convertData(
						"C6 A7 27 5E F8 54 95 D8 7C CD 5D 37 67 05 B7 ED",
						"5F 29 A6 AC 04 F5 EF D4 7B 8F 29 32 70 DC 4A 8D",
						"2A DE 82 2B 29 DE 6C 1E E5 2B DB 8A 47 BF 8F 66"));
		
		System.out.print("\tTEST 2... ");
		test(rabbit,
				convertData("00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"),
				convertIVData("C3 73 F5 75 C1 26 7E 59"),
				convertData(
						"1F CD 4E B9 58 00 12 E2 E0 DC CC 92 22 01 7D 6D",
						"A7 5F 4E 10 D1 21 25 01 7B 24 99 FF ED 93 6F 2E",
						"EB C1 12 C3 93 E7 38 39 23 56 BD D0 12 02 9B A7"));
		
		System.out.print("\tTEST 3... ");
		test(rabbit,
				convertData("00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"),
				convertIVData("A6 EB 56 1A D2 F4 17 27"),
				convertData(
						"44 5A D8 C8 05 85 8D BF 70 B6 AF 23 A1 51 10 4D",
						"96 C8 F2 79 47 F4 2C 5B AE AE 67 C6 AC C3 5B 03",
						"9F CB FC 89 5F A7 1C 17 31 3D F0 34 F0 15 51 CB"));
	}
}

Are you sure the code is correct?
I am looking at the method "setupKey(byte[] key)", which takes every other byte and concatenates the 5th byte to it. According to the specification you should just split the key into 8 parts.