Unpack flash file that was encrypted by DoSWF

DoSWFUnpacker.java

package org.ddth.game;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;

/**
 * Unpack DoSWF encrypted Flash - Useful for hacking dovogame ;-)
 * 
 * @author instcode
 */
public class DoSWFUnpacker {
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		DoSWFUnpacker decrypter = new DoSWFUnpacker();
		try {
			File inputFile = new File(args[0]);
			InputStream inputStream = new FileInputStream(inputFile);
			ByteBuffer buffer = ByteBuffer.allocate((int) inputFile.length());
			inputStream.read(buffer.array());
			decrypter.handle(buffer, new File("./output.swf"));
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	/**
	 * Decompress CWS to FWS
	 * 
	 * CWS (compressed SWF)
	 * FWS (uncompressed SWF)
	 * 
	 * @param data
	 */
	public static void decompress(byte[] data, File outputFile) {
		// Lazily assuming that the compression ratio is about 90%
		byte[] output = new byte[data.length * 10];
		int size = 0;
		try {

			Inflater decompresser = new Inflater();
			String signature = new String(data, 0, 3);
			if (signature.equals("CWS")) {
				decompresser.setInput(data, 8, data.length - 8);
				size = decompresser.inflate(output);
				decompresser.end();

				OutputStream out = new FileOutputStream(outputFile);
				out.write(new byte[] { 'F', 'W', 'S' });
				out.write(data, 3, 5);
				out.write(output, 0, size);
				out.close();
			}
		} catch (DataFormatException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public void unpack(ByteBuffer buffer, File outputFile) {
		ByteBuffer plain = decrypt(buffer);
		handle(plain, outputFile);
	}

	private void handle(ByteBuffer buffer, File outputFile) {
		byte nonce = buffer.get();
		if (nonce > 0) {
			removeNonce(buffer);
		}

		byte bool = buffer.get();
		int size = buffer.getInt();

		byte[] head = new byte[size];
		buffer.get(head, 0, size);

		byte[] tail = new byte[buffer.remaining()];
		buffer.get(tail);

		byte[] data = bool > 0 ? head : tail;
		decompress(data, outputFile);
	}

	private ByteBuffer removeNonce(ByteBuffer buffer) {
		int max = buffer.getInt();
		int mark = buffer.getInt();
		int length = buffer.getInt();

		ByteBuffer data = ByteBuffer.allocate(length);
		data.order(ByteOrder.LITTLE_ENDIAN);
		buffer.get(data.array(), 0, length);

		ByteBuffer output = ByteBuffer.allocate(max * 2);
		while (output.position() < max) {
			data.position(4);
			data.putInt((int) (mark * 3 / 4 + Math.random() * mark / 2));
			output.put(data.array());
		}
		return output;
	}

	private ByteBuffer decrypt(ByteBuffer buffer) {
		byte block_size = (byte) (buffer.get() - 1);
		byte key = (byte) (buffer.get() - 3);

		int offset = buffer.getInt();
		int length = buffer.getInt() - 2;

		byte[] data = new byte[length];
		buffer.position(buffer.limit() - length);
		buffer.get(data);
		for (int count = 0; count < length;) {
			for (int i = 0; i < block_size; i += 4) {
				data[count] = (byte) (data[count] ^ key);
				++count;
				if (count >= length) {
					break;
				}
			}
			count = count + offset;
		}

		ByteBuffer output = ByteBuffer.allocate(data.length * 5);
		try {
			Inflater decompresser = new Inflater();
			decompresser.setInput(data, 0, data.length);
			int size = decompresser.inflate(output.array());
			output.limit(size);
			output.position(0);
			decompresser.end();
		} catch (DataFormatException e) {
			e.printStackTrace();
		}
		return output;
	}
}