Godot Engine open_encrypted_with_pass() reader/writer in C (using the OpenSSL library).

godot_gdec.c

// cc -O2 godot_gdec.c -o godot_gdec -lcrypto

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>

#include <openssl/evp.h>

// unsigned char hash[EVP_MAX_MD_SIZE]
static void md5_simple(const void *buf, size_t size, unsigned char *hash) {
	EVP_MD_CTX *ctx = EVP_MD_CTX_new();
	unsigned len;
	EVP_DigestInit(ctx, EVP_md5());
	EVP_DigestUpdate(ctx, buf, size);
	EVP_DigestFinal(ctx, hash, &len);
	if (len != 16) abort();
	EVP_MD_CTX_free(ctx);
}

static void md5_string(const void *buf, size_t size, char *str) {
	int i; unsigned char hash[16];
	md5_simple(buf, size, hash);
	for (i = 0; i < 32; i++)
		str[i] = "0123456789abcdef"[hash[i >> 1] >> (~i << 2 & 4) & 15];
	str[i] = 0;
}

static uint8_t* loadfile(const char *fn, size_t *num) {
	size_t n, j = 0; uint8_t *buf = 0;
	FILE *fi = fopen(fn, "rb");
	if (fi) {
		fseek(fi, 0, SEEK_END);
		n = ftell(fi);
		if (n) {
			fseek(fi, 0, SEEK_SET);
			buf = (uint8_t*)malloc((n + 15) & -16);
			if (buf) j = fread(buf, 1, n, fi);
		}
		fclose(fi);
	}
	if (num) *num = j;
	return buf;
}

// https://github.com/godotengine/godot/blob/3.x/core/io/file_access_encrypted.cpp

// aes-256-ecb
typedef struct {
	uint8_t magic[4];
	uint32_t mode;
	uint8_t hash[16];
	uint64_t size;
} gdec_t;

// aes-256-cfb128
typedef struct {
	uint8_t magic[4];
	uint8_t hash[16];
	uint32_t size[2];
	uint8_t iv[16];
} gdec4_t;

#define ERR_EXIT(...) \
	do { fprintf(stderr, __VA_ARGS__); return 1; } while (0) \

#define CRYPT(x, src) { \
	uint8_t *dst = src; int outl; \
	if (!EVP_##x##cryptUpdate(ctx, dst, &outl, src, size)) \
		ERR_EXIT("EVP_" #x "cryptUpdate failed\n"); \
	if (!EVP_##x##cryptFinal(ctx, dst + outl, &outl)) \
		ERR_EXIT("EVP_" #x "cryptFinal failed\n"); \
}

int main(int argc, char **argv) {
	uint8_t *mem; size_t size = 0;
	const char *pass, *out; int mode = 0; char md5[33];
	FILE *f; EVP_CIPHER_CTX *ctx;

	if (argc != 5)
		ERR_EXIT("Usage: %s [enc|dec] input pass output\n", argv[0]);
	if (!strcmp(argv[1], "enc")) mode = 1;
	else if (strcmp(argv[1], "dec")) ERR_EXIT("unknown mode\n");

	mem = loadfile(argv[2], &size);
	if (!mem) ERR_EXIT("loadfile failed\n");
	pass = argv[3];
	out = argv[4];

	md5_string(pass, strlen(pass), md5);
	fprintf(stderr, "pass MD5 : %s\n", md5);

	if (mode) {
		gdec_t gdec;
		memcpy(gdec.magic, "GDEC", 4);
		gdec.mode = 1;
		md5_simple(mem, size, gdec.hash);
		gdec.size = size;
		memset(mem + size, 0, -size & 15);
		size = (size + 15) & -16;

		ctx = EVP_CIPHER_CTX_new();
		EVP_EncryptInit_ex(ctx, EVP_aes_256_ecb(), NULL, (uint8_t*)md5, NULL);
		CRYPT(En, mem)

		f = fopen(out, "wb");
		if (!f) ERR_EXIT("fopen(output) failed\n");
		else {
			fwrite(&gdec, 1, sizeof(gdec), f);
			fwrite(mem, 1, size, f);
			fclose(f);
		}
	} else {
		gdec_t *gdec = (gdec_t*)mem;
		uint8_t *src; uint8_t hash[16];

		if (size < sizeof(*gdec)) ERR_EXIT("file is too small\n");
		size -= sizeof(*gdec);
		if (memcmp(gdec->magic, "GDEC", 4)) ERR_EXIT("wrong header\n");
		if (((gdec->size + 15) & -16) != size) ERR_EXIT("wrong file size\n");
		src = (uint8_t*)(gdec + 1);

		ctx = EVP_CIPHER_CTX_new();
		EVP_DecryptInit_ex(ctx, EVP_aes_256_ecb(), NULL, (uint8_t*)md5, NULL);
		EVP_CIPHER_CTX_set_padding(ctx, 0);		/* to fix EVP_DecryptFinal */
		CRYPT(De, src)

		size = gdec->size;
		md5_simple(src, size, hash);
		if (memcmp(gdec->hash, hash, 16)) ERR_EXIT("MD5 mismatch\n");

		f = fopen(out, "wb");
		if (!f) ERR_EXIT("fopen(output) failed\n");
		else {
			fwrite(src, 1, size, f);
			fclose(f);
		}
	}
	return 0;
}