getsubsystem.c

#include <stdio.h>
#include <stdlib.h>

#define BYTE unsigned char
#define WORD unsigned short
#define DWORD unsigned int
#define LONG long
#define ULONGLONG unsigned long long

#define IMAGE_FILE_MACHINE_I386 0x014c
#define IMAGE_FILE_MACHINE_IA64 0x0200
#define IMAGE_FILE_MACHINE_AMD64 0x8664

#define IMAGE_NT_OPTIONAL_HDR32_MAGIC 0x10b
#define IMAGE_NT_OPTIONAL_HDR64_MAGIC 0x20b

#define DOS_MAGIC 0x5A4D
#define PE_MAGIC 0x50450000

typedef struct _IMAGE_DOS_HEADER
{
    WORD e_magic;
    WORD e_cblp;
    WORD e_cp;
    WORD e_crlc;
    WORD e_cparhdr;
    WORD e_minalloc;
    WORD e_maxalloc;
    WORD e_ss;
    WORD e_sp;
    WORD e_csum;
    WORD e_ip;
    WORD e_cs;
    WORD e_lfarlc;
    WORD e_ovno;
    WORD e_res[4];
    WORD e_oemid;
    WORD e_oeminfo;
    WORD e_res2[10];
    LONG e_lfanew;
} IMAGE_DOS_HEADER, * PIMAGE_DOS_HEADER;

typedef struct _IMAGE_FILE_HEADER {
    WORD  Machine;
    WORD  NumberOfSections;
    DWORD TimeDateStamp;
    DWORD PointerToSymbolTable;
    DWORD NumberOfSymbols;
    WORD  SizeOfOptionalHeader;
    WORD  Characteristics;
} IMAGE_FILE_HEADER, * PIMAGE_FILE_HEADER;

typedef struct _IMAGE_OPTIONAL_HEADER {
    WORD                 Magic;
    BYTE                 MajorLinkerVersion;
    BYTE                 MinorLinkerVersion;
    DWORD                SizeOfCode;
    DWORD                SizeOfInitializedData;
    DWORD                SizeOfUninitializedData;
    DWORD                AddressOfEntryPoint;
    DWORD                BaseOfCode;
    DWORD                BaseOfData;
    DWORD                ImageBase;
    DWORD                SectionAlignment;
    DWORD                FileAlignment;
    WORD                 MajorOperatingSystemVersion;
    WORD                 MinorOperatingSystemVersion;
    WORD                 MajorImageVersion;
    WORD                 MinorImageVersion;
    WORD                 MajorSubsystemVersion;
    WORD                 MinorSubsystemVersion;
    DWORD                Win32VersionValue;
    DWORD                SizeOfImage;
    DWORD                SizeOfHeaders;
    DWORD                CheckSum;
    WORD                 Subsystem;
    /* ... */
} IMAGE_OPTIONAL_HEADER32, * PIMAGE_OPTIONAL_HEADER32;

typedef struct _IMAGE_OPTIONAL_HEADER64 {
    WORD                 Magic;
    BYTE                 MajorLinkerVersion;
    BYTE                 MinorLinkerVersion;
    DWORD                SizeOfCode;
    DWORD                SizeOfInitializedData;
    DWORD                SizeOfUninitializedData;
    DWORD                AddressOfEntryPoint;
    DWORD                BaseOfCode;
    ULONGLONG            ImageBase;
    DWORD                SectionAlignment;
    DWORD                FileAlignment;
    WORD                 MajorOperatingSystemVersion;
    WORD                 MinorOperatingSystemVersion;
    WORD                 MajorImageVersion;
    WORD                 MinorImageVersion;
    WORD                 MajorSubsystemVersion;
    WORD                 MinorSubsystemVersion;
    DWORD                Win32VersionValue;
    DWORD                SizeOfImage;
    DWORD                SizeOfHeaders;
    DWORD                CheckSum;
    WORD                 Subsystem;
    /* ... */
} IMAGE_OPTIONAL_HEADER64, * PIMAGE_OPTIONAL_HEADER64;

#define Fread(p, s, h, e) do { 	size_t r = fread(p,1,s,h); if (r != s) {printf(e); exit(0);} } while (0);
#define Fopen(h,n, e) do { errno_t  ee = fopen_s(&h,n,"rb"); if (h == NULL) {printf(e, ee); exit(0); } } while(0);
#define Fsize(h, s) do {fseek(h, 0, SEEK_END); s = ftell(f); fseek(f, 0, SEEK_SET); } while(0);

/* https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#windows-subsystem */
const char* subsystemName[] = {
    "An unknown subsystem",
    "Device drivers and native Windows processes", 
    "The Windows graphical user interface(GUI) subsystem", 
    "The Windows character subsystem",
    NULL, 
    "The OS / 2 character subsystem", 
    NULL, 
    "The Posix character subsystem",
    "Native Win9x driver",
    "Windows CE",
    "An Extensible Firmware Interface(EFI) application",
    "An EFI driver with boot services",
    "An EFI driver with run - time services",
    "An EFI ROM image",
    "XBOX",
    NULL, 
    "Windows boot application"
};

DWORD reverse_dword(DWORD in) {
    DWORD out;
    BYTE* bin = (BYTE *)&in;
    BYTE* bout = (BYTE *)&out;
    bout[0] = bin[3];
    bout[1] = bin[2];
    bout[2] = bin[1];
    bout[3] = bin[0];
    return out;
}

int main(int argc, char **argv) {
    char data[100];
    PIMAGE_DOS_HEADER dh = (PIMAGE_DOS_HEADER)data;
    PIMAGE_FILE_HEADER fh = (PIMAGE_FILE_HEADER)data;
    PIMAGE_OPTIONAL_HEADER64 oh64 = (PIMAGE_OPTIONAL_HEADER64)data;
    PIMAGE_OPTIONAL_HEADER32 oh32 = (PIMAGE_OPTIONAL_HEADER32)data;
    WORD magic;
    WORD subsystem;
    DWORD peSig;
    FILE* f = NULL;
    size_t s;

    if (argc < 2) {
        printf("pass a filename as argument\n");
        exit(0);
    }

    Fopen(f, argv[1], "error opening file (0x%x)\n");
    Fsize(f, s);
    Fread(dh, sizeof(*dh), f, "error reading dos header\n");

    if (dh->e_magic != DOS_MAGIC) {
        printf("dos hdr magic doesn't match\n");
        exit(0);
    }

    if (dh->e_lfanew > s || dh->e_lfanew < 0 || dh->e_lfanew < sizeof(*dh)) {
        printf("malformed e_lfanew\n");
        exit(0);
    }

    fseek(f, dh->e_lfanew - sizeof(*dh), SEEK_CUR);
    Fread(&peSig, sizeof(peSig), f, "error reading PE signature");

    if (reverse_dword(peSig) != PE_MAGIC) {
        printf("PE signature(0x%x) not recognized\n", peSig);
        exit(0);
    }

    Fread(fh, sizeof(*fh), f, "error reading file header\n");
    Fread(oh64, sizeof(*oh64), f, "couldn't read optional header\n");

    if (oh64->Magic != IMAGE_NT_OPTIONAL_HDR32_MAGIC && oh64->Magic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
        printf("magic(0x%x) in optional header not recognized\n", oh64->Magic);
        exit(0);
    }

    if (oh64->Magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
        subsystem = oh32->Subsystem;
    }
    else {
        subsystem = oh64->Subsystem;
    }

    if (subsystem >= sizeof(subsystemName) / sizeof(char*)) {
        printf("subsystem(0x%x) not recognized\n", subsystem);
        exit(0);
    }

    printf("subsystem is: %s (%u)\n", subsystemName[subsystem], subsystem);
    exit(0);
}