openssl server and client using RPK (raw public key) RFC 7250. You must compile against a recent version of libssl that supports RPK.

makefile

all:
	gcc -Wall -o ssl-rpk -I./openssl/include ssl-rpk.c openssl/libssl.a openssl/libcrypto.a -lm -lc

keys:
	openssl ecparam -name prime256v1 -genkey -noout -out server_key.pem
	openssl ec -in server_key.pem -pubout -out server_pub.pem

	openssl ecparam -name prime256v1 -genkey -noout -out client_key.pem
	openssl ec -in client_key.pem -pubout -out client_pub.pem

	openssl ecparam -name prime256v1 -genkey -noout -out alice_key.pem
	openssl ec -in alice_key.pem -pubout -out alice_pub.pem

ssl-rpk.c

#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/evp.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/un.h>

#define CLIENT_SOCK_FILE "con.sock"

void init_openssl() {
    SSL_library_init();
    OpenSSL_add_all_algorithms();
    SSL_load_error_strings();
}

int open_client_socket(void) {
    int fd;
    fd = socket(PF_UNIX, SOCK_STREAM, 0);
    if (fd < 0) {
        perror("socket");
        return -1;
    }
    struct sockaddr_un addr;
    memset(&addr, 0, sizeof(addr));
    addr.sun_family = AF_UNIX;
    addr.sun_path[0] = 0;
    strcpy(&addr.sun_path[1], CLIENT_SOCK_FILE);

    //unlink(CLIENT_SOCK_FILE);
    if (connect(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
        perror("connect");
        return -1;
    }
    return fd;
}

int open_server_socket(void) {
    int fd;
    fd = socket(PF_UNIX, SOCK_STREAM, 0);
    if (fd < 0) {
        perror("socket");
        return -1;
    }
    struct sockaddr_un addr;
    memset(&addr, 0, sizeof(addr));
    addr.sun_family = AF_UNIX;
    addr.sun_path[0] = 0;
    strcpy(&addr.sun_path[1], CLIENT_SOCK_FILE);

    //unlink(CLIENT_SOCK_FILE);
    if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
        perror("bind");
        return -1;
    }

    listen(fd, 5);
    fd = accept(fd, NULL, NULL);
    return fd;
}

EVP_PKEY *load_private_key(const char *key_file) {
    FILE *fp = fopen(key_file, "r");
    if (!fp) return NULL;
    EVP_PKEY *pkey = PEM_read_PrivateKey(fp, NULL, NULL, NULL);
    fclose(fp);
    return pkey;
}

EVP_PKEY *load_peer_pubkey(const char *pubkey_file) {
    FILE *fp = fopen(pubkey_file, "r");
    if (!fp) return NULL;
    EVP_PKEY *pkey = PEM_read_PUBKEY(fp, NULL, NULL, NULL);
    fclose(fp);
    return pkey;
}

EVP_PKEY *peer_pubkey;
int verify_callback(X509_STORE_CTX *ctx, void *arg) {
    // Extract the RPK from the context
    EVP_PKEY *rpk = X509_STORE_CTX_get0_rpk(ctx);
    if (!rpk) {
        // No RPK found, reject
        fprintf(stderr, "REJECT: No RPK found, reject\n");
        return 0;
    }

    if (EVP_PKEY_eq(rpk, peer_pubkey) != 1) {
        fprintf(stderr, "REJECT: I don't know you\n");
        return 0;
    }
    // Perform your custom verification logic here
    // For example, compare against a known key or hash

    fprintf(stderr, "ACCEPT\n");
    return 1; // Return 1 to accept, 0 to reject
}

int main(int argc, char **argv) {
    if (argc < 4) {
        fprintf(stderr, "USAGE: ./PROGRAM [s|c] self.pem peer.pem\n");
        return -1;
    }

    int isserver;
    if (argv[1][0] == 's') {
        isserver = 1;
    } else if (argv[1][0] == 'c') {
        isserver = 0;
    } else {
        return -1;
    }

    const char *self_key_str = argv[2];
    const char *peer_pub_str = argv[3];

    init_openssl();

    EVP_PKEY *self_key = load_private_key(self_key_str);
    if (!self_key) {
        fprintf(stderr, "FAIL load_private_key\n");
        return -1;
    }
    peer_pubkey = load_peer_pubkey(peer_pub_str);
    if (!peer_pubkey) {
        fprintf(stderr, "FAIL load_peer_pubkey\n");
        return -1;
    }

    SSL_CTX *ctx;
    if (isserver) {
        ctx = SSL_CTX_new(TLS_server_method());
    } else {
        ctx = SSL_CTX_new(TLS_client_method());
    }
    if (!ctx) {
        fprintf(stderr, "FAIL SSL_CTX_new\n");
        return -1;
    }
    /* before you call SSL_new() */
    /* ask for the server’s raw‐public‐key and fail if it doesn’t send one */
    SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
    SSL_CTX_set_verify_depth(ctx, 0);  /* depth=0 for RPK only */
    SSL_CTX_set_cert_verify_callback(ctx, verify_callback, NULL);

    SSL *ssl = SSL_new(ctx);
    if (!ssl) {
        fprintf(stderr, "FAIL SSL_new\n");
        return -1;
    }

    unsigned char cert_type[] = { TLSEXT_cert_type_rpk };
    if (!SSL_set1_client_cert_type(ssl, cert_type, sizeof(cert_type))) {
        fprintf(stderr, "FAIL SSL_set1_server_cert_type\n");
        return -1;
    }
    if (!SSL_set1_server_cert_type(ssl, cert_type, sizeof(cert_type))) {
        fprintf(stderr, "FAIL SSL_set1_server_cert_type\n");
        return -1;
    }

    int sock;
    if (isserver) {
        sock = open_server_socket();
    } else {
        sock = open_client_socket();
    }
    if (sock < 0) {
        fprintf(stderr, "FAIL open sock\n");
        return -1;
    }

    if (!SSL_use_PrivateKey(ssl, self_key)) {
        fprintf(stderr, "FAIL SSL_CTX_use_PrivateKey\n");
        return -1;
    }

    SSL_set_fd(ssl, sock);

    if (isserver) {
        if (SSL_accept(ssl) <= 0) {
            fprintf(stderr, "FAIL SSL_accept\n");
            ERR_print_errors_fp(stderr);
            return -1;
        }
        fprintf(stderr, "Connection established with RPK auth\n");
        int verify_result = SSL_get_verify_result(ssl);
        fprintf(stderr, "verify_result: %d\n", verify_result);
        SSL_write(ssl, "Hello, client!", strlen("Hello, client!"));
    } else {
        int ret = SSL_connect(ssl);
        if (ret <= 0) {
            perror("SSL_connect");
            int err = SSL_get_error(ssl, ret);
            fprintf(stderr, "FAIL SSL_connect %d=>%d\n", ret, err);
            ERR_print_errors_fp(stderr);
            return -1;
        }
        fprintf(stderr, "Connection established with RPK auth\n");
        int verify_result = SSL_get_verify_result(ssl);
        fprintf(stderr, "verify_result: %d\n", verify_result);
        char buf[1024];
        int nread = SSL_read(ssl, buf, sizeof(buf));
        fprintf(stderr, "GOT: %.*s\n", nread, buf);
    }

    SSL_free(ssl);
    close(sock);
    SSL_CTX_free(ctx);
    EVP_PKEY_free(peer_pubkey);
    EVP_PKEY_free(self_key);
    return 0;
}