minimal raw ethernet socket send/recv C program

README.md

DESCRIPTION

This implements a custom protocol for sending and receiving files and computes a hash of the file along the way so that the receiving side knows if it received the file correctly or not.

The program focuses on the protocol, not the application, so there are no arguments for customizing operation. Modify the source code for the interface name and send/recv always expect tehe filename to be "filename".

There's plenty of commented code along the way for changing what gets output to screen for debugging.

The behavior here is to print the starting and ending timestamp in microseconds and the difference between the two (in microseconds). Transfer rate is not calculated.

On the receiving side, it prints if the hash of the received file matches the hash sent in the custom protocol.

Set up

$ cd send_recv
$ sudo apt install libssl-dev
$ make
$ head -c10000 > filename

Receiver

$ sudo ./recv
> 1742923963166121
> 1742923963166121 .. 1742923963170107 (3986)
The hashes match.

> 1742924020947468
> 1742924020947468 .. 1742924071412425 (50464957)
The hashes match.

Sender

$ sudo ./send
> 1742923963166121
> 1742923963166121 .. 1742923963170107 (3986)
$ sudo ./send
> 1742924020947468
> 1742924020947468 .. 1742924071412425 (50464957)

Makefile

# requires libssl-dev

all:
	gcc -c -o send_recv.o send_recv.c -Wno-psabi
	gcc -c -o recv.o recv.c -lcrypto
	gcc -o recv send_recv.o recv.o -lm -lcrypto -Wno-psabi
	gcc -c -o send.o send.c -Wno-psabi
	gcc -o send send_recv.o send.o -lm -lcrypto -Wno-psabi

recv.c

#include "send_recv.h"

char *names[]={
	"<", /* incoming */
	"B", /* broadcast */
	"M", /* multicast */
	"P", /* promisc */
	">", /* outgoing */
};

int fd=-1;
char *map;
struct tpacket_req req = {0};
struct iovec *ring;

void sigproc(int sig)
{
	struct tpacket_stats st;
	int len=sizeof(st);

	if (!getsockopt(fd,SOL_PACKET,PACKET_STATISTICS,(char *)&st,&len)) {
		fprintf(stderr, "recieved %u packets, dropped %u\n",
			st.tp_packets, st.tp_drops);
	}
	
	if ( map ) munmap(map, req.tp_block_size * req.tp_block_nr);
	if ( fd>=0 ) close(fd);
	if ( ring ) free(ring);

	exit(0);
}

int main ( int argc, char **argv ) 
{
	struct pollfd pfd;
	struct sockaddr_ll addr;
	int i;
	
	signal(SIGINT, sigproc);

	/* Open the packet socket */
	if ( (fd=socket(PF_PACKET, ETH_P_ALL, 0))<0 ) {
		perror("socket()");
		return 1;
	}

	/* Setup the fd for mmap() ring buffer */
	req.tp_block_size=8192;
	req.tp_frame_size=2048;
	req.tp_block_nr=512;
	req.tp_frame_nr=4*512;
	if ( (setsockopt(fd,
		SOL_PACKET,
		PACKET_RX_RING,
		(char *)&req,
		sizeof(req))) != 0 ) {
		perror("setsockopt()");
		close(fd);
		return 1;
	};

	/* mmap() the sucker */
	map=mmap(NULL,
		req.tp_block_size * req.tp_block_nr,
		PROT_READ|PROT_WRITE|PROT_EXEC, MAP_SHARED, fd, 0);
	if ( map==MAP_FAILED ) {
		perror("mmap()");
		close(fd);
		return 1;
	}

	/* Setup our ringbuffer */
	ring=malloc(req.tp_frame_nr * sizeof(struct iovec));
	for(i=0; i<req.tp_frame_nr; i++) {
		ring[i].iov_base=(void *)((long)map)+(i*req.tp_frame_size);
		ring[i].iov_len=req.tp_frame_size;
	}
	
	/* bind the packet socket */
	memset(&addr, 0, sizeof(addr));
	addr.sll_family=AF_PACKET;
	addr.sll_protocol=htons(ETH_P_ALL);
	addr.sll_ifindex=2;
	addr.sll_hatype=0;
	addr.sll_pkttype=0;
	addr.sll_halen=0;
	if ( bind(fd, (struct sockaddr *)&addr, sizeof(addr)) ) {
		munmap(map, req.tp_block_size * req.tp_block_nr);
		perror("bind()");
		close(fd);
		return 1;
	}

    struct ifreq ifopts;
	char ifname[] = "enp36s0"; // TODO: get from -i
    strncpy(ifopts.ifr_name, ifname, IFNAMSIZ);
    if (ioctl(fd, SIOCGIFFLAGS, &ifopts) == -1) perror("Getting socket flags failed when entering promiscuous mode ");
    ifopts.ifr_flags |= IFF_PROMISC;
    if (ioctl(fd, SIOCSIFFLAGS, &ifopts) == -1) perror("Setting socket flags failed when entering promiscuous mode ");

	long long int c = 0;
	FILE *file;

	// Initialize sha256 digest
	EVP_MD_CTX *mdctx;

	struct tpacket_stats st;
	int len = sizeof(st);

	// Initialize the counter
	if (!getsockopt(fd, SOL_PACKET, PACKET_STATISTICS, &st, &len)) {
		unsigned int total_drops = st.tp_drops;
	}

	for(i=0;;) {
		while(*(unsigned long*)ring[i].iov_base) {
			struct tpacket_hdr *h=ring[i].iov_base;
			struct sockaddr_ll *sll=(void *)h + TPACKET_ALIGN(sizeof(*h));
			unsigned char *bp = (unsigned char *)h + h->tp_mac;
			struct ethhdr *eh = (struct ethhdr *)bp;

			// TODO: sometimes packets arrive and are not dropped, but they are not being handled here
			// this is clear from the exit report of received / dropped packets

			// TODO: syslog dropped packets
			// if (!getsockopt(fd, SOL_PACKET, PACKET_RX_RING, &st, &len)) {
			// 	printf("recieved %u packets, dropped %u\n",	st.tp_packets, st.tp_drops);
			// 	if (st.tp_drops > 0) sigproc(SIGINT);
			// }
	
			// printf("%lld %u.%.6u: if%u %s %u bytes | ",
			// 	++c,
			// 	h->tp_sec, h->tp_usec,
			// 	sll->sll_ifindex,
			// 	names[sll->sll_pkttype],
			// 	h->tp_len);

			// TODO: handle protocols via libraries
			// if compiled with, e.g., ethabac, then handle it with ethabac
			if (ntohs(eh->h_proto) == 0xabac) { // TODO: get from -e
				struct frame frame = parse_frame(bp);
				// if (frame.packet_nr % 10 == 0) { printf("."); fflush(stdout); } // TODO: use a spinner

				// frame 0 is the metadata
				// TODO: sometimes the first packet is dropped, so we need to check for that
				if (frame.packet_nr == 0) { // TODO: time how long it takes to recv all the packets to last_packet_nr
					print_frame(&frame); // TODO: if -v
					// TODO: syslog every recv
					if (frame.data[0] == 0x00) { // command
						// TODO: get command handler dir from -C
						// TODO: verify that the command handler is owned by the same owner as this program or exit
						// TOOD: run the command handler
						if (frame.data[1] == 0xcc && frame.data[2] == 0x01) { // exit
							sigproc(SIGINT);
						}
					}
					else { // file
						// open the file to recv
						file = fopen("filename", "wb"); // TODO: get from frame.data
						if (!file) {
							perror("fopen");
						}
						mdctx = EVP_MD_CTX_new();
						if (!mdctx) {
							perror("EVP_MD_CTX_new");
							return 1;
						}
						if (!EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL)) {
							perror("EVP_DigestInit_ex");
							return 1;
						}
					}
				}
				// frames 1-n are the file data
				else if (frame.packet_nr > 0 && frame.packet_nr <= frame.last_packet_nr) {
					print_frame(&frame); // TODO: if -vv
					if (1) {
						fwrite(frame.data, 1, frame.data_len, file);
						// add the buffer to the sha256 digest
						if (!EVP_DigestUpdate(mdctx, frame.data, frame.data_len)) {
							perror("EVP_DigestUpdate");
						}
					}
				}
				// frame n+1 is the hash (all commands have the same hash)
				// TODO: sometimes the last packet is dropped, so we need to handle for that
				else if (frame.packet_nr > frame.last_packet_nr) {
					print_frame(&frame); // TODO: if -v
					fclose(file);
					// finalize the sha256 digest
					unsigned char hash[EVP_MAX_MD_SIZE];
					unsigned int hash_len;
					if (!EVP_DigestFinal_ex(mdctx, hash, &hash_len)) {
						perror("EVP_DigestFinal_ex");
					}
					// printf("hash exp: %s\n", hex(frame.data, frame.data_len));
					// printf("hash act: %s\n", hex(hash, hash_len));
					if (memcmp(hash, frame.data, hash_len) == 0) {
						printf("The hashes match.\n\n");
						// TODO: verify that the callback script is owned by the same owner as this program or exit
						// TODO: execute the success callback script
					} else {
						printf("The hashes do not match.\n\n");
						// TODO: verify that the callback script is owned by the same owner as this program or exit
						// TODO: execute the error callback script
					}
				}
			}
			else if (ntohs(eh->h_proto) == 0x1234) { // TODO: handle other protocols
				printf("TODO: handle other protocols\n");
			}
			else if (ntohs(eh->h_proto) == 0x5678) { // TODO: handle other protocols
				printf("TODO: handle other protocols\n");
			}
			else {
				// printf("TODO: handle other protocols\n");
			}

			// tell the kernel this packet is done with
			h->tp_status=TP_STATUS_KERNEL;
			__sync_synchronize(); // mb(); // memory barrier
			
			i=(i==req.tp_frame_nr-1) ? 0 : i+1;
		}

		/* sleep when nothing is happening */
		pfd.fd=fd;
		pfd.events=POLLIN|POLLERR;
		pfd.revents=0;
		poll(&pfd, 1, -1);
	}
	
	return 0;
}

send.c

#include "send_recv.h"

int main() {
  struct timespec ts;
  ts.tv_sec = 0;
  ts.tv_nsec = 1000; // TODO: get from -d

  // Initialize sha256 digest
  EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
  if (!mdctx) {
      perror("EVP_MD_CTX_new");
      return 1;
  }
  if (!EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL)) {
      perror("EVP_DigestInit_ex");
      return 1;
  }

  int rawSocket = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
  if (rawSocket == -1) {
    perror("Socket creation error");
    exit(1);
  }

  // TODO: setuid this program to root and allow non-root to run
  // TODO: drop privileges ... is this the right place to do that?

  struct sockaddr_ll socket_address;
  socket_address.sll_ifindex = 2; // ifindex, TODO: get from -i
  socket_address.sll_halen = ETH_ALEN;
  socket_address.sll_addr[0] = 0x00;
  socket_address.sll_addr[1] = 0x00;
  socket_address.sll_addr[2] = 0x00;
  socket_address.sll_addr[3] = 0x00;
  socket_address.sll_addr[4] = 0x00;
  socket_address.sll_addr[5] = 0x00;

  // initialize the frame
  struct frame frame;
  for (int i=0; i<sizeof(frame.dst); i++)
    frame.dst[i] = socket_address.sll_addr[i]; // link dst to sll
  for (int i=0; i<sizeof(frame.src); i++)
    frame.src[i] = 0; // TODO: get from ifindex
  for (int i=0; i<sizeof(frame.eth); i++)
    frame.eth[i] = 0xab+i; // eth, TODO: get from -e 

  // determine what to send
  // TODO: syslog every send
  if (0) { // TODO if -c
    printf("command");
    /*  PROCESS TODO: send a file that contains a TSV of:
        xxxx (file number)
        ssss (file name)
        xxxx (file hash)
        RECV handler will:
        - verify each entry in the TSV
        - if recv handler has no errors, it will finalize the batch and exit
        - report any missing / failed files by number and exit
        - sender will accept stdin for failed numbers and resend, using the same batch id
        - sender will accept stdin 0 to finalize the batch and exit
    */ 

    // frame 0 is the metadata
    frame.packet_nr = 0;
    frame.last_packet_nr = 0;
    frame.unique_id = unique_id(frame.last_packet_nr); // TODO: get from -u
    // frame.data_len = 1;
    // memcpy(&frame.data, &filename, sizeof(filename));
    print_frame(&frame); // print the frame, TODO: if -v
    send_frame(rawSocket, socket_address, &frame);
  }
  else { // TODO: if -f
    char filename[] = "filename"; // TODO: get from -f
    // TODO: run the file against the callback script and exit if it returns false
    // TODO: verify that the callback script is owned by the same owner as this program or exit
    FILE *file = fopen(filename, "rb");
    if (!file) {
        perror("fopen");
        return 1;
    }

    // frame 0 is the metadata
    frame.packet_nr = 0;
    frame.last_packet_nr = (int)ceil((double)file_size(file) / PAYLOAD_DATA_LEN);
    frame.unique_id = unique_id(frame.last_packet_nr); // TODO: get from -u
    frame.data_len = sizeof(filename) - 1;
    memcpy(&frame.data, &filename, sizeof(filename));
    print_frame(&frame); // print the frame, TODO: if -v
    send_frame(rawSocket, socket_address, &frame);

    // frames 1-n are the file data
    unsigned char buffer[PAYLOAD_DATA_LEN];
    int bytesRead = 0;
    while ((bytesRead = fread(buffer, 1, sizeof(buffer), file))) {
      // TODO: slow down transmission based on -d ns
      //nanosleep(&ts, &ts); // sleep in between packets to not overwhelm the kernel or link

      frame.packet_nr++;
      frame.data_len = bytesRead;
      memcpy(&frame.data, &buffer, bytesRead);

      // add the buffer to the sha256 digest
      if (!EVP_DigestUpdate(mdctx, buffer, bytesRead)) {
          perror("EVP_DigestUpdate");
          break;
      }

      // send the frame
      print_frame(&frame); // print the frame, TODO: if -vv
      send_frame(rawSocket, socket_address, &frame);
    }
    fclose(file);
  }

  // finalize the sha256 digest
  unsigned char hash[EVP_MAX_MD_SIZE];
  unsigned int hash_len;
  if (!EVP_DigestFinal_ex(mdctx, hash, &hash_len)) {
      perror("EVP_DigestFinal_ex");
      return 1;
  }

  // frame n+1 is the hash of the 1-n file data frames
  frame.packet_nr++;
  frame.data_len = hash_len;
  memcpy(&frame.data, &hash, frame.data_len);
  print_frame(&frame); // print the frame, TODO: if -v
  send_frame(rawSocket, socket_address, &frame);

  close(rawSocket);
  return 0;
}

send_recv.c

#include "send_recv.h"

struct timeval tv;
long long first_timestamp;

uint64_t htonll(uint64_t value) {
    // Check the endianness of the system
    const int num = 1;
    if(*(char *)&num == 1) {
        // If the system is little endian
        const uint32_t high_part = htonl((uint32_t)(value >> 32));
        const uint32_t low_part = htonl((uint32_t)(value & 0xFFFFFFFFLL));

        return ((uint64_t)low_part << 32) | high_part;
    } else {
        // If the system is big endian
        return value;
    }
}

uint64_t ntohll(uint64_t value) {
    // The operations are the same for both ntohll and htonll
    return htonll(value);
}

char* hex(unsigned char *buffer, int length) {
    char* hexString = malloc(2*length + 1); // Each byte is 2 hex chars, and +1 for null terminator
    if (!hexString) {
        perror("malloc");
        return NULL;
    }

    for (int i = 0; i < length; i++) {
        sprintf(hexString + i*2, "%02x", buffer[i]);
    }
    hexString[2*length] = '\0'; // Null-terminate the string

    return hexString;
}

unsigned long long file_size (FILE *file) {
  fseek(file, 0, SEEK_END);
  unsigned long long size = ftell(file);
  fseek(file, 0, SEEK_SET);
  return size;
}

unsigned long long unique_id(unsigned long long last_packet_nr) {
  gettimeofday(&tv, NULL);
  long long timestamp = tv.tv_sec * 1000000LL + tv.tv_usec; // Get current timestamp in microseconds

  srand(time(NULL));
  unsigned int rand_num = rand(); // Generate a random number

  return (timestamp << 32) | rand_num | last_packet_nr; // Combine timestamp and random number to create a unique 8-byte ID
}

void print_frame(struct frame *frame) {
    int frame_len = FRAME_SIZE(frame->data_len);
    gettimeofday(&tv, NULL);
    long long timestamp = tv.tv_sec * 1000000LL + tv.tv_usec; // Get current timestamp in microseconds
    if (frame->packet_nr == 0) {
        first_timestamp = timestamp;
        printf("> %lld\n", timestamp);
    }
    else if (frame->packet_nr > frame->last_packet_nr) {
        printf("> %lld .. %lld (%lld)\n", first_timestamp, timestamp, (timestamp - first_timestamp));
    }
    return;
    printf("% 5d | %12s %12s %4s %016llx %016llx %016llx %04x ", frame_len, hex(frame->dst, 6), hex(frame->src, 6), hex(frame->eth, 2), frame->unique_id, frame->packet_nr, frame->last_packet_nr, frame->data_len);
    for (int i = 0; i < frame->data_len; i++) {
        if (frame->packet_nr > 0 && frame->packet_nr <= frame->last_packet_nr) {
            if (i < 10 || (i > frame->data_len - 10 && i < frame->data_len)) printf("%02x", frame->data[i]);
            if (i == 9 && i < frame->data_len - 10) printf("...");
        }
        else
            printf("%02x", frame->data[i]);
    }
  printf("\n");
}
// void print_frame(struct frame frame, int frame_len) {
// 	printf("% 5d | %12s %12s %4s %016llx %016llx %016llx %04x ", frame_len, hex(frame.dst, 6), hex(frame.src, 6), hex(frame.eth, 2), frame.unique_id, frame.packet_nr, frame.last_packet_nr, frame.data_len);
// 	for (int i = 0; i < frame.data_len; i++) {
// 		if (frame.packet_nr > 0 && frame.packet_nr <= frame.last_packet_nr) {
// 			if (i < 10 || (i > frame.data_len - 10 && i < frame.data_len)) printf("%02x", frame.data[i]);
// 			if (i == 9 && i < frame.data_len - 10) printf("...");
// 		}
// 		else
// 			printf("%02x", frame.data[i]);
// 	}
// 	printf("\n");
// }

struct frame parse_frame(unsigned char *bp) {
	struct frame frame;
	memcpy(frame.dst, bp, 6);
	memcpy(frame.src, bp + 6, 6);
	memcpy(frame.eth, bp + 12, 2);
	memcpy(&frame.unique_id, bp + 14, 8);
	memcpy(&frame.packet_nr, bp + 14 + 8, 8);
	memcpy(&frame.last_packet_nr, bp + 14 + 8 + 8, 8);
	memcpy(&frame.data_len, bp + 14 + 8 + 8 + 8, 2);
	frame.unique_id = ntohll(frame.unique_id);
	frame.packet_nr = ntohll(frame.packet_nr);
	frame.last_packet_nr = ntohll(frame.last_packet_nr);
	frame.data_len = ntohs(frame.data_len);
	memcpy(frame.data, bp + 14 + 8 + 8 + 8 + 2, frame.data_len);
	return frame;
}

boolean send_frame(int rawSocket, struct sockaddr_ll socket_address, struct frame *frame) {
  unsigned char buffer[sizeof(struct frame)];
  int offset = 0;

  memcpy(buffer + offset, frame->dst, sizeof(frame->dst));
  offset += sizeof(frame->dst);

  memcpy(buffer + offset, frame->src, sizeof(frame->src));
  offset += sizeof(frame->src);

  memcpy(buffer + offset, &frame->eth, sizeof(frame->eth));
  offset += sizeof(frame->eth);

  uint64_t unique_id = htonll(frame->unique_id);
  memcpy(buffer + offset, &unique_id, sizeof(unique_id));
  offset += sizeof(unique_id);

  uint64_t packet_nr = htonll(frame->packet_nr);
  memcpy(buffer + offset, &packet_nr, sizeof(packet_nr));
  offset += sizeof(packet_nr);

  uint64_t last_packet_nr = htonll(frame->last_packet_nr);
  memcpy(buffer + offset, &last_packet_nr, sizeof(last_packet_nr));
  offset += sizeof(last_packet_nr);

  uint16_t data_len = htons(frame->data_len);
  memcpy(buffer + offset, &data_len, sizeof(data_len));
  offset += sizeof(data_len);

  memcpy(buffer + offset, frame->data, frame->data_len);
  offset += frame->data_len;

  if (sendto(rawSocket, buffer, offset, 0, (struct sockaddr*)&socket_address, sizeof(struct sockaddr_ll)) == -1) {
      perror("Send error");
      close(rawSocket);
      exit(1);
  }

  return true;
}

send_recv.h

#ifndef SEND_RECV_H
#define SEND_RECV_H

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <stdint.h>
#include <sys/socket.h>
#include <linux/if_packet.h>
#include <netinet/if_ether.h>
#include <net/ethernet.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/ether.h>
#include <sys/time.h>
#include <time.h>
#include <errno.h>
#include <openssl/evp.h>
#include <math.h>

#ifndef __linux__
#error "Are you loco? This is Linux only!"
#endif

#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>

#define __USE_XOPEN
#include <sys/poll.h>
#include <sys/socket.h>
#include <sys/mman.h>
#include <features.h>    /* for the glibc version number */
#if __GLIBC__ >= 2 && __GLIBC_MINOR >= 1
#include <netpacket/packet.h>
#include <net/ethernet.h>     /* the L2 protocols */
#else
#include <asm/types.h>
#include <linux/if_packet.h>
#include <linux/if_ether.h>   /* The L2 protocols */
#endif
#include <string.h>
#include <netinet/in.h>
//#include <asm/system.h>
#include <signal.h>
//void mb() __attribute__((alias("__sync_synchronize")));
#include <net/if.h>
#include <sys/ioctl.h>
#include <openssl/evp.h>

#define PAYLOAD_DATA_LEN (ETH_FRAME_LEN - ETH_HLEN - 8 - 8 - 8 - 2)
#define FRAME_SIZE(frame_data_len) (ETH_FRAME_LEN - PAYLOAD_DATA_LEN + frame_data_len)

typedef enum { false, true } boolean;

struct frame {
    unsigned char dst[6];
    unsigned char src[6];
    unsigned char eth[2]; // TODO: make this a short int
    unsigned long long unique_id; // TODO: split unique_id into a batch id and transaction id
    unsigned long long packet_nr;
    unsigned long long last_packet_nr;
    unsigned short int data_len;
    unsigned char data[PAYLOAD_DATA_LEN];
};

void sigproc(int sig);
uint64_t htonll(uint64_t value);
uint64_t ntohll(uint64_t value);
char* hex(unsigned char *buffer, int length);
unsigned long long file_size (FILE *file);
unsigned long long unique_id(unsigned long long last_packet_nr);
void print_frame(struct frame *frame);
struct frame parse_frame(unsigned char *bp);
boolean send_frame(int rawSocket, struct sockaddr_ll socket_address, struct frame *frame);

#endif