GDB Remote Protocol Client in Rust + Tokio

gdb.rs

//! Hi. I was going to use this to make some debug tooling for my OS
//! but I instead opted to ditch the direct connection to QEMU's GDB server
//! and instead wrap a GDB executable using the machine-readable interface.
//!
//! I didn't want to throw this away though, so if you can find some use for
//! it, by all means go for it.
//!
//! Released under CC0 or Public Domain or MIT, whatever you want.
//!
//! - Josh

use tokio::{
	io::{self, AsyncReadExt, AsyncWriteExt, BufStream},
	net::{TcpStream, ToSocketAddrs},
	sync::Mutex,
};

#[derive(Debug, thiserror::Error)]
pub enum Error {
	#[error("IO error: {0}")]
	Io(#[from] io::Error),
	#[error("The remote rejected the packet ('-' response)")]
	RemoteRejected,
	#[error("The remote sent an unknown byte in response to a packet: 0x{0:X}")]
	RemoteAckUnknownByte(u8),
	#[error(
		"The remote should have sent '$' to start a response package but instead send 0x{0:X}"
	)]
	MalformedResponseStart(u8),
	#[error("The remote sent a malformed checksum")]
	MalformedResponseChecksum,
	#[error("The remote sent an incorrect checksum: expected 0x{0:X}, got 0x{1:X}")]
	IncorrectResponseChecksum(u8, u8),
	#[error("The remote responded with a non-empty response to vMustReplyEmpty: '{0}'")]
	NonEmptyVMRE(String),
}

pub type Result<T> = std::result::Result<T, Error>;

pub struct GdbConnection {
	stream: BufStream<TcpStream>,
}

impl GdbConnection {
	pub async fn connect<A: ToSocketAddrs>(addr: A) -> Result<Mutex<Self>> {
		let stream = TcpStream::connect(addr).await?;
		Ok(Mutex::new(Self {
			stream: BufStream::new(stream),
		}))
	}

	pub async fn handshake(&mut self) -> Result<()> {
		// Tell GDB to use a modern "unknown vMessage" error format
		let vmre = self.execute_raw("vMustReplyEmpty").await?;
		if vmre != "" {
			return Err(Error::NonEmptyVMRE(vmre));
		}

		Ok(())
	}

	pub async fn execute_raw<B: AsRef<[u8]>>(&mut self, cmd: B) -> Result<String> {
		self.send_packet(cmd).await?;
		let response = self.receive_packet().await?;
		Ok(String::from_utf8_lossy(&response).into())
	}

	async fn receive_packet(&mut self) -> Result<Vec<u8>> {
		let mut result = Vec::new();

		let mut buf = [0; 1];

		self.stream.read_exact(&mut buf).await?;
		if buf[0] != b'$' {
			return Err(Error::MalformedResponseStart(buf[0]));
		}

		let mut sum = 0;
		loop {
			self.stream.read_exact(&mut buf).await?;
			match buf[0] {
				b'#' => break,
				b'}' => {
					sum += b'}' as usize;
					self.stream.read_exact(&mut buf).await?;
					sum += buf[0] as usize;
					let b = buf[0] ^ 0x20;
					result.push(b);
				}
				b => {
					sum += b as usize;
					result.push(buf[0]);
				}
			}
		}

		let mut cs_bytes = [0; 2];
		self.stream.read_exact(&mut cs_bytes).await?;

		let Some(cs_1) = from_hex_digit(cs_bytes[0]) else {
			return Err(Error::MalformedResponseChecksum);
		};
		let Some(cs_2) = from_hex_digit(cs_bytes[1]) else {
			return Err(Error::MalformedResponseChecksum);
		};

		let cs = (cs_1 << 4) | cs_2;
		if (sum & 0xFF) != cs as usize {
			return Err(Error::IncorrectResponseChecksum((sum & 0xFF) as u8, cs));
		}

		self.stream.write_all(&[b'+']).await?;
		self.stream.flush().await?;

		Ok(result)
	}

	async fn send_packet<B: AsRef<[u8]>>(&mut self, packet: B) -> Result<()> {
		let mut sum: usize = 0;

		self.stream.write_all(&[b'$']).await?;

		for byte in packet.as_ref() {
			match byte {
				b'$' | b'#' | b'*' | b'}' => {
					let byte = *byte ^ 0x20;
					sum += b'}' as usize;
					sum += byte as usize;
					self.stream.write_all(&[b'}', byte]).await?;
				}
				b => {
					sum += *b as usize;
					self.stream.write_all(&[*b]).await?;
				}
			}
		}

		let cs = (sum & 0xFF) as u8;
		let cs_1 = to_hex_digit((cs >> 4) & 0xF);
		let cs_2 = to_hex_digit(cs & 0xF);

		self.stream.write_all(&[b'#', cs_1, cs_2]).await?;
		self.stream.flush().await?;

		let mut response = [0; 1];
		self.stream.read_exact(&mut response).await?;

		match response[0] {
			b'+' => Ok(()),
			b'-' => Err(Error::RemoteRejected),
			c => Err(Error::RemoteAckUnknownByte(c)),
		}
	}
}

fn to_hex_digit(n: u8) -> u8 {
	let n = n & 0xF;
	match n {
		0..=9 => b'0' + n,
		10..=15 => b'a' + (n - 10),
		_ => unreachable!(),
	}
}

fn from_hex_digit(n: u8) -> Option<u8> {
	match n {
		b'0'..=b'9' => Some(n - b'0'),
		b'a'..=b'f' => Some(n - b'a' + 10),
		b'A'..=b'F' => Some(n - b'A' + 10),
		_ => None,
	}
}