Boost.spirit based URL parser, and Boost.Beast based HTTP/HTTPS client

easy_http.cpp

#include <iostream>
#include <optional>

#include <variant>
#include <boost/beast.hpp>
#include <boost/asio/ssl/stream.hpp>
#include <boost/asio/ssl/rfc2818_verification.hpp>
#include "easyhttp/easyhttp.hpp"
#include "./url_parser.hpp"

static const char * http_code_str(int http_code)
{
	switch(http_code)
	{
		case 200:
			return "OK";
		case 404:
			return "NOT FOUND";
		default:
			return ""; // TODO
	}
}

easyhttp::error::http_error::http_error(int http_code)
	: runtime_error(http_code_str(http_code))
	, http_code(http_code)
{
}

#ifdef __ANDROID__
#include <android/log.h>

//定义TAG之后，我们可以在LogCat通过TAG过滤出NDK打印的日志
#define TAG "keystore-native"
// 定义info信息
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,TAG,__VA_ARGS__)
// 定义debug信息
#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__)
// 定义error信息
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,TAG,__VA_ARGS__)
#else

#define LOGE(...) do { printf(__VA_ARGS__); printf("\n") ; } while(false)

#endif


static std::string common_http_req(easyhttp::URL url, boost::beast::http::request<boost::beast::http::string_body>& req_)
{
	try
	{
		boost::asio::io_context io;
		boost::asio::ssl::context ssl_ctx(boost::asio::ssl::context::sslv23);

    ssl_ctx.add_certificate_path("/etc/ssl/ca-certs");
		ssl_ctx.set_verify_callback(boost::asio::ssl::rfc2818_verification(url.get_host()));
		ssl_ctx.set_verify_mode(boost::asio::ssl::verify_peer|boost::asio::ssl::verify_fail_if_no_peer_cert);

		boost::asio::ip::tcp::resolver resolver_(io);

		using tcp_socket = boost::asio::ip::tcp::socket;
		using ssl_socket = boost::asio::ssl::stream<boost::asio::ip::tcp::socket>;

		tcp_socket* tcp_layer = nullptr;

		std::optional<std::variant<tcp_socket, ssl_socket>> http_stream;

		boost::beast::flat_buffer buffer_; // (Must persist between reads)
		boost::beast::http::response<boost::beast::http::string_body> res_;

		if (url.schema == "https")  // use SSL here.
		{
			// inplace construct
			http_stream.emplace(std::in_place_type<ssl_socket>, io, ssl_ctx);
			ssl_socket& ssl_sock = std::get<1>(*http_stream);
			// Let SSL have SNI capability
			SSL_set_tlsext_host_name(ssl_sock.native_handle(), url.get_host().c_str());
			tcp_layer = & ssl_sock.next_layer();
		}
		else
		{
			// inplace construct
			http_stream.emplace(std::in_place_type<tcp_socket>, io);
			tcp_layer = &(std::get<0>(*http_stream));
		}

		std::visit([&resolver_, &tcp_layer, &url](auto&& host)
		{
			using T = std::decay_t<decltype(host)>;
			if constexpr (std::is_same_v<T, boost::asio::ip::address>)
			{
				tcp_layer->connect(boost::asio::ip::tcp::endpoint(host, (unsigned short)(url.port ? *url.port : ( url.schema == "https" ? 443 : 80))));
			}
			else
			{
				boost::asio::ip::tcp::resolver::results_type dns_result = resolver_.resolve(url.get_host(), std::to_string(url.port ? *url.port : ( url.schema == "https" ? 443 : 80)));
				boost::asio::connect(*tcp_layer, dns_result.begin(), dns_result.end());
			}

		}, url.host);

		if (url.schema == "https")
		{
			ssl_socket& ssl_stream = std::get<1>(*http_stream);
			boost::system::error_code ec;
			ssl_stream.handshake(boost::asio::ssl::stream_base::client, ec);
			if (ec)
			{
				LOGE("ssl handshake failed: %s", ec.message().c_str());
				return "";
			}
		}

		// Send the HTTP request to the remote host
		std::visit([&req_, &buffer_, &res_](auto && socket_)
		{
			boost::beast::http::write(socket_, req_);
			boost::beast::http::read(socket_, buffer_, res_);
		}, *http_stream);
		return res_.body();
	}
	catch(std::runtime_error& e)
	{
	}
	catch(std::exception& e)
	{
	}

	return "";
}

std::string easyhttp::sync::get(std::string post_url)
{
	// parse post_url to get host
	auto url = parse_url(post_url);

	boost::beast::http::request<boost::beast::http::string_body> req_;

	req_.version(11);
	req_.method(boost::beast::http::verb::get);
	req_.target(url.get_target());
	req_.set(boost::beast::http::field::host, url.get_host());
	req_.set(boost::beast::http::field::user_agent, BOOST_BEAST_VERSION_STRING);

	return common_http_req(url, req_);
}

std::string easyhttp::sync::post(std::string post_url, std::string post_content, std::string content_type)
{
	// parse post_url to get host
	auto url = parse_url(post_url);

	boost::beast::http::request<boost::beast::http::string_body> req_;

	req_.version(11);
	req_.method(boost::beast::http::verb::post);
	req_.target(url.get_target());
	req_.set(boost::beast::http::field::host, url.get_host());
	req_.set(boost::beast::http::field::user_agent, BOOST_BEAST_VERSION_STRING);
	req_.set(boost::beast::http::field::content_type, content_type);


	req_.body() = post_content;
	req_.prepare_payload();
	return common_http_req(url, req_);
}

url_parser.cpp

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>

#include "url_parser.hpp"

namespace qi = boost::spirit::qi;

struct user_cred
{
	std::string username;
	std::string password;
};

BOOST_FUSION_ADAPT_STRUCT(
	user_cred,
	(std::string, username)
	(std::string, password)
)


BOOST_FUSION_ADAPT_STRUCT(
	easyhttp::KV,
	(std::string, k)
	(std::string, v)
)

BOOST_FUSION_ADAPT_STRUCT(
	easyhttp::URL,
	(std::string, schema)
	(easyhttp::URL::host_t, host)
	(std::string, username)
	(std::string, password)
	(std::optional<int>, port)
	(std::string, path)
	(easyhttp::QUERY, query)
	(std::string, fragment)
	(std::string, str_host)
)

static boost::asio::ip::address_v4 from_four_number(unsigned char n1, unsigned char n2, unsigned char n3, unsigned char n4)
{
	boost::asio::ip::address_v4::bytes_type bt;

	bt[0] = n1;
	bt[1] = n2;
	bt[2] = n3;
	bt[3] = n4;

	return boost::asio::ip::address_v4(bt);
}

static boost::asio::ip::address_v6 from_v6_string(std::vector<char> str)
{
	return boost::asio::ip::address_v6::from_string(str.data());
}

BOOST_PHOENIX_ADAPT_FUNCTION(boost::asio::ip::address_v4, v4_from_4number, from_four_number, 4)
BOOST_PHOENIX_ADAPT_FUNCTION(boost::asio::ip::address_v6, v6_from_string, from_v6_string, 1)

template <typename Iterator>
struct uri_grammer : qi::grammar<Iterator, easyhttp::URL()>
{
	uri_grammer() : uri_grammer::base_type(url)
	{
		using namespace boost::phoenix;
		using namespace easyhttp;
		using boost::phoenix::ref;

		url = schema [ at_c<0>(qi::_val) = qi::_1 ]
			>> "://" >> -( username [ at_c<2>(qi::_val) = qi::_1 ] >> -( ':' >> password  [ at_c<3>(qi::_val) = qi::_1 ] ) >> qi::lit('@')[ boost::phoenix::ref(has_user_name) = true ] )
			>> host [ at_c<1>(qi::_val) = qi::_1 ]
			>> -(qi::lit(':') >> qi::int_ [ at_c<4>(qi::_val) = qi::_1 ] )
			>> -(path [ at_c<5>(qi::_val) = qi::_1 ] >> -('?' >> query[ at_c<6>(qi::_val) = qi::_1 ]))
			>> -('#' >> fragment [ at_c<7>(qi::_val) = qi::_1 ]);

		host = ip_host | domain_host;

		domain_host = qi::lexeme[ +(qi::char_("a-zA-Z0-9.\\-")) ];
		ip_host =  ('[' >> ipv6_host >> ']') | ipv4_host;

		ipv6_host = (+qi::char_("0123456789abcdefABCDEF:.")) [ qi::_val = v6_from_string(qi::_1)] ;

		ipv4_host = (
			qi::int_ >> '.' >> qi::int_ >> '.' >> qi::int_ >> '.' >> qi::int_
			) [ qi::_val = v4_from_4number(qi::_1, qi::_2, qi::_3, qi::_4)];

		username = qi::lexeme[ +(qi::char_ - ':' - '@' - '/') ];
		password = qi::lexeme[ +(qi::char_ - '@') ];

		schema = qi::lexeme[ +(qi::char_ - ':' - '/') ];

		path = qi::lexeme[ +(qi::char_ - '?' - '#') ];

		query =  pair >> *((qi::lit(';') | '&') >> pair);
		pair  =  key >> -('=' >> value);
		key = qi::lexeme[ +(qi::char_ - '=' - '#') ];
		value = qi::lexeme[ *(qi::char_ - '&' - '#') ];

		fragment = qi::lexeme[ +(qi::char_) ];
	};

	qi::rule<Iterator, easyhttp::URL()> url;
	qi::rule<Iterator, std::string()> schema, path;

	qi::rule<Iterator, std::variant<std::string, boost::asio::ip::address>()> host;

	qi::rule<Iterator, std::string()> domain_host;
	qi::rule<Iterator, boost::asio::ip::address()> ip_host;

	qi::rule<Iterator, boost::asio::ip::address_v4()> ipv4_host;
	qi::rule<Iterator, boost::asio::ip::address_v6()> ipv6_host;;

	qi::rule<Iterator, std::string()> username, password;

	qi::rule<Iterator, easyhttp::QUERY()> query;
	qi::rule<Iterator, easyhttp::KV()> pair;
	qi::rule<Iterator, std::string()> key, value;

	qi::rule<Iterator, std::string()> fragment;

	bool has_user_name = false;
};

easyhttp::URL easyhttp::parse_url(const std::string_view& url)
{
	easyhttp::URL ast;

	uri_grammer<std::string_view::const_iterator> gramer;

	auto first = url.begin();

	bool r = boost::spirit::qi::parse(first, url.end(), gramer, ast);
	if (!gramer.has_user_name)
		ast.username.clear();
	//ast.str_host = gramer.str_host;
	return ast;
}

struct host_visitor
{
	std::string operator()(std::string v) const
	{
		return v;
	}

	std::string operator()(boost::asio::ip::address v) const
	{
		return v.to_string();
	}
};

std::string easyhttp::URL::get_host() const
{
	return std::visit(host_visitor(), host);
}

std::string easyhttp::URL::get_target() const
{
	if (query.size())
		return path + '?' + query.join();
	return path;
}

std::string easyhttp::QUERY::join() const
{
	std::stringstream ss;
	bool has_pre = false;

	for (const KV& kv : *this)
	{
		if (has_pre)
			ss << '&';

		ss << kv.k << '=' << kv.v;
		has_pre = true;
	}

	return ss.str();
}

url_parser.hpp

#include <optional>
#include <vector>
#include <string>
#include <string_view>
#include <variant>
#include <boost/asio/ip/address.hpp>

namespace easyhttp {
struct KV
{
	std::string k;
	std::string v;
};

struct QUERY : public std::vector<KV>
{
	using std::vector<KV>::vector;

	std::string join() const;
};

struct URL
{
	using host_t = std::variant<std::string, boost::asio::ip::address>;

	std::string schema; // http/https

	host_t host;
	std::string username;
	std::string password;
	std::optional<int> port;
	std::string path = "/";
	QUERY query;
	std::string fragment;

	std::string get_host() const ;

	// the path with query and without fragment
	std::string get_target() const ;
};

URL parse_url(const std::string_view& url);

}

Thanks for sharing this Microcai... If I knew enough pin ying, I'd say thank you in Mandarin. But I appreciate you sharing how to use boost::spirit::qi examples.

Best Regards,
Carl

The only thing that was missing was the easyhttp/easyhttp.hpp file you referenced in easy_http.cpp

just some function def for easyhttp.cpp