HTTP Client Tool - Send costum HTTP request, analyze headers, download content, etc.

xhttp.c

/*!
 *  @file libxutils/examples/xhttp.c
 * 
 *  This source is part of "libxutils" project
 *  2015-2020  Sun Dro ([email protected])
 *
 * @brief Example file for working with the HTTP request/responses.
 * Send costum HTTP request, analyze headers, download content, etc.
 */

#include <xutils/xstd.h>
#include <xutils/sock.h>
#include <xutils/http.h>
#include <xutils/proc.h>
#include <xutils/xfs.h>
#include <xutils/xlog.h>
#include <xutils/xstr.h>
#include <xutils/xver.h>
#include <xutils/xtype.h>

extern char *optarg;

#define XHTTP_VERSION_MAJ 0
#define XHTTP_VERSION_MIN 3

#define XHTTP_DEFAULT_PROTO "http"
#define XHTTP_DEFAULT_PORT  80

typedef struct xhttp_args_ {
    xhttp_method_t eMethod;
    xlog_bar_t progressBar;
    xfile_t *pOutputFile;

    xbool_t nVerbose;
    xbool_t nForce;
    size_t nTimeout;
    size_t nDone;

    char sAddress[XHTTP_URL_MAX];
    char sHeaders[XLINE_MAX];
    char sContent[XPATH_MAX];
    char sOutput[XPATH_MAX];
} xhttp_args_t;

static char *XHTTPApp_WhiteSpace(const int nLength)
{
    static char sRetVal[XHTTP_FIELD_MAX];
    xstrnul(sRetVal);
    int i = 0;

    int nLen = XSTD_MIN(nLength, sizeof(sRetVal) - 1);
    for (i = 0; i < nLen; i++) sRetVal[i] = ' ';

    sRetVal[i] = '\0';
    return sRetVal;
}

void XHTTPApp_DisplayUsage(const char *pName)
{
    int nLength = strlen(pName) + 6;

    printf("===============================================\n");
    printf(" XHTTP client tool - Version %d.%d (%s)\n",
        XHTTP_VERSION_MAJ, XHTTP_VERSION_MIN, __DATE__);
    printf("===============================================\n");

    printf("Usage: %s [-a <address>] [-c <content>] [-f]\n", pName);
    printf(" %s [-m <method>] [-t <seconds>] [-v]\n", XHTTPApp_WhiteSpace(nLength));
    printf(" %s [-o <output>] [-x <headers>] [-h]\n", XHTTPApp_WhiteSpace(nLength));

    printf("Options are:\n");
    printf("  -a <address>          # HTTP/S address (%s*%s)\n", XSTR_CLR_RED, XSTR_CLR_RESET);
    printf("  -c <content>          # Content file path\n");
    printf("  -o <output>           # Output file path\n");
    printf("  -m <method>           # HTTP request method\n");
    printf("  -t <seconds>          # Receive timeout (sec)\n");
    printf("  -x <headers>          # Costum HTTP headers\n");
    printf("  -f                    # Force overwrite output\n");
    printf("  -v                    # Enable verbose logging\n");
    printf("  -h                    # Print version and usage\n\n");
    printf("Examples:\n");
    printf("1) %s -a https://endpoint.com/ -c body.json -m POST\n", pName);
    printf("2) %s -a endpoint.com/test -t 20 -o output.txt -f -v\n", pName);
    printf("2) %s -a endpoint.com/test -x 'X-Is-Costum: True; X-My-Header: Test'\n", pName);
}

int XHTTPApp_ParseArgs(xhttp_args_t *pArgs, int argc, char *argv[])
{
    pArgs->pOutputFile = NULL;
    pArgs->eMethod = XHTTP_GET;
    pArgs->nVerbose = XFALSE;
    pArgs->nForce = XFALSE;
    pArgs->nTimeout = 0;
    pArgs->nDone = 0;

    xstrnul(pArgs->sAddress);
    xstrnul(pArgs->sHeaders);
    xstrnul(pArgs->sContent);
    xstrnul(pArgs->sOutput);
    int nChar = 0;

    while ((nChar = getopt(argc, argv, "a:c:m:x:o:t:f1:v1:h1")) != -1)
    {
        switch (nChar)
        {
            case 'a':
                xstrncpy(pArgs->sAddress, sizeof(pArgs->sAddress), optarg);
                break;
            case 'c':
                xstrncpy(pArgs->sContent, sizeof(pArgs->sContent), optarg);
                break;
            case 'o':
                xstrncpy(pArgs->sOutput, sizeof(pArgs->sOutput), optarg);
                break;
            case 'x':
                xstrncpy(pArgs->sHeaders, sizeof(pArgs->sHeaders), optarg);
                break;
            case 'm':
                pArgs->eMethod = XHTTP_GetMethodType(optarg);
                break;
            case 't':
                pArgs->nTimeout = atoi(optarg);
                break;
            case 'f':
                pArgs->nForce = XTRUE;
                break;
            case 'v':
                pArgs->nVerbose = XTRUE;
                break;
            case 'h':
            default:
                return 0;
        }
    }

    if (!xstrused(pArgs->sAddress)) return XFALSE;
    xlog_bar_t *pBar = &pArgs->progressBar;

    if (xstrused(pArgs->sOutput))
    {
        XLogBar_GetDefaults(pBar);
        xstrncpy(pBar->sPrefix, sizeof(pBar->sPrefix), "Downloading... ");
    }

    if (pArgs->nVerbose)
    {
        xlog_timing(XLOG_TIME_ONLY);
        xlog_enable(XLOG_ALL);
    }

    return XTRUE;
}

int XHTTPApp_AppendArgHeaders(xhttp_t *pHandle, xhttp_args_t *pArgs)
{
    xarray_t *pArr = xstrsplit(pArgs->sHeaders, ";");
    if (pArr == NULL) return XSTDERR;

    pHandle->nAllowUpdate = XTRUE;
    size_t i, nUsed = pArr->nUsed;

    for (i = 0; i < nUsed; i++)
    {
        char *pHeader = (char*)XArray_GetData(pArr, i);
        if (pHeader == NULL) continue;

        char *pSavePtr = NULL;
        char *pOption = xstrtok(pHeader, ":", &pSavePtr);
        if (pOption == NULL) return XSTDERR;
      
        char *pField = xstrtok(NULL, ":", &pSavePtr);
        if (pField == NULL) return XSTDERR;
        
        char *pOptionPtr = pOption;
        char *pFieldPtr = pField;

        while (*pOptionPtr == XSTR_SPACE_CHAR) pOptionPtr++;
        while (*pFieldPtr == XSTR_SPACE_CHAR) pFieldPtr++;

        XHTTP_AddHeader(pHandle, pOptionPtr, "%s", pFieldPtr);
        xlogd("Adding header: %s: %s", pOptionPtr, pFieldPtr);
    }

    XArray_Destroy(pArr);
    return XSTDOK;
}

int XHTTPApp_DisplayRequest(xhttp_t *pHandle)
{
    xlogd("Request RAW:\n%s%s", 
        (char*)pHandle->dataRaw.pData,
        XHTTP_GetBodySize(pHandle) ?
        XSTR_NEW_LINE : XSTR_EMPTY);

    return XSTDOK;
}

void XHTTPApp_UpdateProgress(xhttp_t *pHandle)
{
    xhttp_args_t *pArgs = (xhttp_args_t*)pHandle->pUserCtx;
    xlog_bar_t *pBar = &pArgs->progressBar;

    char sReceivedSize[XHTTP_FIELD_MAX];
    pBar->nPercent = -1;

    XBytesToStr(sReceivedSize, sizeof(sReceivedSize), pArgs->nDone);
    xstrncpyf(pBar->sSuffix, sizeof(pBar->sSuffix), "| %s", sReceivedSize);

    if (pHandle->nContentLength)
    {
        double fPercent = (double)100 / pHandle->nContentLength * pArgs->nDone;
        pBar->nPercent = (int)floor(fPercent);
    }

    XLogBar_Update(pBar);
}

int XHTTPApp_DumpResponse(xhttp_t *pHandle, xhttp_ctx_t *pCbCtx)
{
    xhttp_args_t *pArgs = (xhttp_args_t*)pHandle->pUserCtx;
    if (!XHTTP_IsSuccessCode(pHandle) || !xstrused(pArgs->sOutput)) return XSTDUSR;

    if (pArgs->pOutputFile == NULL)
    {
        pArgs->pOutputFile = XFile_New(pArgs->sOutput, "cwt", NULL);
        if (pArgs->pOutputFile == NULL)
        {
            xloge("Failed to open output file: %s (%d)", pArgs->sOutput, errno);
            return XSTDERR;
        }
    }

    pArgs->nDone += pCbCtx->nLength;
    XHTTPApp_UpdateProgress(pHandle);

    if (XFile_Write(pArgs->pOutputFile, pCbCtx->pData, pCbCtx->nLength) <= 0)
    {
        xloge("Failed to write data to output file: %s (%d)", pArgs->sOutput, errno);
        return XSTDERR;
    }

    return XSTDOK;
}

int XHTTPApp_DumpResponseHdr(xhttp_t *pHandle)
{
    const char *pCntType = XHTTP_GetHeader(pHandle, "Content-Type");
    const char *pStatus = XHTTP_GetCodeStr(pHandle->nStatusCode);

    xlogd("Parsed response:\n"
        "Content Type: %s\n"
        "Header Count: %d\n"
        "Header Length: %d\n"
        "Content Length: %d\n"
        "Status Code: %d\n"
        "Version: %s\n"
        "Status: %s\n\n", 
        pCntType ? pCntType : "Undefined",
        pHandle->nHeaderCount,
        pHandle->nHeaderLength,
        pHandle->nContentLength,
        pHandle->nStatusCode,
        pHandle->sVersion,
        pStatus);

    pHandle->dataRaw.pData[pHandle->nHeaderLength - 1] = '\0';
    xlogd("Response header RAW:\n%s\n", (char*)pHandle->dataRaw.pData);
    pHandle->dataRaw.pData[pHandle->nHeaderLength - 1] = '\n';
    return XSTDOK;
}

int XHTTPApp_Callback(xhttp_t *pHttp, xhttp_ctx_t *pCbCtx)
{
    switch (pCbCtx->eCbType)
    {
        case XHTTP_STATUS:
            if (pCbCtx->eStatus == XHTTP_PARSED) 
                return XHTTPApp_DumpResponseHdr(pHttp);
            xlogd("%s", (const char*)pCbCtx->pData);
            return XSTDOK;
        case XHTTP_READ:
            return XHTTPApp_DumpResponse(pHttp, pCbCtx);
        case XHTTP_WRITE:
            return XHTTPApp_DisplayRequest(pHttp);
        case XHTTP_ERROR:
            xloge("%s", (const char*)pCbCtx->pData);
            return XSTDERR;
        default:
            break;
    }

    return XSTDUSR;
}

int main(int argc, char* argv[])
{
    xlog_defaults();
    xlog_useheap(XTRUE);
    xlog_enable(XLOG_INFO);

    xhttp_args_t args;
    if (!XHTTPApp_ParseArgs(&args, argc, argv))
    {
        XHTTPApp_DisplayUsage(argv[0]);
        return XSTDERR;
    }

    if (XPath_Exists(args.sOutput) && args.nForce == XFALSE)
    {
        xlogw("File already exists: %s", args.sOutput);
        xlogi("Use option -f to overwrite the file");
        return XSTDERR;
    }

    xlink_t link;
    if (XLink_Parse(&link, args.sAddress) == XSTDERR)
    {
        xloge("Unsupported link: %s", args.sAddress);
        return XSTDERR;
    }

    xlogd("Parsed link:\nProtocol: %s\nHost: %s\nAddr: %s\nPort: %d\nUser: %s\nPass: %s\nURL: %s\n\n",
        link.sProtocol, link.sHost, link.sAddr, link.nPort, link.sUser, link.sPass, link.sUrl);

    xhttp_t handle;
    XHTTP_InitRequest(&handle, args.eMethod, link.sUrl, NULL);
    XHTTP_AddHeader(&handle, "Host", "%s", link.sHost);
    XHTTP_AddHeader(&handle, "User-Agent", "xutils/%s", XUtils_VersionShort());
    handle.nTimeout = args.nTimeout;

    uint16_t nCallbacks = XHTTP_ERROR | XHTTP_READ | XHTTP_WRITE | XHTTP_STATUS;
    XHTTP_SetCallback(&handle, XHTTPApp_Callback, &args, nCallbacks);

    if (xstrused(args.sHeaders) && XHTTPApp_AppendArgHeaders(&handle, &args) < 0)
    {
        xloge("Failed to appen costum headers: %s (%d)", args.sHeaders, errno);
        XHTTP_Clear(&handle);
        return XSTDERR;
    }

    xhttp_status_t eStatus;
    xbyte_buffer_t content;
    XByteBuffer_Init(&content, 0, 0);

    if (xstrused(args.sContent) && XPath_LoadBuffer(args.sContent, &content) <= 0)
    {
        xloge("Failed to load content from file: %s (%d)", args.sContent, errno);
        XHTTP_Clear(&handle);
        return XSTDERR;
    }

    eStatus = XHTTP_LinkPerform(&handle, &link, content.pData, content.nUsed);
    if (eStatus != XHTTP_COMPLETE)
    {
        if (eStatus == XHTTP_BIGCNT) xlogi("Try to use output file (-o <file>)");
        if (args.pOutputFile != NULL) XFile_Clean(args.pOutputFile);
        XByteBuffer_Clear(&content);
        return XSTDERR;
    }

    if (args.pOutputFile != NULL) XFile_Clean(args.pOutputFile);
    if (args.nDone && !handle.nContentLength) XLogBar_Finish(&args.progressBar);

    if (!XHTTP_IsSuccessCode(&handle))
    {
        const char *pStatus = XHTTP_GetCodeStr(handle.nStatusCode);
        xlogw("HTTP response: %d (%s)", handle.nStatusCode, pStatus);
    }

    const char *pBody = (const char *)XHTTP_GetBody(&handle);
    if (pBody != NULL && !xstrused(args.sOutput)) printf("%s\n", pBody);

    XByteBuffer_Clear(&content);
    XHTTP_Clear(&handle);
    XSock_DeinitSSL();
    return XSTDNON;
}