commandline file downloader using wininet

wsget.cpp

/*
   Wsget/1.2.3
   Copyright 2022, SASAKI Nobuyuki. Released under the MIT license.
*/

#include <stdio.h>
#include <locale.h>
#include <time.h>
#include <Windows.h>
#include <WinInet.h>

#pragma comment(lib, "wininet.lib")

#define USERAGENT L"Wsget/1.2.3"

#define REDIRECT_MAX		3
#define HEADER_LENGTH		16384

BOOL Download(LPCWSTR url, INT redirect);
BOOL DownloadHTTP(URL_COMPONENTSW *purlc, HINTERNET hConn, INT redirect);
BOOL DownloadFTP(URL_COMPONENTSW *purlc, HINTERNET hConn);
BOOL IfModified(LPCWSTR localpath, LONGLONG size, const FILETIME *pftLastWriteTime);
BOOL MakeDownloadPath(LPCWSTR url, LPWSTR path, DWORD len);
BOOL Retrieve(HINTERNET hFile, LPCWSTR localpath, LONGLONG size, const FILETIME *pftLastModified);
VOID PrintLastResponse();
VOID PrintError(DWORD error, HMODULE hmodule);
VOID PrintError();
VOID PrintWinINetError();

int wmain(int argc, wchar_t *argv[])
{
	_wsetlocale(LC_ALL, L"");

	if (argc < 2)
	{
		wprintf(L"usage : wsget <URL>\n");
		return -1;
	}

	wprintf(L"\n%s\n\n", argv[1]);

	if (Download(argv[1], 0) == FALSE)
	{
		return -1;
	}

	return 0;
}

BOOL Download(LPCWSTR url, INT redirect)
{
	if (url == NULL) return FALSE;

	if (redirect > REDIRECT_MAX)
	{
		wprintf(L"ERROR : over redirect max\n");
		return FALSE;
	}

	WCHAR scheme[INTERNET_MAX_SCHEME_LENGTH] = {};
	WCHAR hostname[INTERNET_MAX_HOST_NAME_LENGTH] = {};
	WCHAR username[INTERNET_MAX_USER_NAME_LENGTH] = {};
	WCHAR password[INTERNET_MAX_PASSWORD_LENGTH] = {};
	WCHAR urlpath[INTERNET_MAX_PATH_LENGTH] = {};
	URL_COMPONENTSW urlc =
	{
		sizeof(urlc),
		scheme, _countof(scheme),
		INTERNET_SCHEME_DEFAULT,
		hostname, _countof(hostname),
		0,
		username, _countof(username),
		password, _countof(password),
		urlpath, _countof(urlpath),
		NULL, 0
	};

	if (InternetCrackUrlW(url, 0, 0, &urlc) == FALSE)
	{
		PrintWinINetError();
		return FALSE;
	}

	DWORD dwService = 0;
	DWORD dwFlags = 0;
	switch (urlc.nScheme)
	{
	case INTERNET_SCHEME_FTP:
		dwService = INTERNET_SERVICE_FTP;
		dwFlags = INTERNET_FLAG_PASSIVE;
		break;
	case INTERNET_SCHEME_HTTP:
	case INTERNET_SCHEME_HTTPS:
		dwService = INTERNET_SERVICE_HTTP;
		break;
	default:
		wprintf(L"ERROR : unsupported scheme \"%s\"\n", urlc.lpszScheme);
		return FALSE;
		break;
	}

	HINTERNET hInet = InternetOpenW(USERAGENT, INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0);
	if (hInet == NULL)
	{
		PrintWinINetError();
		return FALSE;
	}

	HINTERNET hConn = InternetConnectW(hInet, urlc.lpszHostName, urlc.nPort,
		urlc.lpszUserName, urlc.lpszPassword, dwService, dwFlags, 0);
	if (hConn == NULL)
	{
		PrintWinINetError();
		InternetCloseHandle(hInet);
		return FALSE;
	}

	BOOL bRet = FALSE;

	switch (urlc.nScheme)
	{
	case INTERNET_SCHEME_FTP:
		bRet = DownloadFTP(&urlc, hConn);
		break;
	case INTERNET_SCHEME_HTTP:
	case INTERNET_SCHEME_HTTPS:
		bRet = DownloadHTTP(&urlc, hConn, redirect);
		break;
	default:
		break;
	}

	InternetCloseHandle(hConn);
	InternetCloseHandle(hInet);

	return bRet;
}

BOOL DownloadHTTP(URL_COMPONENTSW *purlc, HINTERNET hConn, INT redirect)
{
	static WCHAR szHeader[HEADER_LENGTH] = {};
	DWORD dwQueryLength;

	if (purlc == NULL) return FALSE;

	DWORD dwFlags = (purlc->nScheme == INTERNET_SCHEME_HTTPS ? INTERNET_FLAG_SECURE : 0) |
		INTERNET_FLAG_RELOAD |
		INTERNET_FLAG_NO_CACHE_WRITE |
		INTERNET_FLAG_NO_AUTO_REDIRECT |
		INTERNET_FLAG_NO_COOKIES |
		INTERNET_FLAG_NO_UI;

	LPCWSTR rgpszAcceptTypes[] = { L"*/*", NULL };

	HINTERNET hReq = HttpOpenRequestW(hConn, NULL, purlc->lpszUrlPath, NULL, NULL, rgpszAcceptTypes, dwFlags, 0);
	if (hReq == NULL)
	{
		PrintWinINetError();
		return FALSE;
	}

	if (HttpSendRequestW(hReq, NULL, 0, NULL, 0) == FALSE)
	{
		PrintWinINetError();
		InternetCloseHandle(hReq);
		return FALSE;
	}

	ZeroMemory(szHeader, sizeof(szHeader));
	dwQueryLength = sizeof(szHeader);
	if (HttpQueryInfoW(hReq, HTTP_QUERY_RAW_HEADERS_CRLF | HTTP_QUERY_FLAG_REQUEST_HEADERS, szHeader, &dwQueryLength, 0) == FALSE)
	{
		PrintWinINetError();
		InternetCloseHandle(hReq);
		return FALSE;
	}

	wprintf(L"%s\n", szHeader);

	ZeroMemory(szHeader, sizeof(szHeader));
	dwQueryLength = sizeof(szHeader);
	if (HttpQueryInfoW(hReq, HTTP_QUERY_RAW_HEADERS_CRLF, szHeader, &dwQueryLength, 0) == FALSE)
	{
		PrintWinINetError();
		InternetCloseHandle(hReq);
		return FALSE;
	}

	wprintf(L"%s\n", szHeader);

	DWORD dwStatusCode = 0;
	dwQueryLength = sizeof(dwStatusCode);
	if (HttpQueryInfoW(hReq, HTTP_QUERY_STATUS_CODE | HTTP_QUERY_FLAG_NUMBER, &dwStatusCode, &dwQueryLength, 0) == FALSE)
	{
		PrintWinINetError();
		InternetCloseHandle(hReq);
		return FALSE;
	}

	switch (dwStatusCode)
	{
	case HTTP_STATUS_OK:
		break;
	case HTTP_STATUS_MOVED:
	case HTTP_STATUS_REDIRECT:
	case HTTP_STATUS_REDIRECT_METHOD:
	case HTTP_STATUS_REDIRECT_KEEP_VERB:
		ZeroMemory(szHeader, sizeof(szHeader));
		dwQueryLength = sizeof(szHeader);
		if (HttpQueryInfoW(hReq, HTTP_QUERY_LOCATION, szHeader, &dwQueryLength, 0) == FALSE)
		{
			PrintWinINetError();
			InternetCloseHandle(hReq);
			return FALSE;
		}
		InternetCloseHandle(hReq);

		InternetCloseHandle(hConn);

		return Download(szHeader, ++redirect);
		break;
	default:
		InternetCloseHandle(hReq);
		return FALSE;
		break;
	}

	LONGLONG lContentLength = 0;
	ZeroMemory(szHeader, sizeof(szHeader));
	dwQueryLength = sizeof(szHeader);
	if (HttpQueryInfoW(hReq, HTTP_QUERY_CONTENT_LENGTH, szHeader, &dwQueryLength, 0) == TRUE)
	{
		lContentLength = _wcstoi64(szHeader, NULL, 0);
	}

	WCHAR localpath[MAX_PATH] = {};
	if (MakeDownloadPath(purlc->lpszUrlPath, localpath, _countof(localpath)) == FALSE)
	{
		InternetCloseHandle(hReq);
		return FALSE;
	}

	ZeroMemory(szHeader, sizeof(szHeader));
	dwQueryLength = sizeof(szHeader);
	if (HttpQueryInfoW(hReq, HTTP_QUERY_CONTENT_DISPOSITION, szHeader, &dwQueryLength, 0) == TRUE)
	{
		LPWSTR pattachment = wcsstr(szHeader, L"attachment;");
		if (pattachment != NULL)
		{
			LPWSTR pfilename = wcsstr(pattachment, L"filename");
			if (pfilename != NULL)
			{
				size_t ipath = wcsspn(pfilename + 8, L" =\"");
				LPWSTR pcontent = pfilename + 8 + ipath;
				LPWSTR pch = pcontent;
				pch = wcspbrk(pch, L";\"");
				if (pch != NULL)
				{
					*pch = L'\0';
				}
				pch = pcontent;
				while ((pch = wcspbrk(pch, L"\\/:*?\"<>|")) != NULL)
				{
					*pch = L'_';
				}
				if (wcslen(pcontent) != 0)
				{
					wcsncpy_s(localpath, pcontent, _TRUNCATE);
				}
			}
		}
	}

	FILETIME ftLastModified = {};
	SYSTEMTIME stLastModified = {};
	dwQueryLength = sizeof(stLastModified);
	if (HttpQueryInfoW(hReq, HTTP_QUERY_LAST_MODIFIED | HTTP_QUERY_FLAG_SYSTEMTIME, &stLastModified, &dwQueryLength, 0) == TRUE)
	{
		SystemTimeToFileTime(&stLastModified, &ftLastModified);

		if (IfModified(localpath, lContentLength, &ftLastModified) == FALSE)
		{
			InternetCloseHandle(hReq);
			return TRUE;
		}
	}

	BOOL bRet = Retrieve(hReq, localpath, lContentLength, &ftLastModified);

	InternetCloseHandle(hReq);

	return bRet;
}

BOOL DownloadFTP(URL_COMPONENTSW *purlc, HINTERNET hConn)
{
	if (purlc == NULL) return FALSE;

	WCHAR urlpath[INTERNET_MAX_PATH_LENGTH] = {};
	wcsncpy_s(urlpath, purlc->lpszUrlPath, _TRUNCATE);
	LPWSTR ppath = wcsrchr(urlpath, L'/');
	if (ppath == NULL)
	{
		return FALSE;
	}

	WCHAR localpath[MAX_PATH];
	if (MakeDownloadPath(purlc->lpszUrlPath, localpath, _countof(localpath)) == FALSE)
	{
		return FALSE;
	}

	PrintLastResponse();

	WCHAR pch1 = *(ppath + 1);
	*(ppath + 1) = L'\0';
	LPCWSTR dirname = urlpath;

	if (FtpSetCurrentDirectoryW(hConn, dirname) == FALSE)
	{
		PrintWinINetError();
		return FALSE;
	}

	PrintLastResponse();

	*(ppath + 1) = pch1;
	LPCWSTR filename = ppath + 1;

	WIN32_FIND_DATAW finddata = {};
	HINTERNET hFind = FtpFindFirstFileW(hConn, filename, &finddata, 0, 0);
	if (hFind != NULL)
	{
		InternetCloseHandle(hFind);
	}

	PrintLastResponse();

	HINTERNET hFile = FtpOpenFileW(hConn, filename, GENERIC_READ, FTP_TRANSFER_TYPE_BINARY, 0);
	if (hFile == NULL)
	{
		PrintWinINetError();
		return FALSE;
	}

	PrintLastResponse();

	DWORD dwFileSizeHigh = 0;
	DWORD dwFileSizeLow = FtpGetFileSize(hFile, &dwFileSizeHigh);
	LARGE_INTEGER liFileSize = { dwFileSizeLow, (LONG)dwFileSizeHigh };

	PrintLastResponse();

	if (IfModified(localpath, liFileSize.QuadPart, &finddata.ftLastWriteTime) == FALSE)
	{
		InternetCloseHandle(hFile);
		return TRUE;
	}

	BOOL bRet = Retrieve(hFile, localpath, liFileSize.QuadPart, &finddata.ftLastWriteTime);

	InternetCloseHandle(hFile);

	PrintLastResponse();

	return bRet;
}

BOOL IfModified(LPCWSTR localpath, LONGLONG size, const FILETIME *pftLastWriteTime)
{
	FILETIME ft = {};
	LARGE_INTEGER li = {};

	HANDLE hFileN = CreateFileW(localpath, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
	if (hFileN == INVALID_HANDLE_VALUE)
	{
		return TRUE;
	}

	GetFileTime(hFileN, NULL, NULL, &ft);

	GetFileSizeEx(hFileN, &li);

	CloseHandle(hFileN);

	//SYSTEMTIME st;

	//if(FileTimeToSystemTime(&ft, &st) == TRUE)
	//{
	//	wprintf(L"local  : %04d-%02d-%02dT%02d:%02d:%02dZ %lld bytes\n",
	//		st.wYear, st.wMonth, st.wDay, st.wHour, st.wMinute, st.wSecond, li.QuadPart);
	//}

	//if(FileTimeToSystemTime(pftLastWriteTime, &st) == TRUE)
	//{
	//	wprintf(L"remote : %04d-%02d-%02dT%02d:%02d:%02dZ %lld bytes\n\n",
	//		st.wYear, st.wMonth, st.wDay, st.wHour, st.wMinute, st.wSecond, size);
	//}

	if (size == li.QuadPart && CompareFileTime(pftLastWriteTime, &ft) <= 0)
	{
		wprintf(L"Not Modified\n");
		return FALSE;
	}

	//wprintf(L"Modified\n");
	return TRUE;
}

BOOL MakeDownloadPath(LPCWSTR url, LPWSTR path, DWORD len)
{
	LPCWSTR fnurl = wcsrchr(url, L'/');
	if (fnurl == NULL || wcslen(fnurl) == 1)
	{
		wprintf(L"ERROR : local path\n");

		return FALSE;
	}
	wcsncpy_s(path, len, fnurl + 1, _TRUNCATE);

	LPWSTR ppath = wcschr(path, L'?');
	if (ppath != NULL)
	{
		*ppath = L'\0';
	}

	LPWSTR pfname = path;
	while ((pfname = wcspbrk(pfname, L"\\/:*?\"<>|")) != NULL)
	{
		*pfname = L'_';
	}

	return TRUE;
}

struct {
	double size;
	LPCWSTR unit;
} speedunit[] = {
	{ 1024.0 * 1024.0 * 1024.0 * 1024.0, L"TiB/s" },
	{ 1024.0 * 1024.0 * 1024.0, L"GiB/s" },
	{ 1024.0 * 1024.0, L"MiB/s" },
	{ 1024.0, L"KiB/s" },
	{ 1.0, L"B/s" },
};

BOOL Retrieve(HINTERNET hFile, LPCWSTR localpath, LONGLONG size, const FILETIME *pftLastModified)
{
	static BYTE bufRead[0x10000];	//64KiB
	BOOL retRead;
	DWORD bytesRead, byteWrite;
	int ratio = 0;
	LONGLONG size_whole = size, size_downloaded = 0, size_downloaded_ex = 0;
	FILETIME st, st0, st1;
	WCHAR speed[32] = { L'\0' };

	WCHAR localpathtemp[MAX_PATH];
	_snwprintf_s(localpathtemp, _TRUNCATE, L"%s~", localpath);

	HANDLE hf = CreateFileW(localpath, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
	if (hf == INVALID_HANDLE_VALUE)
	{
		PrintError();
		return FALSE;
	}

	HANDLE hft = CreateFileW(localpathtemp, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
	if (hft == INVALID_HANDLE_VALUE)
	{
		PrintError();
		CloseHandle(hf);
		return FALSE;
	}

	GetSystemTimeAsFileTime(&st);
	st0 = st;
	st1 = st;

	while (true)
	{
		ZeroMemory(bufRead, sizeof(bufRead));
		retRead = InternetReadFile(hFile, bufRead, sizeof(bufRead), &bytesRead);
		if (retRead)
		{
			if (bytesRead == 0)
			{
				break;
			}
		}
		else
		{
			PrintWinINetError();
			CloseHandle(hft);
			CloseHandle(hf);
			return FALSE;
		}

		if (WriteFile(hft, bufRead, bytesRead, &byteWrite, NULL) == FALSE ||
			bytesRead != byteWrite)
		{
			PrintError();
			CloseHandle(hft);
			CloseHandle(hf);
			return FALSE;
		}

		size_downloaded += bytesRead;

		if (size_whole != 0)
		{
			ratio = (int)((size_downloaded * 10000) / size_whole);
		}

		WCHAR progressbar[64];
		ZeroMemory(progressbar, sizeof(progressbar));
		for (int i = 0; i < 40; i++)
		{
			if ((ratio / 250) > i)
			{
				progressbar[i] = L'#';
			}
			else
			{
				progressbar[i] = L'-';
			}
		}

		GetSystemTimeAsFileTime(&st1);
		ULONGLONG difftime = ((PULARGE_INTEGER)&st1)->QuadPart - ((PULARGE_INTEGER)&st0)->QuadPart;
		if ((difftime >= 10000000) || ((ratio != 0) && (ratio % 10000 == 0)))
		{
			double diffsize =
				(double)(size_downloaded - size_downloaded_ex) /
				((double)difftime / (double)10000000);

			ULONGLONG difftimeS = ((PULARGE_INTEGER)&st1)->QuadPart - ((PULARGE_INTEGER)&st)->QuadPart;

			for (int i = 0; i < _countof(speedunit); i++)
			{
				if (diffsize >= speedunit[i].size)
				{
					int diffsec = (int)((difftimeS - (difftimeS % 10000000)) / 10000000);
					_snwprintf_s(speed, _TRUNCATE, L"%02d:%02d:%02d %6.2f%s",
						(diffsec - (diffsec % 3600)) / 3600,
						((diffsec % 3600) - (diffsec % 60)) / 60,
						diffsec % 60,
						diffsize / speedunit[i].size, speedunit[i].unit);
					break;
				}
			}

			st0 = st1;
			size_downloaded_ex = size_downloaded;

			wprintf(L"\r%s %3d.%02d%% %s", progressbar,
				(ratio - (ratio % 100)) / 100, (ratio % 100), speed);
		}
	}

	wprintf(L"\n\n");

	if (pftLastModified != NULL)
	{
		if (SetFileTime(hft, NULL, NULL, pftLastModified) == FALSE)
		{
			PrintError();
			CloseHandle(hft);
			CloseHandle(hf);
			return FALSE;
		}
	}

	CloseHandle(hft);
	CloseHandle(hf);

	if (MoveFileExW(localpathtemp, localpath, MOVEFILE_REPLACE_EXISTING) == FALSE)
	{
		PrintError();
		return FALSE;
	}

	return TRUE;
}

VOID PrintLastResponse()
{
	DWORD dwError = 0;
	PWSTR szBuffer = NULL;
	DWORD dwBufferSize = 0;

	if (InternetGetLastResponseInfoW(&dwError, szBuffer, &dwBufferSize) == FALSE)
	{
		DWORD error = GetLastError();
		if (error == ERROR_INSUFFICIENT_BUFFER)
		{
			dwBufferSize += 1;
			szBuffer = (PWSTR)LocalAlloc(LPTR, dwBufferSize * sizeof(WCHAR));
			if (szBuffer != NULL)
			{
				if (InternetGetLastResponseInfoW(&dwError, szBuffer, &dwBufferSize) == TRUE)
				{
					wprintf(L"%s\n", szBuffer);
				}
				LocalFree(szBuffer);
			}
		}
	}
}

VOID PrintError(DWORD error, HMODULE hmodule)
{
	LPWSTR message = NULL;

	wprintf(L"ERROR : 0x%08X\n", error);

	FormatMessageW(
		FORMAT_MESSAGE_ALLOCATE_BUFFER |
		FORMAT_MESSAGE_IGNORE_INSERTS |
		(hmodule != NULL ? FORMAT_MESSAGE_FROM_HMODULE : FORMAT_MESSAGE_FROM_SYSTEM),
		hmodule, error, MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US),
		(LPWSTR)&message, 0, NULL);

	if (message != NULL)
	{
		wprintf(L"%s\n", message);
		LocalFree(message);
	}
}

VOID PrintError()
{
	DWORD error = GetLastError();

	PrintError(error, NULL);
}

VOID PrintWinINetError()
{
	DWORD error = GetLastError();
	HMODULE hmodule = GetModuleHandleW(L"wininet.dll");

	PrintLastResponse();

	PrintError(error, hmodule);
}

Hello!
DownloadFTP function results to ERROR 0x00000057 when I tried to download a file from FTP server's root like ftp://test.com/test.zip
It happens because of string manipulations. when you're setting *ppath to \0, and then trying to assign a filename variable with (ppath + 1) (which will be empty at this moment due to inserting null-terminating symbol)
Here's a small improvement to fix this:

WCHAR urlpath[INTERNET_MAX_PATH_LENGTH] = {};
WCHAR filename[INTERNET_MAX_PATH_LENGTH] = {};
wcsncpy_s(urlpath, purlc->lpszUrlPath, _TRUNCATE);
LPWSTR ppath = wcsrchr(urlpath, L'/');
  if (ppath == NULL)
  {
    return FALSE;
  }
ppath = ppath + 1;
wcsncpy_s(filename, ppath, _TRUNCATE);
*ppath = L'\0';
LPCWSTR dirname = urlpath;

Anyway, thanks for a good example and hard work!

Thanks for your advice.
Fixed like below for saving memory.

WCHAR urlpath[INTERNET_MAX_PATH_LENGTH] = {};
wcsncpy_s(urlpath, purlc->lpszUrlPath, _TRUNCATE);
LPWSTR ppath = wcsrchr(urlpath, L'/');
if (ppath == NULL)
{
	return FALSE;
}

WCHAR pch1 = *(ppath + 1);
*(ppath + 1) = L'\0';
LPCWSTR dirname = urlpath;

 // use dirname

*(ppath + 1) = pch1;
LPCWSTR filename = ppath + 1;

 // use filename