shipit.erl

-module(shipit).

-define(REGION, "us-east-2").
-define(AWS_ACCESS_KEY_ID, "A11111111...11111111").
-define(AWS_SECRET_ACCESS_KEY, "011111111111111111...1111111111111111111").

-define(HOST, "s3.us-east-2.amazonaws.com").

-include_lib("kernel/include/file.hrl").

-include_lib("ssl/src/ssl_api.hrl").
-include_lib("ssl/src/ssl_cipher.hrl").
-include_lib("ssl/src/ssl_internal.hrl").
-include_lib("ssl/src/ssl_record.hrl").

%% NOTE: need `modprobe tls` on Linux host before upload, at least kernel
%% version 5.2.0.

-export([upload/1]).

upload(File) ->
    Options = [
        inet,
        binary,
        {ktls, true},
        {verify, verify_peer},
        {cacerts, public_key:cacerts_get()}
    ],
    {ok, #sslsocket{connection_handler = Receiver}} =
        ssl:connect(?HOST, 443, Options),
    {ok, KtlsInfo} = ssl_gen_statem:ktls_handover(Receiver),
    Socket = maps:get(socket, KtlsInfo),
    ok = inet_set_ktls(KtlsInfo),

    ok = gen_tcp:send(Socket, header(File)),

    %% {ok, Fd} = file:open(File, [raw, binary, read]),
    {Socket, file:sendfile(File, Socket)}.

-define(ALGORITHM, "AWS4-HMAC-SHA256").
-define(SIGNED_HEADERS, "date;host").
-define(PAYLOAD_HASH, "UNSIGNED-PAYLOAD").

%% erlfmt:ignore
header(File) ->
    %% Write the HTTP/1 header. The main work here is figuring
    %% out the authorization header for AWS.
    %% <https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html>
    {ok, #file_info{size = Size}} = file:read_file_info(File),
    ContentLength = integer_to_binary(Size),
    RequestTimestamp = local_time(),
    [Date | _] = string:split(RequestTimestamp, "T"),

    CanonicalRequest = <<
        %% <HTTPMethod>\n
        "PUT\n"
        %% <CanonicalURI>\n
        "/tiered-storage-dev/upload-test\n"
        %% <CanonicalQueryString>\n
        "\n",
        %% <CanonicalHeaders>\n
        %%   Lowercase(<HeaderName1>) + ":" + Trim(<value>") + "\n" ...
        "date:", Date/binary, "\n"
        "host:" ?HOST "\n"
        "\n"
        %% <SignedHeaders>\n
        ?SIGNED_HEADERS "\n"
        %% <HashedPayload>
        ?PAYLOAD_HASH
    >>,
    HashedCanonicalRequest = sha256(CanonicalRequest),

    StringToSign = <<?ALGORITHM "\n", RequestTimestamp/binary, "\n", Date/binary, "/us-east-2/s3/aws4_request" "\n", HashedCanonicalRequest/binary>>,

    DateKey = hmac_sha256(<<"AWS4" ?AWS_SECRET_ACCESS_KEY>>, Date),
    DateRegionKey = hmac_sha256(DateKey, ?REGION),
    DateRegionServiceKey = hmac_sha256(DateRegionKey, "s3"),
    SigningKey = hmac_sha256(DateRegionServiceKey, "aws4_request"),
    Signature = hmac_sha256(SigningKey, StringToSign),

    Authorization = <<
        ?ALGORITHM " "
        "Credential=" ?AWS_ACCESS_KEY_ID "/", Date/binary, "/us-east-2/s3/aws4_request, "
        "SignedHeaders=" ?SIGNED_HEADERS ", "
        "Signature=", Signature/binary
    >>,

    <<
        "PUT /tiered-storage-dev/upload-test HTTP/1.1\r\n"
        "authorization: ", Authorization/binary, "\r\n"
        "content-length: ", ContentLength/binary, "\r\n"
        "date: ", RequestTimestamp/binary, "\r\n"
        "host: " ?HOST "\r\n"
        "x-amz-content-sha256: " ?PAYLOAD_HASH "\r\n"
        "\r\n"
    >>.

%% --------------------------------------------------------------------
%% header stuff copied from rabbitmq_aws:

-define(ISOFORMAT_BASIC, "~4.10.0b~2.10.0b~2.10.0bT~2.10.0b~2.10.0b~2.10.0bZ").

local_time() ->
    [LocalTime] = calendar:local_time_to_universal_time_dst(
        calendar:local_time()
    ),
    local_time(LocalTime).
local_time({{Y, M, D}, {HH, MM, SS}}) ->
    iolist_to_binary(io_lib:format(?ISOFORMAT_BASIC, [Y, M, D, HH, MM, SS])).

sha256(Value) ->
    iolist_to_binary(
        io_lib:format("~64.16.0b", [
            binary:decode_unsigned(crypto:hash(sha256, Value))
        ])
    ).

hmac_sha256(Key, Message) ->
    crypto:mac(hmac, sha256, Key, Message).

%% --------------------------------------------------------------------
%% inet ktls stuff copied from OTP (from `init_tls_dist`)

inet_set_ktls(
    #{socket := Socket, socket_options := SocketOptions} = KtlsInfo
) ->
    ok =
        inet_tls_dist:set_ktls(
            KtlsInfo#{
                setopt_fun => fun inet_tls_dist:inet_ktls_setopt/3,
                getopt_fun => fun inet_tls_dist:inet_ktls_getopt/3
            }
        ),
    %%
    #socket_options{
        mode = _Mode,
        packet = Packet,
        packet_size = PacketSize,
        header = Header,
        active = Active
    } = SocketOptions,
    case
        inet:setopts(
            Socket,
            [
                list,
                {packet, Packet},
                {packet_size, PacketSize},
                {header, Header},
                {read_ahead, true},
                {active, Active}
            ]
        )
    of
        ok ->
            ok;
        {error, SetoptError} ->
            {error, {ktls_setopt_failed, SetoptError}}
    end.

This is an example of uploading a file to S3 over TLS 1.3 with kTLS and sendfile on Linux hosts. At time of writing, I ran this with a modified OTP source which adds support for the cipher that I found preferred by S3: the-mikedavis/otp@1e0f548