log shipping without sidecars

vector-example.py

import asyncio
import logging
import os
import sys

original_stdout = os.dup(sys.stdout.fileno())
original_stderr = os.dup(sys.stderr.fileno())
# create a temporary buffer in memory
temp_buffer_fd = os.memfd_create("temp_buffer")
# point stdout to that buffer,
# sticking all of our logging until vector starts in that buffer
os.dup2(temp_buffer_fd, sys.stdout.fileno(), inheritable=False)
os.dup2(temp_buffer_fd, sys.stderr.fileno(), inheritable=False)

logging.warning("log from before event loop")


async def init_vector() -> asyncio.subprocess.Process:
    # asyncio.StreamWriter doesn't have a fd, so use an explicit pipe
    read_pipe, write_pipe = os.pipe()
    tee = await asyncio.create_subprocess_shell(
        # use process substitution to copy tee's output to both vectory and original stdout
        # copy the pipeout to both vector's stdin and actual stdout
        "tee 2>&1 >(vector -c ~/.vector/config/vector.toml)",
        # if we just set stdin to just PIPE, it would be a StreamWriter and not have a .fileno()
        stdin=read_pipe,
        stdout=original_stdout,
        shell=True,
        executable="/bin/bash",
    )

    # Cause tee's stdin to get a copy of our stdin/stdout (as well as that
    # of any child processes we spawn)
    os.dup2(write_pipe, sys.stdout.fileno())
    os.dup2(write_pipe, sys.stderr.fileno())
    # print all of the earlier logs for vector to collect
    temp_file = os.fdopen(temp_buffer_fd, mode="r")
    temp_file.seek(0)
    print(temp_file.read(), end="")
    return tee


async def main() -> None:
    vector = await init_vector()
    await (await asyncio.create_subprocess_shell("echo log from subprocess")).wait()
    await vector.wait()

vector.toml

[sources.stdin]
  type = "stdin"

[sinks.honeycomb]
  type = "honeycomb"
  inputs = ["stdin"]
  api_key = "${HONEYCOMB_API_KEY}"
  dataset = "vector-test"