PatrickKalkman

Final Detailed Specification

Overview

• Purpose:
  Develop a Node.js application that runs once per day, retrieves events from a PostgreSQL database older than a configurable threshold (default one year), groups them by their UTC-created date, and exports each group as a CSV file. Each CSV is named in the format events_YYYY-MM-DD.csv and stored in an Azure Data Lake under the folder titan-pulse-event-archive. After successful upload, the corresponding events are deleted from the database in one transaction.

• Deployment Environment:
  The application will run in a Docker container (using a base image node:20-alpine3.18) deployed as a Kubernetes CronJob scheduled daily at 02:00 AM UTC in an Azure Kubernetes Cluster.

PatrickKalkman

PatrickKalkman

PatrickKalkman

resources:
  requests:
    cpu: 1500m
    memory: 1024Mi
  limits:
    cpu: 8000m
    memory: 2524Mi

PatrickKalkman

if (encodingInstructions.videoEncoder === constants.ENCODER_TYPES.X265) {
    const ffmpegCommand = ffmpeg()
      .input(inputAsset)
      .videoBitrate(encodingInstructions.videoBitrate)
      .videoCodec(encodingInstructions.videoEncoder)
      .size(encodingInstructions.videoSize)
      .audioCodec(encodingInstructions.audioEncoder)
      .audioBitrate(encodingInstructions.audioBitrate)
      .audioFrequency(encodingInstructions.audioFrequency)
      .withOutputOptions('-force_key_frames "expr:gte(t,n_forced*2)"')

PatrickKalkman

app.processEncodingTasks = function processEncodingTasks() {
  encoderEngine.searchTasks((searchErr, encoderInstructions) => {
    if (!searchErr && encoderInstructions) {
      log.info(`Got encoder instructions, started encoder for task '${encoderInstructions.name}' id:${encoderInstructions._id}`);
      encoderEngine.startEncoder(encoderInstructions, (startErr) => {
        if (!startErr) {
          encoderEngine.setTaskToFinished(encoderInstructions._id, (finishErr) => {
            if (finishErr) {
              log.error(`An error occurred while trying to set the task to finished. ${finishErr.message}`);
            }

PatrickKalkman

scalingEngine.getNumberOfNeededEncoders = (numberOfEncodingTasks, numberOfEncoders) => {
  if (numberOfEncodingTasks >= 0 && numberOfEncodingTasks <= 10) {
    if (numberOfEncoders >= 1 && numberOfEncodingTasks < 1) {
      return 0;
    }
    return numberOfEncoders;
  }

  if (numberOfEncodingTasks > 10 && numberOfEncodingTasks <= 50) {
    if (numberOfEncoders < 4) {

PatrickKalkman

apiVersion: apps/v1
kind: Deployment
metadata:
  name: encoderscaler
spec:
  replicas: 1
  selector:
    matchLabels:
      app: encoderscaler
  strategy:

PatrickKalkman

import k8s from '@kubernetes/client-node';
import log from './log.js';

const kubernetesClient = {};

kubernetesClient.setNumberOfEncoders = async (numberOfEncoders) => {
  try {
    const kubeconfig = new k8s.KubeConfig();

    if (process.env.NODE_ENV === 'production') {

PatrickKalkman