Highly configurable (auto) file uploader. A util/application which makes HTTP POSTS based on entries of a table from a database and upload specified fields as file-posts. Contact me on twitter (twitter.com/roxlu or by mail info[.......]roxlu[....]com for more info and/or binaries for windows, mac and linux)

uploader.cpp

/**
 * GENERIC UPLOADER / FORM POSTER
 * -------------------------------
 *
 * This util queries a SQLite database and performs a 
 * HTTP POST with information from this table. 
 *
 * To make sure every file gets uploaded, we use a a couple
 * of states: 
 * -----------------------
 * 0 - file has been added 
 * 1 - file is currently being uploaded
 * 2 - file is uploaded.
 * -----------------------
 *
 */

extern "C" {
#  include <uv.h>
}

#include <string>
#include <map>
#include <iostream>
#include <sstream>
#include <vector>
#include <stdio.h>
#include <string>
#include <roxlu/Roxlu.h>
#include <kurl/Kurl.h>
#include <sqlite/Database.h>

/* settings, uploader, db */
INI uploader_ini;
Kurl uploader_kurl;
Database uploader_db;
uv_loop_t* loop;
uv_tty_t tty;

/* settings */
std::string ini_path;                                    /* path where we can find the ini settings, must be passed as argument to this application */
std::string upload_url;                                  /* the url we post the file + field to */
std::string src_files_path;                              /* in the database table you store the filename that we must upload, the files are located in this directory */
std::string database_path;                               /* path to the sqlite database we perform queries on */

/* queries - db */
std::string id_field;                                    /* the primary key of the table, used to update the 'uploaded' field. */
std::string file_field;                                  /* this is the field in the table that is used to fetch the file we need to upload */
std::string get_total_entries_to_upload_query;           /* query that returns how many new entries there are in the database which havent been uploaded, files with a upload state 0 need to be uploaded */
std::string get_entries_to_upload_query;                 /* query that returns all info about the entries that need to be uploaded (state = 0) */
std::string set_upload_state_query;                      /* update query that is used to change the state. we use printf(), the first value is the state the second one the value of the 'id_field', we assume ids are number */
std::string reset_upload_state_query;                    /* when the application restarts after a crash we reset the states from 1 to 0 for the 'upload' state field */

int total_uploading = 0;
int total_uploaded = 0;

char sql_buf[1024];

struct upload_entry {
  int id;
  std::string filepath;
  std::map<std::string, std::string> fields;
};

enum color {
  BLACK = 0,
  RED = 1,
  GREEN = 2,
  YELLOW = 3,
  BLUE = 4,
  MAGENTA = 5,
  CYAN = 6,
  WHITE = 7
};

int spinner_dx = 0;

void log(std::string msg, color fg = WHITE);
void print_spinner();
void print_settings();
int load_settings();

void cb_post_complete(KurlConnection* c, void *user) {

  upload_entry* ue = static_cast<upload_entry*>(user);
  memset(sql_buf, 0, sizeof(sql_buf));
  sprintf(sql_buf, set_upload_state_query.c_str(), 2, ue->id);

  QueryResult update_state(uploader_db);
  uploader_db.query(sql_buf).execute(update_state);
  update_state.finish();

  total_uploaded++;

  delete ue;
  ue = NULL;
}

int main(int argc, char** argv) {

  loop = uv_default_loop();
  uv_tty_init(loop, &tty, 1, 0);
  uv_tty_set_mode(&tty, 0);
  
  if(uv_guess_handle(1) != UV_TTY) {
    printf("Not a compatibly TTY terminal; expect some wierd output\n");
  }

  if(argc != 2) {
    log("ERROR: usage: ./uploader settings.ini\n", RED);
    return EXIT_FAILURE;
  }

  log("uploader v.0.0.1\n", RED);

  ini_path = argv[1];
  if(!uploader_ini.load(ini_path)) {
    ini_path = rx_to_exe_path(argv[1]);
    if(!uploader_ini.load(ini_path)) {
      log("ERROR: cannot open ini file", RED);
    }
    return EXIT_FAILURE;
  }

  if(load_settings() == EXIT_FAILURE) {
    return EXIT_FAILURE;
  }

  print_settings();

  if(!uploader_db.open(database_path)) {
    log("ERROR: cannot open database.", RED); log(database_path, RED);  log("\n", BLACK);
    return EXIT_FAILURE;
  }

  QueryResult state_result(uploader_db);
  bool r = uploader_db.query(reset_upload_state_query).execute(state_result);
  if(!r) {
    log("ERROR: cannot reset files which were being uploaded while application crashed/exited\n", RED);
  }
  state_result.finish();

  log("> reset previously failed upload entries\n", MAGENTA);

  while(true) {
    QueryResult rows_result(uploader_db);
    r =  uploader_db.query(get_total_entries_to_upload_query).execute(rows_result);
    if(!r) {
      log("WARNING: the query to get the total number of new videos did not work.\n", RED);
      rx_sleep_millis(500);
      continue;
    }

    rows_result.next();
    if(rows_result.getInt(0) == 0) {
      print_spinner();
      rows_result.free();

      for(int i = 0; i < 50; ++i) {
        uploader_kurl.update();
      }

      rx_sleep_millis(100);
      continue;
    }
    
    QueryResult fields_result(uploader_db);
    r = uploader_db.query(get_entries_to_upload_query).execute(fields_result);
    if(!r) {
      log("WARNING: cannot execute get_entries_to_upload_query\n", RED);
      rx_sleep_millis(100);
      continue;
    }
    
    std::vector<std::string> fields;
    fields = fields_result.getFieldNames();

    while(fields_result.next()) {
      Form f;
      f.setURL(upload_url);

      /* store all field values + add them to a form */
      upload_entry* e = new upload_entry();
      for(int i = 0; i < fields.size(); ++i) { 
        std::string field_name = fields[i];
        e->fields[field_name] = fields_result.getString(i);
        
        if(field_name == id_field) {
          e->id = fields_result.getInt(i);
          f.addInput(field_name, e->fields[field_name]);
        }
        else if(field_name == file_field) {
          e->filepath = src_files_path + e->fields[field_name];
          f.addFile("file", e->filepath);
        }
        else {
          f.addInput(field_name, e->fields[field_name]);
        }
      }

      /* update the 'upload' status to indicate we're uploading */
      {
        log("ID: ", YELLOW); log(e->fields[id_field]); log(" uploading: ", GREEN);  log(e->filepath, CYAN); log("\n", BLACK);

        memset(sql_buf, 0, sizeof(sql_buf));
        sprintf(sql_buf, set_upload_state_query.c_str(), 1, e->id);

        QueryResult update_state(uploader_db);
        uploader_db.query(sql_buf).execute(update_state);
        update_state.finish();
      }

      if(!uploader_kurl.post(f, cb_post_complete, e)) {
        printf("ERROR: cannot post form.\n");
      }      
      else {
        total_uploading++;
      }
    }
    uploader_kurl.update();
  }

  return EXIT_SUCCESS;
}

int load_settings() {
  upload_url = uploader_ini.getString("upload_url", "none");
  if(upload_url == "none") {
    log("ERROR: cannot find the upload_url in uploader.ini", RED);
    return EXIT_FAILURE;
  }

  src_files_path = uploader_ini.getString("src_files_path", "none");
  if(src_files_path == "none") {
    log("ERROR: cannot find the src_files_path in uploader.ini", RED);
    return EXIT_FAILURE;
  }
  
  database_path = uploader_ini.getString("database_path", "none");
  if(database_path == "none") {
    log("ERROR: cannot get database path.\n", RED);
    return EXIT_FAILURE;
  }
  
  get_total_entries_to_upload_query = uploader_ini.getString("get_total_entries_to_upload_query", "none");
  if(get_total_entries_to_upload_query == "none") {
    log("ERROR: cannot get ini field: get_total_entries_to_upload_query  .\n", RED);
    return EXIT_FAILURE;
  }

  get_entries_to_upload_query = uploader_ini.getString("get_entries_to_upload_query", "none");
  if(get_entries_to_upload_query == "none") {
    log("ERROR: cannot get ini field: get_entries_to_upload_query  .\n", RED);
    return EXIT_FAILURE;
  }

  set_upload_state_query = uploader_ini.getString("set_upload_state_query", "none");
  if(set_upload_state_query == "none") {
    log("ERROR: cannot get ini field: set_upload_state_query  .\n", RED);
    return EXIT_FAILURE;
  }

  reset_upload_state_query = uploader_ini.getString("reset_upload_state_query", "none");
  if(reset_upload_state_query == "none") {
    log("ERROR: cannot get ini field: reset_upload_state_query.\n", RED);
    return EXIT_FAILURE;
  }

  file_field = uploader_ini.getString("file_field", "none");
  if(file_field == "none") {
    log("ERROR: cannot get ini field: file_field\n", RED);
    return EXIT_FAILURE;
  }

  id_field = uploader_ini.getString("id_field", "none");
  if(id_field == "none") {
    log("ERROR: cannot get ini field: id_field\n", RED);
    return EXIT_FAILURE;
  }

  return EXIT_SUCCESS;
}

void print_settings() {
  log("-----------------------------------------------------------------------------------\n", YELLOW);
  log("upload url = ", GREEN); log(upload_url, CYAN); log("\n");
  log("ini path = ", GREEN); log(ini_path, CYAN); log("\n");
  log("source files path = ", GREEN); log(src_files_path, CYAN); log("\n");
  log("database path = ", GREEN); log(database_path, CYAN); log("\n");
  log("-----------------------------------------------------------------------------------\n", YELLOW);
  log("file_field = ", GREEN); log(file_field, CYAN); log("\n");
  log("id_field = ", GREEN); log(id_field, CYAN); log("\n");
  log("get_total_entries_to_upload_query = ", GREEN); log(get_total_entries_to_upload_query, CYAN); log("\n");
  log("get_entries_to_upload_query = ", GREEN); log(get_entries_to_upload_query, CYAN); log("\n");
  log("set_upload_state_query = ", GREEN); log(set_upload_state_query, CYAN); log("\n");
  log("reset_upload_state_query = ", GREEN); log(reset_upload_state_query, CYAN); log("\n");
  log("-----------------------------------------------------------------------------------\n", YELLOW);
}

void log(std::string msg, color fg) {
  uv_buf_t buf;
 
  char str[1024];
  sprintf(str, "\033[1;%d;40m%s\033[37;40m", (30 + fg), msg.c_str());
  buf.base = str;
  buf.len = strlen(str);

  uv_write_t req;
  uv_write(&req, (uv_stream_t*)&tty, &buf, 1, NULL);
  uv_run(loop, UV_RUN_DEFAULT);
}

void print_spinner() {

  int num_dashes = 10;
  ++spinner_dx %= num_dashes;
  std::stringstream ss_spinner;

  ss_spinner << "[";
  for(int i = 0; i < num_dashes; ++i) {
    if(i == spinner_dx) {
      ss_spinner << "##";
    }
    else {
      ss_spinner << '-';
    }
  }
  ss_spinner << "]";
  
  char progress[100];
  sprintf(progress, "%d/%d", total_uploaded, total_uploading);

  char str[512];
  char* info_msg = "> waiting for new entries ";

  sprintf(str, "\033[1;35;40m%s\033[1;37;40m%s \033[36;40m(\033[1;33;40m%s\033[36;40m)\033[%dD",
          info_msg,
          ss_spinner.str().c_str(),
          progress,
          strlen(info_msg) + int(ss_spinner.str().size()) + strlen(progress)+3
          );
  
  uv_buf_t buf;
  buf.base = str;
  buf.len = strlen(str);
  uv_write_t req;
  uv_write(&req, (uv_stream_t*)&tty, &buf, 1, NULL);
  uv_run(loop, UV_RUN_DEFAULT);

}

uploader.ini

# upload_url
# ----------
# The url that we use to post the file + fields to. This can be 
# something like: http://www.yourdomain.com/upload.php?act=new
#
upload_url=http://uploader.localhost/

# database_path
# -------------
# Path to the SQLite database on which we perform the queries. You can 
# use any database/table you want as long as we can get the right data 
# from it. See below for a description of the queries.
#
#        An example table could be:
#        --------------------------
#        id (INTEGER)                   - use this as "id_field" 
#        video_file (STRING)            - use this as "file_field"
#        upload_state (INTEGER)         - use this in the state queries (see below)
#
database_path=/Users/roxlu/roxlu/apps/speech.db

# src_files_path
# --------------
# We assume that you store only a filename in the database. Therefore we 
# prepend this src_files_path to the file_field value and then pass it to 
# the upload handler. 
src_files_path=/Users/roxlu/roxlu/apps/data/

# file_field
# ----------
# The field that contains the name of the file. This is just the name and the file
# must be stored in "src_files_path". 
file_field=video_file

# id_field
# --------
# The primary key field of an entry.
id_field=id

# get_total_entries_to_upload_query
# ----------------------------------
# Query that returns the count(), of the elements that we need to upload. 
# You need to return the entries that have an upload state of 0
#        
#         example
#         ------
#         select count() from videos where uploaded = 0
#         select count() from users where profile_image_upload_state = 0

#  or uploaded = 1
get_total_entries_to_upload_query=select count() from speeches where uploaded = 0

# get_entries_to_upload_query
# ---------------------------
# Almost the same as "get_total_entries_to_upload_query" except this function should
# return all the field/column names that you want to add to the post + which havent 
# been uploaded of course. So add a 'where' clause like: where uploaded = 0
get_entries_to_upload_query=select id, email, video_file from speeches where uploaded = 0

# set_upload_state_query
# -----------------------
# This query is used to change the upload state for a specific entry. We use
# printf() and pass the state as the first entry and the primary key value (id_field)
# as second one, both as integers:
#
#          example
#          -------
#          update videos set uploaded = %d where id %d
#          update users set profile_image_upload_state = %d where id = %d
#
set_upload_state_query=update speeches set uploaded = %d where id = %d

# reset_upload_state_query
# ------------------------
# When the application crashes while one of the files are just being uploaded
# we can retry when the application starts again. If you want to use this feature
# create a query qhich sets the upload state to '0' of the entries which currently
# have a state of 1.
#
#          example
#          -------
#          update videos set uploaded = 0 where uploaded = 1
#          update users set profile_image_upload_state = 0 where profile_image_upload_state = 1
#
reset_upload_state_query=update speeches set uploaded = 0 where uploaded = 1


# fake_insert_query
# -----------------
# We created an auxilary util, the 'fake_uploader' which can be used to automitically
# insert new entries into the the database on a defined timeout, e.g. every second. 
# This can be handy to test performance, or your upload handler.
#
#          example
#          -------
#          insert into videos (name, email, video_file) values ("roxlu", "[email protected]", "test.flv")
#          insert into users (name, profile_image) values ("roxlu", "roxlu.png")

fake_insert_query=insert into speeches (email, video_file) values ("[email protected]", "test.flv")

# fake_insert_timeout_millis
# --------------------------
# Insert a new fake entry ever X-millis. Used by the fake inserter.
fake_insert_timeout_millis=4000

Uploader.php

<?php

/**
 * Version 0.0.1
 * -------------
 * 
 * Uploader handler for uploader util. 
 * ----------------------------------
 * Example class which handles a file upload from the 
 * uploader utility. This class doesn't do a lot, except
 * handling a file upload correctly. It will log verbose
 * information if the 'log' flag is set in the configuration
 * array.
 *
 * Configuration:
 * --------------
 * - destination_path: the full path to the upload directory where you want 
 *                     to store the uploaded files.
 * 
 * - file_field:       the name in the $_FILES array that contains the 
 *                     name of the uploaded file.
 * 
 * - log:              set to true if you want to log
 * - log_file:         append log entries to this file
 * 
 * 
 *
 * Example usage:
 * --------------
 *
 *      $path = pathinfo(__FILE__);
 *      $base_dir = $path['dirname'] .'/';
 *      
 *      $config = array(
 *        'log' => true,
 *        'log_file' => $base_dir .'upload.log',
 *        'destination_path' => $base_dir .'uploaded/',
 *        'file_field' => 'file'
 *      );
 *      
 *      require_once $base_dir .'Uploader.php';
 *      $up = new Uploader($config);
 *      
 *      if($up->execute() !== false) {
 *        echo "yay uploaded!";
 *      }
 *            
 */

class Uploader {
  private $config = NULL;

  function __construct($config) {
    $this->config = $config;
    if(!$this->init()) {
      die('Could not initialize the uploader. Check the logs');
    }
  }

  private function init() {
    if(array_key_exists('log', $this->config)
      && $this->config['log'])
      {
        if(!array_key_exists('log_file', $this->config)) {
          die('ERROR: you asked me to log, but the log_file configuration is not set.');
        }
        else if(!is_writable($this->config['log_file'])) {
          die('ERROR: the log file is not writable (e.g. $ touch logfile.log && chmod 777 logfile.log)');
        }
      }

    if(!array_key_exists('file_field', $this->config)) {
      $this->log('ERROR: the file_field config is not set. ', true);
      return false;
    }

    if(!is_writable($this->config['destination_path'])) {
      $this->log('ERROR: the upload path is not writable or does not exist', true);
      return false;
    }
    
    return true;
  }

  /**
   * Handles the file upload. For now we only handle one
   * file at a time; we do iterate over all the files though, we might
   * return an array of uploaded files in the next versionb but for now 
   * we return just the path of the found file entry.
   *
   * @return mixed false  - We return false on error
   * @return mixed string - The path to the uploaded file on success
   */
  public function execute() {
    $this->log('new request');
    if(!is_array($_FILES) || count($_FILES) == 0) {
      return;
    }

    $uploaded_file = '';
    $ok = true;
    foreach($_FILES as $upname => $file) {
      if($upname != $this->config['file_field']) {
        $this->log('ERROR: we received a file but the name of the file is not the same as in the config.');
        continue;
      }
      
      $dest_file = $this->config['destination_path'] .$file['name'];
      $uploaded_file = $dest_file;
      $result = move_uploaded_file($file['tmp_name'], $dest_file);
      
      if(!$result) {
        $ok = false;
        $str = 'ERROR: ';
        switch($file['error']) {
          case UPLOAD_ERR_INI_SIZE:    $str .= 'The uploaded file exceeds the upload_max_filesize directive in php.ini.'; break;
          case UPLOAD_ERR_FORM_SIZE:   $str .= 'The uploaded file exceeds the MAX_FILE_SIZE directive that was specified in the HTML form'; break;
          case UPLOAD_ERR_PARTIAL:     $str .= 'The uploaded file was only partially uploaded.'; break;
          case UPLOAD_ERR_NO_FILE:     $str .= 'No file was uploaded.'; break;
          case UPLOAD_ERR_NO_TMP_DIR:  $str .= 'Missing a temporary folder. Introduced in PHP 4.3.10 and PHP 5.0.3.'; break;
          case UPLOAD_ERR_CANT_WRITE:  $str .= 'Failed to write file to disk. Introduced in PHP 5.1.0'; break; 
          case UPLOAD_ERR_EXTENSION:   $str .= 'A PHP extension stopped the file upload'; break;
          default: $str .= ' UNKNOWN ERROR '; break;
        };
        $this->log($str);
      }
    }
    if(!$ok) {
      return $ok;
    }
    return $uploaded_file;
  }


  private function log($s, $die = false) {
    if(!$this->config['log']) {
      if($die) {
        die($s);
      }
      return;
    }
    $msg = date('Y-m-d H:i ', time()) .$s ."\n";
    file_put_contents($this->config['log_file'], $msg);
    if($die) {
      die($s);
    }
  }
  
}

Video: http://www.youtube.com/watch?v=LbbG6nm1RVY&feature=youtu.be