csv.js

const fs = require('fs');
const CSVFOLDER = "csv";
const LOGFOLDER = "log";

// updates are changes that haven't been written directly to the csv files,
// logs are changes that have already been written to relevant csv files,
// logs can be saved for historical or audit reasons.
const LOGEXTENSION = ".csvlog";
const UPDATEEXTENSION = ".csvupdate";
const DEFAULTDB = "csvdb";



if(require.main === module){
  Test();
}


// updates file:
//  tablename, a, b, c, d, e, f, g...
// national debt simulator.

function TableSet({ dbname = DEFAULTDB, autoparse = true, quoteall = false, usesinglequotes = false, savelog = false }){
  const folder = dbname;

  const tables = {}; // this is the internal object
  const tableset = {}; // this is the return object
  const pendingupdates = []; // the changes which haven't been flushed to disk yet.

  let updatefile = "";
  let logfile = "";

  tableset.update = (tablename, row, after) =>{
    if(!(tablename in tables)){
      tables[tablename] = Table(); }
    let table = tables[tablename];
    pendingupdates.push([tablename, row]);
  }
  tableset.loadAll = (after) => {
    // todo find a file matching the UPDATEEXTENSION in the LOGFOLDER
  }
  tableset.flush = (after) => {
    if(updatefile.length){
      if(savelog){
        const stream = fs.createWriteStream('asdf', {flags: 'a'});
      } else {
      // fs.appendFile('file.txt', 
      }
    }
    let prefix = getFilenamePrefix();
    logfile = prefix + LOGEXTENSION;
    updatefile = prefix + UPDATEEXTENSION;

  }
  tableset.writeAll = (after) => {
    let waitfor = [];
    for(const tablename of tables){
      table = tables[tablename];
      waitfor.push(new Promise((resolve, reject)=>{
        fs.write('asdlfkj', (err)=>{
          if(err) reject();
          else resolve();
        });
      }));
    }
    Promise.all(waitfor).then(() => {
      if(savelog){
        // move update file to logfile, and write the pendinglog
        fs.rename(updatefile, logfile, after);
      } else{
        // delete old update file
        fs.unlink(updatefile, after);
      }
    })
  }
  return tableset;

  function writeUpdates(stream){
    for(let i=0; i<pendingupdates.length; ++i){
      let update = pendingupdates[i];
      //stream.write(
    }
    stream
  }
  function getFilenamePrefix(){
    return "" + date.getFullYear() + (date.getMonth() + 1) + date.getDate() + dbname;
  }
}

function Table({ filename = null, folder = DEFAULTDB + "/" + CSVFOLDER,
                 autoparse = true, autoid = false }){
  let headers = []; 
  let ids = {};

  // TODO let savedelay = ; // the minimum time between saving
  
  //fs.mkdirSync(folder, {recursive: true});

  // checkType(autoid, "boolean");
  // checkType(autoparse, "boolean");
  //if(filename) checkType(filename, "string");

  // store autoids for live table no matter what,
  // the autoid prop, just determines whether that is saved.
  let currentAutoId = 1;

  let columnquoted = [];
  for(let i=0; i<headers.length; ++i){
    let header = headers[i].trim();
    let n = header.length - 1;
    columnquoted[i] = (header[0] == "'" && header[n] == "'" || header[0] == '"' && header[n] == '"');
  }
  let table = {
    headers: headers,
    rows: [],
    ids: {},
  };


  function checkType(value, type){
    let t = typeof value;
    if(type == "array" || type == "object"){
      t = Array.isArray(value)? "array" : t; }
    if(t != type){
      throw new Error(`csvtable: expected "${type}" got ${value}, with type "${t}".`); }
  }

  function checkRow(row){
    if(!Array.isArray(row)){
      throw new Error("csvtable: row must be an array."); }
    if(row.length == 0){
      throw new Error("csvtable: row must not be empty."); }
  }
  function getRowId(key){
    if(!table.ids.hasOwnProperty(key)){
      return -1;
    }
    return table.ids[key];
  }

  table.addRow = (row)=>{
    checkRow(row);
    let key = autoid? currentAutoId: row[0];
    if(!autoid && table.hasOwnProperty(key)){
      return false;
    }
    
    let rowid = currentAutoId++;
    table.ids[key] = rowid;

    table.rows[rowid] = row;
    return rowid;
  }
  table.setProp = (name, value)=>{
    if(name == "autoid"){
      checkType(value, 'boolean');  
      autoid = value;
    }
    if(name == "autoparse"){
      checkType(value, 'boolean');
      autoparse = value;
    }
    if(name == "filename"){
      checkType(value, 'string');
      filename = value;
    }
  }

  table.getRow = (key)=>{
    let rowid = getRowId(key);
    if(rowid == -1) return null;
    return table.rows[rowid];
    return table.rows[key];
  }
  // id is optional, and only used if autoid is true.
  table.setRow = (row, id)=>{
    checkRow(row);
    let rowid = autoid? id : getRowId(row[0]);
    if(rowid==null || rowid == undefined || rowid < 0){
      rowid = currentAutoId++;
    }
    table.rows[rowid] = row;
  }
  table.deleteRow = (key)=>{
    let rowid = getRowId(key);
    if(rowid < 0) return false;
    if(!table.rows.hasOwnProperty(key)){
      return false;
    }
    delete table.ids[key];
    delete table.rows[rowid];
    return true;
  }
  table.save = (after)=>{
    if(!filename){
      return after(false);
    }
    const stream = fs.createWriteStream(folder + "/" + filename, {encoding: 'utf8'});
    if(after){
      if(typeof after != "function"){
        throw new Error("csvtable table.save(after): after must be a function.");
      }
      stream.on('finish', ()=>after(true));
    }
    const quotecolumn = (x,i)=> columnquoted[i]?'"' + x + '"' : x;
    let headerstr = table.headers.map(quotecolumn).join(",");
    if(autoid) headerstr = "AutoId," + headerstr
    stream.write(headerstr + "\n");
    //console.log(table);
    for(const key in table.ids){
      const rowid = table.ids[key];
      const row = table.rows[rowid];
      //console.log(key, rowid, row);
      let rowstr = rowToString(row);
      if(autoid) rowstr = `${rowid},${rowstr}`;
      stream.write(rowstr + "\n");
    }
  }
  // TODO a field is default quoted if the header is quoted.
  table.load = (after)=>{
    if(!after) after = (success)=> console.log(`Table load ${success? "succeeded" : "failed"}`);
    if(typeof after != "function"){
      throw new Error("csvtable table.load(after): after must be a function.");
    }
    if(!filename){
      return after(false);
    }
    let stream = fs.createReadStream(folder + "/" + filename, {encoding: 'utf8'});
    let line = "";
    let linenumber = 0;
    let n = line.length;

    //console.log("stream", stream);
    stream.on('readable', function(){
      let chunk;
      while((chunk = stream.read()) != null){
        readChunk(chunk);
      }
      readChunk("\n");
    })
    //stream.on('data', readChunk);  
    stream.on('finish', ()=>{ readChunk("\n"); after(true); });
    stream.on('error', (err)=>{
      console.error(`failed to load file '${folder + "/" + filename}`, err);
      after(false);
    })

    console.log(`Reading table ${folder + "/" + filename}`);
    //readChunk('hey');
    
    //readTable(stream);
    //async function readTable(tableStream){
   
    function readChunk(chunk){
      // console.log('chunk', chunk.toString());
      line += chunk; 
      let nextline = "";
      for(let i=n; i<line.length; ++i){
        if(line[i] == "\n"){
          nextline = line.substring(i+1);
          line = line.substring(0, i);
          let row = [];
          if(i > 0){
            row = parseRow(line);
            if(linenumber == 0){
              if(!headers || headers.length == 0){
                headers = row.slice(autoid? 1 : 0);
              } else {
                fileheaders = row.slice(autoid? 1 : 0);
                let match = headers.length == fileheaders.length;
                for(let i=0; match && i<headers.length; ++i){
                  if(headers[i].trim().toLowerCase() != fileheaders[i].trim().toLowerCase()){
                    match = false;
                  }
                }
                if(!match){
                  console.warn(`Table '${tablename}' expected headers do not match fileheaders.\n  ${headers}\n  ${fileheaders}`)
                }
              }
            } else {
              if(autoid){
                let id = parseInt(row[0]);
                row = row.slice(1);
                table.ids[id] = id;
                table.rows[id] = row;
                currentAutoId = Math.max(currentAutoId, id + 1);
              } else {
                if(!table.addRow(row)){
                  console.warn(`Duplicate row key: ${row[0]}, for row ${JSON.stringify(row)}.`);
                }
              }
            }
            ++linenumber;
          }
          line = nextline;
          i = 0;
          n = line.length;
        }
      }
    }
  }
  return table;

  function rowToString(row){
    return CSVRowToString(row, columnquoted);
  }
  function parseRow(s){
    let options = {
      autoparse: autoparse,
    }
    return CSVParseRow(s, options);
  }
}

function CSVRowToString(row, columnquoted, quoteall){
  let s = "";
  if(!columnquoted) columnquoted = [];
  for(let i=0; i<row.length; ++i){
    let entry = (typeof row[i] == "bigint")? row[i] + "n" : "" + row[i];
    let special = entry.indexOf('"') >= 0 || entry.indexOf("\n") >= 0 || entry.indexOf(",") >= 0 || entry.indexOf("'") >= 0;
    if(quoteall || special || columnquoted[i]){
      try{
        entry = JSON.stringify(entry);
      } catch(e){
        console.warn("CSVRowToString: row entry could not be stringified with 'JSON.stringify'.  Falling back on manual escaping.", str);
        entry = "\"" + entry.replace(/\\/g, '\\\\').replace(/\"/g, '\\"') + "\"";
      }
    }
    s += entry;
    if(i < row.length - 1) s += ",";
  }
  return s;
}

// note, this may not handle quoted strings properly according to some CSV specs,
// if they contain unescaped quotes inside a quoted entity.
// quotes may be escaped with a backslash, or two quotes consecutively.
function CSVParseRow(s, options){
  // console.log("parsing line", s);
  if(!options) options = {};
  let row = [];
  let j = 0; // the start of the entry
  let k = 0; // the non whitespace start of the entry
  let entryquoted = false;
  let quotechar = "\"";
  let escapechar = "\\"
  for(let i=0; i<s.length; ++i){
    let c = s[i];
    // trim leading whitespace
    while(i == k && (s[i] == " " || s[i] == "\t")){
      ++i; ++k;
    }
    if(i==k && c == quotechar){
      entryquoted = true;
      j = k; // move start of entry past the whitespace.
      for(++i; i<s.length; ++i){
        if(s[i] == quotechar){
          if(i < s.length - 1 && s[i+1] == quotechar){
            // change a double quote to an escaped quote
            s[i] = escapechar;
            ++i;
            continue;
          } else {
            break;
          }
        }
        if(s[i] == "\n" || i == s.length-1){
          console.warn("CSV quoted entry is unterminated at new line.");
          break;
        }
        if(s[i] == escapechar) ++i;
      }
    }
    if(c == "," || i == s.length-1){
      if(i==s.length-1){
        ++i;
      }
      let entry;
      if(entryquoted){
        // back up to trim whitespace
        while(s[i-1] == " " || s[i-1] == "\t") --i;
        let str = "";
        try{
          str = s.substring(j, i);
          entry = JSON.parse(str)
        } catch(e){
          console.warn("csvtable internal function 'CSVParseRow': row entry could not be parsed with 'JSON.parse'.  Falling back on manual unescaping.", str);
          str = s.substring(j+1, i-1);
          entry = str.replace(/(?:\\([\\\"\']))/g, '$1');
        }
      } else {
        entry = s.substring(j, i); 
      }
      if(options.autoparse){
        if(entry == "true") entry = true;
        else if(entry == "false") entry = false;
        else if(entry == "null") entry = null;
        else if(entry == "undefined") entry = undefined;
        else if(entry.match(FloatRegex)) entry = parseFloat(entry);
        else if(entry.match(BigIntRegex)) entry = BigInt(entry);
        else if(entry.match(HexRegex)) entry = parseInt(entry, 16);
      }
      row.push(entry);
      // we backed up to trim whitespace, go forward to comma again.
      while(entryquoted && i<s.length && (s[i] == " " || s[i] == "\t")) ++i;
      entryquoted = false;
      j = i+1;
    }
    if(c == "\n"){
      throw new Error("csvtable, internal function 'CSVParseRow' encountered unexpected newline character.");
    }
  }
  // console.log("parsed row", row);
  return row;
}


function Test(){
  // combine gui and text user interfaces
  // web command
  console.log('csv database');
}