Last active
August 29, 2015 13:58
-
-
Save agonen/9931313 to your computer and use it in GitHub Desktop.
generate unique number
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ##load data | |
| #!/bin/bash | |
| for i in {2..100}; do | |
| echo gerante data $i | |
| time -p awk -v val=$i 'BEGIN{n=val*1000000}$1=n++' FS=, OFS=, /ebs/tmpdata/tranactions.csv >/run/csv/tranactions.csv | |
| echo load data $i | |
| time -p /home/ubuntu/MonetDB-11.18.0/bin/mclient -d p32 -s "COPY 1000000 OFFSET 2 RECORDS INTO transactions FROM ('/run/csv/tranactions.csv') USING DELIMITERS ',','\n','\"';" | |
| done | |
| # sequence number in first field | |
| for i in {2..10}; do awk -v val=$i 'BEGIN{n=val*100000}$1=n++' FS=, OFS=, a1_$i.csv >>tranactions.csv; done | |
| # replace client code to number | |
| for i in {1..10}; do awk -v val=$i 'BEGIN{n=val*100000}$1=n++,$87=1' FS=, OFS=, A_$i.csv >B_$i.csv; done | |
| #convert dates from dd/mm/yyyy to yyyy-mm-dd | |
| awk '{split($5,a,"/");$5=a[3]"-"a[1]"-"a[2];split($6,a,"/");$6=a[3]"-"a[1]"-"a[2];print}' FS=, OFS=, /ebs/bbl/csv/CLIENTS_LIST.csv > /ebs/bbl/csv/a.csv | |
| #load to monetdb | |
| m -s "COPY 100 OFFSET 2 RECORDS INTO clients_list FROM ('/ebs/bbl/csv/a.csv') USING DELIMITERS ',','\n','""';" | |
| #load multipal files to pytiha | |
| cat B*.csv | psql pythia -c 'SET datestyle = "MDY"; COPY CALCULATED_TAXS from stdin CSV HEADER' | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment