coingecko-etl

coingecko-csv.pl

#!/usr/bin/perl
use strict;
use JSON;
my $JJ = new JSON;
$JJ->canonical(1);
my $id = shift @ARGV;
open(X, qq(curl --silent -X GET "https://api.coingecko.com/api/v3/coins/$id/ohlc?vs_currency=usd&days=max" -H "accept: application/json" |));
my $ohlc = $JJ->decode( join '', <X> );
close(X);
foreach my $t ( @$ohlc ) {
  $t->[0] /= 1000;
  print join(",", ($id, @$t)), "\n";
}
sleep 1.0;

coingecko-metadata.pl

#!/usr/bin/perl
use strict;
use utf8;
use Encode;
use Data::Dumper qw(Dumper);
use JSON;
binmode(STDOUT, ':encoding(utf8)');
my $JJ = new JSON;
$JJ->canonical(1);
my $id = shift @ARGV;
open(X, qq(curl --silent -X GET "https://api.coingecko.com/api/v3/coins/$id" -H "accept: application/json" |));
my $meta = $JJ->decode( join '', map {decode('UTF-8',$_)} <X> );
close(X);
my $symbol = $meta->{symbol};
my $twitter = $meta->{links}->{twitter_screen_name};
my $logo = $meta->{image}->{large};
$symbol = undef if $symbol eq "";
$twitter = undef if $twitter eq "";
$logo = undef if $logo eq "";
my $contract_address = $meta->{contract_address};
print $JJ->encode({id => lc($id), symbol => lc($symbol), twitter => lc($twitter), contract_address => lc($contract_address), logo => $logo});
print "\n";
#print STDERR Dumper($meta);
sleep 1.0;

coingecko.pl

#!/usr/bin/perl
use strict;
use Data::Dumper;
use JSON;
my $JJ = new JSON;
$JJ->canonical(1);
open(X, 'curl --silent -X GET "https://api.coingecko.com/api/v3/coins/list" -H "accept: application/json" |');
my $coins = $JJ->decode( join '', <X> );
close(X);
foreach my $coin ( @$coins ) {
  my $id = $coin->{id};
  print qq(make $id.ohlc.csv;\n);
  print qq(make $id.meta.jsonl;\n);
}

Makefile

%.ohlc.csv :
        perl ./coingecko-csv.pl ${@:.ohlc.csv=} > $@
%.meta.jsonl :
        perl ./coingecko-metadata.pl ${@:.meta.jsonl=} > $@
all.meta.jsonl :
        cat `ls *.meta.jsonl | grep -v $@` | perl -ne 'print lc' | sort > $@
all.ohlc.csv :
        cat `ls *.ohlc.csv | grep -v $@` > $@
all :
        perl ./coingecko.pl | bash
        $(MAKE) all.meta.jsonl
        $(MAKE) all.ohlc.csv
        gsutil cp all.meta.jsonl gs://crypto-public-data/
        gsutil cp all.ohlc.csv   gs://crypto-public-data/
clean :
        rm *.meta.jsonl
        rm *.ohlc.csv