fetch.pl

#!/usr/bin/env perl

use 5.010;
use utf8;
use strict;
use warnings;
use File::Slurp;
use HTTP::Tiny;
use Path::Tiny;
use YAML::Tiny;

my $CONFIG = 'fetch.yml';

my $config = YAML::Tiny->read($CONFIG);

my $base   = $config->[0]{base};
my @majors = @{ $config->[0]{majors} };
my @archs  = @{ $config->[0]{archs} };
my $target = $config->[0]{target};

for my $major (@majors) {
    for my $arch (@archs) {
        my $url = "$base/$major/updates/$arch/repodata";

        my $res = HTTP::Tiny->new->get($url);
        warn("cannot get $major-$arch index\n"), next unless $res->{success};

        my $dir = path($target)->child("$major-$arch");
        $dir->mkpath unless -e $dir;

        my @files = $res->{content} =~ m/"(.*?\.sqlite.bz2)"/g;
        for my $file (@files) {
            my $res = HTTP::Tiny->new->get("$url/$file");

            $file =~ s/^[^-]+-//;
            $dir->child($file)->spew_raw( $res->{content} );
        }
    }
}

__DATA__

http://centos.mirror.cdnetworks.com/5/updates/i386/repodata/
http://centos.mirror.cdnetworks.com/5/updates/x86_64/repodata/


filelists.sqlite.bz2
other.sqlite.bz2
primary.sqlite.bz2

fetch.yml

---
base: http://centos.mirror.cdnetworks.com
majors:
  - 5
  - 6
archs:
  - i386
  - x86_64
target: repodata