get-es-deps.sh

#!/bin/bash

set -euxo pipefail

if ! type -p hub > /dev/null 2>&1 ; then
    echo ERROR: you must use the \"hub\" command line tool
    echo for interacting with github
    echo see https://github.com/github/hub
    echo e.g. on Fedora - dnf -y install hub
    exit 1
fi

gather_dep_gav() {
    local dir subbuild subdir file
    rm -f runtime-deps compile-deps
    for dir in * ; do
	if [ ! -d $dir ] ; then continue ; fi
	case $dir in
	    x-pack|gradle|licenses|qa|test|build|dev-tools|plugins) continue;;
	esac
	local subbuild
	for subbuild in $( find $dir -name build.gradle ) ; do
	    subdir=$( dirname $subbuild )
	    pushd $subdir > /dev/null
	    echo Directory: $subdir >> $BASEDIR/runtime-deps
	    echo Directory: $subdir >> $BASEDIR/compile-deps
	    $BASEDIR/gradlew -q dependencies --configuration runtime >> $BASEDIR/runtime-deps 2>&1 || :
	    $BASEDIR/gradlew -q dependencies --configuration compile >> $BASEDIR/compile-deps 2>&1 || :
	    popd > /dev/null
	done
    done

    for file in runtime-deps compile-deps ; do
	cat $file | \
	    sed -e '/[+\\]---/{
s/^.*[+\\]--- \([^ ][^ ]*\)[ ]*.*$/\1/
s/^org.elasticsearch:jna:/net.java.dev.jna:jna:/
/^org.elasticsearch:/d
p}' -n | sort -u > $file.gav
    done
}

get_gh_tags() {
    local orgrepo=$1
    local tagsfile=$2
    if ! hub api -i repos/$orgrepo/tags | dos2unix > $workdir/tagtmp 2>&1 ; then
	echo ERROR: could not list tags for $orgrepo
	cat $workdir/tagtmp || :
	return 1
    fi
    local code=$( awk '/^Status:/ {print $2; exit 0}' $workdir/tagtmp )
    if [ "${code:-0}" != 200 ] ; then
	echo ERROR: could not list tags for $orgrepo code $code
	cat $workdir/tagtmp || :
	return 1
    fi
    local ii=2
    for ii in $(seq 2 99); do
	sed -n '/^\[/,$ p' $workdir/tagtmp | jq -r '.[].name' >> $tagsfile
	if ! hub api -i repos/$orgrepo/tags?page=$ii | dos2unix > $workdir/tagtmp 2>&1 ; then
	    echo ERROR: could not list tags for $orgrepo
	    cat $workdir/tagtmp || :
	    return 1
	fi
	code=$( awk '/^Status:/ {print $2; exit 0}' $workdir/tagtmp )
	if [ "${code:-0}" != 200 ] ; then
	    echo ERROR: could not list tags for $orgrepo code $code
	    cat $workdir/tagtmp || :
	    return 1
	fi
	if fgrep -x -q '[]' $workdir/tagtmp ; then
	    return 0
	fi
    done
    echo ERROR: could not list tags for $orgrepo code $ii
    return 1
}

BASEDIR=$( pwd )
JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11}
export JAVA_HOME

#gather_dep_gav

# https://search.maven.org/classic/#api
workdir=$( mktemp -d )
#trap "rm -rf ${workdir:-/nosuchdir}" EXIT

while IFS=':' read g a v ; do
    if [ -z "$g" -o -z "$a" -o -z "$v" ] ; then
	echo empty gav $g $a $v
	continue
    fi
    path=$( echo "$g" | sed -e 's,[.],/,g' )
    pom=${a}-${v}.pom
    pomurl=http://repo1.maven.org/maven2/$path/$a/$v/$pom
    if ! code=$( curl -w '%{response_code}' -s -o $workdir/$pom $pomurl ) || [ "$code" != 200 ]  ; then
	echo ERROR: could not retrieve $pomurl: code ${code:-err}
	cat $workdir/$pom || :
	cat $workdir/$pom.err || :
	echo ${g}:${a}:${v} >> $BASEDIR/no-pom
	continue
    fi
    if [ ! -s $workdir/$pom ] ; then
	echo ERROR: $pomurl gave empty pom file
	cat $workdir/$pom.err || :
	echo ${g}:${a}:${v} >> $BASEDIR/no-pom
	continue
    fi
    sed -i -e 's/xmlns=".*"//g' -e 's,xsi:schemaLocation="[^"][^"]*",,' $workdir/$pom
    scm_conn=$( xmllint --xpath "string(/project/scm/connection)" $workdir/$pom 2> $workdir/conn.err )
    scm_devconn=$( xmllint --xpath "string(/project/scm/developerConnection)" $workdir/$pom 2> $workdir/devconn.err )
    scm_url=$( xmllint --xpath "string(/project/scm/url)" $workdir/$pom 2> $workdir/url.err )
    scm_tag=$( xmllint --xpath "string(/project/scm/tag)" $workdir/$pom 2> $workdir/tag.err )
    proj_url=$( xmllint --xpath "string(/project/url)" $workdir/$pom 2> $workdir/projurl.err )
    if [ -z "$scm_conn" -a -z "$scm_devconn" -a -z "$scm_url" -a -z "$proj_url" ] ; then
	echo ERROR: could not extract useful SCM information from $pomurl
	cat $workdir/conn.err || :
	cat $workdir/devconn.err || :
	cat $workdir/url.err || :
	cat $workdir/projurl.err || :
	echo ${g}:${a}:${v} $pomurl >> $BASEDIR/no-scm-information
	continue
    fi
    # prefer conn - then devconn then url
    # if none use git, find one that uses http or https
    found=0
    for scm in "$scm_conn" "$scm_devconn" "$scm_url" "$proj_url" ; do
	if [ -z "$scm" ] ; then continue ; fi
	ghorgrepo=
	case "$scm" in
	    https://github.com/*) ghorgrepo=${scm#https://github.com/} ;;
	    http://github.com/*) ghorgrepo=${scm#http://github.com/} ;;
	esac
	if [ -z "${ghorgrepo}" ] ; then
	    IFS=":" read key type suburl <<< "$scm"
	    if [ "$key" != "scm" ] ; then
		echo ERROR: I do not know what $scm is
		echo ${g}:${a}:${v} $pomurl >> $BASEDIR/pom-error
		continue
	    fi
	    # ignore type for now - try to derive from url
	    case "$suburl" in
		[email protected]:*) ghorgrepo=${suburl#[email protected]:} ;;
		git://github.com/*) ghorgrepo=${suburl#git://github.com/} ;;
		git://github.com:maxmind/*) ghorgrepo=maxmind/${suburl#git://github.com:maxmind/} ;;
		http://github.com/*) ghorgrepo=${suburl#http://github.com/} ;;
		https://github.com/*) ghorgrepo=${suburl#https://github.com/} ;;
		http://svn.apache.org/*) ghorgrepo=apache/$a ;;
		*) echo I do not know what to do with scm $scm; echo ${g}:${a}:${v} $pomurl >> $BASEDIR/unknown-scm ; continue ;;
	    esac
	fi
	tagprefix=$a
	try=${tagprefix#common-}
	# apache commons $a looks like common-something, and the git tags
	# look like something-$v
	if [ -n "$try" -a "$try" != "$tagprefix" ] ; then
	    tagprefix=$try
	fi
	if [ -n "$ghorgrepo" -a "$ghorgrepo" != "$suburl" ] ; then
	    ghorgrepo=${ghorgrepo%.git}
	    # see if repo exists and get list of tags if so
	    if ! get_gh_tags $ghorgrepo $workdir/${a}-${v}.tags 2>&1 ; then
		echo ERROR: could not list tags for $suburl
		cat $workdir/${a}-${v}.tags.json || :
		continue
	    fi
	    if fgrep -x -q "${tagprefix}-${v}" $workdir/${a}-${v}.tags ; then
		echo INFO: Found tag ${tagprefix}-${v} for $scm
		tag="${tagprefix}-${v}"
	    elif fgrep -x -q "${v}" $workdir/${a}-${v}.tags ; then
		echo INFO: Found tag $v for $scm
		tag=$v
	    elif fgrep -x -q "v${v}" $workdir/${a}-${v}.tags ; then
		echo INFO: Found tag v$v for $scm
		tag=v$v
	    elif [ -n "$scm_tag" ] ; then
		tag="$scm_tag"
	    else
		echo ERROR: could not find tag for $scm for ${a}-${v}
		cat $workdir/${a}-${v}.tags || :
		continue
	    fi
	    # "tarball_url": "https://api.github.com/repos/lemire/javaewah/tarball/JavaEWAH-1.1.5"
	    if hub api repos/$ghorgrepo/tarball/$tag > $workdir/${a}-${v}.tar.gz ; then
		found=1
		break
	    else
		echo ERROR: could not find tarball for $ghorgrepo/tarball/$tag
		continue
	    fi
	    found=1
	    break
	else
	    echo INFO: not sure what to do with $suburl
	fi
    done
    if [ $found = 0 ] ; then
	echo ${g}:${a}:${v} $pomurl >> $BASEDIR/no-download-link
    fi
done < $BASEDIR/compile-deps.gav

ls -alrtF $workdir || :

new and improved