Last active
May 19, 2022 00:11
-
-
Save dmsimard/bcab919e6b5818eda68ca54d2843826f to your computer and use it in GitHub Desktop.
git repos for collections in ansible package
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
# Query an ansible.in file to retrieve the list of collections included in the Ansible | |
# package so we can query galaxy and retrieve their git repositories. | |
# ansible.in files can be cloned from ansible-build-data: | |
# - https://github.com/ansible-community/ansible-build-data/blob/main/2.10/ansible.in | |
# - https://github.com/ansible-community/ansible-build-data/blob/main/3/ansible.in | |
import argparse | |
import json | |
import logging | |
import requests | |
# from requests.adapters import HTTPAdapter | |
import time | |
GALAXY = "https://galaxy.ansible.com" | |
NAMESPACES = f"{GALAXY}/api/v1/namespaces" | |
REPOSITORIES = f"{GALAXY}/api/v1/repositories" | |
COLLECTIONS_API = f"{GALAXY}/api/v2/collections" | |
# Retry Galaxy API calls if they fail | |
RETRY_COUNT = 15 | |
def get_args(): | |
parser = argparse.ArgumentParser() | |
parser.add_argument("--infile", help="Path to an ansible.in file", required=True) | |
parser.add_argument( | |
"--ignore", | |
help=( | |
"Omit repositories belonging to specified git organizations (not collection namespaces), can be repeated.\n" | |
"For example: --ignore ansible --ignore ansible-community --ignore ansible-collections\n" | |
), | |
action="append", | |
default=[], | |
required=False, | |
) | |
args = parser.parse_args() | |
return args | |
def query_galaxy(session, url): | |
""" | |
Queries galaxy and re-tries a few times on failures which can improve the odds | |
of succeeding past rate-limiting or unreliability. | |
""" | |
logging.info(f"Querying galaxy: {url}") | |
retries = 0 | |
query = None | |
while query is None: | |
try: | |
query = session.get(url).json() | |
except Exception as e: | |
retries += 1 | |
if retries >= RETRY_COUNT: | |
logging.error(f"giving up retrying on {url}: {e}") | |
break | |
logging.warning(f"retrying call to galaxy: {url}: {e}") | |
time.sleep(2) | |
query = None | |
return query | |
def main(): | |
args = get_args() | |
logging.basicConfig(level="INFO", format="%(asctime)s %(levelname)s %(name)s: %(message)s") | |
session = requests.Session() | |
# session.mount(GALAXY, HTTPAdapter(max_retries=5)) | |
with open(args.infile) as f: | |
collections = f.readlines() | |
collections = [x.strip() for x in collections] | |
repositories = [] | |
for collection in collections: | |
if not collection.startswith("#"): | |
namespace, repository = collection.split(".") | |
# Galaxy knows about the git repository for a collection and we can find it in the latest_version of a collection | |
# ex: https://galaxy.ansible.com/api/v2/collections/community/general/ | |
collection = query_galaxy(session, f"{COLLECTIONS_API}/{namespace}/{repository}/") | |
if collection is None: | |
logging.error("couldn't get the latest version for {collection}") | |
continue | |
# Now query the latest version so we can get the git repo | |
# ex: https://galaxy.ansible.com/api/v2/collections/community/general/versions/2.2.0/ | |
version = query_galaxy(session, collection["latest_version"]["href"]) | |
if version is None: | |
logging.error("couldn't get the git repository for {collection}") | |
continue | |
repository = version["metadata"]["repository"] | |
# We might want to exclude organizations or namespaces for repositories we otherwise already know about | |
ignored = False | |
for ignored_ns in args.ignore: | |
if f"/{ignored_ns}/" in repository: | |
ignored = True | |
if ignored: | |
logging.info(f"ignoring {repository}, in ignored namespace") | |
continue | |
repositories.append(repository) | |
print(json.dumps(repositories, indent=2)) | |
if __name__ == "__main__": | |
main() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[ | |
"https://github.com/ansible-collections/amazon.aws", | |
"https://github.com/ansible-collections/ansible.netcommon", | |
"https://github.com/ansible-collections/ansible.posix", | |
"https://github.com/ansible-collections/ansible.windows", | |
"https://github.com/ansible-collections/arista.eos", | |
"https://github.com/ansible/awx", | |
"https://github.com/ansible-collections/azure", | |
"https://github.com/CheckPointSW/CheckPointAnsibleMgmtCollection", | |
"https://github.com/chocolatey/chocolatey-ansible", | |
"https://github.com/CiscoDevNet/ansible-aci", | |
"https://github.com/ansible-collections/cisco.asa", | |
"https://github.com/CiscoDevNet/intersight-ansible", | |
"https://github.com/ansible-collections/cisco.ios", | |
"https://github.com/ansible-collections/cisco.iosxr", | |
"https://github.com/CiscoDevNet/ansible-meraki", | |
"https://github.com/CiscoDevNet/ansible-mso", | |
"https://github.com/CiscoDevNet/ansible-nso", | |
"https://github.com/ansible-collections/nxos", | |
"https://github.com/CiscoDevNet/ansible-ucs", | |
"https://github.com/cloudscale-ch/ansible-collection-cloudscale", | |
"https://github.com/ansible-collections/community.aws", | |
"https://github.com:ansible-collections/community.azure", | |
"https://github.com/ansible-collections/community.crypto", | |
"https://github.com/ansible-collections/community.digitalocean", | |
"https://github.com/ansible-collections/community.docker", | |
"https://github.com/ansible-collections/community.fortios", | |
"https://github.com/ansible-collections/community.general", | |
"https://github.com/ansible-collections/community.google", | |
"https://github.com/ansible-collections/grafana.git", | |
"https://github.com/ansible-collections/community.hashi_vault", | |
"https://github.com/ansible-collections/community.hrobot", | |
"https://github.com/ansible-collections/community.kubernetes", | |
"https://github.com/ansible-collections/community.kubevirt", | |
"https://github.com/ansible-collections/community.libvirt", | |
"https://github.com/ansible-collections/community.mongodb", | |
"https://github.com/ansible-collections/community.mysql", | |
"https://github.com/ansible-collections/community.network", | |
"https://github.com/ansible-collections/community.okd", | |
"https://github.com/ansible-collections/community.postgresql", | |
"https://github.com/ansible-collections/community.proxysql", | |
"https://github.com/ansible-collections/community.rabbitmq", | |
"https://github.com/ansible-collections/community.routeros", | |
"https://github.com/ansible-collections/skydive", | |
"https://github.com/ansible-collections/community.vmware.git", | |
"https://github.com/ansible-collections/community.windows", | |
"https://github.com/ansible-collections/community.zabbix.git", | |
"https://github.com/containers/ansible-podman-collections.git", | |
"https://github.com/cyberark/ansible-conjur-collection", | |
"https://github.com/cyberark/ansible-security-automation-collection", | |
"https://github.com/ansible-collections/dellemc.os10", | |
"https://github.com/ansible-collections/dellemc.os6", | |
"https://github.com/ansible-collections/dellemc.os9", | |
"https://github.com/F5Networks/f5-ansible", | |
"https://github.com/fortinet-ansible-dev/ansible-galaxy-fortimanager-collection/tree/galaxy/2.0.1", | |
"https://github.com/fortinet-ansible-dev/ansible-galaxy-fortios-collection/tree/fos_v6.0.0/galaxy_1.1.9", | |
"https://github.com/ansible-collections/frr.frr", | |
"https://github.com/gluster/gluster-ansible-collection", | |
"http://github.com/ansible/ansible_collections_google", | |
"https://github.com/ansible-collections/hetzner.hcloud", | |
"https://github.com/ansible-collections/ibm.qradar", | |
"https://www.github.com/infinidat/ansible-infinidat-collection", | |
"https://github.com/ansible-collections/junipernetworks.junos", | |
"https://github.com/ansible-collections/mellanox.onyx", | |
"https://github.com/ansible-collections/netapp", | |
"https://github.com/ansible-collections/netapp", | |
"https://www.github.com/netapp-eseries/santricity", | |
"https://github.com/ansible-collections/netapp", | |
"https://github.com/netbox-community/ansible_modules", | |
"https://github.com/ngine-io/ansible-collection-cloudstack", | |
"https://github.com/ngine-io/ansible-collection-exoscale", | |
"https://github.com/ngine-io/ansible-collection-vultr", | |
"https://opendev.org/openstack/ansible-collections-openstack", | |
"https://github.com/ansible-collections/openvswitch.openvswitch", | |
"https://github.com/ovirt/ovirt-ansible-collection", | |
"https://github.com/Pure-Storage-Ansible/FlashArray-Collection", | |
"https://github.com/Pure-Storage-Ansible/FlashBlade-Collection", | |
"https://github.com/ServiceNowITOM/servicenow-ansible", | |
"https://github.com/ansible-collections/splunk.es", | |
"https://github.com/theforeman/foreman-ansible-modules", | |
"https://github.com/ansible-collections/vyos.vyos", | |
"https://github.com/wtinetworkgear/wti-collection" | |
] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[ | |
"https://github.com/ansible-collections/amazon.aws", | |
"https://github.com/ansible-collections/ansible.netcommon", | |
"https://github.com/ansible-collections/ansible.posix", | |
"https://github.com/ansible-collections/ansible.windows", | |
"https://github.com/ansible-collections/arista.eos", | |
"https://github.com/ansible/awx", | |
"https://github.com/ansible-collections/azure", | |
"https://github.com/CheckPointSW/CheckPointAnsibleMgmtCollection", | |
"https://github.com/chocolatey/chocolatey-ansible", | |
"https://github.com/CiscoDevNet/ansible-aci", | |
"https://github.com/ansible-collections/cisco.asa", | |
"https://github.com/CiscoDevNet/intersight-ansible", | |
"https://github.com/ansible-collections/cisco.ios", | |
"https://github.com/ansible-collections/cisco.iosxr", | |
"https://github.com/CiscoDevNet/ansible-meraki", | |
"https://github.com/CiscoDevNet/ansible-mso", | |
"https://github.com/CiscoDevNet/ansible-nso", | |
"https://github.com/ansible-collections/nxos", | |
"https://github.com/CiscoDevNet/ansible-ucs", | |
"https://github.com/cloudscale-ch/ansible-collection-cloudscale", | |
"https://github.com/ansible-collections/community.aws", | |
"https://github.com:ansible-collections/community.azure", | |
"https://github.com/ansible-collections/community.crypto", | |
"https://github.com/ansible-collections/community.digitalocean", | |
"https://github.com/ansible-collections/community.docker", | |
"https://github.com/ansible-collections/community.fortios", | |
"https://github.com/ansible-collections/community.general", | |
"https://github.com/ansible-collections/community.google", | |
"https://github.com/ansible-collections/grafana.git", | |
"https://github.com/ansible-collections/community.hashi_vault", | |
"https://github.com/ansible-collections/community.hrobot", | |
"https://github.com/ansible-collections/community.kubernetes", | |
"https://github.com/ansible-collections/community.kubevirt", | |
"https://github.com/ansible-collections/community.libvirt", | |
"https://github.com/ansible-collections/community.mongodb", | |
"https://github.com/ansible-collections/community.mysql", | |
"https://github.com/ansible-collections/community.network", | |
"https://github.com/ansible-collections/community.okd", | |
"https://github.com/ansible-collections/community.postgresql", | |
"https://github.com/ansible-collections/community.proxysql", | |
"https://github.com/ansible-collections/community.rabbitmq", | |
"https://github.com/ansible-collections/community.routeros", | |
"https://github.com/ansible-collections/skydive", | |
"https://github.com/ansible-collections/community.sops", | |
"https://github.com/ansible-collections/community.vmware.git", | |
"https://github.com/ansible-collections/community.windows", | |
"https://github.com/ansible-collections/community.zabbix.git", | |
"https://github.com/containers/ansible-podman-collections.git", | |
"https://github.com/cyberark/ansible-conjur-collection", | |
"https://github.com/cyberark/ansible-security-automation-collection", | |
"https://github.com/dell/dellemc-openmanage-ansible-modules/tree/collections", | |
"https://github.com/ansible-collections/dellemc.os10", | |
"https://github.com/ansible-collections/dellemc.os6", | |
"https://github.com/ansible-collections/dellemc.os9", | |
"https://github.com/F5Networks/f5-ansible", | |
"https://github.com/fortinet-ansible-dev/ansible-galaxy-fortimanager-collection/tree/galaxy/2.0.1", | |
"https://github.com/fortinet-ansible-dev/ansible-galaxy-fortios-collection/tree/fos_v6.0.0/galaxy_1.1.9", | |
"https://github.com/ansible-collections/frr.frr", | |
"https://github.com/gluster/gluster-ansible-collection", | |
"http://github.com/ansible/ansible_collections_google", | |
"https://github.com/ansible-collections/hetzner.hcloud", | |
"https://github.com/ansible-collections/ibm.qradar", | |
"https://www.github.com/infinidat/ansible-infinidat-collection", | |
"https://github.com/ansible-collections/junipernetworks.junos", | |
"https://github.com/ansible-collections/mellanox.onyx", | |
"https://github.com/ansible-collections/netapp", | |
"https://github.com/ansible-collections/netapp", | |
"https://www.github.com/netapp-eseries/santricity", | |
"https://github.com/ansible-collections/netapp", | |
"https://github.com/netbox-community/ansible_modules", | |
"https://github.com/ngine-io/ansible-collection-cloudstack", | |
"https://github.com/ngine-io/ansible-collection-exoscale", | |
"https://github.com/ngine-io/ansible-collection-vultr", | |
"https://opendev.org/openstack/ansible-collections-openstack", | |
"https://github.com/ansible-collections/openvswitch.openvswitch", | |
"https://github.com/ovirt/ovirt-ansible-collection", | |
"https://github.com/Pure-Storage-Ansible/FlashArray-Collection", | |
"https://github.com/Pure-Storage-Ansible/FlashBlade-Collection", | |
"https://github.com/sensu/sensu-go-ansible", | |
"https://github.com/ServiceNowITOM/servicenow-ansible", | |
"https://github.com/ansible-collections/splunk.es", | |
"https://github.com/theforeman/foreman-ansible-modules", | |
"https://github.com/T-Systems-MMS/ansible-collection-icinga-director", | |
"https://github.com/ansible-collections/vyos.vyos", | |
"https://github.com/wtinetworkgear/wti-collection" | |
] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Update from ansible 6.0.0a3: