[Distribution of number of candidates that go into making ZTF objects] #ztf #lasair

0_REAME.md

Distribution of number of candidates that go into making ZTF objects

In ZTF speak a candidate is a single detection and an object is a collection of associated candidates. I wanted to visualise (see plot above) the distribution of the number of candidates (ncand) that go into making a ZTF object.

To generate the CANDIDATE table distribution data in candidate_ncand_dist.csv I run the following query on Lasair (April 8, 2020):

SELECT 
    ncand, COUNT(*)
FROM
    (SELECT 
        objectId, COUNT(*) AS ncand
    FROM
        candidates
    GROUP BY objectId) AS a
GROUP BY ncand;

To generate the OBJECT table distribution data in object_ncand_dist.csv I run the following query on Lasair (April 8, 2020):

SELECT 
    ncand, COUNT(*)
FROM
    objects
GROUP BY ncand;

Then to generate this plot run python plot_ncand_histogram.py candidate_ncand_dist.csv object_ncand_dist.csv. There are many source where ncand > 20 (not visualised above) but I'm interested in the low-end to determine which objects we should limit sherlock to run on.

candidates_ncand_dist.csv

objects_ncand_dist.csv

plot_ncand_histogram.py

#!/usr/bin/env python
# encoding: utf-8
"""
*plot a histogram of number of candidates that go into making ZTF objects*

:Author:
    David Young

:Date Created:
    April  8, 2020

Usage:
    plot_ncand_histogram.py <candidate_file> <object_file>

Options:
    candidate_file        path to the csv export of ZTF ncand distribution - from candidate table
    object_file           path to the csv export of ZTF ncand distribution - from object table

    -h, --help            show this help message
    -v, --version         show version
    -s, --settings        the settings file
"""
################# GLOBAL IMPORTS ####################
import sys
import os
from fundamentals import tools
import unicodecsv as csv
import matplotlib.pyplot as plt
import numpy as np


def main(arguments=None):
    """
    *The main function used when ``plot_ncand_histogram.py`` is run as a single script from the cl*
    """

    # SETUP THE COMMAND-LINE UTIL SETTINGS
    su = tools(
        arguments=arguments,
        docString=__doc__,
        logLevel="WARNING",
        options_first=False,
        projectName=False
    )
    arguments, settings, log, dbConn = su.setup()

    # UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
    # AUTOMATICALLY
    a = {}
    for arg, val in list(arguments.items()):
        if arg[0] == "-":
            varname = arg.replace("-", "") + "Flag"
        else:
            varname = arg.replace("<", "").replace(">", "")
        a[varname] = val
        if arg == "--dbConn":
            dbConn = val
            a["dbConn"] = val
        log.debug('%s = %s' % (varname, val,))

    # OPEN THE OBJECTS TABLE DISTRIBUTION
    with open(a["object_file"], 'rb') as csvFile:
        csvReader = csv.DictReader(
            csvFile, dialect='excel', delimiter=',', quotechar='"')
        object_ncand = []
        object_count = []
        for row in csvReader:
            thisCount = int(row["count(*)"])
            nc = int(row["ncand"])

            if nc <= 20:
                object_ncand.append(int(row["ncand"]))
                object_count.append(int(row["count(*)"]))

    with open(a["candidate_file"], 'rb') as csvFile:
        csvReader = csv.DictReader(
            csvFile, dialect='excel', delimiter=',', quotechar='"')
        cand_ncand = []
        cand_count = []
        total = 0
        moreone = 0
        moretwo = 0
        morethree = 0
        for row in csvReader:
            thisCount = int(row["count(*)"])
            nc = int(row["ncand"])
            if nc:
                total += thisCount
            if nc > 1:
                moreone += thisCount
            if nc > 2:
                moretwo += thisCount
            if nc > 3:
                morethree += thisCount
            if nc <= 20:
                cand_ncand.append(int(row["ncand"]))
                cand_count.append(int(row["count(*)"]))

    print("Total number of objects = %(total)s" % locals())
    print("Objects with ncand > 1 : %(moreone)s" % locals())
    print("Objects with ncand > 2 : %(moretwo)s" % locals())
    print("Objects with ncand > 3 : %(morethree)s" % locals())

    # CANDIDATE DATASET
    height = cand_count
    bars = cand_ncand
    tick_pos = np.arange(len(bars))
    bar_pos = np.arange(len(bars)) - 0.2
    plt.bar(bar_pos, height, width=0.4, label='Candidates')

    # OBJECT DATASET
    height = object_count
    bars = object_ncand
    tick_pos = np.arange(len(bars)) - 1
    bar_pos = np.arange(len(bars)) - 0.8
    plt.bar(bar_pos, height, width=0.4, label='Objects')

    # CREATE LABELS ON THE X-AXIS
    plt.xticks(tick_pos, bars)
    plt.yscale('log')
    plt.ylabel("Number of ZTF Objects (log)")
    plt.xlabel("ncand")

    legend = plt.legend(loc="upper right", framealpha=1.0)
    # Set the legend face color
    frame = legend.get_frame()
    frame.set_facecolor('#D2D1D1')

    # Show graphic
    # plt.show()

    plt.savefig('ztf_ncand_dist.png',  dpi=300)

    return

if __name__ == '__main__':
    main()