clean opat data

clean_opat_data.py

import reversion
import json
import datetime
from django.db import models
from opal import models as opal_models
from elcid import models as elcid_models


COMMIT = True
opat_team = opal_models.Team.objects.get(name="opat")

def filter_rejected(episodes):
    acceptance = episodes.annotate(models.Count("opatrejection"))
    return acceptance.filter(opatrejection__count=0)

def get_completed_opat_episodes():
    opat_episodes = opal_models.Episode.objects.filter(tagging__team=opat_team)

    for subtag in ["opat_referrals", "opat_current", "opat_review", "opat_followup"]:
        existing_tags = opal_models.Tagging.objects.filter(archived=False).filter(team__name=subtag)
        existing_episodes = existing_tags.values_list("episode__id", flat=True)
        opat_episodes = opat_episodes.exclude(id__in=existing_episodes)

    return filter_rejected(opat_episodes)


def clean_opat_data():
    complete_episodes = get_completed_opat_episodes()
    no_outcome_stage = complete_episodes.exclude(opatoutcome__outcome_stage="Completed Therapy")

    no_outcome_stage_ids = no_outcome_stage.values_list("id", flat=True)
    result = get_tag_removal_for_episodes(set(no_outcome_stage_ids), "opat_current")
    result[5341] = datetime.datetime(2015, 10, 2)
    updated = []
    rejected = []

    for row in no_outcome_stage:
        review_stage = row.opatoutcome_set.filter(outcome_stage="OPAT Review").first()

        if not review_stage:
            rejected.append(row.id)
            continue

        # this clones the episode
        review_stage.id = None
        review_stage.outcome_stage = "Completed Therapy"

        if review_stage.episode_id not in result:
            print "unable to find a completed date for {}".format(review_stage.episode_id)
        else:
            review_stage.created = result[review_stage.episode_id]
            review_stage.updated = result[review_stage.episode_id]

        if COMMIT:
            review_stage.save()
        updated.append(review_stage.episode.id)

    print "we have updated {} with no outcome stage".format(updated)
    print "we were unable to update {} as they have no review_stage".format(rejected)


def get_tag_removal_for_episodes(episode_ids, tag_name):
    deleted = reversion.get_deleted(opal_models.Tagging)
    result = {}
    tag = opal_models.Team.objects.get(name=tag_name)

    for i in deleted:
        loaded_json = i.field_dict
        if loaded_json["episode"] in episode_ids:
            if loaded_json["team"] == tag.id:
                result[loaded_json["episode"]] = i.revision.date_created

    missing = set(episode_ids) - set(result.keys())
    recent_tags = opal_models.Tagging.objects.filter(
        episode_id__in=missing,
        team__name=tag_name,
        archived=True
    ).values("episode_id", "updated", "created")

    for recent_tag in recent_tags:
        result[recent_tag["episode_id"]] = recent_tag["updated"] or recent_tag["created"]

    return result


def get_versions_for_episode(episode_id):
    deleted = reversion.get_deleted(opal_models.Tagging)
    for_episode = [i for i in deleted if i.field_dict["episode"] == episode_id]
    return for_episode


def get_discharge_date_for_episodes(episode_ids):
    episode_ids = set(episode_ids)
    result = {}

    current_results = get_tag_removal_for_episodes(episode_ids, "opat_current")
    result.update(current_results)

    missing = set(episode_ids) - set(current_results.keys())

    if len(missing):
        print "we are missing results for some removal dates for {}".format(missing)

    return result


def update_empty_discharge_date():
    complete_episodes = get_completed_opat_episodes()
    complete_episodes = complete_episodes.filter(discharge_date=None)
    removal_dates = get_discharge_date_for_episodes(complete_episodes.values_list("id", flat=True))
    updated = []
    rejected = []

    for i in complete_episodes:
        removal_date = removal_dates.get(i.id)

        if not removal_date:
            rejected.append(i.id)
            continue
        else:
            i.discharge_date = removal_date.date()

            if COMMIT:
                i.save()
            updated.append(i.id)

    print "we have updated {} for empty discharge date".format(updated)
    print "we have been unable to update {}".format(rejected)


def update_acceptance_date():
    acceptance = opal_models.Episode.objects.filter(tagging__team=opat_team)
    acceptance = acceptance.filter(location__opat_acceptance=None)
    acceptance = filter_rejected(acceptance)
    acceptance_episode_ids = set(acceptance.values_list("id", flat=True))
    result_dict = get_tag_removal_for_episodes(acceptance_episode_ids, "opat_referrals")
    locations = elcid_models.Location.objects.filter(episode__id__in=result_dict.keys())

    updated = []
    rejected = []
    for location in locations:
        location.opat_acceptance = result_dict[location.episode_id]
        if location.opat_acceptance:
            updated.append(location.episode_id)
        else:
            rejected.append(location.episode_id)

        if COMMIT:
            location.save()

    print "we have updated {} locations".format(updated)
    print "we have been unable to update {} locations".format(rejected)

    for i in [5241, 9157, 9164, 7773, 7774]:
        pass



def get_episodes_with_multiple_outcomes_per_stage():
    stages = [
        "Completed Therapy Post Follow Up",
        "OPAT Review",
        "Completed Therapy"
    ]
    for stage in stages:
        episodes = opal_models.Episode.objects.filter(opatoutcome__outcome_stage=stage)

        for episode in episodes:
            outcome_count = episode.opatoutcome_set.filter(outcome_stage=stage).count()

            if outcome_count > 1:
                if outcome_count == 2:
                    outcomes = episode.opatoutcome_set.filter(outcome_stage=stage)

                    if not outcomes[0].updated and not outcomes[1].updated:
                        if outcomes[1].created and outcomes[0].created:
                            delta = outcomes[1].created = outcomes[0].created

                            if delta.seconds < 60:
                                b = outcomes[1]
                                b.delete()
                print "duplicate outcomes for {0} with {1}".format(episode.id, episode.get_tag_names(None))
                for i in episode.opatoutcome_set.filter(outcome_stage=stage):
                    print "stage {0} on {1}".format(i.outcome_stage, i.created)



def run_cleanup():
    get_episodes_with_multiple_outcomes_per_stage()
    update_acceptance_date()
    update_empty_discharge_date()
    clean_opat_data()
    e = opal_models.Episode.objects.get(id=7773)
    location = e.location_set.first()
    location.opat_referral = location.opat_acceptance
    location.save()