Skip to content

Instantly share code, notes, and snippets.

View snopoke's full-sized avatar

Simon Kelly snopoke

View GitHub Profile
cat icds_cas_commcare-nginx_access.log | cut -d' ' -f 10,14 > cas_logs.txt
cat icds_cas_commcare-nginx_access.log.1 | cut -d' ' -f 10,14 >> cas_logs.txt
cat icds_commcare-nginx_access.log | cut -d' ' -f 10,14 >> cas_logs.txt
zcat icds_cas_commcare-nginx_access.log.*.gz | cut -d' ' -f 10,14 >> cas_logs.txt
echo -n 'Total,'; echo awk '{s+=$2}END{print s}' cas_logs.txt
echo -n 'App audio downloads,'; cat cas_logs.txt | grep CommCareAudio | awk '{s+=$2}END{print s}'
echo -n 'App video downloads,'; cat cas_logs.txt | grep CommCareVideo | awk '{s+=$2}END{print s}'
echo -n 'App image downloads,'; cat cas_logs.txt | grep CommCareImage | awk '{s+=$2}END{print s}'
echo -n 'Static files,'; cat cas_logs.txt | grep static | awk '{s+=$2}END{print s}'
@snopoke
snopoke / drop_child_tables.py
Last active March 15, 2019 11:25
Script to drop child tables of intermediate dashboard tables (ICDS)
# Drop child tables of intermediate aggregation tables.
# Each table must have a 'month' column.
# If tables have usage as reported by pg_stat_user_tables
# they will not be dropped.
import itertools
from datetime import date
from collections import defaultdict
from corehq.sql_db.connections import connection_manager
from sqlalchemy.exc import ProgrammingError
copy_files:
- couch6:
- source_host: couch4
source_user: couchdb
source_dir: /opt/data/couchdb2/
target_dir: /opt/data/couchdb2/
files:
- shards/00000000-1fffffff/commcarehq__fixtures.1495821829.couch
- .shards/00000000-1fffffff/commcarehq__fixtures.1495821829_design
- couch7:
@snopoke
snopoke / log_sql_from_mobile_ucr.py
Last active April 1, 2019 08:49
Mobile UCR SQL Log
from casexml.apps.phone.tests.utils import call_fixture_generator
from corehq.apps.app_manager.fixtures.mobile_ucr import report_fixture_generator
from casexml.apps.phone.models import OTARestoreCommCareUser
from corehq.apps.users.models import CommCareUser
from corehq.apps.app_manager.models import Application
from datetime import date
import logging
from corehq.apps.app_manager.models import ReportModule
from corehq.apps.app_manager.fixtures.mobile_ucr import ReportFixturesProvider
import json
@snopoke
snopoke / show_licenses.py
Created October 30, 2018 10:21
Print licenses of packages in requirements file.
# based on https://github.com/dhatim/python-license-check
import argparse
try:
from configparser import ConfigParser, NoOptionError
except ImportError:
from ConfigParser import ConfigParser, NoOptionError
import re
import sys
cchq icds-cas run-shell-command couchdb2 -b "service couchdb2 stop && rm -rf /opt/data/couchdb2/* && rm -f /usr/local/couchdb2/couchdb/etc/local.d/z_admin.ini && rm -f /usr/local/couchdb2/couchdb/etc/local.d/auto_gen.ini && > /usr/local/couchdb2/couchdb/etc/local.d/ssl.ini && service couchdb2 start"
cchq icds-cas ap deploy_db.yml --limit couchdb2 --branch=je/icds-cas-deploy --skip-check --start-at-task="Create system databases"
@snopoke
snopoke / add_snapshot.py
Last active August 2, 2018 10:45
Size comparison for storing case block data in SQL alongside case transactions
import cStringIO
import json
import psycopg2
from django.db import connections
from corehq.apps.dump_reload.sql.serialization import JsonLinesSerializer
from corehq.form_processor.models import CommCareCaseSQL
@snopoke
snopoke / first_form_from_location.py
Created June 26, 2017 13:11
Min Form Date from Location
from dimagi.utils.chunked import chunked
from corehq.apps.locations.dbaccessors import user_ids_at_locations
from corehq.apps.es.aggregations import *
from corehq.apps.es import FormES
from corehq.apps.locations.models import *
from elasticsearch.exceptions import *
from iso8601 import iso8601
domain = 'domain'
@snopoke
snopoke / report.py
Last active August 24, 2017 11:21 — forked from javierwilson/report.py
Generates CSV for last week cpu, mem and disk utilization
#!/usr/bin/env python
import datetime
import os
from datadog import initialize, api
from collections import defaultdict
options = {
@snopoke
snopoke / create_functions.sql
Created June 22, 2016 18:10
Bash and SQL scripts for testing plproxy server configuration
drop function if exists make_new_user(TEXT);
create function make_new_user(p_username TEXT, user_id OUT INTEGER ) as $$
begin
INSERT INTO users (username, shard, last_updated) VALUES (p_username, (hashtext(p_username)&15), now())
RETURNING users.user_id INTO user_id;
end;
$$ language plpgsql;
drop function if exists get_all_users_with_db();