blazetopher

#!/bin/sh

DO_POP=False
if [ "$1" = "pop" ]; then
    DO_POP=True
fi

LAST_NAME=$(git log -1 --pretty=%B 2> /dev/null | awk "{ print \$1 }")
if [ "$LAST_NAME" = "WIP" ]; then
    if $DO_POP; then

blazetopher

# Fix the ipython path
project_path = '/Users/cmueller/Development/OOI/Dev/code/coi-services'
virtualenv_path = '/Users/cmueller/Development/OOI/Dev/virtenvs/clean27'

import IPython
IPython.sys.path = ['',
 '{0}/eggs/gevent-0.13.7-py2.7-macosx-10.6-intel.egg'.format(project_path),
 '{0}/eggs/coverage-3.5.2-py2.7-macosx-10.6-intel.egg'.format(project_path),
 '{0}'.format(project_path),
 '{0}/eggs/objgraph-1.7.2-py2.7.egg'.format(project_path),

blazetopher

# IPython log file

from ion.services.dm.utility.granule_utils import time_series_domain
from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient
from interface.services.dm.idataset_management_service import DatasetManagementServiceClient
from interface.services.dm.iingestion_management_service import IngestionManagementServiceClient
from interface.services.sa.idata_product_management_service import DataProductManagementServiceClient
from interface.services.dm.idata_retriever_service import DataRetrieverServiceClient
from ion.services.dm.inventory.dataset_management_service import DatasetManagementService
from ion.services.dm.utility.granule import RecordDictionaryTool

blazetopher

#!/usr/bin/env python

"""
@package 
@file memory_trials.py
@author Christopher Mueller
@brief 
"""

from coverage_model import *

blazetopher

#!/usr/bin/env python

# https://gist.github.com/4438441
import os

from coverage_model import *
import numpy as np
import time
import objgraph
import gevent

blazetopher

Outstanding R2 DM Items

##Other

• Consider a technology (i.e. vagrant) for enabling a common environment among developers

##Coverage Model

• Current usage of attributes in HDF may be a bad choice for a few reasons, should be reevaluated and/or replaced with another mechanism of storing attributes:
  • After many ingestion events, metadata files end up very large and mostly 'unallocated' (i.e. empty) due to B-Tree change history
  • May leak memory (unverified)
• May not be particularly fast (unverified)

blazetopher

import gevent
import gc
from load_datasets import *

def get_mem():
    import resource
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024.0**2)
    
data_product_id, stream_id, route, stream_def_id, dataset_id = create_data_product('ctd_parsed_param_dict')
populate_data_product(data_product_id,1)

blazetopher

#!/usr/bin/env python
import gevent
import gc
from load_datasets import *
from coverage_model import AbstractCoverage, ViewCoverage, ParameterContext, ParameterFunctionType, create_guid

def get_mem():
    import resource
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024.0**2)

blazetopher

#!/usr/bin/env python

"""
@package 
@file memory_trials.py
@author Christopher Mueller
@brief 
"""

from coverage_model import *

blazetopher

quality_flag: !!python/tuple
- 15
- variability: BOTH
  _derived_from_name: quality_flag
  display_name: ''
  description: ''
  extension: {}
  name: quality_flag
  reference_urls: ''
  precision: ''