Convert MOD14 hdf into csv

mod14-process.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import with_statement
from optparse import OptionParser
import os, sys, fnmatch, shutil, datetime

def echo_err(parser,msg):
    parser.print_help()
    print "*** " + msg
    sys.exit(1)

def main():
    usage = 'usage: %prog --directory=PATH --sattelite=MOD|MYD'
    parser = OptionParser(usage=usage)
    parser.add_option('-d', '--directory', action='store', dest='rootPath')
    parser.add_option('-s', '--sattelite', action='store', dest='sattelite', help='MOD|MYD')
    (options, args) = parser.parse_args()
    
    rootPath = options.rootPath
    sattelite = options.sattelite
    
    if rootPath == None:
        rootPath = os.getcwd()
    if not os.path.exists(rootPath):
        echo_err(parser,"Path not found")
    if sattelite == None:
        echo_err(parser,"Sattelite name is required")
    if ((sattelite != 'MOD') and (sattelite != 'MYD')):
        echo_err(parser,"Sattelite name is invalid")
    
    if sattelite == 'MOD':
        satPattern = 'MOD14*.hdf'
        sensor = 'TERRA'
    if sattelite == 'MYD':
        satPattern = 'MYD14*.hdf'
        sensor = 'AQUA'

    fldList = 'FP_line,FP_sample,FP_latitude,FP_longitude,FP_R2,FP_T21,FP_T31,FP_MeanT21,FP_MeanT31,FP_MeanDT,FP_MAD_T21,FP_MAD_T31,FP_MAD_DT,FP_power,FP_AdjCloud,FP_AdjWater,FP_WinSize,FP_NumValid,FP_confidence'
    fldlistout = 'line,sample,latitude,longitude,R2,T21,T31,MeanT21,MeanT31,MeanDT,MAD_T21,MAD_T31,MAD_DT,power,AdjCloud,AdjWater,WinSize,NumValid,confidence,sat,granule,year,month,day,daynumber,date,time,version'
    tempRoot = 'temp'
    csvRoot = 'csv'
    tempExt = 'txt'
    csvExt = 'csv'
    csvtExt = 'csvt'
    tempPath = os.path.join(rootPath,tempRoot)
    csvPath = os.path.join(rootPath,csvRoot)
    tempPattern = '*.'+tempExt
    
    '''Create directory for temporary files'''
    if os.path.exists(tempPath):
        shutil.rmtree(tempPath)
    if os.path.exists(csvPath):
        shutil.rmtree(csvPath)
    os.makedirs(tempPath)
    os.makedirs(csvPath)

    '''Extract data into temporary ASCII files'''
    for root, dirs, files in os.walk(rootPath):
        hdfFiles = fnmatch.filter(files, satPattern)
        if (len(hdfFiles) != 0):
            for fileName in hdfFiles:
                inputFile = os.path.join(root,fileName)
                tempFile = os.path.join(tempPath, fileName[0:-3]+tempExt)
                print "extracting from ......" + inputFile
                os.system('hdp dumpsds -n %s -o %s -d %s' % (fldList, tempFile, inputFile))
                if os.path.getsize(tempFile) == 0:
                    os.remove(tempFile)
     
    '''Convert temporary files into CSV'''
    for root, dirs, files in os.walk(tempPath):
        tempFiles = fnmatch.filter(files, tempPattern)
        if (len(tempFiles) != 0):
            for fileName in tempFiles:
                groups = []
                group = []
                inputFile = os.path.join(root,fileName)
                csvFile = os.path.join(csvPath, fileName[0:-3]+csvExt)
                csvtFile = os.path.join(csvPath, fileName[0:-3]+csvtExt)
                
                csvt = open(csvtFile,'wb')
                csvt.write('"Integer","Integer","Real","Real","Real","Real","Real","Real","Real","Real","Real","Real","Real","Real","Integer","Integer","Integer","Integer","Integer","String","String","Integer","Integer","Integer","Integer","Date","String","String"')
                csvt.close()
                
                for line in (x.strip() for x in file(inputFile).readlines()) :
                    if line :
                        group.append(line)
                    else :
                        cnt = len(" ".join(group).split())
                        groups.append(" ".join(group).split())
                        group[:] = []
                
                '''Manual adding additional fields'''
                '''Sensor name'''
                group.append(" ".join([sensor]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []
                
                '''Granule name'''
                group.append(" ".join([fileName[0:-4]]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []
                
                '''Year'''
                group.append(" ".join([datetime.datetime.strptime(fileName[7:14], '%Y%j').strftime('%Y')]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []
                
                '''Month'''
                group.append(" ".join([datetime.datetime.strptime(fileName[7:14], '%Y%j').strftime('%m')]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []
                
                '''Day'''
                group.append(" ".join([datetime.datetime.strptime(fileName[7:14], '%Y%j').strftime('%d')]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []
                
                '''Day number'''
                group.append(" ".join([datetime.datetime.strptime(fileName[7:14], '%Y%j').strftime('%j')]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []
                
                '''Date'''
                group.append(" ".join([datetime.datetime.strptime(fileName[7:14], '%Y%j').strftime('%Y-%m-%d')]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []
                
                '''Time'''
                group.append(" ".join([datetime.datetime.strptime(fileName[15:19], '%H%M').strftime('%H:%M')]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []
                
                '''Version'''
                group.append(" ".join([fileName[20:23]]*cnt))
                groups.append(" ".join(group).split(' '))
                group[:] = []

                with file(csvFile, "wb") as out:
                    out.write(fldlistout+'\n')
                    for group in zip(*groups) :
                        out.write(", ".join(group) + "\n")
    
    if os.path.exists(tempPath):
        shutil.rmtree(tempPath)

if __name__=="__main__":
    main()