Skip to content

Instantly share code, notes, and snippets.

@rischanlab

rischanlab/av_layer_better.py

Created November 27, 2015 13:14
Show Gist options
  • Save rischanlab/3f96ffdd305531a4d80c to your computer and use it in GitHub Desktop.
Save rischanlab/3f96ffdd305531a4d80c to your computer and use it in GitHub Desktop.
Download ZIP
script
Raw
av_layer_better.py
__author__ = 'rischan'
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from agri_mapper.utilities.script import CallWMS, CallWFS, UpdateModel
import logging
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'This script is for synchronizing available layer data'
def handle(self, *args, **options):
""" Do your work here """
wms1 = CallWMS()
updatemodel = UpdateModel()
projects = updatemodel.get_project_names()
for project in projects:
try:
wms = wms1.get_wms(project)
wms_lists = (list(wms.contents))
except Exception as e:
print "Oops! That project name {} was not available. " \
" Try again...".format(project)
continue
farm = updatemodel.get_farm_by_project_name(project)
for layer_name in wms_lists:
layer_name_has_date = updatemodel.layer_name_contain_date(
layer_name)
if layer_name_has_date:
layer_data = updatemodel.get_dates_by_layer_name(
layer_name)
layer_filter = updatemodel.get_layer_filter(
layer_data['layer_name'])
# get layer_filter by clean layer_name
if layer_filter is None:
print "The layer {} is not " \
"in layer_filter table".format(layer_name)
continue
if updatemodel.check_layer_in_available_layer(
farm, layer_filter,
layer_data['date'],
layer_data['week_number']):
print "The layer {} has been " \
"in the available layer table".format(
layer_name)
continue
try:
updatemodel.insert_available_layer(
farm,
layer_filter,
layer_data['date'],
layer_data['week_number'])
print "Success insert available layer " \
"for layer {}".format(layer_name)
except ObjectDoesNotExist as e:
print "Error in insert layer {} with date {} " \
"by date information in layer name".format(
layer_name, date['date'])
continue
else:
layer_filter = updatemodel.get_layer_filter(layer_name)
wfs_layer_data = updatemodel.get_dates_by_wfs(
project, layer_name)
if layer_filter is None:
print "The layer {} is not in layer_filter table".format(
layer_name)
continue
if wfs_layer_data is not None:
layer_data = updatemodel.get_dates_by_wfs(
project, layer_name)
for date in layer_data:
if updatemodel.check_wfs_layer_in_available_layer(
layer_filter,
date=date['date'],
week_number=date['week_number']):
print "The layer {} with date {} has been " \
"in the available layer table".format(
layer_name, date['date'])
continue
try:
updatemodel.insert_available_layer(
farm,
layer_filter,
date['date'],
date['week_number'])
print "Success insert available layer " \
"for layer {} with date {} ".format(
layer_name, date['date'])
except ObjectDoesNotExist as e:
print "Error in insert layer {} with date {} " \
"by wfs".format(layer_name, date['date'])
continue
else:
layer_data = updatemodel.get_date_from_current_date(
layer_name)
if updatemodel.check_layer_in_available_layer(
farm,
layer_filter,
layer_data['date'],
layer_data['week_number']):
print "The layer {} has been " \
"in the available layer table".format(layer_name)
continue
try:
updatemodel.insert_available_layer(
farm,
layer_filter,
layer_data['date'],
layer_data['week_number'])
print "Success insert available layer " \
"for layer {} with date {} ".format(
layer_name, layer_data['date'])
except ObjectDoesNotExist as e:
print "Error in insert layer {} with date {} " \
"by current_date".format(
layer_name, layer_data['date'])
continue
Raw
av_layer_too_deep.py
__author__ = 'rischan'
from django.core.management.base import BaseCommand
from agri_mapper.utilities.script import CallWMS, CallWFS, UpdateModel
import logging
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'This script is for synchronizing available layer data'
def handle(self, *args, **options):
""" Do your work here """
wms1 = CallWMS()
updatemodel = UpdateModel()
projects = updatemodel.get_project_names()
for project in projects:
try:
wms = wms1.get_wms(project)
wms_lists = (list(wms.contents))
farm = updatemodel.get_farm_by_project_name(project)
for layer_name in wms_lists:
layer_name_has_date = updatemodel.layer_name_contain_date(layer_name)
if layer_name_has_date:
layer_data = updatemodel.get_dates_by_layer_name(layer_name)
layer_filter = updatemodel.get_layer_filter(layer_data['layer_name'])
# get layer_filter by clean layer_name
if layer_filter is not None:
try:
if not updatemodel.check_layer_in_available_layer(farm, layer_filter, layer_data['date'], layer_data['week_number']):
updatemodel.insert_available_layer(farm, layer_filter, layer_data['date'], layer_data['week_number'])
print "Success insert available layer for layer {}".format(layer_name)
else:
print "The layer {} has been in the available layer table".format(layer_name)
except Exception as e:
print "Error in insert layer {} with date {} by date information in layer name".format(layer_name, date['date'])
continue
else:
print "The layer {} is not in layer_filter table".format(layer_name)
else:
layer_filter = updatemodel.get_layer_filter(layer_name)
wfs_layer_data = updatemodel.get_dates_by_wfs(project, layer_name)
# in this block layer_name is clean (does not has any date information)
if layer_filter is not None:
if wfs_layer_data is not None:
layer_data = updatemodel.get_dates_by_wfs(project, layer_name)
try:
for date in layer_data:
if not updatemodel.check_wfs_layer_in_available_layer(layer_filter, date=date['date'], week_number=date['week_number']):
updatemodel.insert_available_layer(farm, layer_filter, date['date'], date['week_number'])
print "Success insert available layer for layer {} with date {} ".format(layer_name, date['date'])
else:
print "The layer {} with date {} has been in the available layer table".format(layer_name, date['date'])
except Exception as e:
print "Error in insert layer {} with date {} by wfs".format(layer_name, date['date'])
continue
else:
layer_data = updatemodel.get_date_from_current_date(layer_name)
try:
if not updatemodel.check_layer_in_available_layer(farm, layer_filter, layer_data['date'], layer_data['week_number']):
updatemodel.insert_available_layer(farm, layer_filter, layer_data['date'], layer_data['week_number'])
print "Success insert available layer for layer {} with date {} ".format(layer_name, layer_data['date'])
else:
print "The layer {} with date {} has been in the available layer table".format(layer_name, layer_data['date'])
except Exception as e:
print "Error in insert layer {} with date {} by current_date".format(layer_name, layer_data['date'])
continue
else:
print "The layer {} is not in layer_filter table".format(layer_name)
# except ValueError:
# # If the project name is not valid.
except Exception as e:
print "Oops! That project name {} was not available. Try again...".format(project)
continue
Raw
script.py
__author__ = 'rischan'
import urllib
import datetime
from owslib.wms import WebMapService
from agri_mapper.models.farm import Farm
from agri_mapper.models.layer_filter import LayerFilter
from agri_mapper.models.available_layer import AvailableLayer
from xml.etree import ElementTree as ET
static_url = 'http://129.232.164.58:6003/cgi-bin/qgis_mapserv.fcgi?map=/qgis/docs/projects/'
extension = '.qgs'
wms_version = '1.1.1'
wfs_version = '1.0.0'
farm_extent_layer = 'Plase'
ns = {
'gml': 'http://www.opengis.net/gml',
'qgs': 'http://www.qgis.org/gml',
}
class CallWFS():
def wfs_url(self, project, layer_name):
service = 'WFS'
request = 'GetFeature'
propertyname = 'datum'
url = static_url + project + extension
path = '&SERVICE=' + service + '&REQUEST=' + request + '&typeName=' + layer_name + '&propertyName=' + propertyname + ''
xml_url = url+path
return xml_url
def get_wfs_dates(self, project, layer_name):
try:
url = self.wfs_url(project, layer_name)
dates = []
root = ET.parse(urllib.urlopen(url)).getroot()
for featureMember in root.findall('gml:featureMember', ns):
for itest_leaf in featureMember:
dates.append(itest_leaf.find('qgs:datum', ns).text)
date_list = [x for x in dates if x is not None]
return date_list
except Exception as e:
return None
class CallWMS():
def get_capabilities_url(self):
data = {
'URL': static_url,
'extension': extension,
'version': wms_version
}
return data
def get_wms(self, project_name):
data = self.get_capabilities_url()
wms_url = data['URL'] + project_name + data['extension']
wms = WebMapService(wms_url, version=data['version'])
return wms
class UpdateModel(CallWMS, CallWFS):
def get_project_names(self):
return [farm.project_name for farm in Farm.objects.all()]
def get_week_number(self, date):
return date.isocalendar()[1]
def check_layer_in_available_layer(self, farm, layer_filter, date, week_number):
layer_is_there = False
try:
layer = AvailableLayer.objects.get(farm=farm, layer_filter=layer_filter, date_added=date, week_number=week_number)
layer_is_there = True
except Exception as e:
layer_is_there = False
return layer_is_there
def check_wfs_layer_in_available_layer(self, layer_filter, date, week_number):
layer_is_there = False
try:
layer = AvailableLayer.objects.get(layer_filter=layer_filter, date_added=date, week_number=week_number)
layer_is_there = True
except Exception as e:
layer_is_there = False
return layer_is_there
def insert_available_layer(
self, farm, layer_filter, date_added, week_number):
layer = AvailableLayer(
farm=farm, layer_filter=layer_filter,
date_added=date_added, week_number=week_number)
layer.save()
def get_farm_by_project_name(self, project_name):
farm = Farm.objects.get(project_name=project_name)
return farm
def get_layer_filter(self, layer_name):
layer_filter = None
try:
layer_filter = LayerFilter.objects.get(layer_name=layer_name)
return layer_filter
except Exception as e:
return layer_filter
def layer_name_contain_date(self, layer_name):
l = layer_name.split("_")
if l[len(l)-1].isdigit() and l[len(l)-2].isdigit():
return True
else:
return False
def get_dates_by_layer_name(self, layer_name):
#layer_name = "Valskodlingmot_Larwes_2015_43"
l = layer_name.split("_")
length = len(l)
year = l[length-2]
wn = l[length-1]
d = year+"-W"+wn
date = datetime.datetime.strptime(d + '-0', "%Y-W%W-%w")
name_length = length - 2
if name_length >= 2:
name = l[:len(l)-2]
layer_name = " ".join(name)
else:
layer_name = l[0]
data = {
'layer_name' : layer_name,
'date' : date,
'week_number' : wn
}
return data
def get_dates_by_wfs(self, project, layer_name):
date = self.get_wfs_dates(project, layer_name)
if date:
unique_date = list(set(date))
data = []
for d in unique_date:
week_number = self.get_week_number(datetime.datetime.strptime(d, "%Y/%m/%d"))
data.append({
'layer_name' : layer_name,
'date' : datetime.datetime.strptime(d, "%Y/%m/%d"),
'week_number' : week_number
})
return data
else:
return None
def get_date_from_current_date(self, layer_name):
date = datetime.date.today()
week_number = self.get_week_number(date)
data = {
'layer_name' : layer_name,
'date' : date,
'week_number' : week_number
}
return data
def get_farm_id(self, project_name):
farm_id = None
farm = Farm.objects.get(project_name=project_name)
farm_id = farm.id
return farm_id
def check_extent(self, wms):
if farm_extent_layer in wms:
return True
else:
return False
def get_extent(self, project_name):
wms = self.get_wms(project_name)
bbox = wms[farm_extent_layer].boundingBoxWGS84
data = {
'minx': bbox[0],
'miny': bbox[1],
'maxx': bbox[2],
'maxy': bbox[3]
}
return data
def check_farm_extent_in_farm(self, project_name):
bbox_is_there = False
try:
layer = Farm.objects.get(project_name=project_name)
bbox = (layer.bbox_minx is None) or (
layer.bbox_miny is None) or (
layer.bbox_maxx is None) or (
layer.bbox_maxy is None)
if bbox:
bbox_is_there = False
else:
bbox_is_there = True
except Exception as e:
bbox_is_there = False
return bbox_is_there
def update_farm_extent(self, project_name, minx, miny, maxx, maxy):
farm_id = self.get_farm_id(project_name)
farm = Farm.objects.get(pk=farm_id)
farm.bbox_minx = minx
farm.bbox_miny = miny
farm.bbox_maxx = maxx
farm.bbox_maxy = maxy
farm.save()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment