import sys
import json
import csv
import shapely.wkb
import shapely.geometry
import shapely.ops
import os
import copy
from osgeo import ogr
from osgeo import osr
from booleano.parser import Grammar, EvaluableParseManager, SymbolTable, Bind
from booleano.operations import Variable
class JQVMap:
def __init__(self, name, language):
self.paths = {}
self.name = name
self.language = language
self.width = 0
self.height = 0
self.bbox = []
def addPath(self, path, code, name):
self.paths[code] = {"path": path, "name": name}
def getJSCode(self):
map = {"paths": self.paths, "width": self.width, "height": self.height, "insets": self.insets, "projection": self.projection}
header = "/** JQVMap " + self.projection['type'] + " map for " + self.name + " */"
js = "jQuery.fn.vectorMap('addMap', '" + self.name + "'," + json.dumps(map) + ");"
return header + "\n" + js
class Converter:
def __init__(self, config):
args = {
'buffer_distance': -0.4,
'simplify_tolerance': 0.2,
'longitude0': 0,
'projection': 'mill',
'name': 'world',
'width': 900,
'left': 0,
'top': 0,
'language': 'en',
'precision': 2,
'insets': []
}
args.update(config)
self.config = args
self.map = JQVMap(args['name'], args.get('language'))
if args.get('sources'):
self.sources = args['sources']
else:
self.sources = [{
'input_file': args.get('input_file'),
'where': args.get('where'),
'name_field': args.get('name_field'),
'code_field': args.get('code_field'),
'input_file_encoding': args.get('input_file_encoding')
}]
default_source = {
'where': '',
'name_field': 0,
'code_field': 1,
'input_file_encoding': 'iso-8859-1'
}
for index in range(len(self.sources)):
for key in default_source:
if self.sources[index].get(key) is None:
self.sources[index][key] = default_source[key]
self.features = {}
self.width = args.get('width')
self.left = args.get('left')
self.top = args.get('top')
self.minimal_area = args.get('minimal_area')
self.longitude0 = float(args.get('longitude0'))
self.projection = args.get('projection')
self.precision = args.get('precision')
self.buffer_distance = args.get('buffer_distance')
self.simplify_tolerance = args.get('simplify_tolerance')
self.for_each = args.get('for_each')
self.emulate_longitude0 = args.get('emulate_longitude0')
if args.get('emulate_longitude0') is None and (self.projection == 'merc' or self.projection =='mill') and self.longitude0 != 0:
self.emulate_longitude0 = True
if args.get('viewport'):
self.viewport = map(lambda s: float(s), args.get('viewport').split(' '))
else:
self.viewport = False
# spatial reference to convert to
self.spatialRef = osr.SpatialReference()
projString = '+proj='+str(self.projection)+' +a=6381372 +b=6381372 +lat_0=0'
if not self.emulate_longitude0:
projString += ' +lon_0='+str(self.longitude0)
self.spatialRef.ImportFromProj4(projString)
# handle map insets
if args.get('insets'):
self.insets = args.get('insets')
else:
self.insets = []
def convert(self, data_source, output_file):
codes = map(lambda g: g.properties[self.config['code_field']], data_source.geometries)
main_codes = copy.copy(codes)
self.map.insets = []
envelope = []
for inset in self.insets:
insetBbox = self.renderMapInset(data_source, inset['codes'], inset['left'], inset['top'], inset['width'])
insetHeight = (insetBbox[3] - insetBbox[1]) * (inset['width'] / (insetBbox[2] - insetBbox[0]))
self.map.insets.append({
"bbox": [{"x": insetBbox[0], "y": -insetBbox[3]}, {"x": insetBbox[2], "y": -insetBbox[1]}],
"left": inset['left'],
"top": inset['top'],
"width": inset['width'],
"height": insetHeight
})
envelope.append(
shapely.geometry.box(
inset['left'], inset['top'], inset['left'] + inset['width'], inset['top'] + insetHeight
)
)
for code in inset['codes']:
main_codes.remove(code)
insetBbox = self.renderMapInset(data_source, main_codes, self.left, self.top, self.width)
insetHeight = (insetBbox[3] - insetBbox[1]) * (self.width / (insetBbox[2] - insetBbox[0]))
envelope.append( shapely.geometry.box( self.left, self.top, self.left + self.width, self.top + insetHeight ) )
mapBbox = shapely.geometry.MultiPolygon( envelope ).bounds
self.map.width = mapBbox[2] + mapBbox[0]
self.map.height = mapBbox[3] + mapBbox[1]
self.map.insets.append({
"bbox": [{"x": insetBbox[0], "y": -insetBbox[3]}, {"x": insetBbox[2], "y": -insetBbox[1]}],
"left": self.left,
"top": self.top,
"width": self.width,
"height": insetHeight
})
self.map.projection = {"type": self.projection, "centralMeridian": float(self.longitude0)}
open(output_file, 'w').write( self.map.getJSCode() )
if self.for_each is not None:
for code in codes:
childConfig = copy.deepcopy(self.for_each)
for param in ('input_file', 'output_file', 'where', 'name'):
childConfig[param] = childConfig[param].replace('{{code}}', code.lower())
converter = Converter(childConfig)
converter.convert(childConfig['output_file'])
def renderMapInset(self, data_source, codes, left, top, width):
envelope = []
geometries = filter(lambda g: g.properties[self.config['code_field']] in codes, data_source.geometries)
for geometry in geometries:
envelope.append( geometry.geom.envelope )
bbox = shapely.geometry.MultiPolygon( envelope ).bounds
scale = (bbox[2]-bbox[0]) / width
# generate SVG paths
for geometry in geometries:
geom = geometry.geom
if self.buffer_distance:
geom = geom.buffer(self.buffer_distance*scale, 1)
if geom.is_empty:
continue
if self.simplify_tolerance:
geom = geom.simplify(self.simplify_tolerance*scale, preserve_topology=True)
if isinstance(geom, shapely.geometry.multipolygon.MultiPolygon):
polygons = geom.geoms
else:
polygons = [geom]
path = ''
for polygon in polygons:
rings = []
rings.append(polygon.exterior)
rings.extend(polygon.interiors)
for ring in rings:
for pointIndex in range( len(ring.coords) ):
point = ring.coords[pointIndex]
if pointIndex == 0:
path += 'M'+str( round( (point[0]-bbox[0]) / scale + left, self.precision) )
path += ','+str( round( (bbox[3] - point[1]) / scale + top, self.precision) )
else:
path += 'l' + str( round(point[0]/scale - ring.coords[pointIndex-1][0]/scale, self.precision) )
path += ',' + str( round(ring.coords[pointIndex-1][1]/scale - point[1]/scale, self.precision) )
path += 'Z'
self.map.addPath(path, geometry.properties[self.config['code_field']].lower(), geometry.properties[self.config['name_field']])
return bbox
class Geometry:
def __init__(self, geometry, properties):
self.geom = geometry
self.properties = properties
class GeometryProperty(Variable):
operations = set(["equality", "membership"])
def __init__(self, name):
self.name = name
def equals(self, value, context):
return context[self.name] == value
def belongs_to(self, value, context):
return value in context[self.name]
def is_subset(self, value, context):
return set(value).issubset(set(context[self.name]))
def to_python(self, value):
return unicode(value[self.name])
class DataSource:
def __init__(self, config):
default_config = {
"projection": "merc",
"longitude0": 0
}
default_config.update(config)
self.config = default_config
self.spatialRef = osr.SpatialReference()
projString = '+proj='+str(self.config['projection'])+' +a=6381372 +b=6381372 +lat_0=0'
#if 'emulate_longitude0' in self.config and not self.config['emulate_longitude0']:
projString += ' +lon_0='+str(self.config['longitude0'])
self.spatialRef.ImportFromProj4(projString)
def load_data(self):
self.source = ogr.Open( self.config['file_name'], update = 0 )
self.layer = self.source.GetLayer(0)
if 'filter' in self.config and self.config['filter'] is not None:
self.layer.SetAttributeFilter( self.config['filter'].encode('ascii') )
self.layer_dfn = self.layer.GetLayerDefn()
self.fields = []
field_count = self.layer_dfn.GetFieldCount()
for field_index in range(field_count):
field = self.layer_dfn.GetFieldDefn( field_index )
self.fields.append({
'name': field.GetName(),
'type': field.GetType(),
'width': field.GetWidth(),
'precision': field.GetPrecision()
})
self.geometries = []
for feature in self.layer:
geometry = feature.GetGeometryRef()
geometry.TransformTo( self.spatialRef )
geometry = shapely.wkb.loads( geometry.ExportToWkb() )
if not geometry.is_valid:
geometry = geometry.buffer(0)
properties = {}
for field in self.fields:
properties[field['name']] = feature.GetFieldAsString(field['name']).decode('utf-8')
self.geometries.append( Geometry(geometry, properties) )
self.layer.ResetReading()
self.create_grammar()
def create_grammar(self):
root_table = SymbolTable("root",
map( lambda f: Bind(f['name'], GeometryProperty(f['name'])), self.fields )
)
tokens = {
'not': 'not',
'eq': '==',
'ne': '!=',
'belongs_to': 'in',
'is_subset': 'are included in',
'or': "or",
'and': 'and'
}
grammar = Grammar(**tokens)
self.parse_manager = EvaluableParseManager(root_table, grammar)
def output(self, output):
if output.get('format') == 'jqvmap':
self.output_jvm(output)
else:
self.output_ogr(output)
def output_ogr(self, output):
driver = ogr.GetDriverByName( 'ESRI Shapefile' )
if os.path.exists( output['file_name'] ):
driver.DeleteDataSource( output['file_name'] )
source = driver.CreateDataSource( output['file_name'] )
layer = source.CreateLayer( self.layer_dfn.GetName(),
geom_type = self.layer_dfn.GetGeomType(),
srs = self.layer.GetSpatialRef() )
for field in self.fields:
fd = ogr.FieldDefn( str(field['name']), field['type'] )
fd.SetWidth( field['width'] )
if 'precision' in field:
fd.SetPrecision( field['precision'] )
layer.CreateField( fd )
for geometry in self.geometries:
if geometry.geom is not None:
feature = ogr.Feature( feature_def = layer.GetLayerDefn() )
for index, field in enumerate(self.fields):
if field['name'] in geometry.properties:
feature.SetField( index, geometry.properties[field['name']].encode('utf-8') )
else:
feature.SetField( index, '' )
feature.SetGeometryDirectly(
ogr.CreateGeometryFromWkb(
shapely.wkb.dumps(
geometry.geom
)
)
)
layer.CreateFeature( feature )
feature.Destroy()
source.Destroy()
def output_jvm(self, output):
params = copy.deepcopy(output['params'])
params.update({
"projection": self.config["projection"],
"longitude0": self.config["longitude0"]
})
converter = Converter(params)
converter.convert(self, output['file_name'])
class PolygonSimplifier:
def __init__(self, geometries):
self.format = '%.8f %.8f'
self.tolerance = 0.05
self.geometries = geometries
connections = {}
counter = 0
for geom in geometries:
counter += 1
polygons = []
if isinstance(geom, shapely.geometry.Polygon):
polygons.append(geom)
else:
for polygon in geom:
polygons.append(polygon)
for polygon in polygons:
if polygon.area > 0:
lines = []
lines.append(polygon.exterior)
for line in polygon.interiors:
lines.append(line)
for line in lines:
for i in range(len(line.coords)-1):
indexFrom = i
indexTo = i+1
pointFrom = self.format % line.coords[indexFrom]
pointTo = self.format % line.coords[indexTo]
if pointFrom == pointTo:
continue
if not (pointFrom in connections):
connections[pointFrom] = {}
connections[pointFrom][pointTo] = 1
if not (pointTo in connections):
connections[pointTo] = {}
connections[pointTo][pointFrom] = 1
self.connections = connections
self.simplifiedLines = {}
self.pivotPoints = {}
def simplifyRing(self, ring):
coords = list(ring.coords)[0:-1]
simpleCoords = []
isPivot = False
pointIndex = 0
while not isPivot and pointIndex < len(coords):
pointStr = self.format % coords[pointIndex]
pointIndex += 1
isPivot = ((len(self.connections[pointStr]) > 2) or (pointStr in self.pivotPoints))
pointIndex = pointIndex - 1
if not isPivot:
simpleRing = shapely.geometry.LineString(coords).simplify(self.tolerance)
if len(simpleRing.coords) <= 2:
return None
else:
self.pivotPoints[self.format % coords[0]] = True
self.pivotPoints[self.format % coords[-1]] = True
simpleLineKey = self.format % coords[0]+':'+self.format % coords[1]+':'+self.format % coords[-1]
self.simplifiedLines[simpleLineKey] = simpleRing.coords
return simpleRing
else:
points = coords[pointIndex:len(coords)]
points.extend(coords[0:pointIndex+1])
iFrom = 0
for i in range(1, len(points)):
pointStr = self.format % points[i]
if ((len(self.connections[pointStr]) > 2) or (pointStr in self.pivotPoints)):
line = points[iFrom:i+1]
lineKey = self.format % line[-1]+':'+self.format % line[-2]+':'+self.format % line[0]
if lineKey in self.simplifiedLines:
simpleLine = self.simplifiedLines[lineKey]
simpleLine = list(reversed(simpleLine))
else:
simpleLine = shapely.geometry.LineString(line).simplify(self.tolerance).coords
lineKey = self.format % line[0]+':'+self.format % line[1]+':'+self.format % line[-1]
self.simplifiedLines[lineKey] = simpleLine
simpleCoords.extend( simpleLine[0:-1] )
iFrom = i
if len(simpleCoords) <= 2:
return None
else:
return shapely.geometry.LineString(simpleCoords)
def simplifyPolygon(self, polygon):
simpleExtRing = self.simplifyRing(polygon.exterior)
if simpleExtRing is None:
return None
simpleIntRings = []
for ring in polygon.interiors:
simpleIntRing = self.simplifyRing(ring)
if simpleIntRing is not None:
simpleIntRings.append(simpleIntRing)
return shapely.geometry.Polygon(simpleExtRing, simpleIntRings)
def simplify(self):
results = []
for geom in self.geometries:
polygons = []
simplePolygons = []
if isinstance(geom, shapely.geometry.Polygon):
polygons.append(geom)
else:
for polygon in geom:
polygons.append(polygon)
for polygon in polygons:
simplePolygon = self.simplifyPolygon(polygon)
if not (simplePolygon is None or simplePolygon._geom is None):
simplePolygons.append(simplePolygon)
if len(simplePolygons) > 0:
results.append(shapely.geometry.MultiPolygon(simplePolygons))
else:
results.append(None)
return results
class Processor:
def __init__(self, config):
self.config = config
def process(self):
self.data_sources = {}
for action in self.config:
getattr(self, action['name'])( action, self.data_sources.get(".") )
def read_data(self, config, data_source):
self.data_sources["."] = DataSource( config )
self.data_sources["."].load_data()
def write_data(self, config, data_source):
data_source.output( config )
def union(self, config, data_source):
groups = {}
geometries = []
for geometry in data_source.geometries:
if geometry.properties[config['by']] in groups:
groups[geometry.properties[config['by']]]['geoms'].append(geometry.geom)
else:
groups[geometry.properties[config['by']]] = {
'geoms': [geometry.geom],
'properties': geometry.properties
}
for key in groups:
geometries.append( Geometry(shapely.ops.cascaded_union( groups[key]['geoms'] ), groups[key]['properties']) )
data_source.geometries = geometries
def merge(self, config, data_source):
new_geometries = []
for rule in config['rules']:
expression = data_source.parse_manager.parse( rule['where'] )
geometries = filter(lambda g: expression(g.properties), data_source.geometries)
geometries = map(lambda g: g.geom, geometries)
new_geometries.append( Geometry(shapely.ops.cascaded_union( geometries ), rule['fields']) )
data_source.fields = config['fields']
data_source.geometries = new_geometries
def join_data(self, config, data_source):
field_names = [f['name'] for f in config['fields']]
if 'data' in config:
data_col = config['data']
else:
data_file = open(config['file_name'], 'rb')
data_col = csv.reader(data_file, delimiter='\t', quotechar='"')
data = {}
for row in data_col:
row_dict = dict(zip(field_names, row))
data[row_dict.pop(config['on'])] = row_dict
for geometry in data_source.geometries:
if geometry.properties[config['on']] in data:
geometry.properties.update( data[geometry.properties[config['on']]] )
field_names = map(lambda f: f['name'], data_source.fields)
data_source.fields = data_source.fields + filter(lambda f: f['name'] not in field_names, config['fields'])
def remove(self, config, data_source):
expression = data_source.parse_manager.parse( config['where'] )
data_source.geometries = filter(lambda g: not expression(g.properties), data_source.geometries)
def remove_fields(self, config, data_source):
data_source.fields = filter(lambda f: f.name not in config['fields'], data_source.fields)
def remove_other_fields(self, config, data_source):
data_source.fields = filter(lambda f: f['name'] in config['fields'], data_source.fields)
def buffer(self, config, data_source):
for geometry in data_source.geometries:
geometry.geom = geometry.geom.buffer(config['distance'], config['resolution'])
def simplify_adjancent_polygons(self, config, data_source):
simple_geometries = PolygonSimplifier( map( lambda g: g.geom, data_source.geometries ) ).simplify()
for i in range(len(data_source.geometries)):
data_source.geometries[i].geom = simple_geometries[i]
def intersect_rect(self, config, data_source):
transform = osr.CoordinateTransformation( data_source.layer.GetSpatialRef(), data_source.spatialRef )
point1 = transform.TransformPoint(config['rect'][0], config['rect'][1])
point2 = transform.TransformPoint(config['rect'][2], config['rect'][3])
rect = shapely.geometry.box(point1[0], point1[1], point2[0], point2[1])
for geometry in data_source.geometries:
geometry.geom = geometry.geom.intersection(rect)
def remove_small_polygons(self, config, data_source):
for geometry in data_source.geometries:
if isinstance(geometry.geom, shapely.geometry.multipolygon.MultiPolygon):
polygons = geometry.geom.geoms
else:
polygons = [geometry.geom]
polygons = filter(lambda p: p.area > config['minimal_area'], polygons)
if len(polygons) > 0:
geometry.geom = shapely.geometry.multipolygon.MultiPolygon(polygons)
args = {}
if len(sys.argv) > 1:
paramsJson = open(sys.argv[1], 'r').read()
else:
paramsJson = sys.stdin.read()
paramsJson = json.loads(paramsJson)
processor = Processor(paramsJson)
processor.process()
|
Download
