Major refactoring.

This commit is contained in:
Sergey Vartanov 2020-08-22 10:28:30 +03:00
parent 5c8a49555f
commit 99b88bb341
12 changed files with 393 additions and 57 deletions

15
.idea/Roentgen.iml generated Normal file
View file

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="pytest" />
</component>
</module>

View file

@ -3,10 +3,10 @@ Extract icons from SVG file.
Author: Sergey Vartanov (me@enzet.ru).
"""
import re
import xml.dom.minidom
class IconExtractor:
def __init__(self, svg_file_name):
self.icons = {}
@ -27,7 +27,7 @@ class IconExtractor:
path = node.attributes['d'].value
m = re.match('[Mm] ([0-9.e-]*)[, ]([0-9.e-]*)', path)
if not m:
print 'Error path: ' + path
print(f"Error path: {path}.")
else:
x = float(m.group(1))
y = float(m.group(2))
@ -36,11 +36,11 @@ class IconExtractor:
self.icons[node.attributes['id'].value] = \
(node.attributes['d'].value, x, y)
else:
for subnode in node.childNodes:
self.parse(subnode)
for sub_node in node.childNodes:
self.parse(sub_node)
def get_path(self, id):
def get_path(self, id_):
if id in self.icons:
return self.icons[id]
return self.icons[id_]
else:
return None, 0, 0

View file

@ -103,7 +103,7 @@ def get_d_from_file(file_name):
if path:
return path, x, y
else:
# print 'No such icon: ' + file_name
# print('No such icon: ' + file_name)
# TODO: add to missed icons
return 'M 4,4 L 4,10 10,10 10,4 z', 0, 0
@ -896,7 +896,7 @@ def construct_nodes(drawing):
"""
Draw nodes.
"""
print 'Draw nodes...'
print('Draw nodes...')
start_time = datetime.now()
@ -964,10 +964,10 @@ def construct_nodes(drawing):
'tags': tags})
ui.write_line(-1, len(node_map))
print 'Nodes drawed in ' + str(datetime.now() - start_time) + '.'
print 'Tags processed: ' + str(processed_tags) + ', tags skipped: ' + \
str(skipped_tags) + ' (' + \
str(processed_tags / float(processed_tags + skipped_tags) * 100) + ' %).'
print('Nodes drawed in ' + str(datetime.now() - start_time) + '.')
print('Tags processed: ' + str(processed_tags) + ', tags skipped: ' +
str(skipped_tags) + ' (' +
str(processed_tags / float(processed_tags + skipped_tags) * 100) + ' %).')
def way_sorter(element):
if 'layer' in element:
@ -1048,7 +1048,7 @@ if options.mode in ['user-coloring', 'time']:
input_file_name = options.input_file_name
if not os.path.isfile(input_file_name):
print 'Fatal: no such file: ' + input_file_name + '.'
print('Fatal: no such file: ' + input_file_name + '.')
sys.exit(1)
full = False # Full keys getting
@ -1093,10 +1093,10 @@ for color_name in w3c_colors:
if len(sys.argv) > 3:
flinger = GeoFlinger(min1, max1, Vector(0, 0), Vector(w, h))
else:
print 'Compute borders...'
print('Compute borders...')
minimum, maximum = get_min_max(node_map)
flinger = GeoFlinger(minimum, maximum, Vector(25, 25), Vector(975, 975))
print 'Done.'
print('Done.')
icons = extract_icon.IconExtractor(icons_file_name)
@ -1116,8 +1116,8 @@ if flinger.space.x == 0:
output_file.rect(0, h - flinger.space.y, w, flinger.space.y, color='FFFFFF')
if options.show_index:
print min1.lon, max1.lon
print min1.lat, max1.lat
print(min1.lon, max1.lon)
print(min1.lat, max1.lat)
lon_step = 0.001
lat_step = 0.001
@ -1180,5 +1180,5 @@ sys.exit(0)
top_authors = sorted(authors.keys(), key=lambda x: -authors[x])
for author in top_authors:
print author + ': ' + str(authors[author])
print(author + ': ' + str(authors[author]))

86
roentgen/network.py Normal file
View file

@ -0,0 +1,86 @@
"""
Utility for network connections.
Author: Sergey Vartanov (me@enzet.ru)
"""
import json
import os
import urllib
import urllib3
import time
from datetime import datetime, timedelta
from typing import Dict, List
def get_data(address: str, parameters: Dict[str, str], is_secure: bool=False, name: str=None) -> bytes:
"""
Construct Internet page URL and get its descriptor.
:param address: first part of URL without "http://"
:param parameters: URL parameters
:param is_secure: https or http
:param name: name to display in logs
:return: connection descriptor
"""
url = "http" + ("s" if is_secure else "") + "://" + address
if len(parameters) > 0:
url += "?" + "&".join(parameters) # urllib.parse.urlencode(parameters)
if not name:
name = url
print("getting " + name)
pool_manager = urllib3.PoolManager()
url = url.replace(" ", "_")
urllib3.disable_warnings()
result = pool_manager.request("GET", url)
pool_manager.clear()
time.sleep(2)
return result.data
def get_content(address, parameters, cache_file_name, kind, is_secure, name=None, exceptions=None, update_cache=False):
"""
Read content from URL or from cached file.
:param address: first part of URL without "http://"
:param parameters: URL parameters
:param cache_file_name: name of cache file
:param kind: type of content: "html" or "json"
:return: content if exist
"""
if exceptions and address in exceptions:
return None
if os.path.isfile(cache_file_name) and \
datetime(1, 1, 1).fromtimestamp(os.stat(cache_file_name).st_mtime) > \
datetime.now() - timedelta(days=90) and \
not update_cache:
with open(cache_file_name) as cache_file:
if kind == "json":
try:
return json.load(cache_file)
except ValueError:
return None
if kind == "html":
return cache_file.read()
else:
try:
data = get_data(address, parameters, is_secure=is_secure, name=name)
if kind == "json":
try:
obj = json.loads(data)
with open(cache_file_name, "w+") as cached:
cached.write(json.dumps(obj, indent=4))
return obj
except ValueError:
print("cannot get " + address + " " + str(parameters))
return None
if kind == "html":
with open(cache_file_name, "w+") as cached:
cached.write(data)
return data
except Exception as e:
print("during getting JSON from " + address + " with parameters " + str(parameters))
print(e)
if exceptions:
exceptions.append(address)
return None

View file

@ -5,10 +5,11 @@ sys.path.append('../lib')
import network
usage = '<box coordinates: left,bottom,right,top>'
if len(sys.argv) < 2:
print 'Usage: python ' + sys.argv[0] + ' ' + usage
print('Usage: python ' + sys.argv[0] + ' ' + usage)
sys.exit(0)
boundary_box = sys.argv[1]
@ -18,13 +19,15 @@ result_file_name = '../map/' + boundary_box + '.xml'
matcher = re.match('(?P<left>[0-9\\.-]*),(?P<bottom>[0-9\\.-]*),' + \
'(?P<right>[0-9\\.-]*),(?P<top>[0-9\\.-]*)', boundary_box)
def error(message=None):
if message:
print 'Error: ' + message + '.'
print('Error: ' + message + '.')
else:
print 'Error.'
print('Error.')
sys.exit(1)
if not matcher:
error('invalid boundary box')
else:

View file

@ -3,21 +3,23 @@ Reading OpenStreetMap data from XML file.
Author: Sergey Vartanov
"""
import datetime
import ui
import re
import sys
from typing import Dict
def parse_node_full(node_data):
"""
Parse full node parameters using regular expressions: id, visible, version,
etc. For faster parsing use parse_node().
"""
m = re.match('id="(.*)" visible="(.*)" version="(.*)" changeset="(.*)" ' + \
'timestamp="(.*)" user="(.*)" uid="(.*)" ' + \
'lat="(.*)" lon="(.*)"', node_data)
m = re.match(
'id="(.*)" visible="(.*)" version="(.*)" changeset="(.*)" '
'timestamp="(.*)" user="(.*)" uid="(.*)" lat="(.*)" lon="(.*)"',
node_data)
if m:
return {'id': int(m.group(1)), 'visible': m.group(2),
'version': m.group(3),
@ -25,9 +27,10 @@ def parse_node_full(node_data):
'user': m.group(6), 'uid': m.group(7),
'lat': float(m.group(8)), 'lon': float(m.group(9))}
else:
print 'Error: cannot parse node data: ' + node_data + '.'
print(f"Error: cannot parse node data: {node_data}.")
return None
def parse_node(text):
"""
Just parse node identifier, latitude, and longitude.
@ -39,6 +42,7 @@ def parse_node(text):
lon = text[lon_index + 5:text.find('"', lon_index + 7)]
return {'id': int(node_id), 'lat': float(lat), 'lon': float(lon)}
def parse_way_full(way_data):
"""
Parse full way parameters using regular expressions: id, visible, version,
@ -51,9 +55,10 @@ def parse_way_full(way_data):
'changeset': m.group(4), 'timestamp': m.group(5),
'user': m.group(6), 'uid': m.group(7)}
else:
print 'Error: cannot parse way data: ' + way_data + '.'
print(f"Error: cannot parse way data: {way_data}.")
return None
def parse_way(text):
"""
Just parse way identifier.
@ -61,6 +66,7 @@ def parse_way(text):
id = text[4:text.find('"', 6)]
return {'id': int(id)}
def parse_relation(text):
"""
Just parse relation identifier.
@ -68,7 +74,8 @@ def parse_relation(text):
id = text[4:text.find('"', 6)]
return {'id': int(id)}
def parse_member(text):
def parse_member(text) -> Dict[str, Any]:
"""
Parse member type, reference, and role.
"""
@ -82,51 +89,56 @@ def parse_member(text):
role = text[role_index + 6:text.find('"', role_index + 8)]
return {'type': type, 'ref': int(ref), 'role': role}
def parse_tag(text):
def parse_tag(text: str) -> (str, str):
v_index = text.find('v="')
k = text[3:text.find('"', 4)]
v = text[v_index + 3:text.find('"', v_index + 4)]
return k, v
def parse_osm_file(file_name, parse_nodes=True, parse_ways=True,
parse_relations=True, full=False):
def parse_osm_file(
file_name: str, parse_nodes: bool = True, parse_ways: bool = True,
parse_relations: bool = True, full: bool = False):
start_time = datetime.datetime.now()
try:
node_map, way_map, relation_map = parse_osm_file_fast(file_name,
parse_nodes=parse_nodes, parse_ways=parse_ways,
parse_relations=parse_relations, full=full)
except Exception as e:
print e
print '\033[31mFast parsing failed.\033[0m'
a = raw_input('Do you want to use slow but correct XML parsing? [y/n] ')
print(e)
print("\033[31mFast parsing failed.\033[0m")
a = input("Do you want to use slow but correct XML parsing? [y/n] ")
if a in ['y', 'yes']:
start_time = datetime.datetime.now()
print 'Opening OSM file ' + file_name + '...'
input_file = open(input_file_name)
print 'Done.'
print 'Parsing OSM file ' + file_name + '...'
print(f"Opening OSM file {file_name}...")
with open(file_name) as input_file:
print('Done.')
print('Parsing OSM file ' + file_name + '...')
content = xml.dom.minidom.parse(input_file)
input_file.close()
print 'Done.'
print('Done.')
else:
sys.exit(0)
print 'File readed in ' + \
str(datetime.datetime.now() - start_time) + '.'
print 'Nodes: ' + str(len(node_map)) + ', ways: ' + \
str(len(way_map)) + ', relations: ' + str(len(relation_map)) + '.'
print('File readed in ' + \
str(datetime.datetime.now() - start_time) + '.')
print('Nodes: ' + str(len(node_map)) + ', ways: ' + \
str(len(way_map)) + ', relations: ' + str(len(relation_map)) + '.')
return node_map, way_map, relation_map
def parse_osm_file_fast(file_name, parse_nodes=True, parse_ways=True,
parse_relations=True, full=False):
node_map = {}
way_map = {}
relation_map = {}
print 'Line number counting for ' + file_name + '...'
print('Line number counting for ' + file_name + '...')
with open(file_name) as f:
for lines_number, l in enumerate(f):
pass
print 'Done.'
print 'Parsing OSM file ' + file_name + '...'
print('Done.')
print('Parsing OSM file ' + file_name + '...')
input_file = open(file_name)
line = input_file.readline()
line_number = 0

View file

@ -61,7 +61,7 @@ def get_icon(tags, scheme, fill='444444'):
if fill != '444444':
processed.add(color_name)
else:
print 'No color ' + tags[color_name] + '.'
print("No color {tags[color_name]}.")
if main_icon:
returned = [main_icon] + extra_icons, fill, processed
else:

217
roentgen/svg.py Normal file
View file

@ -0,0 +1,217 @@
"""
Very simple SVG library.
Author: Sergey Vartanov (me@enzet.ru)
"""
import math
class SVG:
def __init__(self, file_):
self.file = file_
self.index = 0
def begin(self, width, height):
self.file.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n\n')
self.file.write('''<svg version = "1.1" baseProfile = "full"
xmlns = "http://www.w3.org/2000/svg"
xmlns:xlink = "http://www.w3.org/1999/xlink"
xmlns:ev = "http://www.w3.org/2001/xml-events"
bgcolor = "#e5e9e6" width = "''' + str(width) +
'" height = "' + str(height) + '">\n')
def end(self):
self.file.write('</svg>\n')
def path(self, path, color='black', width=1, fill='none', end='butt', id=None, color2=None,
gx1=0, gy1=0, gx2=0, gy2=0, dash=None, dashoffset=None, opacity=1, transform=None):
if color2:
self.index += 1
self.file.write('<defs><linearGradient id = "grad' + str(self.index) + '" x1 = "' + str(gx1) + '" y1 = "' +
str(gy1) + ' " x2 = "' + str(gx2) + '" y2 = "' + str(gy2) +
'" gradientUnits="userSpaceOnUse"> <stop style="stop-color:' + self.get_color(color) +
';stop-opacity:1;" offset="0" /><stop style="stop-color:' + self.get_color(color2) +
';stop-opacity:1;" offset="1" /></linearGradient></defs>\n')
self.file.write(' <path d = "' + path + '" ')
if id:
self.file.write('id = "' + id + '" ')
if transform:
self.file.write('transform="' + transform + '" ')
self.file.write('style = "')
if not color2:
self.file.write('stroke:' + self.get_color(color) + '; ')
else:
self.file.write('stroke:url(#grad' + str(self.index) + '); ')
self.file.write('stroke-width:' + str(width) + '; ')
self.file.write('fill:' + self.get_color(fill) + '; ')
self.file.write('stroke-linecap:' + end + '; ')
if opacity != 1:
self.file.write('opacity:' + str(opacity) + '; ')
if dash:
self.file.write('stroke-dasharray:' + dash + '; ')
if dashoffset:
self.file.write('stroke-dashoffset:' + str(dashoffset) + '; ')
self.file.write('" />\n')
def line(self, x1, y1, x2, y2, width=1, color='black', end='butt', id=None, color2=None,
gx1=None, gy1=None, gx2=None, gy2=None, dash=None, dashoffset=None, opacity=None):
if color2:
if not gx1:
gx1 = x1
if not gy1:
gy1 = y1
if not gx2:
gx2 = x2
if not gy2:
gy2 = y2
self.index += 1
self.file.write('<defs><linearGradient id = "grad' + str(self.index) + '" x1 = "' + str(gx1) + '" y1 = "' + str(gy1) + '" x2 = "' + str(gx2) + '" y2 = "' + str(gy2) + '" gradientUnits="userSpaceOnUse">\n<stop\nstyle="stop-color:#' + str(color) + ';stop-opacity:1;"\noffset="0"\n /><stop\nstyle="stop-color:#' + str(color2) + ';stop-opacity:1;"\noffset="1"\n /></linearGradient>\n</defs>\n')
self.file.write(' <path d = "M ' + str(x1) + ',' + str(y1) + ' ' + str(x2) + ',' + str(y2) + '" ')
if id:
self.file.write('id = "' + id + '" ')
self.file.write('style = "')
if not color2:
self.file.write('stroke:' + self.get_color(color) + '; ')
else:
self.file.write('stroke:url(#grad' + str(self.index) + '); ')
self.file.write('stroke-width:' + str(width) + '; ')
self.file.write('stroke-linecap:' + end + '; ')
if dash:
self.file.write('stroke-dasharray:' + dash + '; ')
if dashoffset:
self.file.write('stroke-dashoffset:' + str(dashoffset) + '; ')
if opacity:
self.file.write('opacity: ' + str(opacity) + '; ')
self.file.write('" />\n')
def rect(self, x, y, width, height, color='black', id=None, rx=0, ry=0, opacity=1.0, stroke_color='none',
stroke_width=1.0):
self.file.write(' <rect x = "' + str(x) + '" y = "' + str(y) + '" rx = "' + str(rx) + '" ry = "' + str(ry) +
'" ')
self.file.write(' width = "' + str(width) + '" height = "' + str(height) + '" ')
if id:
self.file.write('id = "' + id + '" ')
self.file.write('style = "')
if opacity != 1:
self.file.write('opacity:' + str(opacity) + '; ')
self.file.write('fill:' + self.get_color(color) + '; ')
self.file.write('stroke:' + self.get_color(stroke_color) + '; ')
self.file.write('stroke-width:' + str(stroke_width) + '; ')
self.file.write('" />\n')
def curve(self, x1, y1, x2, y2, x3, y3, x4, y4, id=None, width=1, color='black'):
self.file.write(' <path d = "M ' + str(x1) + ',' + str(y1) + ' C ' + str(x2) + ',' + str(y2) + ' ')
self.file.write(str(x3) + ',' + str(y3) + ' ' + str(x4) + ',' + str(y4) + '" ')
self.file.write('style = "')
self.file.write('stroke:' + self.get_color(color) + '; ')
self.file.write('stroke-width:' + str(width) + '; ')
self.file.write('fill: none; ')
self.file.write('" />\n')
def rhombus(self, x, y, width, height, color='black', id=None):
self.file.write(''' <path d = "M %5.1f %5.1f L %5.1f %5.1f L %5.1f %5.1f
L %5.1f %5.1f L %5.1f %5.1f" ''' % (x, y - height, x + width, y, x, y + height, x - width, y, x,
y - height))
if id:
self.file.write('id = "' + id + '" ')
self.file.write('style = "')
self.file.write('fill:' + self.get_color(color) + '; ')
self.file.write('" />\n')
def circle(self, x, y, d, color='black', color2='white', fill='none',
opacity=None, width=1, id=None, gx=0, gy=0, gr=0, fx=0, fy=0):
is_grad = gx != 0 or gy != 0 or gr != 0
if is_grad:
self.index += 1
self.file.write('''<defs>
<radialGradient id="grad''' + str(self.index) + '" cx="' + str(gx) + '%" cy="' + str(gy) + '%" r="' + str(gr) + '%" fx="' + str(fx) + '%" fy="' + str(fy) + '''%">
<stop offset="0%" style="stop-color:rgb(255,255,255);
stop-opacity:0" />
<stop offset="100%" style="stop-color:''' + self.get_color(color) + ''';stop-opacity:1" />
</radialGradient>
</defs>''')
c = 0.577
self.file.write(''' <path d = "M %5.1f %5.1f C %5.1f %5.1f
%5.1f %5.1f %5.1f %5.1f C %5.1f %5.1f %5.1f %5.1f %5.1f
%5.1f C %5.1f %5.1f %5.1f %5.1f %5.1f %5.1f C %5.1f %5.1f
%5.1f %5.1f %5.1f %5.1f" ''' % (
x, y + d, x - d * c, y + d, x - d, y + d * c, x - d, y, x - d,
y - d * c, x - d * c, y - d, x, y - d, x + d * c, y - d, x + d,
y - d * c, x + d, y, x + d, y + d * c, x + d * c, y + d, x,
y + d))
if id:
self.file.write('id = "' + id + '" ')
self.file.write('style = "')
if is_grad:
self.file.write('fill:url(#grad' + str(self.index) + '); ')
else:
self.file.write('fill:' + self.get_color(fill) + '; ')
self.file.write('stroke-width:' + str(width) + '; ')
self.file.write('stroke:' + self.get_color(color) + '; ')
if opacity:
self.file.write('opacity:' + str(opacity) + '; ')
self.file.write('" />\n')
# text-align:center;text-anchor:middle
# font-weight:bold;font-stretch:normal;text-align:start;line-height:125%;letter-spacing:2px;
def text(self, x, y, text, font='Myriad Pro', size='10', align='left',
color='black', id=None, weight=None, letter_spacing=None, angle=None,
opacity=None):
"""
Drawing SVG <text> element.
"""
if angle is None:
self.file.write(' <text x = "' + str(x) + '" y = "' + str(y) + '" ')
else:
self.file.write(' <text x = "0" y = "0" ')
self.file.write('font-size = "' + str(size) + '" ')
if id:
self.file.write('id = "' + id + '" ')
if not (angle is None) and angle <= 0:
align = 'end'
if align == 'right':
align = 'end'
if align == 'center':
align = 'middle'
self.file.write('style = "')
self.file.write('text-anchor:' + align + '; ')
if opacity:
self.file.write('opacity:' + str(opacity) + '; ')
self.file.write('font-family: ' + font + '; ')
self.file.write('fill: ' + self.get_color(color) + '; ')
if weight == 'bold':
self.file.write('font-weight:bold; ')
if letter_spacing:
self.file.write('letter-spacing:' + str(letter_spacing) + '; ')
self.file.write('"')
if not (angle is None):
if math.sin(angle) > 0:
trans = 'transform = "matrix(' + str(math.sin(angle)) + ',' + str(math.cos(angle)) + ',' + \
str(-math.cos(angle)) + ',' + str(math.sin(angle)) + ',' + str(x) + ',' + str(y) + ')"'
else:
trans = 'transform = "matrix(' + str(math.sin(angle + math.pi)) + ',' + str(math.cos(angle + math.pi)) + ',' + \
str(-math.cos(angle + math.pi)) + ',' + str(math.sin(angle + math.pi)) + ',' + str(x) + ',' + str(y) + ')"'
self.file.write(' ' + trans)
self.file.write('>')
self.file.write(text)
self.file.write('</text>\n')
@staticmethod
def get_color(color):
if color == 'none':
return 'none'
if color == 'black':
return 'black'
return '#' + str(color)
def begin_layer(self, name):
self.file.write('<g id="' + name + '" label="' + name + '" ')
self.file.write('inkscape:groupmode="layer">\n')
def end_layer(self):
self.file.write('</g>\n')
def write(self, raw_code):
self.file.write(raw_code)

View file

@ -89,10 +89,10 @@ for icons_to_draw in to_draw:
if path:
icon_set['icons'].append({'path': path,
'x': (- 8.0 - xx * 16),
'y': (- 8.0 - yy * 16)});
'y': (- 8.0 - yy * 16)})
drawed = True
else:
print '\033[31m' + icon + '\033[0m'
print('\033[31m' + icon + '\033[0m')
if drawed:
icons.append(icon_set)
number += 1
@ -117,6 +117,6 @@ for icon in icons:
y += step
height += step
print 'Icons: ' + str(number) + '.'
print(f"Icons: {number}.")
output_file.end()

View file

@ -1,9 +1,6 @@
# -*- coding: utf-8 -*- from __future__ import unicode_literals
"""
Author: Sergey Vartanov (me@enzet.ru).
"""
import argparse
import sys

6
run.py Normal file
View file

@ -0,0 +1,6 @@
from roentgen.osm_getter import main
from roentgen.mapper import main
if __name__ == "__main__":
# main("2.374,48.843,2.378,48.846")
main()