Freifunk-Troisdorf/yanic
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[BUGFIX] yanic-import-timestamp: properly parse timestamps with strptime (#67)

Browse Source 
Also some additional refactoring and more verbosity.
This commit is contained in:
hexa- 2017-06-14 09:47:42 +02:00 committed by Geno
parent 7f554bd6d6
commit de968473cd
1 changed files with 66 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
contrib/yanic-import-timestamp
View File

@ -1,44 +1,82 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import json
import argparse import argparse
import json
import os import os
import sys import sys
from datetime import datetime
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-n', '--nodesjson', action='store', parser.add_argument('-n', '--nodesjson', action='store', required=True,
help='old nodes.json file you want to read firstseen from',required=True) help='old nodes.json file you want to read firstseen from')
parser.add_argument('-s', '--state', action='store', parser.add_argument('-s', '--state', action='store', required=True,
help='state.json you want to store',required=True) help='state.json you want to store')
args = parser.parse_args() args = parser.parse_args()
options = vars(args)
oldnodes_fn = os.path.realpath(options['nodesjson']) with open(os.path.realpath(args.nodesjson), encoding='UTF-8') as handle:
newnodes_fn = os.path.realpath(options['state']) legacy_db = json.load(handle)
with open(oldnodes_fn, 'r', encoding=('UTF-8')) as oldnodedb_handle: # w/o a version tag we cannot decide how to walk and interpret the nodes.json
oldnodedb = json.load(oldnodedb_handle) assert ('version' in legacy_db)
with open(newnodes_fn, 'r', encoding=('UTF-8')) as newnodedb_handle:
newnodedb = json.load(newnodedb_handle)
count = 0 yanic_db_path = os.path.realpath(args.state)
if oldnodedb['version'] == 1: with open(yanic_db_path, encoding='UTF-8') as handle:
for nodeid, node in newnodedb['nodes'].items(): yanic_db = json.load(handle)
for oldnodeid, oldnode in oldnodedb['nodes'].items():
if oldnodeid == nodeid:
node['firstseen'] = "{}+0100".format(oldnode['firstseen'])
count+=1
if oldnodedb['version'] == 2: total = 0
for nodeid, node in newnodedb['nodes'].items(): updated = 0
for oldnode in oldnodedb['nodes']: yanic_date_format = '%Y-%m-%dT%H:%M:%S+0000'
if oldnode['nodeinfo']['node_id'] == nodeid: v1_date_format = '%Y-%m-%dT%H:%M:%S.%f' # 2017-05-31T18:30:19.759610
node['firstseen'] = "{}+0100".format(oldnode['firstseen']) v2_date_format = '%Y-%m-%dT%H:%M:%S.%fZ' # 2015-08-22T16:05:02.000Z
count+=1 version = legacy_db['version']
with open(newnodes_fn, 'w') as f: print('nodes.json is in v{} format'.format(version))
json.dump(newnodedb, f)
print('firstseen updated on %d nodes' % count) fallback_date_format = None
if version == 1:
legacy_date_format = v1_date_format # ffmap-backend
elif version == 2:
legacy_date_format = v2_date_format # hopglass
fallback_date_format = v1_date_format # ffmap-backend
else:
print('unhandled nodes.json version number!', file=sys.stderr)
sys.exit(1)
for nodeid, node in yanic_db.get('nodes', {}).items():
legacy_node = None
if version == 1:
# v1 nodes.json is a dict, so lookups are cheap
legacy_node = legacy_db.get('nodes', {}).get(nodeid, None)
elif version == 2:
# v2 nodes.json however carries nodes as a list of dicts …
legacy_node = next((
candidate for candidate in legacy_db.get('nodes', [])
if candidate['nodeinfo']['node_id'] == nodeid), None)
if legacy_node is not None:
try:
dt = datetime.strptime(legacy_node['firstseen'], legacy_date_format)
except ValueError as ex:
# time format mismatch, try fallback format
if fallback_date_format is not None:
dt = datetime.strptime(legacy_node['firstseen'],
fallback_date_format)
else:
# or if none is set, re-raise the original exception
raise ex
dts = dt.strftime(yanic_date_format)
if node['firstseen'] != dts:
node['firstseen'] = dts
updated += 1
total += 1
with open(yanic_db_path, 'w') as f:
json.dump(yanic_db, f)
print('found {} nodes, {} changed.'.format(total, updated))