From d5f6013b420b81006ef19fb26c48e3eda269682e Mon Sep 17 00:00:00 2001 From: "christian.demmler" Date: Wed, 24 Jan 2024 12:11:19 +0100 Subject: [PATCH 1/2] Fixed bug around spaces in avalanche names --- avaframeExport.py | 96 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 80 insertions(+), 16 deletions(-) diff --git a/avaframeExport.py b/avaframeExport.py index 52d3203..b0bb30a 100644 --- a/avaframeExport.py +++ b/avaframeExport.py @@ -2,12 +2,15 @@ import os from os import path import geopandas, pandas as pd -import gdal +from osgeo import gdal from shapely import wkb from datetime import datetime +import csv +import numpy as np def checkDir(dir): + dir=dir.replace(' ','') if not os.path.isdir(dir): - os.mkdir(dir) + os.makedirs(dir, exist_ok=True) return dir def checkPath(filePath): @@ -24,12 +27,13 @@ def checkPath(filePath): outPath='' return outPath -def grabRaster(config, outdir, design_event=False, constraint=''): +def grabRaster(config, outdir, design_event=False, projstr ='', constraint=''): if (design_event): schema = 'design' # Design refers to reference avalanches meant for designing mitigation measures etc. else: schema = 'public' # Public refers to actual, real-life avalanches - projstr = amaConnect.query("select * from getclosestconf('%s','projection_raster')"%config)['getclosestconf'][0] + if projstr == '': + projstr = amaConnect.query("select * from getclosestconf('%s','projection_raster')"%config)['getclosestconf'][0] #getting the output projection config as stored in the config table epsg = int(projstr.lower().replace('epsg:','').strip()) @@ -68,13 +72,62 @@ def grabRaster(config, outdir, design_event=False, constraint=''): print('skipping.') return len(pathList) +def writeConfig(eng, configfile, configname): + #reads CSV file from path , expects columns ['key', 'value'], using DB engine + #will enter the presented configuration value-pairs into DB table ext_project, using configuration + df = pd.read_csv(configfile, quoting=csv.QUOTE_ALL) + res = None + cols = df.columns.tolist() + if 'key' in cols and 'value' in cols: #seems like the reading worked + df['conf_group'] = configname + df.rename({'key': 'conf_key', 'value': 'conf_textvalue'}) + if not 'conf_numvalue' in cols: + df['conf_numvalue'] = np.nan + df['cfg_id'] = 'default' + if 'conf_key' in cols and 'conf_textvalue' in cols: + df['conf_textvalue']=df['conf_textvalue'].astype(str) + res = configEnter(eng,df, configname) -def grabEvents(config, outpath,design_event=False, constraint = '' ): + return res + +def configEnter(eng,df, configname): + #upserts configuration values from dataframe to ext_project table + qry = '' + schema = 'public' + table = 'ext_project' + valrep = {"'default'":"default", 'nan': 'NULL', "'NULL'": "NULL"} + for index, row in df.iterrows(): + colstr = amaConnector.listToPGstr(df.columns.tolist()) + valstr = amaConnector.listToPGstr(row.tolist(),r"'") + for key in valrep.keys(): + valstr = valstr.replace(key, valrep[key]) + + qry += "INSERT INTO %s.%s %s VALUES %s ON CONFLICT (""conf_group"", ""conf_key"", ""conf_numvalue"") DO UPDATE SET conf_textvalue = excluded.conf_textvalue;\n" %(schema, table, colstr, valstr) + eng.insertquery(qry) + return getconfig(eng, configname) + + +def getconfig(eng, configname): + return eng.query("with abc as (select getconfparents('%s') as configs) select conf_group, conf_key, conf_textvalue, conf_numvalue from ext_project, abc where conf_group = any(abc.configs)"%(configname)) + +def saveConfig(eng, configfile, configname): + #writes CSV file to path with columns ['key', 'value'], using DB engine + #will store the used configuration to disk for reference + configs = getconfig(eng, configname) + configs.to_csv(configfile, quoting=csv.QUOTE_ALL, index=False) + return configs + + + + + +def grabEvents(config, outpath,design_event=False, projstr = 'epsg:31287', constraint = '' ): if (design_event): schema = 'design' # Design refers to reference avalanches meant for designing mitigation measures etc. else: schema = 'public' # Public refers to actual, real-life avalanches - projstr = amaConnect.query("select * from getclosestconf('%s','projection_raster')" % config)['getclosestconf'][0] + if projstr == '': + projstr = amaConnect.query("select * from getclosestconf('%s','projection_raster')" % config)['getclosestconf'][0] # getting the output projection config as stored in the config table epsg = int(projstr.lower().replace('epsg:', '').strip()) @@ -84,6 +137,7 @@ def grabEvents(config, outpath,design_event=False, constraint = '' ): geometryCols = amaConnect.query("select * from getclosestconf('%s','exportgeometry')"%config) + #geometryCols = {'getclosestconf': ['geom_event_pt, geom_path_ln, geom_event_ln, geom_rel_pt, geom_rel_ln']} #this asks the database for the stored geometry column names which should get exported in the current configuration for index, event in eventList.iterrows(): @@ -99,8 +153,7 @@ def grabEvents(config, outpath,design_event=False, constraint = '' ): if not (geometry['geom'].isnull()[0]): #only work with existing geometries geometry['geom'] = geometry['geom'].apply(wkb.loads, hex=True) - attr = amaConnect.query( - "select * from %s.extractattributes(%d, '%s', '%s')" % (schema, event_id, col, config)) + attr = amaConnect.query("select * from %s.extractattributes(%d, '%s', '%s')" % (schema, event_id, col, config)) #this retrieves the selected set of attributes according to the current configuration and the current geometry column # one may set different (or none at all) attributes to be included in paths, release lines etc. attributes = pd.DataFrame([attr['value']]).rename(columns=attr['key']).reset_index(drop=True) @@ -124,45 +177,56 @@ def grabEvents(config, outpath,design_event=False, constraint = '' ): global amaConnect -outdir = '' -#outdir = r'c:\temp\my_export_dir' +#outdir = '' +outdir = r'c:\temp\my_export_dir' #if you want to, uncomment and set your own export dir above -this should point to an already created directory. #otherwise, the following function will create a valid, individual export directory based on the current time: exportdate = datetime.now().strftime('export_%Y-%m-%d_%H-%M-%S') if not os.path.isdir(outdir): outdir = path.join(os.path.dirname(os.getcwd()),exportdate) print('Using output directory %s'%outdir) - os.mkdir(outdir) + os.makedirs(outdir, exist_ok=True) #fill in configurations if necessary: configuration = 'avaframe' # this refers to the configuration variables, as set in the DB table "ext_project" ("External Project configuration" in QGIS) useDesignEvents = False #set to True if you want to export Design Events instead of real recorded Events -constraint = "where not(st_isempty(geom_rel_event_ln) or st_isempty(geom_event_ln))" #this constraint delivers all avalanches that have filled out release event line and event deposition line geometries. +constraint = "WHERE event_id = 582" #this constraint delivers only Event 582 (Gaiskogel Nordwest) + +accessFile = path.join(os.getcwd(),'fullaccess_ama.txt') +conffile = path.join(os.getcwd(), 'config.ini') +amaConnect = amaConnector.amaAccess(accessFile) -accessFile = path.join(os.getcwd(),'access.txt') +#writeConfig(amaConnect, conffile, 'tmpconf2' ) -amaConnect = amaConnector.amaAccess(accessFile) + + +# + +conf = saveConfig(amaConnect, conffile, configuration) try: with open(path.join(outdir,'log.txt'),'w') as file: file.write('Time: %s\n'%exportdate) file.write('Exporting db using export configuration "%s"\n'%configuration) + file.write('configuration saved to "%s"' % conffile) file.write('Design Events: %s\n' %useDesignEvents) file.write('Using constraint: "%s"\n'%constraint) except: print ("ERROR --------- no writing access to directory '%s'!"%outdir) #initAma(outpath, accessFile, ':') -eventsRes= grabEvents(configuration,outdir, useDesignEvents,constraint) -rasterRes= grabRaster(configuration,outdir, useDesignEvents,constraint) +projstr='' #possible override, use format 'epsg:4326' +eventsRes= grabEvents(configuration,outdir, useDesignEvents, projstr, constraint) +rasterRes= grabRaster(configuration,outdir, useDesignEvents,projstr, constraint) try: with open(path.join(outdir,'log.txt'),'a') as file: file.write('====Results====\n') file.write('Written %d events"\n'%eventsRes) file.write('Written %d path rasters\n'%rasterRes) + except: print ("ERROR --------- no writing access to directory '%s'!"%outdir) From 9ff3e42b76a7e661411779a9b610d3a61f096f72 Mon Sep 17 00:00:00 2001 From: "christian.demmler" Date: Wed, 24 Jan 2024 12:17:15 +0100 Subject: [PATCH 2/2] Fixed bug around spaces in avalanche names --- avaframeExport.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/avaframeExport.py b/avaframeExport.py index e6ed9ba..b0f917e 100644 --- a/avaframeExport.py +++ b/avaframeExport.py @@ -197,12 +197,6 @@ def grabEvents(amaConnect, config, outpath,design_event=False, projstr = 'epsg:3 conffile = path.join(os.getcwd(), 'config.ini') amaConnect = amaConnector.amaAccess(accessFile) -#writeConfig(amaConnect, conffile, 'tmpconf2' ) - - - - -# conf = saveConfig(amaConnect, conffile, configuration) try: