Skip to content

Commit c5feadd

Browse files
CharlesPikachuCharlesPikachu
committed
update configparser and loggerhandle
1 parent b075332 commit c5feadd

25 files changed

+274
-84
lines changed

‎csseg/configs/__init__.py

-2
This file was deleted.

‎csseg/configs/builder.py

-50
This file was deleted.

‎csseg/configs/ilt/base_cfg.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
'eval_interval_epochs': 10,
5252
'log_interval_iterations': 10,
5353
'choose_best_segmentor_by_metric': 'mean_iou',
54-
'logfilepath': '',
54+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': ''},
5555
'num_total_classes': -1,
5656
'fp16_cfg': {'type': 'pytorch', 'autocast': {}, 'grad_scaler': {}},
5757
'segmentor_cfg': SEGMENTOR_CFG,

‎csseg/configs/ilt/ilt_r101iabnd16_aspp_512x512_vocaug10-1_disjoint.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
'num_tasks': 11,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})

‎csseg/configs/ilt/ilt_r101iabnd16_aspp_512x512_vocaug10-1_overlap.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
'num_tasks': 11,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})

‎csseg/configs/ilt/ilt_r101iabnd16_aspp_512x512_vocaug15-1_disjoint.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
'num_tasks': 6,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})

‎csseg/configs/ilt/ilt_r101iabnd16_aspp_512x512_vocaug15-1_overlap.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
'num_tasks': 6,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})

‎csseg/configs/ilt/ilt_r101iabnd16_aspp_512x512_vocaug15-5_disjoint.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
'num_tasks': 2,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})

‎csseg/configs/ilt/ilt_r101iabnd16_aspp_512x512_vocaug15-5_overlap.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
'num_tasks': 2,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})

‎csseg/configs/mib/base_cfg.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
'eval_interval_epochs': 10,
5151
'log_interval_iterations': 10,
5252
'choose_best_segmentor_by_metric': 'mean_iou',
53-
'logfilepath': '',
53+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': ''},
5454
'num_total_classes': -1,
5555
'fp16_cfg': {'type': 'pytorch', 'autocast': {}, 'grad_scaler': {}},
5656
'segmentor_cfg': SEGMENTOR_CFG,

‎csseg/configs/mib/mib_r101iabnd16_aspp_512x512_vocaug10-1_disjoint.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@
2626
'num_tasks': 11,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})
3131
RUNNER_CFG['segmentor_cfg']['losses_cfgs']['distillation']['scale_factor'] = 100

‎csseg/configs/mib/mib_r101iabnd16_aspp_512x512_vocaug10-1_overlap.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@
2626
'num_tasks': 11,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})
3131
RUNNER_CFG['segmentor_cfg']['losses_cfgs']['distillation']['scale_factor'] = 100

‎csseg/configs/mib/mib_r101iabnd16_aspp_512x512_vocaug15-1_disjoint.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@
2626
'num_tasks': 6,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})
3131
RUNNER_CFG['segmentor_cfg']['losses_cfgs']['distillation']['scale_factor'] = 100

‎csseg/configs/mib/mib_r101iabnd16_aspp_512x512_vocaug15-1_overlap.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@
2626
'num_tasks': 6,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})
3131
RUNNER_CFG['segmentor_cfg']['losses_cfgs']['distillation']['scale_factor'] = 100

‎csseg/configs/mib/mib_r101iabnd16_aspp_512x512_vocaug15-5_disjoint.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
'num_tasks': 2,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})

‎csseg/configs/mib/mib_r101iabnd16_aspp_512x512_vocaug15-5_overlap.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@
2626
'num_tasks': 2,
2727
'num_total_classes': 21,
2828
'work_dir': os.path.split(__file__)[-1].split('.')[0],
29-
'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log",
29+
'logger_handle_cfg': {'type': 'LocalLoggerHandle', 'logfilepath': f"{os.path.split(__file__)[-1].split('.')[0]}/{os.path.split(__file__)[-1].split('.')[0]}.log"},
3030
})

‎csseg/modules/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
)
77
from .utils import (
88
setrandomseed, saveckpts, loadckpts, touchdir, saveaspickle, loadpicklefile, symlink, loadpretrainedweights,
9-
BaseModuleBuilder, Logger
9+
BaseModuleBuilder, EnvironmentCollector, ConfigParser, LoggerHandleBuilder, BuildLoggerHandle
1010
)
1111
from .models import (
1212
BuildLoss, LossBuilder, BuildDecoder, DecoderBuilder, BuildOptimizer, OptimizerBuilder, BuildParamsConstructor, ParamsConstructorBuilder,

‎csseg/modules/runners/base.py

+3-5
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from ..models import BuildSegmentor, BuildOptimizer, BuildScheduler
2020
from ..parallel import BuildDistributedDataloader, BuildDistributedModel
2121
from torch.distributed.algorithms.ddp_comm_hooks import default as comm_hooks
22-
from ..utils import Logger, touchdir, loadckpts, saveckpts, saveaspickle, symlink, loadpicklefile
22+
from ..utils import BuildLoggerHandle, touchdir, loadckpts, saveckpts, saveaspickle, symlink, loadpicklefile
2323

2424

2525
'''BaseRunner'''
@@ -45,7 +45,7 @@ def __init__(self, mode, cmd_args, runner_cfg):
4545
touchdir(dirname=self.root_work_dir)
4646
touchdir(dirname=self.task_work_dir)
4747
# build logger handle
48-
self.logger_handle = Logger(logfilepath=runner_cfg['logfilepath'])
48+
self.logger_handle = BuildLoggerHandle(logger_handle_cfg=runner_cfg['logger_handle_cfg'])
4949
# build datasets
5050
dataset_cfg = runner_cfg['dataset_cfg']
5151
train_set = BuildDataset(mode='TRAIN', task_name=runner_cfg['task_name'], task_id=runner_cfg['task_id'], dataset_cfg=dataset_cfg) if mode == 'TRAIN' else None
@@ -250,9 +250,7 @@ def test(self, cur_epoch):
250250
def state(self):
251251
state_dict = self.scheduler.state()
252252
state_dict.update({
253-
'best_score': self.best_score,
254-
'segmentor': self.segmentor.state_dict(),
255-
'task_id': self.runner_cfg['task_id'],
253+
'best_score': self.best_score, 'segmentor': self.segmentor.state_dict(), 'task_id': self.runner_cfg['task_id'],
256254
})
257255
if self.fp16_type in ['pytorch']:
258256
state_dict.update({'grad_scaler': self.grad_scaler.state_dict()})

‎csseg/modules/utils/__init__.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
'''initialize'''
2-
from .logger import Logger
32
from .misc import setrandomseed
3+
from .env import EnvironmentCollector
4+
from .configparser import ConfigParser
45
from .modulebuilder import BaseModuleBuilder
6+
from .logger import LoggerHandleBuilder, BuildLoggerHandle
57
from .io import saveckpts, loadckpts, touchdir, saveaspickle, loadpicklefile, symlink, loadpretrainedweights

‎csseg/modules/utils/configparser.py

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
'''
2+
Function:
3+
Implementation of ConfigParser
4+
Author:
5+
Zhenchao
6+
'''
7+
import os
8+
import sys
9+
import importlib
10+
import importlib.util
11+
import dill as pickle
12+
from .io import touchdir
13+
14+
15+
'''ConfigParser'''
16+
class ConfigParser():
17+
def __init__(self, library_name='csseg'):
18+
self.library_name = library_name
19+
'''parsefrompy'''
20+
def parsefrompy(self, cfg_file_path):
21+
# assert
22+
assert cfg_file_path.endswith('.py')
23+
# obtain module path
24+
module_path = cfg_file_path[len(os.getcwd()):].replace('\\', '/')
25+
module_path = module_path.replace('/', '.')
26+
module_path = module_path.strip('.')[:-3]
27+
# load cfg
28+
spec = importlib.util.spec_from_file_location(module_path, cfg_file_path)
29+
cfg = importlib.util.module_from_spec(spec)
30+
sys.modules[module_path] = cfg
31+
spec.loader.exec_module(cfg)
32+
# return cfg
33+
return cfg, cfg_file_path
34+
'''parsefrompkl'''
35+
def parsefrompkl(self, cfg_file_path):
36+
cfg = pickle.load(open(cfg_file_path, 'rb'))
37+
return cfg, cfg_file_path
38+
'''parse'''
39+
def parse(self, cfg_file_path):
40+
# ext to parse method
41+
ext_to_parse_method = {
42+
'.py': self.parsefrompy, '.pkl': self.parsefrompkl,
43+
}
44+
# config ext
45+
cfg_ext = os.path.splitext(cfg_file_path)[-1]
46+
# assert
47+
assert cfg_ext in ext_to_parse_method, f'unable to parse config with extension {cfg_ext}'
48+
# parse
49+
return ext_to_parse_method[cfg_ext](cfg_file_path=cfg_file_path)
50+
'''save'''
51+
def save(self, work_dir=''):
52+
work_dir = os.path.join(work_dir, 'configs')
53+
touchdir(work_dir)
54+
savepath = os.path.join(work_dir, os.path.basename(self.cfg_file_path) + '.pkl')
55+
return pickle.dump(self.cfg, open(savepath, 'wb'))
56+
'''call'''
57+
def __call__(self, cfg_file_path):
58+
cfg_file_path = os.path.abspath(os.path.expanduser(cfg_file_path))
59+
self.cfg, self.cfg_file_path = self.parse(cfg_file_path=cfg_file_path)
60+
return self.cfg, self.cfg_file_path

0 commit comments

Comments
 (0)