diff --git a/README.md b/README.md index 7fe76ba..96fca45 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,10 @@ An **unofficial** PyTorch implementation of the paper "Skeleton-Based Action Rec ## Dependencies -- Python >= 3.5 +- Python >= 3.6 - scipy >= 1.3.0 - numpy >= 1.16.4 -- PyTorch >= 1.1.0 +- PyTorch >= 1.4.0 - tensorboardX >= 1.8 (For logging) ## Directory Structure @@ -35,8 +35,7 @@ Most of the interesting stuff can be found in: 3. Generate the joint dataset first: ```bash -cd data_gen -python3 ntu_gen_joint_data.py +python3 ./data_gen/ntu_gen_joint_data.py ``` Specify the data location if the raw skeletons data are placed somewhere else. The default looks at `./data/nturgbd_raw/`. @@ -44,13 +43,13 @@ Specify the data location if the raw skeletons data are placed somewhere else. T 4. Then, in `data_gen/`, generate the bone dataset: ```bash -python3 ntu_gen_bone_data.py +python3 ./data_gen/ntu_gen_bone_data.py ``` 5. Finally, generate the motion data from joints/bones: ```bash -python3 ntu_gen_motion_data.py +python3 ./data_gen/ntu_gen_motion_data.py ``` The generation scripts look for generated data in previous step. By default they look at `./data`; change dir configs if needed. diff --git a/config/node_config.yaml b/config/node_config.yaml new file mode 100644 index 0000000..8c30432 --- /dev/null +++ b/config/node_config.yaml @@ -0,0 +1,4 @@ +# node_mask: [1, 2, 5, 9, 13, 14, 17, 18, 21] # quarter joint +# node_mask: [1, 2, 3, 4, 5, 6, 9, 10, 13, 14, 17, 18, 21] # harf joint +# selected_action: ['A004', 'A005', 'A006', 'A007', 'A008', 'A009', 'A010', 'A027', 'A031', 'A059'] +selected_action: ['A004', 'A005'] \ No newline at end of file diff --git a/data_gen/ntu_gen_bone_data.py b/data_gen/ntu_gen_bone_data.py index 53bffcb..0ca04bf 100644 --- a/data_gen/ntu_gen_bone_data.py +++ b/data_gen/ntu_gen_bone_data.py @@ -4,27 +4,29 @@ from tqdm import tqdm +# Joint 연관 관계 paris = { - 'ntu/xview': ( + 'ntu/xview': [ (1, 2), (2, 21), (3, 21), (4, 3), (5, 21), (6, 5), (7, 6), (8, 7), (9, 21), (10, 9), (11, 10), (12, 11), (13, 1), (14, 13), (15, 14), (16, 15), (17, 1), (18, 17), (19, 18), (20, 19), (22, 23), (21, 21), (23, 8), (24, 25), (25, 12) - ), - 'ntu/xsub': ( + ], + + 'ntu/xsub': [ (1, 2), (2, 21), (3, 21), (4, 3), (5, 21), (6, 5), (7, 6), (8, 7), (9, 21), (10, 9), (11, 10), (12, 11), (13, 1), (14, 13), (15, 14), (16, 15), (17, 1), (18, 17), (19, 18), (20, 19), (22, 23), (21, 21), (23, 8), (24, 25), (25, 12) - ), + ], - 'kinetics': ( + 'kinetics': [ (0, 0), (1, 0), (2, 1), (3, 2), (4, 3), (5, 1), (6, 5), (7, 6), (8, 2), (9, 8), (10, 9), (11, 5), (12, 11), (13, 12), (14, 0), (15, 0), (16, 14), (17, 15) - ) + ] } sets = {'train', 'val'} @@ -32,24 +34,32 @@ def gen_bone_data(): """Generate bone data from joint data for NTU skeleton dataset""" - for dataset in datasets: - for set in sets: + for dataset in datasets: # 'ntu/xview', 'ntu/xsub' + for set in sets: # 'train', 'val' print(dataset, set) - data = np.load('../data/{}/{}_data_joint.npy'.format(dataset, set)) - N, C, T, V, M = data.shape + + # 앞에서 생성한 Joint data Load + data = np.load('./data/{}/{}_data_joint.npy'.format(dataset, set)) + N, C, T, V, M = data.shape # channels (C), frames (T), nodes (V), persons (M) + + # Bone 데이터 저장할 파일 생성. Joint 데이터와 Shape를 같게. fp_sp = open_memmap( - '../data/{}/{}_data_bone.npy'.format(dataset, set), + './data/{}/{}_data_bone.npy'.format(dataset, set), dtype='float32', mode='w+', shape=(N, 3, T, V, M)) - - # Copy the joints data to bone placeholder tensor - fp_sp[:, :C, :, :, :] = data - for v1, v2 in tqdm(paris[dataset]): + + # Joint data를 fp_sp에 복사 + fp_sp[:, :C, :, :, :] = data # Deep copy + + for v1, v2 in tqdm(paris[dataset]): # dataset : 'ntu/xview', 'ntu/xsub' # Reduce class index for NTU datasets + + # Joint 번호는 1부터 시작하지만, 인덱스는 0부터 시작하기 때문 if dataset != 'kinetics': v1 -= 1 v2 -= 1 + # Assign bones to be joint1 - joint2, the pairs are pre-determined and hardcoded # There also happens to be 25 bones fp_sp[:, :, :, v1, :] = data[:, :, :, v1, :] - data[:, :, :, v2, :] diff --git a/data_gen/ntu_gen_joint_data.py b/data_gen/ntu_gen_joint_data.py index eedc31e..b8f464e 100644 --- a/data_gen/ntu_gen_joint_data.py +++ b/data_gen/ntu_gen_joint_data.py @@ -3,11 +3,12 @@ import pickle import argparse import numpy as np +import yaml from tqdm import tqdm import sys sys.path.extend(['../']) -from data_gen.preprocess import pre_normalization +from preprocess import pre_normalization # For Cross-Subject benchmark "xsub" training_subjects = [1, 2, 4, 5, 8, 9, 13, 14, 15, 16, 17, 18, 19, 25, 27, 28, 31, 34, 35, 38] @@ -72,8 +73,8 @@ def get_nonzero_std(s): s = 0 return s - -def read_xyz(file, max_body=4, num_joint=25): # 取了前两个body +# 관절의 x,y,z 읽어오기 +def read_xyz(file, max_body=4, num_joint=25): seq_info = read_skeleton_filter(file) # Create data tensor of shape: (# persons (M), # frames (T), # nodes (V), # channels (C)) data = np.zeros((max_body, seq_info['numFrame'], num_joint, 3)) @@ -87,34 +88,53 @@ def read_xyz(file, max_body=4, num_joint=25): # 取了前两个body energy = np.array([get_nonzero_std(x) for x in data]) index = energy.argsort()[::-1][0:max_body_true] data = data[index] - # Data new shape: (C, T, V, M) + + # Data new shape: (C, T, V, M) (# channels (C), # frames (T), # nodes (V), # persons (M)) data = data.transpose(3, 1, 2, 0) return data -def gendata(data_path, out_path, ignored_sample_path=None, benchmark='xview', part='eval'): +def gendata(data_path, out_path, node_mask=None, attention_action=None, ignored_sample_path=None, + benchmark='xview', part='eval'): + + ########################################### 데이터 선별 ########################################### + # attention action, ignored_sample_path, benchmark, part에 따라 Train 데이터와 Test 데이터를 선별 + # 선별된 데이터의 파일 이름과 Label을 List로 저장 + ################################################################################################### + # ignored_sample_path if ignored_sample_path != None: with open(ignored_sample_path, 'r') as f: ignored_samples = [line.strip() + '.skeleton' for line in f.readlines()] else: ignored_samples = [] - sample_name = [] - sample_label = [] + sample_name = [] # benchmark와 part에 따라 선택되는 데이터 파일의 이름을 저장하는 List + sample_label = [] # benchmark와 part에 따라 선택되는 데이터 파일의 라벨을 저장하는 List + + # Label data 생성 for filename in os.listdir(data_path): + # ignored_sample_path가 입력으로 들어오면 그 데이터는 Pass if filename in ignored_samples: continue + + # attention_action에 입력이 있으면 그 action class만 데이터 생성 + if attention_action != None : + if filename[filename.find('A'):filename.find('A') + 4] not in attention_action : + continue + action_class = int(filename[filename.find('A') + 1:filename.find('A') + 4]) subject_id = int(filename[filename.find('P') + 1:filename.find('P') + 4]) camera_id = int(filename[filename.find('C') + 1:filename.find('C') + 4]) + # View와 Sub의 Train 데이터셋 분리하는 부분 if benchmark == 'xview': - istraining = (camera_id in training_cameras) + istraining = (camera_id in training_cameras) # istraining은 bool type elif benchmark == 'xsub': - istraining = (subject_id in training_subjects) + istraining = (subject_id in training_subjects) # istraining은 bool type else: raise ValueError('Invalid benchmark provided: {}'.format(benchmark)) + # Train과 Val에 따라 데이터 선택 if part == 'train': issample = istraining elif part == 'val': @@ -122,36 +142,91 @@ def gendata(data_path, out_path, ignored_sample_path=None, benchmark='xview', pa else: raise ValueError('Invalid data part provided: {}'.format(part)) - if issample: + # 선택된 Sample들의 Filename과 라벨 데이터를 list로 저장 + if issample: sample_name.append(filename) - sample_label.append(action_class - 1) - + + # attention action이 있으면, 라벨들이 앞으로 당겨지게끔 설정 + if attention_action != None : + sample_label.append(attention_action.index(filename[filename.find('A'):filename.find('A') + 4])) + + # attention action이 없으면, filename 안의 Action number가 라벨 + else: + sample_label.append(action_class - 1) + + # 선택된 Sample들의 이름과 라벨 저장 with open('{}/{}_label.pkl'.format(out_path, part), 'wb') as f: pickle.dump((sample_name, list(sample_label)), f) - # Create data tensor with shape (# examples (N), C, T, V, M) + ########################################## 데이터 전처리 ########################################## + # 위에서 선별된 File name에 맞는 데이터를 Load하여 N, C, T, V, M의 joint 데이터를 Load + # Joint data Normalize + # node_mask로 선택한 node만 npy 파일로 데이터 저장 + ################################################################################################### + # Joint data 생성 + # Joint data를 저장할 임시 변수 생성. tensor with shape (# examples (N), C=3, T=300, V=25, M=2) fp = np.zeros((len(sample_label), 3, max_frame, num_joint, max_body_true), dtype=np.float32) - # Fill in the data tensor `fp` one training example a time + # Sample File에서 Joint Data를 읽어와서 임시 변수(fp)에 저장 for i, s in enumerate(tqdm(sample_name)): - data = read_xyz(os.path.join(data_path, s), max_body=max_body_kinect, num_joint=num_joint) - fp[i, :, :data.shape[1], :, :] = data - - fp = pre_normalization(fp) + # Data new shape: (C, T, V, M) + data = read_xyz(os.path.join(data_path, s), max_body=max_body_kinect, num_joint=num_joint) # (C, T, V, M) + fp[i, :, :data.shape[1], :, :] = data # (N, C, T, V, M) + + # fp normalize + fp = pre_normalization(fp) # (N, C=3, T=300, V=25, M=2) + + + # 특정 노드만 가져오기 + if node_mask != None : + node_mask = node_mask - np.ones([len(node_mask)], dtype=int) + fp = fp[:, :, :, node_mask, :] + + # Joint data 저장 np.save('{}/{}_data_joint.npy'.format(out_path, part), fp) + if __name__ == '__main__': parser = argparse.ArgumentParser(description='NTU-RGB-D Data Converter.') - parser.add_argument('--data_path', default='../data/nturgbd_raw/nturgb+d_skeletons/') + parser.add_argument('--data_path', default='/mnt/disk2/data/private_data/NTU_RGB+D/nturgb+d_skeletons/nturgbd_skeletons_s001_to_s017/') parser.add_argument('--ignored_sample_path', - default='../data/nturgbd_raw/samples_with_missing_skeletons.txt') - parser.add_argument('--out_folder', default='../data/ntu/') + default='./data/nturgbd_raw/samples_with_missing_skeletons.txt') + parser.add_argument('--out_folder', default='./data/ntu/') + parser.add_argument( + '--node_mask', + default=None, + help='사용할 노드') # 사용할 노드 + parser.add_argument( + '--selected_action', + default=None, + help='action to train') # 사용할 Action 번호 + parser.add_argument( + '--config', + default='./config/node_config.yaml', + help='path to the configuration file') # config 파일이 위치하는 디렉토리 benchmarks = ['xsub', 'xview'] parts = ['train', 'val'] arg = parser.parse_args() + if arg.config is not None: + with open(arg.config, 'r') as f: + default_arg = yaml.load(f) # config 파일에 들어있는 keys. config key로 명명 + + # 예외 처리 + key = vars(arg).keys() # parser에 들어있는 key값. default key로 명명 + for k in default_arg.keys(): + if k not in key: # config key가 default key에 들어있지 않으면 + print('WRONG ARG: {}'.format(k)) # default key에 해당 키가 없음을 알림. + assert (k in key) + + parser.set_defaults(**default_arg) # 임의의 개수의 Keyword arguments를 받아서 default key -> config key로 변경 + + arg = parser.parse_args() # config key가 반영된 argument + + print("arg : ", arg) + for b in benchmarks: for p in parts: out_path = os.path.join(arg.out_folder, b) @@ -159,8 +234,10 @@ def gendata(data_path, out_path, ignored_sample_path=None, benchmark='xview', pa os.makedirs(out_path) print(b, p) gendata( - arg.data_path, - out_path, - arg.ignored_sample_path, - benchmark=b, - part=p) + arg.data_path, # 전처리할 Skeleton data가 위치한 경로. NTU RGB 데이터를 의미 + out_path, # 전처리된 데이터가 위치할 공통 경로. + node_mask=arg.node_mask, # node mask. 25개의 Joint를 다 사용하지 않을 경우 사용할 Joint를 List 형태로 넣으면 된다. + attention_action = arg.selected_action, # 특정 Action에 대해서만 데이터 생성. 원하는 Action을 List 형태로 넣으면 된다. + ignored_sample_path = arg.ignored_sample_path, # 무시할 Sample 데이터가 적혀진 txt파일의 경로. + benchmark=b, # xsub / xview + part=p) # train할지, validation할지 diff --git a/data_gen/ntu_gen_motion_data.py b/data_gen/ntu_gen_motion_data.py index 1f0b873..e1dbc28 100644 --- a/data_gen/ntu_gen_motion_data.py +++ b/data_gen/ntu_gen_motion_data.py @@ -8,30 +8,35 @@ datasets = {'ntu/xview', 'ntu/xsub'} parts = {'joint', 'bone'} - def gen_motion_data(): - for dataset in datasets: - for set in sets: - for part in parts: - fn = '../data/{}/{}_data_{}.npy'.format(dataset, set, part) + for dataset in datasets: # 'ntu/xview', 'ntu/xsub' + for set in sets: # 'train', 'val' + for part in parts: # 'joint', 'bone' + fn = './data/{}/{}_data_{}.npy'.format(dataset, set, part) + + # Joint, Bone 파일 존재하지 않으면 for문 continue 실행 if not os.path.exists(fn): print('Joint/bone data does not exist for {} {} set'.format(dataset, set)) continue + # Joint, Bone 파일이 존재하면 print('Generating motion data for', dataset, set, part) + + # Joint, Bone 데이터 Load data = np.load(fn) (N, C, T, V, M) = data.shape fp_sp = open_memmap( - '../data/{}/{}_data_{}_motion.npy'.format(dataset, set, part), + './data/{}/{}_data_{}_motion.npy'.format(dataset, set, part), dtype='float32', mode='w+', shape=data.shape) + # Motion데이터는 dt동안 Joint가 움직인 거리. m = v(t+1) - v(t) # Loop through frames and insert motion difference for t in tqdm(range(T - 1)): fp_sp[:, :, t, :, :] = data[:, :, t + 1, :, :] - data[:, :, t, :, :] - # Pad last frame with 0 + # 마지막 한 프레임은 0으로 채움 fp_sp[:, :, T - 1, :, :] = 0 diff --git a/data_gen/preprocess.py b/data_gen/preprocess.py index 3e2dac8..39d7b65 100644 --- a/data_gen/preprocess.py +++ b/data_gen/preprocess.py @@ -2,62 +2,98 @@ sys.path.extend(['../']) import numpy as np -from data_gen.rotation import angle_between, rotation_matrix +from rotation import angle_between, rotation_matrix from tqdm import tqdm - def pre_normalization(data, zaxis=[0, 1], xaxis=[8, 4]): + # examples (N), channels (C), frames (T), nodes (V), persons (M)) N, C, T, V, M = data.shape s = np.transpose(data, [0, 4, 2, 3, 1]) # to (N, M, T, V, C) - + + ################################################################################################################### + # 1. 스켈레톤 데이터에서 중간 Frame부터 동작이 있는 경우 데이터 Shift + # 2. 스켈레톤 데이터에서 뒷부분에 동작이 없는 경우, 유효한 데이터를 여러번 반복 print('pad the null frames with the previous frames') - for i_s, skeleton in enumerate(tqdm(s)): # Dimension N - if skeleton.sum() == 0: + for i_s, skeleton in enumerate(tqdm(s)): # Dimension N # skeleton : (M, T, V, C) (2, 300, 25, 3) + + # skeleton 데이터가 전부 0으로 되어 있는 데이터 찾기. - 잘못된 데이터 검출 + # continue가 빠진 것 같음 + if skeleton.sum() == 0: print(i_s, ' has no skeleton') - for i_p, person in enumerate(skeleton): # Dimension M (# person) - # `person` has shape (T, V, C) - if person.sum() == 0: + + # 전체 스켈레톤 데이터에 대해 + for i_p, person in enumerate(skeleton): # Dimension M (# person) # person : (T, V, C) (300, 25, 3) + # Person 데이터가 전부 0인 경우(Person 1과 Person 2가 전부 0)이면 넘기기 + if person.sum() == 0: # person continue + + # 1. Person의 첫번째 프레임이 0인 경우. 즉, 프레임 초반에는 데이터가 없는 경우. if person[0].sum() == 0: # `index` of frames that have non-zero nodes + # frames의 합 != 0인 인덱스 찾기. 즉, T가 30일 때, Frame이 5부터 10까지인 데이터는 5~10의 인덱스 반환. index = (person.sum(-1).sum(-1) != 0) - tmp = person[index].copy() + tmp = person[index].copy() # 프레임 임시 저장 # Shift non-zero nodes to beginning of frames person *= 0 - person[:len(tmp)] = tmp - for i_f, frame in enumerate(person): - # Each frame has shape (V, C) + person[:len(tmp)] = tmp # shift + + # 전체 스켈레톤 데이터의 Node와 Channel에 대해 + for i_f, frame in enumerate(person): # Each frame : (V, C) + # 한 프레임에 대해서 데이터가 0이면(스켈레톤 데이터가 없으면) if frame.sum() == 0: + # 그 프레임의 데이터부터 끝까지 데이터가 0이면(스켈레톤 데이터가 없으면) if person[i_f:].sum() == 0: - # Repeat all the frames up to now (`i_f`) till the max seq len - rest = len(person) - i_f - reps = int(np.ceil(rest / i_f)) - pad = np.concatenate([person[:i_f] for _ in range(reps)], 0)[:rest] + # 2. Repeat all the frames up to now (`i_f`) till the max seq len + rest = len(person) - i_f # 나머지 프레임 갯수 + reps = int(np.ceil(rest / i_f)) # 같은 동작을 몇번 반복할 것인지. + pad = np.concatenate([person[:i_f] for _ in range(reps)], 0)[:rest] # 프레임 수가 넘어가면 뒷부분 자르기 s[i_s, i_p, i_f:] = pad break - - print('sub the center joint #1 (spine joint in ntu and neck joint in kinetics)') + ################################################################################################################### + + # Joint 데이터를 x, y, z = 0, 0, 0 좌표 근처로 이동 + print('subtract the center joint #1 (spine joint in ntu and neck joint in kinetics)') for i_s, skeleton in enumerate(tqdm(s)): + # Person 데이터가 전부 0인 경우(Person 1과 Person 2가 전부 0)이면 넘기기 if skeleton.sum() == 0: continue - # Use the first skeleton's body center (`1:2` along the nodes dimension) - main_body_center = skeleton[0][:, 1:2, :].copy() # Shape (T, 1, C) - for i_p, person in enumerate(skeleton): - if person.sum() == 0: + + # Use the first skeleton's body center (`1:2` along the nodes dimension) + # 첫번째 Person의 복부 부분의 Joint를 Main body center로 저장(2번 Joint) + ''' + 이상한 게 첫번째 프레임의 2번 Joint 좌표를 중심으로 보는 것이 아니라 + 모든 프레임에서 2번 Joint 좌표를 중심으로 바라봄. + 이 경우, 달리기는 제자리기 뛰기가 되는 단점이 존재할 것 같음. + ''' + + main_body_center = skeleton[0][:, 1:2, :].copy() # skeleton: (M, T, V, C) (2, 300, 25, 3) -> main body center: (T, 1, C) + + # 전체 스켈레톤에 대해 + for i_p, person in enumerate(skeleton): # Dimension M (# person) # person : (T, V, C) + + # Person 데이터가 전부 0인 경우(Person 1과 Person 2가 전부 0)이면 넘기기(continue) + if person.sum() == 0: # person : (T, V, C) frames (T), nodes (V), channels (C) continue + # For all `person`, compute the `mask` which is the non-zero channel dimension - mask = (person.sum(-1) != 0).reshape(T, V, 1) + # Person 데이터가 존재하는 경우, mask는 X, Y, Z 값의 합이 0이 아니면(Action이 있으면) True, 0이면 False + mask = (person.sum(-1) != 0).reshape(T, V, 1) + # Subtract the first skeleton's centre joint + # Skeleton data의 2번째 Joint를 모두 0, 0, 0으로 이동시킴. s[i_s, i_p] = (s[i_s, i_p] - main_body_center) * mask + # 척추 부분을 Z축과 Parallel하게 만들음. print('parallel the bone between hip(jpt 0) and spine(jpt 1) of the first person to the z axis') - for i_s, skeleton in enumerate(tqdm(s)): + for i_s, skeleton in enumerate(tqdm(s)): # skeleton: (M, T, V, C) (2, 300, 25, 3) # persons (M), frames (T), nodes (V), channels (C) + # Person 데이터가 전부 0인 경우(Person 1과 Person 2가 전부 0)이면 넘기기 if skeleton.sum() == 0: continue + # Shapes: (C,) - joint_bottom = skeleton[0, 0, zaxis[0]] - joint_top = skeleton[0, 0, zaxis[1]] - axis = np.cross(joint_top - joint_bottom, [0, 0, 1]) + joint_bottom = skeleton[0, 0, zaxis[0]] # zxais = [0, 1]이면, 1번 Joint + joint_top = skeleton[0, 0, zaxis[1]] # zxais = [0, 1]이면, 2번 Joint + axis = np.cross(joint_top - joint_bottom, [0, 0, 1]) # bottom to top vector. XY평면이 지평면 angle = angle_between(joint_top - joint_bottom, [0, 0, 1]) matrix_z = rotation_matrix(axis, angle) for i_p, person in enumerate(skeleton): @@ -69,11 +105,12 @@ def pre_normalization(data, zaxis=[0, 1], xaxis=[8, 4]): for i_j, joint in enumerate(frame): s[i_s, i_p, i_f, i_j] = np.dot(matrix_z, joint) + # 양쪽 어깨선을 X축과 나란하게 만들음. print('parallel the bone between right shoulder(jpt 8) and left shoulder(jpt 4) of the first person to the x axis') for i_s, skeleton in enumerate(tqdm(s)): if skeleton.sum() == 0: continue - joint_rshoulder = skeleton[0, 0, xaxis[0]] + joint_rshoulder = skeleton[0, 0, xaxis[0]] # xaxis = [8, 4] joint_lshoulder = skeleton[0, 0, xaxis[1]] axis = np.cross(joint_rshoulder - joint_lshoulder, [1, 0, 0]) angle = angle_between(joint_rshoulder - joint_lshoulder, [1, 0, 0]) @@ -90,7 +127,6 @@ def pre_normalization(data, zaxis=[0, 1], xaxis=[8, 4]): data = np.transpose(s, [0, 4, 2, 3, 1]) return data - if __name__ == '__main__': data = np.load('../data/ntu/xview/val_data.npy') pre_normalization(data) diff --git a/data_gen/rotation.py b/data_gen/rotation.py index 6e8aaa0..a9219f3 100644 --- a/data_gen/rotation.py +++ b/data_gen/rotation.py @@ -21,13 +21,12 @@ def rotation_matrix(axis, theta): def unit_vector(vector): - """ Returns the unit vector of the vector. """ + # 벡터의 단위벡터 반환 return vector / np.linalg.norm(vector) def angle_between(v1, v2): """ Returns the angle in radians between vectors 'v1' and 'v2':: - >>> angle_between((1, 0, 0), (0, 1, 0)) 1.5707963267948966 >>> angle_between((1, 0, 0), (1, 0, 0)) @@ -35,8 +34,10 @@ def angle_between(v1, v2): >>> angle_between((1, 0, 0), (-1, 0, 0)) 3.141592653589793 """ + # 벡터가 0에 가까우면 두 벡터의 각 = 0으로 처리 if np.abs(v1).sum() < 1e-6 or np.abs(v2).sum() < 1e-6: return 0 + v1_u = unit_vector(v1) v2_u = unit_vector(v2) return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0)) diff --git a/main.py b/main.py index 2a14dcd..b3eda40 100644 --- a/main.py +++ b/main.py @@ -20,7 +20,6 @@ import inspect import torch.backends.cudnn as cudnn - def init_seed(_): torch.cuda.manual_seed_all(1) torch.manual_seed(1) @@ -30,141 +29,141 @@ def init_seed(_): torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False - def get_parser(): # parameter priority: command line > config > default parser = argparse.ArgumentParser( description='Directed Graph Neural Net for Skeleton Action Recognition') + parser.add_argument( '--work-dir', default='./work_dir/temp', - help='the work folder for storing results') + help='the work folder for storing results') # work_dir. 추가로 주석 달기 parser.add_argument( - '--model-saved-name', default='') + '--model-saved-name', default='') # 추가로 주석 달기 parser.add_argument( '--config', - default='./config/nturgbd-cross-view/test_bone.yaml', - help='path to the configuration file') + default='./config/nturgbd-cross-subject/train_spatial.yaml', + help='path to the configuration file') # config 파일이 위치하는 디렉토리 # processor parser.add_argument( - '--phase', default='train', help='must be train or test') + '--phase', default='train', help='must be train or test') # Train할 지, Test할 지 정하는 인자. parser.add_argument( '--save-score', type=str2bool, default=False, - help='if ture, the classification score will be stored') + help='if ture, the classification score will be stored') # 추가로 주석 달기 # visulize and debug parser.add_argument( - '--seed', type=int, default=1, help='random seed for pytorch') + '--seed', type=int, default=1, help='random seed for pytorch') # random seed parser.add_argument( '--log-interval', type=int, default=100, - help='the interval for printing messages (#iteration)') + help='the interval for printing messages (#iteration)') # log massage 출력할 Interval parser.add_argument( '--save-interval', type=int, default=2, - help='the interval for storing models (#iteration)') + help='the interval for storing models (#iteration)') # model을 저장할 Interval parser.add_argument( '--eval-interval', type=int, default=5, - help='the interval for evaluating models (#iteration)') + help='the interval for evaluating models (#iteration)') # Evaluate할 Interval parser.add_argument( '--print-log', type=str2bool, default=True, - help='print logging or not') + help='print logging or not') # Log 출력할지에 대한 여부 parser.add_argument( '--show-topk', type=int, - default=[1, 5], + default=[1, 2], nargs='+', - help='which Top K accuracy will be shown') + help='which Top K accuracy will be shown') # 추후에 주석 달기 # feeder parser.add_argument( - '--feeder', default='feeder.feeder', help='data loader will be used') + '--feeder', default='feeder.feeder', help='data loader will be used') # 추후에 주석 달기 parser.add_argument( '--num-worker', type=int, default=os.cpu_count(), - help='the number of worker for data loader') + help='the number of worker for data loader') # 추후에 주석 달기 parser.add_argument( '--train-feeder-args', default=dict(), - help='the arguments of data loader for training') + help='the arguments of data loader for training') # 추후에 주석 달기 parser.add_argument( '--test-feeder-args', default=dict(), - help='the arguments of data loader for test') + help='the arguments of data loader for test') # 추후에 주석 달기 # model parser.add_argument( - '--model', default=None, help='the model will be used') + '--model', default=None, help='the model will be used') # 사용할 Model parser.add_argument( '--model-args', type=dict, default=dict(), - help='the arguments of model') + help='the arguments of model') # 사용할 Model의 argument # 추후에 주석 추가 parser.add_argument( '--weights', default=None, - help='the weights for network initialization') + help='the weights for network initialization') # Initialize할 weights parser.add_argument( '--ignore-weights', type=str, default=[], nargs='+', - help='the name of weights which will be ignored in the initialization') + help='the name of weights which will be ignored in the initialization') # 추후에 주석 달기. # optim parser.add_argument( - '--base-lr', type=float, default=0.01, help='initial learning rate') + '--base-lr', type=float, default=0.01, help='initial learning rate') # 초기 Learning rate parser.add_argument( '--step', type=int, default=[60, 90], nargs='+', - help='the epoch where optimizer reduce the learning rate') + help='the epoch where optimizer reduce the learning rate') # Learning rate를 감소시킬 Epoch parser.add_argument( '--device', type=int, default=0, nargs='+', - help='the indexes of GPUs for training or testing') + help='the indexes of GPUs for training or testing') # 사용할 GPU 번호. 여러개 입력 가능. parser.add_argument( - '--optimizer', default='SGD', help='type of optimizer') + '--optimizer', default='SGD', help='type of optimizer') # Optimizer 종류 parser.add_argument( - '--nesterov', type=str2bool, default=True, help='use nesterov or not') + '--nesterov', type=str2bool, default=True, help='use nesterov or not') # Nesterov 사용 여부 parser.add_argument( - '--batch-size', type=int, default=32, help='training batch size') + '--batch-size', type=int, default=32, help='training batch size') # Training시 Batch 사이즈 parser.add_argument( - '--test-batch-size', type=int, default=32, help='test batch size') + '--test-batch-size', type=int, default=32, help='test batch size') # Test시 Batch 사이즈 parser.add_argument( '--start-epoch', type=int, default=0, - help='start training from which epoch') + help='start training from which epoch') # Training 시 시작할 Epoch parser.add_argument( '--num-epoch', type=int, default=120, - help='stop training in which epoch') + help='stop training in which epoch') # Epoch 크기 parser.add_argument( '--weight-decay', type=float, default=0.0001, - help='weight decay for optimizer') + help='weight decay for optimizer') # weight decay parser.add_argument( '--freeze-graph-until', type=int, default=10, - help='number of epochs before making graphs learnable') + help='number of epochs before making graphs learnable') # Graph Freeze를 할 Epoch # parser.add_argument('--only_train_part', default=False) # parser.add_argument('--only_train_epoch', default=0) @@ -173,19 +172,27 @@ def get_parser(): class Processor(): - """Processor for Skeleton-based Action Recgnition""" + """Processor for Skeleton-based Action Recognition""" def __init__(self, arg): self.arg = arg + + # work_dir에 config 파일 생성 및 저장 self.save_arg() + + # phase가 Train일 때 if arg.phase == 'train': + # train_feeder_args['debug']가 False 이고 if not arg.train_feeder_args['debug']: + # 모델 파라미터를 저장할 model_saved_name 디렉토리가 존재하면 if os.path.isdir(arg.model_saved_name): print('log_dir: ', arg.model_saved_name, 'already exist') answer = input('delete it? [y]/n:') + # 삭제를 선택하면 if answer.lower() in ('y', ''): - shutil.rmtree(arg.model_saved_name) - print('Dir removed: ', arg.model_saved_name) + shutil.rmtree(arg.model_saved_name) # 지정된 디렉토리의 모든 파일 삭제 + print('Dir removed: ', arg.model_saved_name) input('Refresh the website of tensorboard by pressing any keys') + # 삭제하지 않을거면 else: print('Dir not removed: ', arg.model_saved_name) @@ -193,47 +200,64 @@ def __init__(self, arg): self.val_writer = SummaryWriter(os.path.join(arg.model_saved_name, 'val'), 'val') # self.writer = SummaryWriter(os.path.join(arg.model_saved_name, 'training'), 'both') + # num_point 수 데이터에서 가져오기 + if self.arg.phase == 'train': + joint_data = np.load(self.arg.train_feeder_args['joint_data_path']) + elif self.arg.phase == 'test': + joint_data = np.load(self.arg.test_feeder_args['joint_data_path']) + + self.arg.model_args['num_point'] = joint_data.shape[3] + del joint_data + self.global_step = 0 - self.load_model() - self.load_param_groups() # Group parameters to apply different learning rules - self.load_optimizer() - self.load_data() + self.load_model() # Model 선언 / Parameter Load / GPU 설정 + self.load_param_groups() # Group parameters to apply different learning rules # Parameter 그룹 분할 + self.load_optimizer() # Optimizer 설정 + self.load_data() # data load self.lr = self.arg.base_lr self.best_acc = 0 self.best_acc_epoch = 0 + # data load def load_data(self): - Feeder = import_class(self.arg.feeder) + Feeder = import_class(self.arg.feeder) # self.arg.feeder : feeders.feeder.Feeder self.data_loader = dict() if self.arg.phase == 'train': - self.data_loader['train'] = torch.utils.data.DataLoader( - dataset=Feeder(**self.arg.train_feeder_args), - batch_size=self.arg.batch_size, - shuffle=True, - num_workers=self.arg.num_worker, - drop_last=True, - worker_init_fn=init_seed) + self.data_loader['train'] = torch.utils.data.DataLoader(dataset=Feeder(**self.arg.train_feeder_args), + batch_size=self.arg.batch_size, + shuffle=True, + num_workers=self.arg.num_worker, + drop_last=True, + worker_init_fn=init_seed) # Load test data regardless - self.data_loader['test'] = torch.utils.data.DataLoader( - dataset=Feeder(**self.arg.test_feeder_args), - batch_size=self.arg.test_batch_size, - shuffle=False, - num_workers=self.arg.num_worker, - drop_last=False, - worker_init_fn=init_seed) - + self.data_loader['test'] = torch.utils.data.DataLoader(dataset=Feeder(**self.arg.test_feeder_args), + batch_size=self.arg.test_batch_size, + shuffle=False, + num_workers=self.arg.num_worker, + drop_last=False, + worker_init_fn=init_seed) + + # Model 선언 / Parameter Load / GPU 설정 def load_model(self): - output_device = self.arg.device[0] if type(self.arg.device) is list else self.arg.device + # 출력으로 사용할 device + output_device = self.arg.device[0] if type(self.arg.device) is list else self.arg.device self.output_device = output_device - Model = import_class(self.arg.model) + Model = import_class(self.arg.model) # In here, self.arg.model = model.dgnn.Model + # Copy model file to output dir - shutil.copy2(inspect.getfile(Model), self.arg.work_dir) - print(Model) - self.model = Model(**self.arg.model_args).cuda(output_device) + # Model Class 파일을 work_dir에 복사 + shutil.copy2(inspect.getfile(Model), self.arg.work_dir) # inspect.getfile(Model): ./model/dgnn.py + + # argument 값을 이용해서 model 객체를 생성하고 model과 loss를 device로 이동 + # self.arg.model_args['graph']: graph.directed_ntu_rgb_d.Graph + # 'num_class': 10, 'num_point': 25, 'num_person': 2, 'graph': 'graph.directed_ntu_rgb_d.Graph' + self.model = Model(**self.arg.model_args).cuda(output_device) self.loss = nn.CrossEntropyLoss().cuda(output_device) # Load weights + # 저장해두었던 모델의 weights가 있으면 weight load. + # scratch train할 때는 코드가 작동하지 않음. if self.arg.weights: self.global_step = int(arg.weights[:-3].split('-')[-1]) self.print_log('Load weights from {}.'.format(self.arg.weights)) @@ -265,14 +289,17 @@ def load_model(self): self.model.load_state_dict(state) # Parallelise data if mulitple GPUs - if type(self.arg.device) is list: - if len(self.arg.device) > 1: + # GPU가 여러개 있으면 Parallelise data + if type(self.arg.device) is list: # config GPU가 List 값으로 되어있고 + if len(self.arg.device) > 1: # 길이가 1보다 크면 + # 모델이 DataParallel을 사용하게 설정 self.model = nn.DataParallel( self.model, device_ids=self.arg.device, output_device=output_device) + # Optimizer 설정 def load_optimizer(self): p_groups = list(self.optim_param_groups.values()) if self.arg.optimizer == 'SGD': @@ -292,11 +319,16 @@ def load_optimizer(self): self.lr_scheduler = MultiStepLR(self.optimizer, milestones=self.arg.step, gamma=0.1) + # self.arg를 work_dir에 config 파일로 저장 def save_arg(self): # save arg arg_dict = vars(self.arg) + + # work_dir 없으면 폴더 생성 if not os.path.exists(self.arg.work_dir): os.makedirs(self.arg.work_dir) + + # work_dir에 config 파일 생성 및 저장 with open('{}/config.yaml'.format(self.arg.work_dir), 'w') as f: yaml.dump(arg_dict, f) @@ -322,9 +354,11 @@ def split_time(self): self.record_time() return split_time + # Parameter 그룹 분할 def load_param_groups(self): - self.param_groups = defaultdict(list) - for name, params in self.model.named_parameters(): + self.param_groups = defaultdict(list) # list 형태의 아무것도 존재하지 않는 dictionary 생성 + for name, params in self.model.named_parameters(): + # parameter가 Adaptive Graph인지, 그 이외것인지 구분 if ('source_M' in name) or ('target_M' in name): self.param_groups['graph'].append(params) else: @@ -348,6 +382,7 @@ def update_graph_freeze(self, epoch): def train(self, epoch, save_model=False): self.print_log('Training epoch: {}'.format(epoch + 1)) self.model.train() + loader = self.data_loader['train'] loss_values = [] self.train_writer.add_scalar('epoch', epoch, self.global_step) @@ -377,6 +412,7 @@ def train(self, epoch, save_model=False): joint_data = joint_data.float().cuda(self.output_device) bone_data = bone_data.float().cuda(self.output_device) label = label.long().cuda(self.output_device) + timer['dataloader'] += self.split_time() # Clear gradients @@ -538,6 +574,7 @@ def eval(self, epoch, save_score=False, loader_name=['test'], wrong_file=None, r pickle.dump(score_dict, f) def start(self): + # phase가 Train일 때 if self.arg.phase == 'train': self.print_log('Parameters:\n{}\n'.format(str(vars(self.arg)))) self.global_step = self.arg.start_epoch * len(self.data_loader['train']) / self.arg.batch_size @@ -553,6 +590,7 @@ def start(self): print('Best accuracy: {}, epoch: {}, model_name: {}' .format(self.best_acc, self.best_acc_epoch, self.arg.model_saved_name)) + # phase가 Test일 때 elif self.arg.phase == 'test': if not self.arg.test_feeder_args['debug']: wf = self.arg.model_saved_name + '_wrong.txt' @@ -588,19 +626,25 @@ def import_class(name): if __name__ == '__main__': parser = get_parser() - # load arg form config file + # load arg from config file p = parser.parse_args() + + p.config = './config/nturgbd-cross-subject/train_spatial.yaml' + if p.config is not None: with open(p.config, 'r') as f: - default_arg = yaml.load(f) - key = vars(p).keys() + default_arg = yaml.load(f) # config 파일에 들어있는 keys. config key로 명명 + + # 예외 처리 + key = vars(p).keys() # parser에 들어있는 key값. default key로 명명 for k in default_arg.keys(): - if k not in key: - print('WRONG ARG: {}'.format(k)) - assert (k in key) - parser.set_defaults(**default_arg) - - arg = parser.parse_args() - init_seed(0) - processor = Processor(arg) + if k not in key: # config key가 default key에 들어있지 않으면 + print('WRONG ARG: {}'.format(k)) # default key에 해당 키가 없음을 알림. + assert (k in key) + + parser.set_defaults(**default_arg) # 임의의 개수의 Keyword arguments를 받아서 default key -> config key로 변경 + + arg = parser.parse_args() # config key가 반영된 argument + init_seed(0) # 시드 초기화 + processor = Processor(arg) # processor.start() diff --git a/model/dgnn.py b/model/dgnn.py index 3cb761d..79f43c6 100644 --- a/model/dgnn.py +++ b/model/dgnn.py @@ -60,12 +60,13 @@ class DGNBlock(nn.Module): def __init__(self, in_channels, out_channels, source_M, target_M): super().__init__() self.num_nodes, self.num_edges = source_M.shape + # Adaptive block with learnable graphs; shapes (V_node, V_edge) self.source_M = nn.Parameter(torch.from_numpy(source_M.astype('float32'))) self.target_M = nn.Parameter(torch.from_numpy(target_M.astype('float32'))) # Updating functions - self.H_v = nn.Linear(3 * in_channels, out_channels) + self.H_v = nn.Linear(3 * in_channels, out_channels) self.H_e = nn.Linear(3 * in_channels, out_channels) self.bn_v = nn.BatchNorm2d(out_channels) @@ -82,18 +83,20 @@ def forward(self, fv, fe): _, _, _, V_edge = fe.shape # Reshape for matmul, shape: (N, CT, V) - fv = fv.view(N, -1, V_node) - fe = fe.view(N, -1, V_edge) + fv = fv.reshape([N, -1, V_node]) + fe = fe.reshape([N, -1, V_edge]) # Compute features for node/edge updates + # 두 행렬의 곱 fe_in_agg = torch.einsum('nce,ev->ncv', fe, self.source_M.transpose(0,1)) fe_out_agg = torch.einsum('nce,ev->ncv', fe, self.target_M.transpose(0,1)) fvp = torch.stack((fv, fe_in_agg, fe_out_agg), dim=1) # Out shape: (N,3,CT,V_nodes) fvp = fvp.view(N, 3 * C, T, V_node).contiguous().permute(0,2,3,1) # (N,T,V_node,3C) - fvp = self.H_v(fvp).permute(0,3,1,2) # (N,C_out,T,V_node) + fvp = self.H_v(fvp).permute(0,3,1,2) # (N, C_out, T, V_node) fvp = self.bn_v(fvp) fvp = self.relu(fvp) + # 두 행렬의 곱 fv_in_agg = torch.einsum('ncv,ve->nce', fv, self.source_M) fv_out_agg = torch.einsum('ncv,ve->nce', fv, self.target_M) fep = torch.stack((fe, fv_in_agg, fv_out_agg), dim=1) # Out shape: (N,3,CT,V_edges) @@ -103,7 +106,6 @@ def forward(self, fv, fe): fep = self.relu(fep) return fvp, fep - class GraphTemporalConv(nn.Module): def __init__(self, in_channels, out_channels, source_M, target_M, temp_kernel_size=9, stride=1, residual=True): super(GraphTemporalConv, self).__init__() @@ -111,6 +113,7 @@ def __init__(self, in_channels, out_channels, source_M, target_M, temp_kernel_si self.tcn = BiTemporalConv(out_channels, out_channels, kernel_size=temp_kernel_size, stride=stride) self.relu = nn.ReLU(inplace=True) + # residual if not residual: self.residual = lambda fv, fe: (0, 0) elif (in_channels == out_channels) and (stride == 1): @@ -131,13 +134,14 @@ class Model(nn.Module): def __init__(self, num_class=60, num_point=25, num_person=2, graph=None, graph_args=dict(), in_channels=3): super(Model, self).__init__() - if graph is None: + # 입력인자 graph에 따라 + if graph is None: # self.arg.model_args['graph']: graph.directed_ntu_rgb_d.Graph raise ValueError() else: # KV config pairs should be supplied with the config file - Graph = import_class(graph) - self.graph = Graph(**graph_args) - + Graph = import_class(graph) + self.graph = Graph(**graph_args) # graph 개체 생성 + source_M, target_M = self.graph.source_M, self.graph.target_M self.data_bn_v = nn.BatchNorm1d(num_person * in_channels * num_point) self.data_bn_e = nn.BatchNorm1d(num_person * in_channels * num_point) @@ -161,24 +165,25 @@ def __init__(self, num_class=60, num_point=25, num_person=2, graph=None, graph_a def count_params(m): return sum(p.numel() for p in m.parameters() if p.requires_grad) + for module in self.modules(): print('Module:', module) print('# Params:', count_params(module)) print() print('Model total number of params:', count_params(self)) - def forward(self, fv, fe): - N, C, T, V_node, M = fv.shape + def forward(self, fv, fe): # fv : batch_joint_data, fe : batch_bone_data + N, C, T, V_node, M = fv.shape # examples (N), channels (C), frames (T), nodes (V), persons (M) _, _, _, V_edge, _ = fe.shape # Preprocessing - fv = fv.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V_node * C, T) - fv = self.data_bn_v(fv) - fv = fv.view(N, M, V_node, C, T).permute(0, 1, 3, 4, 2).contiguous().view(N * M, C, T, V_node) + fv = fv.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V_node * C, T) # N, M*V*C, T + fv = self.data_bn_v(fv) # batch norm + fv = fv.view(N, M, V_node, C, T).permute(0, 1, 3, 4, 2).contiguous().view(N * M, C, T, V_node) # N*M, C, T, V - fe = fe.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V_edge * C, T) - fe = self.data_bn_e(fe) - fe = fe.view(N, M, V_edge, C, T).permute(0, 1, 3, 4, 2).contiguous().view(N * M, C, T, V_edge) + fe = fe.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V_edge * C, T) # N, M*V*C, T + fe = self.data_bn_e(fe) # batch norm + fe = fe.view(N, M, V_edge, C, T).permute(0, 1, 3, 4, 2).contiguous().view(N * M, C, T, V_edge) # N*M, C, T, V fv, fe = self.l1(fv, fe) fv, fe = self.l2(fv, fe) @@ -191,10 +196,11 @@ def forward(self, fv, fe): fv, fe = self.l9(fv, fe) fv, fe = self.l10(fv, fe) - # Shape: (N*M,C,T,V), C is same for fv/fe + # Shape: (N*M, C, T, V), C is same for fv/fe out_channels = fv.size(1) # Performs pooling over both nodes and frames, and over number of persons + # Global average pooling? fv = fv.view(N, M, out_channels, -1).mean(3).mean(1) fe = fe.view(N, M, out_channels, -1).mean(3).mean(1) diff --git a/runs/ntu_cs_dgnn_spatial/train/events.out.tfevents.1625558227.node01 b/runs/ntu_cs_dgnn_spatial/train/events.out.tfevents.1625558227.node01 new file mode 100644 index 0000000..e69de29 diff --git a/runs/ntu_cs_dgnn_spatial/val/events.out.tfevents.1625558227.node01 b/runs/ntu_cs_dgnn_spatial/val/events.out.tfevents.1625558227.node01 new file mode 100644 index 0000000..e69de29 diff --git a/work_dir/ntu/xsub/dgnn_spatial/config.yaml b/work_dir/ntu/xsub/dgnn_spatial/config.yaml new file mode 100644 index 0000000..ebb0252 --- /dev/null +++ b/work_dir/ntu/xsub/dgnn_spatial/config.yaml @@ -0,0 +1,51 @@ +base_lr: 0.1 +batch_size: 32 +config: ./config/nturgbd-cross-subject/train_spatial.yaml +device: +- 0 +eval_interval: 5 +feeder: feeders.feeder.Feeder +freeze_graph_until: 10 +ignore_weights: [] +log_interval: 100 +model: model.dgnn.Model +model_args: + graph: graph.directed_ntu_rgb_d.Graph + num_class: 60 + num_person: 2 + num_point: 25 +model_saved_name: ./runs/ntu_cs_dgnn_spatial +nesterov: true +num_epoch: 120 +num_worker: 48 +optimizer: SGD +phase: train +print_log: true +save_interval: 2 +save_score: false +seed: 1 +show_topk: +- 1 +- 2 +start_epoch: 0 +step: +- 60 +- 90 +test_batch_size: 32 +test_feeder_args: + bone_data_path: ./data/ntu/xsub/val_data_bone.npy + joint_data_path: ./data/ntu/xsub/val_data_joint.npy + label_path: ./data/ntu/xsub/val_label.pkl +train_feeder_args: + bone_data_path: ./data/ntu/xsub/train_data_bone.npy + debug: false + joint_data_path: ./data/ntu/xsub/train_data_joint.npy + label_path: ./data/ntu/xsub/train_label.pkl + normalization: false + random_choose: false + random_move: false + random_shift: false + window_size: -1 +weight_decay: 0.0005 +weights: null +work_dir: ./work_dir/ntu/xsub/dgnn_spatial diff --git a/work_dir/ntu/xsub/dgnn_spatial/dgnn.py b/work_dir/ntu/xsub/dgnn_spatial/dgnn.py new file mode 100644 index 0000000..79f43c6 --- /dev/null +++ b/work_dir/ntu/xsub/dgnn_spatial/dgnn.py @@ -0,0 +1,224 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +import numpy as np +import math + + +def import_class(name): + components = name.split('.') + mod = __import__(components[0]) + for comp in components[1:]: + mod = getattr(mod, comp) + return mod + + +def conv_init(conv): + nn.init.kaiming_normal_(conv.weight, mode='fan_out') + nn.init.constant_(conv.bias, 0) + + +def bn_init(bn, scale): + nn.init.constant_(bn.weight, scale) + nn.init.constant_(bn.bias, 0) + + +class TemporalConv(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size=9, stride=1): + super().__init__() + pad = int((kernel_size - 1) / 2) + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=(kernel_size, 1), # Conv along the temporal dimension only + padding=(pad, 0), + stride=(stride, 1) + ) + + self.bn = nn.BatchNorm2d(out_channels) + conv_init(self.conv) + bn_init(self.bn, 1) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + return x + + +class BiTemporalConv(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size=9, stride=1): + super().__init__() + # NOTE: assuming that temporal convs are shared between node/edge features + self.tempconv = TemporalConv(in_channels, out_channels, kernel_size, stride) + + def forward(self, fv, fe): + return self.tempconv(fv), self.tempconv(fe) + + +class DGNBlock(nn.Module): + def __init__(self, in_channels, out_channels, source_M, target_M): + super().__init__() + self.num_nodes, self.num_edges = source_M.shape + + # Adaptive block with learnable graphs; shapes (V_node, V_edge) + self.source_M = nn.Parameter(torch.from_numpy(source_M.astype('float32'))) + self.target_M = nn.Parameter(torch.from_numpy(target_M.astype('float32'))) + + # Updating functions + self.H_v = nn.Linear(3 * in_channels, out_channels) + self.H_e = nn.Linear(3 * in_channels, out_channels) + + self.bn_v = nn.BatchNorm2d(out_channels) + self.bn_e = nn.BatchNorm2d(out_channels) + bn_init(self.bn_v, 1) + bn_init(self.bn_e, 1) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, fv, fe): + # `fv` (node features) has shape (N, C, T, V_node) + # `fe` (edge features) has shape (N, C, T, V_edge) + N, C, T, V_node = fv.shape + _, _, _, V_edge = fe.shape + + # Reshape for matmul, shape: (N, CT, V) + fv = fv.reshape([N, -1, V_node]) + fe = fe.reshape([N, -1, V_edge]) + + # Compute features for node/edge updates + # 두 행렬의 곱 + fe_in_agg = torch.einsum('nce,ev->ncv', fe, self.source_M.transpose(0,1)) + fe_out_agg = torch.einsum('nce,ev->ncv', fe, self.target_M.transpose(0,1)) + fvp = torch.stack((fv, fe_in_agg, fe_out_agg), dim=1) # Out shape: (N,3,CT,V_nodes) + fvp = fvp.view(N, 3 * C, T, V_node).contiguous().permute(0,2,3,1) # (N,T,V_node,3C) + fvp = self.H_v(fvp).permute(0,3,1,2) # (N, C_out, T, V_node) + fvp = self.bn_v(fvp) + fvp = self.relu(fvp) + + # 두 행렬의 곱 + fv_in_agg = torch.einsum('ncv,ve->nce', fv, self.source_M) + fv_out_agg = torch.einsum('ncv,ve->nce', fv, self.target_M) + fep = torch.stack((fe, fv_in_agg, fv_out_agg), dim=1) # Out shape: (N,3,CT,V_edges) + fep = fep.view(N, 3 * C, T, V_edge).contiguous().permute(0,2,3,1) # (N,T,V_edge,3C) + fep = self.H_e(fep).permute(0,3,1,2) # (N,C_out,T,V_edge) + fep = self.bn_e(fep) + fep = self.relu(fep) + return fvp, fep + +class GraphTemporalConv(nn.Module): + def __init__(self, in_channels, out_channels, source_M, target_M, temp_kernel_size=9, stride=1, residual=True): + super(GraphTemporalConv, self).__init__() + self.dgn = DGNBlock(in_channels, out_channels, source_M, target_M) + self.tcn = BiTemporalConv(out_channels, out_channels, kernel_size=temp_kernel_size, stride=stride) + self.relu = nn.ReLU(inplace=True) + + # residual + if not residual: + self.residual = lambda fv, fe: (0, 0) + elif (in_channels == out_channels) and (stride == 1): + self.residual = lambda fv, fe: (fv, fe) + else: + self.residual = BiTemporalConv(in_channels, out_channels, kernel_size=temp_kernel_size, stride=stride) + + def forward(self, fv, fe): + fv_res, fe_res = self.residual(fv, fe) + fv, fe = self.dgn(fv, fe) + fv, fe = self.tcn(fv, fe) + fv += fv_res + fe += fe_res + return self.relu(fv), self.relu(fe) + + +class Model(nn.Module): + def __init__(self, num_class=60, num_point=25, num_person=2, graph=None, graph_args=dict(), in_channels=3): + super(Model, self).__init__() + + # 입력인자 graph에 따라 + if graph is None: # self.arg.model_args['graph']: graph.directed_ntu_rgb_d.Graph + raise ValueError() + else: + # KV config pairs should be supplied with the config file + Graph = import_class(graph) + self.graph = Graph(**graph_args) # graph 개체 생성 + + source_M, target_M = self.graph.source_M, self.graph.target_M + self.data_bn_v = nn.BatchNorm1d(num_person * in_channels * num_point) + self.data_bn_e = nn.BatchNorm1d(num_person * in_channels * num_point) + + self.l1 = GraphTemporalConv(3, 64, source_M, target_M, residual=False) + self.l2 = GraphTemporalConv(64, 64, source_M, target_M) + self.l3 = GraphTemporalConv(64, 64, source_M, target_M) + self.l4 = GraphTemporalConv(64, 64, source_M, target_M) + self.l5 = GraphTemporalConv(64, 128, source_M, target_M, stride=2) + self.l6 = GraphTemporalConv(128, 128, source_M, target_M) + self.l7 = GraphTemporalConv(128, 128, source_M, target_M) + self.l8 = GraphTemporalConv(128, 256, source_M, target_M, stride=2) + self.l9 = GraphTemporalConv(256, 256, source_M, target_M) + self.l10 = GraphTemporalConv(256, 256, source_M, target_M) + + self.fc = nn.Linear(256 * 2, num_class) + + nn.init.normal_(self.fc.weight, 0, math.sqrt(2. / num_class)) + bn_init(self.data_bn_v, 1) + bn_init(self.data_bn_e, 1) + + def count_params(m): + return sum(p.numel() for p in m.parameters() if p.requires_grad) + + for module in self.modules(): + print('Module:', module) + print('# Params:', count_params(module)) + print() + print('Model total number of params:', count_params(self)) + + def forward(self, fv, fe): # fv : batch_joint_data, fe : batch_bone_data + N, C, T, V_node, M = fv.shape # examples (N), channels (C), frames (T), nodes (V), persons (M) + _, _, _, V_edge, _ = fe.shape + + # Preprocessing + fv = fv.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V_node * C, T) # N, M*V*C, T + fv = self.data_bn_v(fv) # batch norm + fv = fv.view(N, M, V_node, C, T).permute(0, 1, 3, 4, 2).contiguous().view(N * M, C, T, V_node) # N*M, C, T, V + + fe = fe.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V_edge * C, T) # N, M*V*C, T + fe = self.data_bn_e(fe) # batch norm + fe = fe.view(N, M, V_edge, C, T).permute(0, 1, 3, 4, 2).contiguous().view(N * M, C, T, V_edge) # N*M, C, T, V + + fv, fe = self.l1(fv, fe) + fv, fe = self.l2(fv, fe) + fv, fe = self.l3(fv, fe) + fv, fe = self.l4(fv, fe) + fv, fe = self.l5(fv, fe) + fv, fe = self.l6(fv, fe) + fv, fe = self.l7(fv, fe) + fv, fe = self.l8(fv, fe) + fv, fe = self.l9(fv, fe) + fv, fe = self.l10(fv, fe) + + # Shape: (N*M, C, T, V), C is same for fv/fe + out_channels = fv.size(1) + + # Performs pooling over both nodes and frames, and over number of persons + # Global average pooling? + fv = fv.view(N, M, out_channels, -1).mean(3).mean(1) + fe = fe.view(N, M, out_channels, -1).mean(3).mean(1) + + # Concat node and edge features + out = torch.cat((fv, fe), dim=-1) + + return self.fc(out) + + +if __name__ == "__main__": + import sys + sys.path.append('..') + model = Model(graph='graph.directed_ntu_rgb_d.Graph') + + # for name, param in model.named_parameters(): + # print('name is:', name) + # print('type(name):', type(name)) + # print('param:', type(param)) + # print() + + print('Model total # params:', sum(p.numel() for p in model.parameters() if p.requires_grad)) diff --git a/work_dir/ntu/xsub/dgnn_spatial/log.txt b/work_dir/ntu/xsub/dgnn_spatial/log.txt new file mode 100644 index 0000000..f30318e --- /dev/null +++ b/work_dir/ntu/xsub/dgnn_spatial/log.txt @@ -0,0 +1,5 @@ +[ Tue Jul 6 16:57:12 2021 ] Parameters: +{'seed': 1, 'test_feeder_args': {'joint_data_path': './data/ntu/xsub/val_data_joint.npy', 'bone_data_path': './data/ntu/xsub/val_data_bone.npy', 'label_path': './data/ntu/xsub/val_label.pkl'}, 'test_batch_size': 32, 'phase': 'train', 'ignore_weights': [], 'feeder': 'feeders.feeder.Feeder', 'num_worker': 48, 'log_interval': 100, 'config': './config/nturgbd-cross-subject/train_spatial.yaml', 'num_epoch': 120, 'save_score': False, 'weights': None, 'freeze_graph_until': 10, 'base_lr': 0.1, 'batch_size': 32, 'device': [0], 'eval_interval': 5, 'model_args': {'num_person': 2, 'num_class': 60, 'num_point': 25, 'graph': 'graph.directed_ntu_rgb_d.Graph'}, 'nesterov': True, 'optimizer': 'SGD', 'save_interval': 2, 'model': 'model.dgnn.Model', 'start_epoch': 0, 'step': [60, 90], 'print_log': True, 'train_feeder_args': {'debug': False, 'random_shift': False, 'label_path': './data/ntu/xsub/train_label.pkl', 'random_choose': False, 'joint_data_path': './data/ntu/xsub/train_data_joint.npy', 'random_move': False, 'bone_data_path': './data/ntu/xsub/train_data_bone.npy', 'window_size': -1, 'normalization': False}, 'show_topk': [1, 2], 'work_dir': './work_dir/ntu/xsub/dgnn_spatial', 'model_saved_name': './runs/ntu_cs_dgnn_spatial', 'weight_decay': 0.0005} + +[ Tue Jul 6 16:57:12 2021 ] Training epoch: 1 +[ Tue Jul 6 16:57:12 2021 ] Graphs are frozen at epoch 1