DCN 40M fp32

[MARD1NO]

fp32: 
================ Test Evaluation ================
Rank[0], Epoch 7, Step 75000, AUC 0.802074, LogLoss 0.125882, Eval_time 19.48 s, Metrics_time 6.48 s, Eval_samples 89140000, GPU_Memory 15074 MiB, Host_Memory 10726 MiB, 2022-07-27 11:03:38

对应HugeCTR脚本：

import hugectr
from mpi4py import MPI

data_dir = "/RAID0/liujuncheng/criteo1t_parquet_40M_long"

print(f"{data_dir}/train/_file_list.txt")

solver = hugectr.CreateSolver(batchsize_eval = 55296,# real value
                              batchsize = 55296, # 55296 or 69120
                              lr = 0.0025, # 对齐
                              warmup_steps = 2750, 
                              decay_start = 40000, 
                              decay_steps = 40000, 
                              decay_power = 2.0,
                              end_lr = 1e-6,
                              enable_tf32_compute = True,
                              #use_mixed_precision = True,
                              #scaler = 1024,
                              vvgpu = [[0,1,2,3]], # 8 gpus
                              repeat_dataset = True,
                              use_algorithm_search=False,
                              i64_input_key = True) # in32, False

reader = hugectr.DataReaderParams(data_reader_type = hugectr.DataReaderType_t.Parquet,
                                  source = [f"{data_dir}/train/_file_list.txt"],
                                  eval_source = f"{data_dir}/test/_file_list.txt",
                                  slot_size_array = [62774, 8001, 2901, 74279, 7513, 3369, 1392, 21627, 7919, 21, 276, 1231236, 9643, 39873199, 38853, 17240, 7421, 20263, 3, 7103, 1540, 63, 38457188, 2929249, 400771, 10, 2209, 11910, 152, 4, 976, 14, 39976779, 25414584, 39639858, 583095, 12929, 108, 36],  # real value
                                  check_type = hugectr.Check_t.Non)
optimizer = hugectr.CreateOptimizer(optimizer_type = hugectr.Optimizer_t.Adam,
                                    update_type = hugectr.Update_t.Local, #有可能会影响性能
                                    beta1 = 0.9,
                                    beta2 = 0.999,
                                    epsilon = 1e-8)

dropout_rate = 0.05

model = hugectr.Model(solver, reader, optimizer)
model.add(hugectr.Input(label_dim = 1, label_name = "labels",
                        dense_dim = 0, 
                        dense_name = "dense",
                        data_reader_sparse_param_array = 
                        [hugectr.DataReaderSparseParam("data1", 2, False, 39)])) # 2 False 的含义

model.add(hugectr.SparseEmbedding(embedding_type = hugectr.Embedding_t.LocalizedSlotSparseEmbeddingHash, #有三种可以选 
                            workspace_size_per_gpu_in_mb = 15000,#bigger enough
                            embedding_vec_size = 16,
                            combiner = "sum",
                            sparse_embedding_name = "sparse_embedding1",
                            bottom_name = "data1",
                            slot_size_array = [62774, 8001, 2901, 74279, 7513, 3369, 1392, 21627, 7919, 21, 276, 1231236, 9643, 39873199, 38853, 17240, 7421, 20263, 3, 7103, 1540, 63, 38457188, 2929249, 400771, 10, 2209, 11910, 152, 4, 976, 14, 39976779, 25414584, 39639858, 583095, 12929, 108, 36], 
                            optimizer = optimizer))

model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Reshape,
                            bottom_names = ["sparse_embedding1"],
                            top_names = ["reshape_sparse_embedding"],
                            leading_dim=16 * 39))  

model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.MultiCross,
                            bottom_names = ["reshape_sparse_embedding"],
                            top_names = ["multicross1"],
                            num_layers=4))

# layer1
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
                            bottom_names = ["reshape_sparse_embedding"],
                            top_names = ["fc1"],
                            num_output=1000))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.ReLU,
                            bottom_names = ["fc1"],
                            top_names = ["relu1"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Dropout,
                            bottom_names = ["relu1"],
                            top_names = ["dropout1"],
                            dropout_rate=dropout_rate))

# layer2
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
                            bottom_names = ["dropout1"],
                            top_names = ["fc2"],
                            num_output=1000))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.ReLU,
                            bottom_names = ["fc2"],
                            top_names = ["relu2"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Dropout,
                            bottom_names = ["relu2"],
                            top_names = ["dropout2"],
                            dropout_rate=dropout_rate))

# layer3
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
                            bottom_names = ["dropout2"],
                            top_names = ["fc3"],
                            num_output=1000))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.ReLU,
                            bottom_names = ["fc3"],
                            top_names = ["relu3"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Dropout,
                            bottom_names = ["relu3"],
                            top_names = ["dropout3"],
                            dropout_rate=dropout_rate))

# layer4
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
                            bottom_names = ["dropout3"],
                            top_names = ["fc4"],
                            num_output=1000))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.ReLU,
                            bottom_names = ["fc4"],
                            top_names = ["relu4"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Dropout,
                            bottom_names = ["relu4"],
                            top_names = ["dropout4"],
                            dropout_rate=dropout_rate))

# layer5
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
                            bottom_names = ["dropout4"],
                            top_names = ["fc5"],
                            num_output=1000))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.ReLU,
                            bottom_names = ["fc5"],
                            top_names = ["relu5"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Dropout,
                            bottom_names = ["relu5"],
                            top_names = ["dropout5"],
                            dropout_rate=dropout_rate))

model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Concat,
                            bottom_names = ["dropout5", "multicross1"],
                            top_names = ["concat2"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
                            bottom_names = ["concat2"],
                            top_names = ["fc6"],
                            num_output=1))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.BinaryCrossEntropyLoss,
                            bottom_names = ["fc6", "labels"],
                            top_names = ["loss"]))
model.compile()
model.summary()
# model.fit(max_iter = 2300, display = 200, eval_interval = 1000, snapshot = 1000000, snapshot_prefix = "dcn")
model.fit(max_iter = 75000, display = 1000, eval_interval = 4999, snapshot = 1000000, snapshot_prefix = "dcn")

[HCTR][02:21:10.156][INFO][RK0][main]: Evaluation, AUC: 0.804863
[HCTR][02:21:10.156][INFO][RK0][main]: Eval Time for 100 iters: 3.16018s