Intemittent memory errors when trying to load model.

[balisujohn]

Sometimes the model correctly loads, and sometimes it errors out with the following output. Does anyone have any idea what's going on here?

output (ggml commit 98875cd):

length: 187diffusion_model_load: loading model from '../models/ggml-diffusion-model.bin'
diffusion_model_load: num_heads = 16
diffusion_model_load: out_channels = 200
diffusion_model_load: ggml tensor size    = 368 bytes
diffusion_model_load: backend buffer size =  76.12 MB
lol
lol2
diffusion_model_load: using CUDA backend
created backend
diffusion_model_load: model size  =    76.12 MB
reached
corrupted size vs. prev_size
Aborted (core dumped)

code:

bool diffusion_model_load(const std::string & fname, diffusion_model & model)
{
    printf("%s: loading model from '%s'\n", __func__, fname.c_str());

    auto fin = std::ifstream(fname, std::ios::binary);
    if (!fin) {
        fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
        return false;
    }

      // verify magic
    {
        uint32_t magic;
        fin.read((char *) &magic, sizeof(magic));
        if (magic != GGML_FILE_MAGIC) {
            fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
            return false;
        }
    }

    // load hparams
    {
        auto & hparams = model.hparams;


        int32_t num_heads;
        int32_t out_channels;
       

        fin.read((char *) &hparams.num_heads, sizeof(hparams.num_heads));
        fin.read((char *) &hparams.out_channels, sizeof(hparams.out_channels));
    
        printf("%s: num_heads = %d\n", __func__, hparams.num_heads);
        printf("%s: out_channels = %d\n", __func__, hparams.out_channels);

    
    }    

    size_t buffer_size = 0;

    buffer_size += 1 * 2048 * ggml_type_sizef(GGML_TYPE_F32); // conditioning latent

    buffer_size += 1024 * 1024 * 3 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioning weight
    buffer_size += 1024 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioning bias

    for (int i = 0; i < 4; i++ )
    {
        buffer_size += 1024 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioner attention block norm weight
        buffer_size += 1024 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioner attention block norm bias
        buffer_size += 3072 * 1024 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioner key value query weight
        buffer_size += 3072 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioner key value query bias
        buffer_size += 1024 * 1024 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioner projection out weight
        buffer_size += 1024 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioner projection out bias
        buffer_size += 16* 32 * ggml_type_sizef(GGML_TYPE_F32); // latent conditioner relative position embeddings relative attention bias weight 

    
    }

    buffer_size += 1024; // code norm weight
    buffer_size += 1024; // code norm bias




    printf("%s: ggml tensor size    = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
    printf("%s: backend buffer size = %6.2f MB\n", __func__, buffer_size/(1024.0*1024.0));

     struct ggml_init_params params = {
          ggml_tensor_overhead() * (size_t)(5 + 7*4), //mem size
           NULL, //mem buffer
            true, //no alloc
        };

        std::cout << "lol" << std::endl;
        model.ctx = ggml_init(params);
        std::cout << "lol2" << std::endl;

        if (!model.ctx) {
            fprintf(stderr, "%s: ggml_init() failed\n", __func__);
            return false;
        }


    // initialize the backend
#ifdef GGML_USE_CUBLAS
        fprintf(stderr, "%s: using CUDA backend\n", __func__);
        model.backend = ggml_backend_cuda_init(0);
        std::cout << "created backend" << std::endl;
        if (!model.backend) {
            fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);

        }
#endif

#ifdef GGML_USE_METAL
        fprintf(stderr, "%s: using Metal backend\n", __func__);
        ggml_metal_log_set_callback(ggml_log_callback_default, nullptr);
        model.backend = ggml_backend_metal_init();
        if (!model.backend) {
            fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
        }
        
#endif

        if (!model.backend) {
            // fallback to CPU backend
            fprintf(stderr, "%s: using CPU backend\n", __func__);
            model.backend = ggml_backend_cpu_init();
        }

        if (!model.backend) {
            fprintf(stderr, "%s: ggml_backend_cpu_init() failed\n", __func__);
            return false;
        }


        model.buffer_w = ggml_backend_alloc_buffer(model.backend, buffer_size);


        auto & ctx = model.ctx;

        model.diffusion_conditioning_latent = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2048,1);
        model.latent_conditioner_convolution_weight = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 3,1024,1024);
        model.latent_conditioner_convolution_bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1024);
        model.code_norm_weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1024);
        model.code_norm_bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1024);



        model.latent_conditioner_attention_blocks.resize(1);
        for (int i = 1; i < 5; i ++)
        {

            auto & block = model.latent_conditioner_attention_blocks[i-1];


            block.norm_weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1024);
            block.norm_bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1024);
            block.qkv_weight = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1024, 3072);
            block.qkv_bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3072);
            block.projection_out_weight = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1024,1024);
            block.projection_out_bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1024);
            block.relative_position_embeddings_relative_attention_bias_weight = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 16,32);


            model.tensors["latent_conditioner." + std::to_string(i) + ".norm.weight"] =  block.norm_weight;
            model.tensors["latent_conditioner." + std::to_string(i) + ".norm.bias"] =  block.norm_bias;
            model.tensors["latent_conditioner." + std::to_string(i) + ".qkv.weight"] =  block.qkv_weight;
            model.tensors["latent_conditioner." + std::to_string(i) + ".qkv.bias"] =  block.qkv_bias;
            model.tensors["latent_conditioner." + std::to_string(i) + ".proj_out.weight"] =  block.projection_out_weight;
            model.tensors["latent_conditioner." + std::to_string(i) + ".proj_out.bias"] =  block.projection_out_bias;
            model.tensors["latent_conditioner." + std::to_string(i) + ".relative_pos_embeddings.relative_attention_bias.weight"] =  block.relative_position_embeddings_relative_attention_bias_weight;

    
        }



        model.tensors["diffusion_conditioning_latent"] = model.diffusion_conditioning_latent;
        model.tensors["latent_conditioner.0.bias"] = model.latent_conditioner_convolution_bias;
        model.tensors["latent_conditioner.0.weight"] = model.latent_conditioner_convolution_weight;

        model.tensors["code_norm.weight"] = model.code_norm_weight;
        model.tensors["code_norm.bias"] = model.code_norm_bias;





        {
        //ggml_allocr * alloc = ggml_allocr_new_from_buffer(model.buffer_w);
        model.buffer_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);

        size_t total_size = 0;

        bool has_lm_head = false;

        std::vector<char> read_buf;

     while (true) {
            int32_t n_dims;
            int32_t length;
            int32_t ttype;

            fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
            fin.read(reinterpret_cast<char *>(&length), sizeof(length));
            fin.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));

            if (fin.eof()) {
                break;
            }

            int32_t nelements = 1;
            int32_t ne[2] = { 1, 1 };
            for (int i = 0; i < n_dims; ++i) {
                fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
                nelements *= ne[i];
            }

            std::string name(length, 0);
            fin.read(&name[0], length);

            if (model.tensors.find(name) == model.tensors.end()) {
                fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
                return false;
            }

            auto tensor = model.tensors[name];
            ggml_set_name(tensor, name.c_str());
            if (ggml_nelements(tensor) != nelements) {
                fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
                return false;
            }

            if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
                fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
                        __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
                return false;
            }

            // for debugging
            if (0) {
                printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
            }

            const size_t bpe = ggml_type_size(ggml_type(ttype));

            if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
                fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
                        __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
                return false;
            }

            if (ggml_backend_buffer_is_host(model.buffer_w)) {
                // for some backends such as CPU and Metal, the tensor data is in system memory and we can read directly into it
                fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
            } else {
                // read into a temporary buffer first, then copy to device memory
                read_buf.resize(ggml_nbytes(tensor));
                fin.read(read_buf.data(), ggml_nbytes(tensor));
                ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor));
            }

    

            total_size += ggml_nbytes(tensor);
        }


        printf("%s: model size  = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
    }

    fin.close();

    return true;


}

[balisujohn]

It's weird because my gpt2 model always seems to load correctly, but when I load this model afterwards, using as far as I can tell the same loading strategy and the same file format, the load seems to non-deterministically fail.

[balisujohn]

I think I fixed it lol, weird that it worked even intermittently before. If it's anything other than user error, I'll make a note of it here.

[balisujohn]

model.latent_conditioner_attention_blocks.resize(1);

This is the offending line, in case anyone encounters something similar; super strange that it does sometimes successfully load the model, even with the vector at size 1. Setting the vector size to 4 fixes the problem.