Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ggml_compute_forward_concat() for arbitrary tensor type #1118

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

ggml_compute_forward_concat() for arbitrary tensor type #1118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4cc7679
ggml_compute_forward_concat() for arbitrary tensor type
vmobilis Feb 20, 2025
d02df79
Check that tensors' type match
vmobilis Feb 21, 2025
1478f33
ggml-cpu.c: check type of source tensors
vmobilis Feb 21, 2025
5f4fde3
ggml-cpu.c: move tensor type check to ggml_compute_forward_concat()
vmobilis Feb 21, 2025
9a3041d
ggml.c: check concatenated tensor type
vmobilis Feb 21, 2025
e0832b7
Remove tensor type check from ggml_compute_forward_concat() in ggml-c…
vmobilis Feb 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 141 additions & 2 deletions src/ggml-cpu/ggml-cpu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6094,14 +6094,143 @@ static void ggml_compute_forward_repeat_back(

// ggml_compute_forward_concat

static void ggml_compute_forward_concat_any(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {

const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];

const size_t len = ggml_type_size(src0->type);

const int ith = params->ith;
const int nth = params->nth;

GGML_TENSOR_BINARY_OP_LOCALS

const int32_t dim = ggml_get_op_params_i32(dst, 0);

GGML_ASSERT(dim >= 0 && dim < 4);

int64_t o[4] = {0, 0, 0, 0};
o[dim] = src0->ne[dim];

const char * x;

// TODO: smarter multi-theading
for (int i3 = 0; i3 < ne3; i3++) {
for (int i2 = ith; i2 < ne2; i2 += nth) {
for (int i1 = 0; i1 < ne1; i1++) {
for (int i0 = 0; i0 < ne0; i0++) {
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03;
} else {
x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13;
}

char * y = (char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3;

memcpy(y, x, len);
}
}
}
}
}

static void ggml_compute_forward_concat_i8(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {

const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];

GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t));

const int ith = params->ith;
const int nth = params->nth;

GGML_TENSOR_BINARY_OP_LOCALS

const int32_t dim = ggml_get_op_params_i32(dst, 0);

GGML_ASSERT(dim >= 0 && dim < 4);

int64_t o[4] = {0, 0, 0, 0};
o[dim] = src0->ne[dim];

const int8_t * x;

// TODO: smarter multi-theading
for (int i3 = 0; i3 < ne3; i3++) {
for (int i2 = ith; i2 < ne2; i2 += nth) {
for (int i1 = 0; i1 < ne1; i1++) {
for (int i0 = 0; i0 < ne0; i0++) {
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
} else {
x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
}

int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);

*y = *x;
}
}
}
}
}

static void ggml_compute_forward_concat_f16(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {

const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];

GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t));

const int ith = params->ith;
const int nth = params->nth;

GGML_TENSOR_BINARY_OP_LOCALS

const int32_t dim = ggml_get_op_params_i32(dst, 0);

GGML_ASSERT(dim >= 0 && dim < 4);

int64_t o[4] = {0, 0, 0, 0};
o[dim] = src0->ne[dim];

const ggml_fp16_t * x;

// TODO: smarter multi-theading
for (int i3 = 0; i3 < ne3; i3++) {
for (int i2 = ith; i2 < ne2; i2 += nth) {
for (int i1 = 0; i1 < ne1; i1++) {
for (int i0 = 0; i0 < ne0; i0++) {
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
} else {
x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
}

ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);

*y = *x;
}
}
}
}
}

static void ggml_compute_forward_concat_f32(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {

const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];

GGML_ASSERT(src0->nb[0] == sizeof(float));
GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float));

const int ith = params->ith;
const int nth = params->nth;
Expand Down Expand Up @@ -6144,14 +6273,24 @@ static void ggml_compute_forward_concat(
const struct ggml_tensor * src0 = dst->src[0];

switch (src0->type) {
case GGML_TYPE_F16:
case GGML_TYPE_BF16:
case GGML_TYPE_I16:
{
ggml_compute_forward_concat_f16(params, dst);
} break;
case GGML_TYPE_I8:
{
ggml_compute_forward_concat_i8(params, dst);
} break;
case GGML_TYPE_F32:
case GGML_TYPE_I32:
{
ggml_compute_forward_concat_f32(params, dst);
} break;
default:
{
GGML_ABORT("fatal error");
ggml_compute_forward_concat_any(params, dst);
}
}
}
Expand Down
1 change: 1 addition & 0 deletions src/ggml.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2328,6 +2328,7 @@ struct ggml_tensor * ggml_concat(
struct ggml_tensor * b,
int dim) {
GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS);
GGML_ASSERT(a->type == b->type);

int64_t ne[GGML_MAX_DIMS];
for (int d = 0; d < GGML_MAX_DIMS; ++d) {
Expand Down