Newer
Older
// South-West
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_x = (coordinate_x - 1) >= 0 ? coordinate_x - 1 : dc->network_dimensions[1] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, 0);
int send_count = (int) inner_envelope->front->south_west->count;
MPI_Isend(inner_envelope->front->south_west->data, send_count, dc->datatype, dest_source, FRONT_SOUTH_WEST_TAG,
/* Neighbour send north-east cell, which correspond to south-west cell */
int dimensions[3] = {thickness, thickness, 1};
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
chunk_info_init(outer_envelope->front->south_west, dc->type, dimensions, start_y, start_x, 0, true);
int recv_count = (int) outer_envelope->front->south_west->count;
MPI_Irecv(outer_envelope->front->south_west->data, recv_count, dc->datatype, dest_source, FRONT_NORTH_EAST_TAG,
}
// North-West
{
int dest_source_y = (coordinate_y - 1) >= 0 ? coordinate_y - 1 : dc->network_dimensions[0] - 1;
int dest_source_x = (coordinate_x - 1) >= 0 ? coordinate_x - 1 : dc->network_dimensions[1] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, 0);
int send_count = (int) inner_envelope->front->north_west->count;
MPI_Isend(inner_envelope->front->north_west->data, send_count, dc->datatype, dest_source, FRONT_NORTH_WEST_TAG,
/* Neighbour send south-east cell, which correspond to north-west cell */
int dimensions[3] = {thickness, thickness, 1};
int start_y = dc->chunks_info[dest_source].y + dc->chunks_info[dest_source].dimensions[0] - dimensions[0];
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
chunk_info_init(outer_envelope->front->north_west, dc->type, dimensions, start_y, start_x, 0, true);
int recv_count = (int) outer_envelope->front->north_west->count;
MPI_Irecv(outer_envelope->front->north_west->data, recv_count, dc->datatype, dest_source, FRONT_SOUTH_EAST_TAG,
}
MPI_Waitall(i_request, requests, MPI_STATUSES_IGNORE);
return outer_envelope;
}
static envelope_t *get_outer_envelope_3d(struct dispatch_context *dc, int thickness, envelope_t *inner_envelope) {
int coordinate_y = dc->coordinates[0];
int coordinate_x = dc->coordinates[1];
int coordinate_z = dc->coordinates[2];
MPI_Request requests[52] = {0};
int i_request = 0;
envelope_t *outer_envelope = envelope_new();
// Back
{
// Surface
{
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, coordinate_y, coordinate_x, dest_source_z);
int send_count = (int) inner_envelope->back->surface->count;
MPI_Isend(inner_envelope->back->surface->data, send_count, dc->datatype, dest_source_z, BACK_SURFACE_TAG,
dc->communicators[DEPTH_COMMUNICATOR], &requests[i_request++]);
// Neighbour send front-surface, which correspond to back-surface
int dimensions[3] = {
dc->chunks_info[dest_source].dimensions[0],
dc->chunks_info[dest_source].dimensions[1],
min(thickness, dc->chunks_info[dest_source].dimensions[2]),
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->back->surface, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->back->surface->data, (int) outer_envelope->back->surface->count, dc->datatype,
dest_source_z, FRONT_SURFACE_TAG, dc->communicators[DEPTH_COMMUNICATOR], &requests[i_request++]);
}
// East
{
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, coordinate_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->back->east->count;
MPI_Isend(inner_envelope->back->east->data, send_count, dc->datatype, dest_source, BACK_EAST_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send front-west, which correspond to back-east
int dimensions[3] = {
dc->chunks_info[dest_source].dimensions[0],
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2]),
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->back->east, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->back->east->data, (int) outer_envelope->back->east->count, dc->datatype,
dest_source, FRONT_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// West
{
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, coordinate_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->back->west->count;
MPI_Isend(inner_envelope->back->west->data, send_count, dc->datatype, dest_source, BACK_WEST_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send front-east, which correspond to back-west
int dimensions[3] = {
dc->chunks_info[dest_source].dimensions[0],
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2]),
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->back->west, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->back->west->data, (int) outer_envelope->back->west->count, dc->datatype,
dest_source, FRONT_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, coordinate_x, coordinate_z);
int send_count = (int) inner_envelope->bottom->surface->count;
MPI_Isend(inner_envelope->bottom->surface->data, send_count, dc->datatype, dest_source_y,
BOTTOM_SURFACE_TAG, dc->communicators[COLUMN_COMMUNICATOR], &requests[i_request++]);
// Neighbour send top-surface, which correspond to bottom-surface
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
dc->chunks_info[dest_source].dimensions[1],
dc->chunks_info[dest_source].dimensions[2],
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->bottom->surface, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->surface->data, (int) outer_envelope->bottom->surface->count, dc->datatype,
dest_source_y, TOP_SURFACE_TAG, dc->communicators[COLUMN_COMMUNICATOR], &requests[i_request++]);
}
// North-West
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->bottom->north_west->count;
MPI_Isend(inner_envelope->bottom->north_west->data, send_count, dc->datatype, dest_source,
BOTTOM_NORTH_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send top-south-east, which correspond to bottom-north-west
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->bottom->north_west, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->north_west->data, (int) outer_envelope->bottom->north_west->count,
dc->datatype, dest_source, TOP_SOUTH_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// North-East
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->bottom->north_east->count;
MPI_Isend(inner_envelope->bottom->north_east->data, send_count, dc->datatype, dest_source,
BOTTOM_NORTH_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send top-south-west, which correspond to bottom-north-east
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->bottom->north_east, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->north_east->data, (int) outer_envelope->bottom->north_east->count,
dc->datatype, dest_source, TOP_SOUTH_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// South-West
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->bottom->south_west->count;
MPI_Isend(inner_envelope->bottom->south_west->data, send_count, dc->datatype, dest_source,
BOTTOM_SOUTH_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send top-north-east, which correspond to bottom-south-west
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z + dc->chunks_info[dest_source].dimensions[2] - dimensions[2];
chunk_info_init(outer_envelope->bottom->south_west, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->south_west->data, (int) outer_envelope->bottom->south_west->count,
dc->datatype, dest_source, TOP_NORTH_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// South-East
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->bottom->south_east->count;
MPI_Isend(inner_envelope->bottom->south_east->data, send_count, dc->datatype, dest_source,
BOTTOM_SOUTH_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send top-north-west, which correspond to bottom-south-east
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z + dc->chunks_info[dest_source].dimensions[2] - dimensions[2];
chunk_info_init(outer_envelope->bottom->south_east, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->south_east->data, (int) outer_envelope->bottom->south_east->count,
dc->datatype, dest_source, TOP_NORTH_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// East
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, coordinate_z);
int send_count = (int) inner_envelope->bottom->east->count;
MPI_Isend(inner_envelope->bottom->east->data, send_count, dc->datatype, dest_source, BOTTOM_EAST_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send top-west, which correspond to bottom-east
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
dc->chunks_info[dest_source].dimensions[2]
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->bottom->east, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->east->data, (int) outer_envelope->bottom->east->count, dc->datatype,
dest_source, TOP_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// West
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, coordinate_z);
int send_count = (int) inner_envelope->bottom->west->count;
MPI_Isend(inner_envelope->bottom->west->data, send_count, dc->datatype, dest_source, BOTTOM_WEST_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send top-east, which correspond to bottom-west
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
dc->chunks_info[dest_source].dimensions[2]
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->bottom->west, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->west->data, (int) outer_envelope->bottom->west->count, dc->datatype,
dest_source, TOP_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// North
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, coordinate_x, dest_source_z);
int send_count = (int) inner_envelope->bottom->north->count;
MPI_Isend(inner_envelope->bottom->north->data, send_count, dc->datatype, dest_source, BOTTOM_NORTH_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send top-south, which correspond to bottom-north
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
dc->chunks_info[dest_source].dimensions[1],
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->bottom->north, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->north->data, (int) outer_envelope->bottom->north->count, dc->datatype,
dest_source, TOP_SOUTH_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// South
{
int dest_source_y = (coordinate_y + 1) % dc->network_dimensions[0];
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, coordinate_x, dest_source_z);
int send_count = (int) inner_envelope->bottom->south->count;
MPI_Isend(inner_envelope->bottom->south->data, send_count, dc->datatype, dest_source, BOTTOM_SOUTH_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send top-north, which correspond to bottom-south
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
dc->chunks_info[dest_source].dimensions[1],
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z + dc->chunks_info[dest_source].dimensions[2] - dimensions[2];
chunk_info_init(outer_envelope->bottom->south, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->bottom->south->data, (int) outer_envelope->bottom->south->count, dc->datatype,
dest_source, TOP_NORTH_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
}
// Front
{
// Surface
{
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, coordinate_y, coordinate_x, dest_source_z);
int send_count = (int) inner_envelope->front->surface->count;
MPI_Isend(inner_envelope->front->surface->data, send_count, dc->datatype, dest_source_z, FRONT_SURFACE_TAG,
dc->communicators[DEPTH_COMMUNICATOR], &requests[i_request++]);
// Neighbour send back-surface, which correspond to front-surface
int dimensions[3] = {
dc->chunks_info[dest_source].dimensions[0],
dc->chunks_info[dest_source].dimensions[1],
min(thickness, dc->chunks_info[dest_source].dimensions[2]),
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z + dc->chunks_info[dest_source].dimensions[2] - dimensions[2];
chunk_info_init(outer_envelope->front->surface, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->front->surface->data, (int) outer_envelope->front->surface->count, dc->datatype,
dest_source_z, BACK_SURFACE_TAG, dc->communicators[DEPTH_COMMUNICATOR], &requests[i_request++]);
}
// East
{
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, coordinate_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->front->east->count;
MPI_Isend(inner_envelope->front->east->data, send_count, dc->datatype, dest_source, FRONT_EAST_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send back-west, which correspond to front-east
int dimensions[3] = {
dc->chunks_info[dest_source].dimensions[0],
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z + dc->chunks_info[dest_source].dimensions[2] - dimensions[2];
chunk_info_init(outer_envelope->front->east, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->front->east->data, (int) outer_envelope->front->east->count, dc->datatype,
dest_source, BACK_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// West
{
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, coordinate_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->front->west->count;
MPI_Isend(inner_envelope->front->west->data, send_count, dc->datatype, dest_source, FRONT_WEST_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send back-east, which correspond to front-west
int dimensions[3] = {
dc->chunks_info[dest_source].dimensions[0],
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2]),
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z + dc->chunks_info[dest_source].dimensions[2] - dimensions[2];
chunk_info_init(outer_envelope->front->west, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->front->west->data, (int) outer_envelope->front->west->count, dc->datatype,
dest_source, BACK_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
}
// Left
{
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source = cart_rank_to_comm_rank(dc, coordinate_y, dest_source_x, coordinate_z);
int send_count = (int) inner_envelope->left->surface->count;
MPI_Isend(inner_envelope->left->surface->data, send_count, dc->datatype, dest_source_x, LEFT_SURFACE_TAG,
dc->communicators[ROW_COMMUNICATOR], &requests[i_request++]);
// Neighbour send right-surface, which correspond to left-surface
int dimensions[3] = {
dc->chunks_info[dest_source].dimensions[0],
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
dc->chunks_info[dest_source].dimensions[2],
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->left->surface, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->left->surface->data, (int) outer_envelope->left->surface->count, dc->datatype,
dest_source_x, RIGHT_SURFACE_TAG, dc->communicators[ROW_COMMUNICATOR], &requests[i_request++]);
}
// Right
{
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source = cart_rank_to_comm_rank(dc, coordinate_y, dest_source_x, coordinate_z);
int send_count = (int) inner_envelope->right->surface->count;
MPI_Isend(inner_envelope->right->surface->data, send_count, dc->datatype, dest_source_x, RIGHT_SURFACE_TAG,
dc->communicators[ROW_COMMUNICATOR], &requests[i_request++]);
// Neighbour send left-surface, which correspond to right-surface
int dimensions[3] = {
dc->chunks_info[dest_source].dimensions[0],
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
dc->chunks_info[dest_source].dimensions[2],
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->right->surface, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->right->surface->data, (int) outer_envelope->right->surface->count, dc->datatype,
dest_source_x, LEFT_SURFACE_TAG, dc->communicators[ROW_COMMUNICATOR], &requests[i_request++]);
}
// Top
{
// Surface
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, coordinate_x, coordinate_z);
int send_count = (int) inner_envelope->top->surface->count;
MPI_Isend(inner_envelope->top->surface->data, send_count, dc->datatype, dest_source_y, TOP_SURFACE_TAG,
dc->communicators[COLUMN_COMMUNICATOR], &requests[i_request++]);
// Neighbour send bottom-surface, which correspond to top-surface
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
dc->chunks_info[dest_source].dimensions[1],
dc->chunks_info[dest_source].dimensions[2],
};
int start_y = dc->chunks_info[dest_source].y + dc->chunks_info[dest_source].dimensions[0] - dimensions[0];
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->top->surface, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->surface->data, (int) outer_envelope->top->surface->count, dc->datatype,
dest_source_y, BOTTOM_SURFACE_TAG, dc->communicators[COLUMN_COMMUNICATOR],
&requests[i_request++]);
}
// North-West
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->top->north_west->count;
MPI_Isend(inner_envelope->top->north_west->data, send_count, dc->datatype, dest_source,
TOP_NORTH_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send bottom-south-east, which correspond to top-north-west
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y + dc->chunks_info[dest_source].dimensions[0] - dimensions[0];
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->top->north_west, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->north_west->data, (int) outer_envelope->top->north_west->count, dc->datatype,
dest_source, BOTTOM_SOUTH_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// North-East
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->top->north_east->count;
MPI_Isend(inner_envelope->top->north_east->data, send_count, dc->datatype, dest_source,
TOP_NORTH_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send bottom-south-west, which correspond to top-north-east
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y + dc->chunks_info[dest_source].dimensions[0] - dimensions[0];
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->top->north_east, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->north_east->data, (int) outer_envelope->top->north_east->count, dc->datatype,
dest_source, BOTTOM_SOUTH_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// South-West
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->top->south_west->count;
MPI_Isend(inner_envelope->top->south_west->data, send_count, dc->datatype, dest_source,
TOP_SOUTH_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send bottom-north-east, which correspond to top-south-west
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z + dc->chunks_info[dest_source].dimensions[2] - dimensions[2];
chunk_info_init(outer_envelope->top->south_west, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->south_west->data, (int) outer_envelope->top->south_west->count, dc->datatype,
dest_source, BOTTOM_NORTH_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// South-East
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, dest_source_z);
int send_count = (int) inner_envelope->top->south_east->count;
MPI_Isend(inner_envelope->top->south_east->data, send_count, dc->datatype, dest_source,
TOP_SOUTH_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send bottom-north-west, which correspond to top-south-east
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
min(thickness, dc->chunks_info[dest_source].dimensions[2])
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->top->south_east, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->south_east->data, (int) outer_envelope->top->south_east->count, dc->datatype,
dest_source, BOTTOM_NORTH_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// East
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source_x = (coordinate_x + 1) % dc->network_dimensions[1];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, coordinate_z);
int send_count = (int) inner_envelope->top->east->count;
MPI_Isend(inner_envelope->top->east->data, send_count, dc->datatype, dest_source, TOP_EAST_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send bottom-west, which correspond to top-east
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
dc->chunks_info[dest_source].dimensions[2]
};
int start_y = dc->chunks_info[dest_source].y + dc->chunks_info[dest_source].dimensions[0] - dimensions[0];
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->top->east, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->east->data, (int) outer_envelope->top->east->count, dc->datatype,
dest_source,
BOTTOM_WEST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// West
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source_x = (coordinate_x - 1) >= 0 ? (coordinate_x - 1) : dc->network_dimensions[1] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, dest_source_x, coordinate_z);
int send_count = (int) inner_envelope->top->west->count;
MPI_Isend(inner_envelope->top->west->data, send_count, dc->datatype, dest_source, TOP_WEST_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send bottom-east, which correspond to top-west
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
min(thickness, dc->chunks_info[dest_source].dimensions[1]),
dc->chunks_info[dest_source].dimensions[2]
};
int start_y = dc->chunks_info[dest_source].y + dc->chunks_info[dest_source].dimensions[0] - dimensions[0];
int start_x = dc->chunks_info[dest_source].x + dc->chunks_info[dest_source].dimensions[1] - dimensions[1];
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->top->west, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->west->data, (int) outer_envelope->top->west->count, dc->datatype,
dest_source,
BOTTOM_EAST_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// North
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source_z = (coordinate_z + 1) % dc->network_dimensions[2];
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, coordinate_x, dest_source_z);
int send_count = (int) inner_envelope->top->north->count;
MPI_Isend(inner_envelope->top->north->data, send_count, dc->datatype, dest_source, TOP_NORTH_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send bottom-south, which correspond to top-north
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
dc->chunks_info[dest_source].dimensions[1],
min(thickness, dc->chunks_info[dest_source].dimensions[1])
};
int start_y = dc->chunks_info[dest_source].y + dc->chunks_info[dest_source].dimensions[0] - dimensions[0];
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z;
chunk_info_init(outer_envelope->top->north, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->north->data, (int) outer_envelope->top->north->count, dc->datatype,
dest_source, BOTTOM_SOUTH_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
// South
{
int dest_source_y = (coordinate_y - 1) >= 0 ? (coordinate_y - 1) : dc->network_dimensions[0] - 1;
int dest_source_z = (coordinate_z - 1) >= 0 ? (coordinate_z - 1) : dc->network_dimensions[2] - 1;
int dest_source = cart_rank_to_comm_rank(dc, dest_source_y, coordinate_x, dest_source_z);
int send_count = (int) inner_envelope->top->south->count;
MPI_Isend(inner_envelope->top->south->data, send_count, dc->datatype, dest_source, TOP_SOUTH_TAG,
MPI_COMM_WORLD, &requests[i_request++]);
// Neighbour send bottom-north, which correspond to top-south
int dimensions[3] = {
min(thickness, dc->chunks_info[dest_source].dimensions[0]),
dc->chunks_info[dest_source].dimensions[1],
min(thickness, dc->chunks_info[dest_source].dimensions[2]),
};
int start_y = dc->chunks_info[dest_source].y;
int start_x = dc->chunks_info[dest_source].x;
int start_z = dc->chunks_info[dest_source].z + dc->chunks_info[dest_source].dimensions[2] - dimensions[2];
chunk_info_init(outer_envelope->top->south, dc->type, dimensions, start_y, start_x, start_z, true);
MPI_Irecv(outer_envelope->top->south->data, (int) outer_envelope->top->south->count, dc->datatype,
dest_source, BOTTOM_NORTH_TAG, MPI_COMM_WORLD, &requests[i_request++]);
}
}
MPI_Waitall(i_request, requests, MPI_STATUSES_IGNORE);
return outer_envelope;
extern envelope_t *get_outer_envelope(struct dispatch_context *dc, struct futhark_context *fc, int thickness) {
envelope_t *inner_envelope = get_inner_envelope(dc, fc, thickness);
envelope_t *outer_envelope = NULL;
switch (dc->n_dimensions) {
case 1:
outer_envelope = get_outer_envelope_1d(dc, thickness, inner_envelope);
outer_envelope = get_outer_envelope_2d(dc, thickness, inner_envelope);
outer_envelope = get_outer_envelope_3d(dc, thickness, inner_envelope);
break;
default:
fprintf(stderr, "Invalid dimensions size.");
MPI_Abort(MPI_COMM_WORLD, 1);
break;
}
envelope_free(inner_envelope);
return outer_envelope;
}
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
static uint8_t *
chunk_info_to_futhark_struct(chunk_info_t *ci, struct dispatch_context *dc, uint8_t *out, char *futhark_type) {
/* Taken from Futhark generated code */
*out++ = 'b';
*out++ = 2;
*out++ = dc->n_dimensions;
memcpy(out, futhark_type, 4);
out += 4;
int64_t dimensions64[3];
dimensions64[0] = (int64_t) ci->dimensions[0];
dimensions64[1] = (int64_t) ci->dimensions[1];
dimensions64[2] = (int64_t) ci->dimensions[2];
if (dc->n_dimensions == 1) {
memcpy(out, &dimensions64[1], 1 * sizeof(int64_t));
} else if (dc->n_dimensions == 2) {
memcpy(out, dimensions64, 2 * sizeof(int64_t));
} else {
memcpy(out, dimensions64, 3 * sizeof(int64_t));
}
out += dc->n_dimensions * sizeof(int64_t);
memcpy(out, ci->data, ci->count * dc->type);
out += ci->count * dc->type;
return out;
}
static void *
futhark_outer_envelope_1d_new(struct dispatch_context *dc, struct futhark_context *fc, envelope_t *outer_envelope,
void *f(struct futhark_context *, const void *), char *futhark_type) {
/* Taken from Futhark generated code */
int64_t size_0 = 7 + 1 * sizeof(int64_t) + outer_envelope->front->east->count * dc->type;
int64_t size_1 = 7 + 1 * sizeof(int64_t) + outer_envelope->front->west->count * dc->type;
void *opaque_struct = calloc(size_0 + size_1, sizeof(uint8_t));
uint8_t *opaque_struct8 = (uint8_t *) opaque_struct;
// East
opaque_struct8 = chunk_info_to_futhark_struct(outer_envelope->front->east, dc, opaque_struct8, futhark_type);
// West
chunk_info_to_futhark_struct(outer_envelope->front->west, dc, opaque_struct8, futhark_type);
void *fut_opaque_struct = f(fc, opaque_struct);
futhark_context_sync(fc);
free(opaque_struct);
return fut_opaque_struct;
}
static void *
futhark_outer_envelope_2d_new(struct dispatch_context *dc, struct futhark_context *fc, envelope_t *outer_envelope,
void *f(struct futhark_context *, const void *), char *futhark_type) {
size_t total_size = 0;
for (int i = 0; i < NB_CHUNKS; ++i) {
int j = FUTHARK_CHUNKS_ORDER[i];
if (j != INDEX_CHUNK_SURFACE) {
total_size += (7 + 2 * sizeof(int64_t) + outer_envelope->front->chunks[j].count * dc->type);
}
}
void *opaque_struct = calloc(total_size, sizeof(uint8_t));
uint8_t *opaque_struct8 = (uint8_t *) opaque_struct;
for (int i = 0; i < NB_CHUNKS; ++i) {
int j = FUTHARK_CHUNKS_ORDER[i];
if (j != INDEX_CHUNK_SURFACE) {
opaque_struct8 = chunk_info_to_futhark_struct(&outer_envelope->front->chunks[j], dc, opaque_struct8,
futhark_type);
}
}
void *fut_opaque_struct = f(fc, opaque_struct);
futhark_context_sync(fc);
free(opaque_struct);
return fut_opaque_struct;
}
static void *
futhark_outer_envelope_3d_new(struct dispatch_context *dc, struct futhark_context *fc, envelope_t *outer_envelope,
void *f(struct futhark_context *, const void *), char *futhark_type) {
size_t total_size = 0;
for (int i = 0; i < NB_SIDES; ++i) {
for (int j = 0; j < NB_CHUNKS; ++j) {
int k = FUTHARK_CHUNKS_ORDER[j];
total_size += (7 + 3 * sizeof(int64_t) + outer_envelope->sides[i].chunks[k].count * dc->type);
}
}
void *opaque_struct = calloc(total_size, sizeof(uint8_t));
uint8_t *opaque_struct8 = (uint8_t *) opaque_struct;
for (int i = 0; i < NB_SIDES; ++i) {
for (int j = 0; j < NB_CHUNKS; ++j) {
int k = FUTHARK_CHUNKS_ORDER[j];
opaque_struct8 = chunk_info_to_futhark_struct(&outer_envelope->sides[i].chunks[k], dc, opaque_struct8,
futhark_type);
}
}
void *fut_opaque_struct = f(fc, opaque_struct);
futhark_context_sync(fc);
free(opaque_struct);
return fut_opaque_struct;
}
extern void *futhark_outer_envelope_new(struct dispatch_context *dc, struct futhark_context *fc,
envelope_t *outer_envelope, void *f(struct futhark_context *, const void *),
char *futhark_type) {
switch (dc->n_dimensions) {
case 1:
return futhark_outer_envelope_1d_new(dc, fc, outer_envelope, f, futhark_type);
case 2:
return futhark_outer_envelope_2d_new(dc, fc, outer_envelope, f, futhark_type);
case 3:
return futhark_outer_envelope_3d_new(dc, fc, outer_envelope, f, futhark_type);
default:
fprintf(stderr, "Invalid dimensions size.");
MPI_Abort(MPI_COMM_WORLD, 1);
return NULL;
}
}
static void chunk_data_to_data(struct dispatch_context *dc, void *chunk_data, void *data, int rank) {
int y = dc->chunks_info[rank].y;
int x = dc->chunks_info[rank].x;
int z = dc->chunks_info[rank].z;
uint8_t *data8 = (uint8_t *) data;
uint8_t *chunk_data8 = (uint8_t *) chunk_data;
for (int i = 0; i < dc->chunks_info[rank].dimensions[0]; ++i) {
for (int j = 0; j < dc->chunks_info[rank].dimensions[1]; ++j) {
for (int k = 0; k < dc->chunks_info[rank].dimensions[2]; ++k) {
uint8_t *src = chunk_data8 + (INDEX_3D_TO_1D(i, j, k, dc->chunks_info[rank].dimensions[1],
dc->chunks_info[rank].dimensions[2]) * dc->type);
uint8_t *dst = data8
+ (INDEX_3D_TO_1D(y + i, x + j, z + k,
dc->data_dimensions[1], dc->data_dimensions[2])
* dc->type);
memcpy(dst, src, (size_t) dc->type);
}
extern void *get_data(struct dispatch_context *dc) {
void *data = NULL;
if (dc->my_rank == ROOT_RANK) {
chunk_data_to_data(dc, dc->chunk_info->data, data, dc->my_rank);
for (int i = 0; i < dc->world_size; ++i) {
if (i != dc->my_rank) {
chunk_info_allocate_data(&dc->chunks_info[i], dc->type);
MPI_Recv(dc->chunks_info[i].data, (int) dc->chunks_info[i].count, dc->datatype, i, CHUNK_DATA_TAG,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
chunk_data_to_data(dc, dc->chunks_info[i].data, data, i);
MPI_Send(dc->chunk_info->data, (int) dc->chunk_info->count, dc->datatype, ROOT_RANK, CHUNK_DATA_TAG,
MPI_COMM_WORLD);
}
return data;
}
extern chunk_info_t get_chunk_info(struct dispatch_context *dc) {
return *dc->chunk_info;
}
extern void dispatch_context_free(struct dispatch_context *dc) {
MPI_Comm_free(&dc->communicators[2]);
MPI_Comm_free(&dc->communicators[1]);
MPI_Comm_free(&dc->communicators[0]);
}
static void side_envelope_free(side_envelope_t *side_envelope) {
for (int i = 0; i < NB_CHUNKS; ++i) {
chunk_info_free(&side_envelope->chunks[i]);
}
}
for (int i = 0; i < NB_SIDES; ++i) {
side_envelope_free(&envelope->sides[i]);
}