Matrix Multiplication : Performance decreases after coalescing global memory access in CUDA - parallel-processing

Question

I have started recently started working with GPUs using CUDA. As a starter programme I tried to implement a simple matrix multiplication efficiently
C = AB
, Starting with the naive matrix multiplication (each thread loads all the elements of A and B for an element in C), the tiled implementation (threads collaboratively load a tile of elements from A and B in a tile in shared memory to reduce global memory traffic) provides good speed up.
However, in the tiled implementation too the access to the global memory is not in coalesced order. So, to increase performance it is better to transpose matrix B and then multiply. Below is my code,
#include<stdio.h>
#include<stdlib.h>
#include<cuda_runtime.h>
#include <time.h>
#include <sys/time.h>
void querydeviceprop();
void allocate_matrix(float *h_a, float *h_b, int matDim);
void verify(float *h_c, float *h_c_check, int matDim);
void print_matrix(float *ha, int m,int n);
void transpose_matrix(float *ha, int matDim);
void mat_mul();
#define TILE_WIDTH 16 //should be equal to numThread for tiling implementation
__global__ void MatrixMult_tiling(float *d_a,float *d_b,float *d_c, int dim){
__shared__ float ta[TILE_WIDTH][TILE_WIDTH]; //to load one tile of A
__shared__ float tb[TILE_WIDTH][TILE_WIDTH]; //to load one tile of A
int bx,by,tx,ty,i,j;
float res;
int row, col;
bx=blockIdx.x; by=blockIdx.y;
tx=threadIdx.x; ty=threadIdx.y;
row=by*TILE_WIDTH+ty;
col=bx*TILE_WIDTH+tx;
res=0;
for(i=0;i<dim/TILE_WIDTH;i++){
//collaboratively load the elements. Each thread loads a single element.
ta[ty][tx]=d_a[row*dim+TILE_WIDTH*i+tx];
tb[ty][tx]=d_b[(ty+i*TILE_WIDTH)*dim+col];
__syncthreads();
for(j=0;j<TILE_WIDTH;j++){
res=res+ta[ty][j]*tb[j][tx];
}
__syncthreads();
}
d_c[row*dim+col]=res;
}
__global__ void MatrixMult_tiling_coalesced(float *d_a,float *d_b,float *d_c, int dim){
__shared__ float ta[TILE_WIDTH][TILE_WIDTH]; //to load one tile of A
__shared__ float tb[TILE_WIDTH][TILE_WIDTH]; //to load one tile of A
int bx,by,tx,ty,i,j;
float res;
int row, col;
bx=blockIdx.x; by=blockIdx.y;
tx=threadIdx.x; ty=threadIdx.y;
row=by*TILE_WIDTH+ty;
col=bx*TILE_WIDTH+tx;
res=0;
for(i=0;i<dim/TILE_WIDTH;i++){
//collaboratively load the elements. Each thread loads a single element.
ta[ty][tx]=d_a[row*dim+TILE_WIDTH*i+tx];
tb[ty][tx]=d_b[bx*TILE_WIDTH*dim + TILE_WIDTH*i+ty*dim+tx];
__syncthreads();
for(j=0;j<TILE_WIDTH;j++){
res=res+ta[ty][j]*tb[tx][j];
}
__syncthreads();
}
d_c[row*dim+col]=res;
}
__global__ void MatrixMult_naive(float *d_a,float *d_b,float *d_c, int dim){
int row,col,i;
col=blockIdx.y*blockDim.y+threadIdx.y;
row=blockIdx.x*blockDim.x+threadIdx.x;
float res;
if(row<dim && col<dim){
res=0;
for(i=0;i<dim;i++){
res=res+(d_a[row*dim+i]*d_b[i*dim+col]);
}
d_c[row*dim+col]=res;
}
}
int main(){
mat_mul();
return 0;
}
void mat_mul(){
cudaSetDevice(0);
time_t t;
cudaError_t err = cudaSuccess;
srand((unsigned) time(&t));
cudaEvent_t start, stop;
float milliseconds=0;
cudaEventCreate(&start);
cudaEventCreate(&stop);
int matDim = 8192;
float *h_a, *h_b, *h_c, *h_c_check;
/*declare the host memories*/
h_a=(float *)malloc(matDim*matDim*sizeof(float));
h_b=(float *)malloc(matDim*matDim*sizeof(float));
h_c=(float *)malloc(matDim*matDim*sizeof(float));
h_c_check=(float *)malloc(matDim*matDim*sizeof(float));
// Verify that allocations succeeded
if (h_a == NULL || h_b == NULL || h_c == NULL || h_c_check ==NULL)
{
fprintf(stderr, "Failed to allocate host vectors!\n");
exit(EXIT_FAILURE);
}
allocate_matrix(h_a,h_b,matDim); // allocate memory to hold the matrix
//allocate cuda memory
float *d_a=NULL;
float *d_b=NULL;
float *d_c=NULL;
err=cudaMalloc((void **)&d_a, matDim*matDim*sizeof(float));
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to allocate device matrix A (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
err=cudaMalloc((void **)&d_b, matDim*matDim*sizeof(float));
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to allocate device matrix A (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
err=cudaMalloc((void **)&d_c, matDim*matDim*sizeof(float));
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to allocate device matrix A (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
printf("Matrix dimension is : %d\n",matDim);
// Copy the host input matrix A and B in host memory to the device matrix in device memory
//printf("Copy input data from the host memory to the CUDA device\n");
cudaEventRecord(start);
err = cudaMemcpy(d_a, h_a, matDim*matDim*sizeof(float), cudaMemcpyHostToDevice);
cudaEventRecord(stop);
cudaEventSynchronize(stop);
cudaEventElapsedTime(&milliseconds, start, stop);
//printf("GPU memcpy matrix A %10.10f ms\n",milliseconds);
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to copy vector A from host to device (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
cudaEventRecord(start);
err = cudaMemcpy(d_b, h_b, matDim*matDim*sizeof(float), cudaMemcpyHostToDevice);
cudaEventRecord(stop);
cudaEventSynchronize(stop);
cudaEventElapsedTime(&milliseconds, start, stop);
//printf("GPU memcpy matrix B %10.10f ms\n",milliseconds);
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to copy vector B from host to device (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
/*constants for kernel launch*/
int numThread=16; //number of threads per Block axis
int numBlocks=matDim/numThread;
if(matDim%numThread)
numBlocks++;
dim3 dimGrid(numBlocks,numBlocks);
dim3 dimBlock(numThread,numThread);
//-------------------------------------------------------------
//-------transpose and copy to GPU-------
transpose_matrix(h_b, matDim);//transpose first the b matrix
err = cudaMemcpy(d_b, h_b, matDim*matDim*sizeof(float), cudaMemcpyHostToDevice);
cudaEventSynchronize(stop);
if (err != cudaSuccess){
fprintf(stderr, "Failed to copy vector A from host to device (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
//--------transpose and copy ends-------------
cudaEventRecord(start);
MatrixMult_tiling_coalesced<<<dimGrid,dimBlock>>>(d_a, d_b, d_c, matDim);
cudaEventRecord(stop);
err = cudaGetLastError();
if (err != cudaSuccess){
fprintf(stderr, "Failed to launch vector matrix kernel (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
cudaEventSynchronize(stop);
cudaEventElapsedTime(&milliseconds, start, stop);
printf("GPU time tiled & coalesced %10.10f ms\n",milliseconds);
//printf("Copy output data from the CUDA device to the host memory\n");
cudaEventRecord(start);
err = cudaMemcpy(h_c_check, d_c, matDim*matDim*sizeof(float), cudaMemcpyDeviceToHost);
cudaEventRecord(stop);
cudaEventSynchronize(stop);
cudaEventElapsedTime(&milliseconds, start, stop);
//printf("GPU memcpy time tiled & coalesced %10.10f ms\n",milliseconds);
//------------transpose back the original B matrix----------------
transpose_matrix(h_b, matDim);//transpose first the b matrix
err = cudaMemcpy(d_b, h_b, matDim*matDim*sizeof(float), cudaMemcpyHostToDevice);
cudaEventSynchronize(stop);
if (err != cudaSuccess){
fprintf(stderr, "Failed to copy vector A from host to device (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
//------------transpose back the original matrix ends-------------
//-------------------------------------------------------------
cudaEventRecord(start);
MatrixMult_tiling<<<dimGrid,dimBlock>>>(d_a, d_b, d_c, matDim);
cudaEventRecord(stop);
err = cudaGetLastError();
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to launch vector matrix kernel (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
cudaEventSynchronize(stop);
cudaEventElapsedTime(&milliseconds, start, stop);
printf("GPU time tiled %10.10f ms\n",milliseconds);
//printf("Copy output data from the CUDA device to the host memory\n");
cudaEventRecord(start);
err = cudaMemcpy(h_c, d_c, matDim*matDim*sizeof(float), cudaMemcpyDeviceToHost);
cudaEventRecord(stop);
cudaEventSynchronize(stop);
cudaEventElapsedTime(&milliseconds, start, stop);
//printf("GPU memcpy time tiled %10.10f ms\n",milliseconds);
//-------------------------------------------------------------
/*
cudaEventRecord(start);
MatrixMult_naive<<<dimGrid,dimBlock>>>(d_a, d_b, d_c, matDim);
cudaEventRecord(stop);
err = cudaGetLastError();
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to launch vector matrix kernel (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
cudaEventSynchronize(stop);
cudaEventElapsedTime(&milliseconds, start, stop);
printf("GPU time naive %10.10f ms\n",milliseconds);
printf("Copy output data from the CUDA device to the host memory\n");
cudaEventRecord(start);
err = cudaMemcpy(h_c, d_c, matDim*matDim*sizeof(float), cudaMemcpyDeviceToHost);
cudaEventRecord(stop);
cudaEventSynchronize(stop);
cudaEventElapsedTime(&milliseconds, start, stop);
printf("GPU memcpy time naive %10.10f ms\n",milliseconds);
*/
//-------------------------------------------------------------
verify(h_c, h_c_check, matDim);
// Free device global memory
err = cudaFree(d_a);
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to free device vector A (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
err = cudaFree(d_b);
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to free device vector B (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
err = cudaFree(d_c);
if (err != cudaSuccess)
{
fprintf(stderr, "Failed to free device vector C (error code %s)!\n", cudaGetErrorString(err));
exit(EXIT_FAILURE);
}
// Free host memory
free(h_a);
free(h_b);
free(h_c);
printf("Done\n");
}
void allocate_matrix(float *h_a, float *h_b, int matDim){
int i,j;
// Initialize the host input vectors
for (i = 0; i < matDim; i++)
{
for(j=0;j< matDim;j++){
h_a[i*matDim+j] = rand()%10;
h_b[i*matDim+j] = rand()%10;
}
}
}
void print_matrix(float *ha, int m,int n){
int i, j;
for(i=0;i<m;i++){
for(j=0;j<n;j++){
printf(" %.1f ",ha[i*m+j]);
}
printf("\n");
}
}
void transpose_matrix(float *h_a, int matDim){
int i, j;
int temp;
for(i=0;i<matDim;i++)
{
for(j=0;j<i;j++)
{
temp=h_a[i*matDim+j];
h_a[i*matDim+j]=h_a[j*matDim+i];
h_a[j*matDim+i]=temp;
}
}
}
void verify(float *h_c, float *h_c_check, int matDim){
int i,j;
//check the code
for (i = 0; i < matDim; i++)
{
for(j=0;j<matDim;j++){
if (fabs(h_c[i*matDim+j] - h_c_check[i*matDim+j]) > 1e-5)
{
printf("cpu : %f , gpu : %f\t",h_c[i*matDim+j],h_c_check[i*matDim+j]);
fprintf(stderr, "Result verification failed at element %d,%d !\n\n", i,j);
exit(EXIT_FAILURE);
}
}
}
printf("Test PASSED\n");
}
MatrixMult_tiling_coalesced and void MatrixMult_tiling are the functions with and without colalesced memroy access of the elements of B respectively.
Now, the problem is the time taken by MatrixMult_tiling_coalesced is almost double of the time taken by MatrixMult_tiling.
I understand that in the MatrixMult_tiling the eleemnts are loaded in the tiles in coalesced manner(i.e in row major order) for each tile, but the tiles are arranged in along a column whereas the the tiles in MatrixMult_tiling_coalesced are arranged along a row, so the function MatrixMult_tiling_coalesced should be faster than the other one.
But in practice I can see the opposite is true.
I will appreciate if someone can point out the reason.
Thnaks in advance..
EDIT 1:
After the answer of Robert (see below) I understand the problem was in the load operation during elemntwise multiplication.
tb[ty][tx]=d_b[bx*TILE_WIDTH*dim + TILE_WIDTH*i+ty*dim+tx];]
to
tb[tx][ty]=d_b[bx*TILE_WIDTH*dim + TILE_WIDTH*i+ty*dim+tx];
and
res=res+ta[ty][j]*tb[tx][j];
to
res=res+ta[ty][j]*tb[j][tx];
This inclreased performance of the MatrixMult_tiling_coalesced function from 1500 ms to 1000 ms. However, the function MatrixMult_tiling takes only 879 ms. So, the coalesced routine is still slower. I don't understand where is the problem now.
EDIT 2 :
I realized that in the EDIT 1, I had just moved the bank conflict problem from the elementwise multiplication to the element loading section. The folling changes in the code has no bank conflict,
tb[tx][ty]=d_b[bx*TILE_WIDTH*dim + TILE_WIDTH*i+ty*dim+tx];
to
tb[ty][tx]=d_b[bx*TILE_WIDTH*dim + TILE_WIDTH*i+ty*dim+tx];
And
res=res+ta[ty][j]*tb[j][tx];
to
res=res+ta[ty][j]*tb[ty][j];
But it is still slightly slower than the MatrixMult_tiling function. The MatrixMult_tiling_coalesced function takes 982 ms vs 870 ms of MatrixMult_tiling function. It should be at least similar to MatrixMult_tiling if not faster.
FINAL EDIT :
Edit 2 will not produce correct result. So the code with edit 1 will be optimum. Transposing the one of the multiplicand matrix is probably not a good idea. :-(
Thanks all for helping.

Answer 1

B certainly isn't the matrix I would transpose in C=AB. But that is neither here nor there.
I'm not sure why you think:
in the tiled implementation too the access to the global memory is not in coalesced order
I don't see any lines of code in your MatrixMult_tiling that result in uncoalesced access.
Just to make sure we don't trip over terminology, "coalesced" or "uncoalesced" are terms that we apply to access patterns to global memory (not shared memory). Your global memory access patterns are in these lines in your tiled kernel:
ta[ty][tx]=d_a[row*dim+TILE_WIDTH*i+tx];
tb[ty][tx]=d_b[(ty+i*TILE_WIDTH)*dim+col];
...
d_c[row*dim+col]=res;
and none of those patterns to global memory are uncoalesced. In each of the generated indices into d_a, d_b and d_c, if you perform the substitutions, you will find that the threadIdx.x variable is present in all of them and is not multiplied by any value, constant or otherwise. Therefore these patterns will all coalesce (nicely).
I will appreciate if someone can point out the reason.
You have done something bad when it comes to shared memory.
In your tiled kernel, your multiplication operation looks like this:
res=res+ta[ty][j]*tb[j][tx];
For this case:
ta[ty][j]
we have a situation where all threads in the warp (which have linearly increasing tx values but the same ty value) are reading the same location in shared memory. This is an "optimal" access pattern - it does not present any bank conflicts, and will be serviced in the shortest possible time.
For this case:
tb[j][tx]
we have a situation where adjacent threads in the warp are reading adjacent locations in shared memory. This is also an "optimal", un-bank-conflicted pattern, and will be serviced in the shortest possible time.
However in your MatrixMult_tiling_coalesced kernel, the corresponding multiplication operation is:
res=res+ta[ty][j]*tb[tx][j];
Again, with this case:
ta[ty][j]
we have a shared memory "broadcast" pattern (all threads in a warp read from the same location) which is optimal and fast. But in this case:
tb[tx][j]
you have actually created columnar access into shared memory. This is the worst possible access pattern for shared memory, and it will result in a 32-way serialization (or possibly 16-way serialization, in the case of your 16x16 threadblocks) of the load process, and definitely worse performance. Why? Remember that for a given load, j is constant across the warp, and tx increases linearly across the warp. Therefore, let's say j is 1 on a particular loop iteration. The threads in warp 0 will read:
tb[0][1], tb[1][1], tb[2][1], tb[3][1], ...
and these locations all belong to a particular "column" of shared memory, i.e. they all belong to the same shared memory bank. This is the worst-case pattern for shared memory.
For completeness, I claim that all of your global memory access patterns in your MatrixMult_tiling_coalesced kernel are also coalesced:
ta[ty][tx]=d_a[row*dim+TILE_WIDTH*i+tx];
tb[ty][tx]=d_b[bx*TILE_WIDTH*dim + TILE_WIDTH*i+ty*dim+tx];
...
d_c[row*dim+col]=res;
so there should be no major difference in the global memory access pattern/activity/efficiency, between your two kernel implementations.
As a side note, I assume this is all a learning exercise. If you are interested in high-performance matrix multiply on the GPU, I would encourage you to consider using CUBLAS.