windows socket multithreading I/O Completion ports - windows

Question

I wrote a windows socket server with I/O Completion port to handle heavy data transmission. It works smoothly when there is only one client connected. when more than one client connect, other threads just seem blocked while only one thread works fine.
I changed my design to use select() with each thread for each connection, the problem seems still the same. By the way, the same design works fine in OSX.
However, when I run multiple instances of my process, a process serves each connection, it works great.
Anyone kind enough to enlighten me? I still prefer multithread design. Core code as following:
other part of code basically comes from: http://www.codeproject.com/Articles/13382/A-simple-application-using-I-O-Completion-Ports-an
//Worker thread will service IOCP requests
DWORD WINAPI WorkerThread(LPVOID lpParam)
{
int nThreadNo = (int)lpParam;
void *lpContext = NULL;
OVERLAPPED *pOverlapped = NULL;
CClientContext *pClientContext = NULL;
DWORD dwBytesTransfered = 0;
int nBytesRecv = 0;
int nBytesSent = 0;
DWORD dwBytes = 0, dwFlags = 0;
//Worker thread will be around to process requests, until a Shutdown event is not Signaled.
while (WAIT_OBJECT_0 != WaitForSingleObject(g_hShutdownEvent, 0))
{
BOOL bReturn = GetQueuedCompletionStatus(
g_hIOCompletionPort,
&dwBytesTransfered,
(LPDWORD)&lpContext,
&pOverlapped,
INFINITE);
//WriteToConsole("\nThread %d: GetQueuedCompletionStatus.", nThreadNo);
if (NULL == lpContext)
{
//We are shutting down
break;
}
//Get the client context
pClientContext = (CClientContext *)lpContext;
if ((FALSE == bReturn) || ((TRUE == bReturn) && (0 == dwBytesTransfered)))
{
//Client connection gone, remove it.
RemoveFromClientListAndFreeMemory(pClientContext);
continue;
}
WSABUF *p_wbuf = pClientContext->GetWSABUFPtr();
OVERLAPPED *p_ol = pClientContext->GetOVERLAPPEDPtr();
switch (pClientContext->GetOpCode())
{
case OP_READ:
pClientContext->SetTransLen(dwBytesTransfered);
if(!pClientContext->IsComplete())
{
pClientContext->SetOpCode(OP_READ);
dwFlags = 0;
//Overlapped send
nBytesSent = WSASend(pClientContext->GetSocket(), p_wbuf, 1,
&dwBytes, dwFlags, p_ol, NULL);
WriteToConsole("\nThread %d: WSASend continue bytes = %d.", nThreadNo, dwBytes);
if ((SOCKET_ERROR == nBytesSent) && (WSA_IO_PENDING != WSAGetLastError()))
{
//Let's not work with this client
RemoveFromClientListAndFreeMemory(pClientContext);
WriteToConsole("\nThread %d: WSASend failed.", nThreadNo);
}
}
else
{
WriteToConsole("\nsocket %d: WSASend complete.", pClientContext->GetSocket());
//clear cache
pClientContext->ResetWSABUF();
//for the next recv, must be triggered.
pClientContext->SetOpCode(OP_WRITE);
//Get the data.
nBytesRecv = WSARecv(pClientContext->GetSocket(), p_wbuf, 1,
&dwBytes, &dwFlags, p_ol, NULL);
if ((SOCKET_ERROR == nBytesRecv) && (WSA_IO_PENDING != WSAGetLastError()))
{
WriteToConsole("\nThread %d: Error occurred while executing WSARecv().", nThreadNo);
//Let's not work with this client
RemoveFromClientListAndFreeMemory(pClientContext);
}
}
break;
case OP_WRITE:
pClientContext->SetTransLen(dwBytesTransfered);
if(pClientContext->IsComplete())
{
if(!pClientContext->ProcessCmd())
{
WriteToConsole("\nThread %d: ProcessCmd failed.", nThreadNo);
//Let's not work with this client
RemoveFromClientListAndFreeMemory(pClientContext);
}
WriteToConsole("\nThread %d: receive completed.", nThreadNo);
//Send the message back to the client.
pClientContext->SetOpCode(OP_READ);
dwFlags = 0;
//Overlapped send
nBytesSent = WSASend(pClientContext->GetSocket(), p_wbuf, 1,
&dwBytes, dwFlags, p_ol, NULL);
WriteToConsole("\nThread %d: WSASend bytes = %d.", nThreadNo, dwBytes);
if ((SOCKET_ERROR == nBytesSent) && (WSA_IO_PENDING != WSAGetLastError()))
{
WriteToConsole("\nThread %d: Error occurred while executing WSASend().", nThreadNo);
//Let's not work with this client
RemoveFromClientListAndFreeMemory(pClientContext);
}
}
else //continue receiving
{
pClientContext->SetOpCode(OP_WRITE);
//Get the data.
nBytesRecv = WSARecv(pClientContext->GetSocket(), p_wbuf, 1,
&dwBytes, &dwFlags, p_ol, NULL);
WriteToConsole("\nThread %d: WSARecv continue bytes = %d.", nThreadNo, dwBytes);
if ((SOCKET_ERROR == nBytesRecv) && (WSA_IO_PENDING != WSAGetLastError()))
{
WriteToConsole("\nThread %d: Error occurred while executing WSARecv().", nThreadNo);
//Let's not work with this client
RemoveFromClientListAndFreeMemory(pClientContext);
}
}
break;
default:
//We should never be reaching here, under normal circumstances.
break;
} // switch
} // while
return 0;
}
Updates: since my server app need to send large amount of data to multiple clients, it works fine maybe for some minutes, but somehow for no reason, some threads just block without responding. Does it have anything to do with the size of data?