I implement game server where I need to both read and write. So I accept incoming connection and start reading from it using aio_read() but when I need to send something, I stop reading using aio_cancel() and then use aio_write(). Within write's callback I resume reading. So, I do read all the time but when I need to send something - I pause reading.

It works for ~20% of time - in other case call to aio_cancel() fails with "Operation now in progress" - and I cannot cancel it (even within permanent while cycle). So, my added write operation never happens.

How to use these functions well? What did I missed?

EDIT: Used under Linux 2.6.35. Ubuntu 10 - 32 bit.

Example code:

void handle_read(union sigval sigev_value) { /* handle data or disconnection */ }
void handle_write(union sigval sigev_value) { /* free writing buffer memory */ }
void start()
{
    const int acceptorSocket = socket(AF_INET, SOCK_STREAM, 0);
    struct sockaddr_in addr;
memset(&addr, 0, sizeof(struct sockaddr_in));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = INADDR_ANY;
addr.sin_port = htons(port);
    bind(acceptorSocket, (struct sockaddr*)&addr, sizeof(struct sockaddr_in));

    listen(acceptorSocket, SOMAXCONN);

    struct sockaddr_in address;
socklen_t addressLen = sizeof(struct sockaddr_in);

    for(;;)
    {
         const int incomingSocket = accept(acceptorSocket, (struct sockaddr*)&address, &addressLen);
         if(incomingSocket == -1)
         { /* handle error ... */}
         else
         {
              //say socket to append outcoming messages at writing:
              const int currentFlags = fcntl(incomingSocket, F_GETFL, 0);
              if(currentFlags < 0) { /* handle error ... */ }
              if(fcntl(incomingSocket, F_SETFL, currentFlags | O_APPEND) == -1) { /* handle another error ... */ }

              //start reading:
              struct aiocb* readingAiocb = new struct aiocb;
              memset(readingAiocb, 0, sizeof(struct aiocb));
              readingAiocb->aio_nbytes = MY_SOME_BUFFER_SIZE;
              readingAiocb->aio_fildes = socketDesc;
              readingAiocb->aio_buf = mySomeReadBuffer;
              readingAiocb->aio_sigevent.sigev_notify = SIGEV_THREAD;
              readingAiocb->aio_sigevent.sigev_value.sival_ptr = (void*)mySomeData;
              readingAiocb->aio_sigevent.sigev_notify_function = handle_read;
              if(aio_read(readingAiocb) != 0) { /* handle error ... */ }
          }
    }
}

//called at any time from server side:
send(void* data, const size_t dataLength)
{
    //... some thread-safety precautions not needed here ...

    const int cancellingResult = aio_cancel(socketDesc, readingAiocb);
    if(cancellingResult != AIO_CANCELED)
    {
        //this one happens ~80% of the time - embracing previous call to permanent while cycle does not help:
        if(cancellingResult == AIO_NOTCANCELED)
        {
            puts(strerror(aio_return(readingAiocb))); // "Operation now in progress"
            /* don't know what to do... */
        }
    }
    //otherwise it's okay to send:
    else
    {
        aio_write(...);
    }
}