Very different IO performance in C/C++
- by Roberto Tirabassi
Hi all, I'm a new user and my english is not so good so I hope to be clear.
We're facing a performance problem using large files (1GB or more) expecially (as it seems) when you try to grow them in size.
Anyway... to verify our sensations we tryed the following (on Win 7 64Bit, 4core, 8GB Ram, 32 bit code compiled with VC2008)
a) Open an unexisting file. Write it from the beginning up to 1Gb in 1Mb slots.
Now you have a 1Gb file.
Now randomize 10000 positions within that file, seek to that position and write 50 bytes in each position, no matter what you write.
Close the file and look at the results.
Time to create the file is quite fast (about 0.3"), time to write 10000 times is fast all the same (about 0.03").
Very good, this is the beginnig.
Now try something else...
b) Open an unexisting file, seek to 1Gb-1byte and write just 1 byte.
Now you have another 1Gb file.
Follow the next steps exactly same way of case 'a', close the file and look at the results.
Time to create the file is the faster you can imagine (about 0.00009") but write time is something you can't believe.... about 90"!!!!!
b.1) Open an unexisting file, don't write any byte.
Act as before, ramdomizing, seeking and writing, close the file and look at the result.
Time to write is long all the same: about 90"!!!!!
Ok... this is quite amazing. But there's more!
c) Open again the file you crated in case 'a', don't truncate it... randomize again 10000 positions and act as before. You're fast as before, about 0,03" to write 10000 times.
This sounds Ok... try another step.
d) Now open the file you created in case 'b', don't truncate it... randomize again 10000 positions and act as before. You're slow again and again, but the time is reduced to... 45"!! Maybe, trying again, the time will reduce.
I actually wonder why...
Any Idea?
The following is part of the code I used to test what I told in previuos cases (you'll have to change someting in order to have a clean compilation, I just cut & paste from some source code, sorry).
The sample can read and write, in random, ordered or reverse ordered mode, but write only in random order is the clearest test.
We tryed using std::fstream but also using directly CreateFile(), WriteFile() and so on the results are the same (even if std::fstream is actually a little slower).
Parameters for case 'a' = -f_tempdir_\casea.dat -n10000 -t -p -w
Parameters for case 'b' = -f_tempdir_\caseb.dat -n10000 -t -v -w
Parameters for case 'b.1' = -f_tempdir_\caseb.dat -n10000 -t -w
Parameters for case 'c' = -f_tempdir_\casea.dat -n10000 -w
Parameters for case 'd' = -f_tempdir_\caseb.dat -n10000 -w
Run the test (and even others) and see...
// iotest.cpp : Defines the entry point for the console application.
//
#include <windows.h>
#include <iostream>
#include <set>
#include <vector>
#include "stdafx.h"
double RealTime_Microsecs()
{
LARGE_INTEGER fr = {0, 0};
LARGE_INTEGER ti = {0, 0};
double time = 0.0;
QueryPerformanceCounter(&ti);
QueryPerformanceFrequency(&fr);
time = (double) ti.QuadPart / (double) fr.QuadPart;
return time;
}
int main(int argc, char* argv[])
{
std::string sFileName ;
size_t stSize, stTimes, stBytes ;
int retval = 0 ;
char *p = NULL ;
char *pPattern = NULL ;
char *pReadBuf = NULL ;
try {
// Default
stSize = 1<<30 ; // 1Gb
stTimes = 1000 ;
stBytes = 50 ;
bool bTruncate = false ;
bool bPre = false ;
bool bPreFast = false ;
bool bOrdered = false ;
bool bReverse = false ;
bool bWriteOnly = false ;
// Comsumo i parametri
for(int index=1; index < argc; ++index)
{
if ( '-' != argv[index][0] ) throw ;
switch(argv[index][1])
{
case 'f': sFileName = argv[index]+2 ;
break ;
case 's': stSize = xw::str::strtol(argv[index]+2) ;
break ;
case 'n': stTimes = xw::str::strtol(argv[index]+2) ;
break ;
case 'b':stBytes = xw::str::strtol(argv[index]+2) ;
break ;
case 't': bTruncate = true ;
break ;
case 'p' : bPre = true, bPreFast = false ;
break ;
case 'v' : bPreFast = true, bPre = false ;
break ;
case 'o' : bOrdered = true, bReverse = false ;
break ;
case 'r' : bReverse = true, bOrdered = false ;
break ;
case 'w' : bWriteOnly = true ;
break ;
default: throw ;
break ;
}
}
if ( sFileName.empty() )
{
std::cout << "Usage: -f<File Name> -s<File Size> -n<Number of Reads and Writes> -b<Bytes per Read and Write> -t -p -v -o -r -w" << std::endl ;
std::cout << "-t truncates the file, -p pre load the file, -v pre load 'veloce', -o writes in order mode, -r write in reverse order mode, -w Write Only" << std::endl ;
std::cout << "Default: 1Gb, 1000 times, 50 bytes" << std::endl ;
throw ;
}
if ( !stSize || !stTimes || !stBytes )
{
std::cout << "Invalid Parameters" << std::endl ;
return -1 ;
}
size_t stBestSize = 0x00100000 ;
std::fstream fFile ;
fFile.open(sFileName.c_str(), std::ios_base::binary|std::ios_base::out|std::ios_base::in|(bTruncate?std::ios_base::trunc:0)) ;
p = new char[stBestSize] ;
pPattern = new char[stBytes] ;
pReadBuf = new char[stBytes] ;
memset(p, 0, stBestSize) ;
memset(pPattern, (int)(stBytes&0x000000ff), stBytes) ;
double dTime = RealTime_Microsecs() ;
size_t stCopySize, stSizeToCopy = stSize ;
if ( bPre )
{
do {
stCopySize = std::min(stSizeToCopy, stBestSize) ;
fFile.write(p, stCopySize) ;
stSizeToCopy -= stCopySize ;
} while (stSizeToCopy) ;
std::cout << "Creating time is: " << xw::str::itoa(RealTime_Microsecs()-dTime, 5, 'f') << std::endl ;
}
else if ( bPreFast )
{
fFile.seekp(stSize-1) ;
fFile.write(p, 1) ;
std::cout << "Creating Fast time is: " << xw::str::itoa(RealTime_Microsecs()-dTime, 5, 'f') << std::endl ;
}
size_t stPos ;
::srand((unsigned int)dTime) ;
double dReadTime, dWriteTime ;
stCopySize = stTimes ;
std::vector<size_t> inVect ;
std::vector<size_t> outVect ;
std::set<size_t> outSet ;
std::set<size_t> inSet ;
// Prepare vector and set
do {
stPos = (size_t)(::rand()<<16) % stSize ;
outVect.push_back(stPos) ;
outSet.insert(stPos) ;
stPos = (size_t)(::rand()<<16) % stSize ;
inVect.push_back(stPos) ;
inSet.insert(stPos) ;
} while (--stCopySize) ;
// Write & read using vectors
if ( !bReverse && !bOrdered )
{
std::vector<size_t>::iterator outI, inI ;
outI = outVect.begin() ;
inI = inVect.begin() ;
stCopySize = stTimes ;
dReadTime = 0.0 ;
dWriteTime = 0.0 ;
do {
dTime = RealTime_Microsecs() ;
fFile.seekp(*outI) ;
fFile.write(pPattern, stBytes) ;
dWriteTime += RealTime_Microsecs() - dTime ;
++outI ;
if ( !bWriteOnly )
{
dTime = RealTime_Microsecs() ;
fFile.seekg(*inI) ;
fFile.read(pReadBuf, stBytes) ;
dReadTime += RealTime_Microsecs() - dTime ;
++inI ;
}
} while (--stCopySize) ;
std::cout << "Write time is " << xw::str::itoa(dWriteTime, 5, 'f') << " (Ave: " << xw::str::itoa(dWriteTime/stTimes, 10, 'f') << ")" << std::endl ;
if ( !bWriteOnly )
{
std::cout << "Read time is " << xw::str::itoa(dReadTime, 5, 'f') << " (Ave: " << xw::str::itoa(dReadTime/stTimes, 10, 'f') << ")" << std::endl ;
}
} // End
// Write in order
if ( bOrdered )
{
std::set<size_t>::iterator i = outSet.begin() ;
dWriteTime = 0.0 ;
stCopySize = 0 ;
for(; i != outSet.end(); ++i)
{
stPos = *i ;
dTime = RealTime_Microsecs() ;
fFile.seekp(stPos) ;
fFile.write(pPattern, stBytes) ;
dWriteTime += RealTime_Microsecs() - dTime ;
++stCopySize ;
}
std::cout << "Ordered Write time is " << xw::str::itoa(dWriteTime, 5, 'f') << " in " << xw::str::itoa(stCopySize) << " (Ave: " << xw::str::itoa(dWriteTime/stCopySize, 10, 'f') << ")" << std::endl ;
if ( !bWriteOnly )
{
i = inSet.begin() ;
dReadTime = 0.0 ;
stCopySize = 0 ;
for(; i != inSet.end(); ++i)
{
stPos = *i ;
dTime = RealTime_Microsecs() ;
fFile.seekg(stPos) ;
fFile.read(pReadBuf, stBytes) ;
dReadTime += RealTime_Microsecs() - dTime ;
++stCopySize ;
}
std::cout << "Ordered Read time is " << xw::str::itoa(dReadTime, 5, 'f') << " in " << xw::str::itoa(stCopySize) << " (Ave: " << xw::str::itoa(dReadTime/stCopySize, 10, 'f') << ")" << std::endl ;
}
}// End
// Write in reverse order
if ( bReverse )
{
std::set<size_t>::reverse_iterator i = outSet.rbegin() ;
dWriteTime = 0.0 ;
stCopySize = 0 ;
for(; i != outSet.rend(); ++i)
{
stPos = *i ;
dTime = RealTime_Microsecs() ;
fFile.seekp(stPos) ;
fFile.write(pPattern, stBytes) ;
dWriteTime += RealTime_Microsecs() - dTime ;
++stCopySize ;
}
std::cout << "Reverse ordered Write time is " << xw::str::itoa(dWriteTime, 5, 'f') << " in " << xw::str::itoa(stCopySize) << " (Ave: " << xw::str::itoa(dWriteTime/stCopySize, 10, 'f') << ")" << std::endl ;
if ( !bWriteOnly )
{
i = inSet.rbegin() ;
dReadTime = 0.0 ;
stCopySize = 0 ;
for(; i != inSet.rend(); ++i)
{
stPos = *i ;
dTime = RealTime_Microsecs() ;
fFile.seekg(stPos) ;
fFile.read(pReadBuf, stBytes) ;
dReadTime += RealTime_Microsecs() - dTime ;
++stCopySize ;
}
std::cout << "Reverse ordered Read time is " << xw::str::itoa(dReadTime, 5, 'f') << " in " << xw::str::itoa(stCopySize) << " (Ave: " << xw::str::itoa(dReadTime/stCopySize, 10, 'f') << ")" << std::endl ;
}
}// End
dTime = RealTime_Microsecs() ;
fFile.close() ;
std::cout << "Flush/Close Time is " << xw::str::itoa(RealTime_Microsecs()-dTime, 5, 'f') << std::endl ;
std::cout << "Program Terminated" << std::endl ;
}
catch(...)
{
std::cout << "Something wrong or wrong parameters" << std::endl ;
retval = -1 ;
}
if ( p ) delete []p ;
if ( pPattern ) delete []pPattern ;
if ( pReadBuf ) delete []pReadBuf ;
return retval ;
}
