I am a new user, and my English is not so good, so I hope to be clear. We encounter a performance issue when using large files (1 GB or more), especially (when possible) when you are trying to increase their size.
In any case ... to test our feelings, we tried the following (on Win 7 64Bit, 4core, 8GB Ram, 32-bit code compiled with VC2008)
a) Open an unused file. Write it from the beginning to 1 GB in 1 MB slots.
You now have a 1Gb file.
Now randomize 10,000 positions in this file, look for that position and write 50 bytes at each position, no matter what you write.
Close the file and see the results.
The time to create the file is pretty fast (about 0.3 inches), the recording time 10,000 times fast anyway (about 0.03 ").
Very good, this is the beginning.
Try something else ...
b) Open an unused file, find 1Gb-1byte and write only 1 byte. You now have another 1 GB file.
Follow the next steps in the same way as case "a", close the file and see the results.
File creation time is faster than you can imagine (about 0.00009), but recording time is something you can’t believe .... about 90 "!!!!!
b.1) Open an unused file, do not write a single byte.
Act as before, ramdomizing, search and write, close the file and look at the result.
Time to write is long anyway: about 90 "!!!!!
Okay ... that's pretty amazing. But there is more!
c) , "a", ... 10000 -. -, 0,03 ", 10000 .
... .
d) , "b", ... 10000 -. , ... 45 "!! , , .
, ...
?
, , ( someting, , - , ). < > , .
std:: fstream, CreateFile(), WriteFile() .. ( std:: fstream ).
'a' = > -f_tempdir_\casea.dat -n10000 -t -p -w
'b' = > -f_tempdir_\caseb.dat -n10000 -t -v -w
'b.1' = > -f_tempdir_\caseb.dat -n10000 -t -w
'c' = > -f_tempdir_\casea.dat -n10000 -w
'd' = > -f_tempdir_\caseb.dat -n10000 -w
( ) ...
#include <windows.h>
#include <iostream>
#include <set>
#include <vector>
#include "stdafx.h"
double RealTime_Microsecs()
{
LARGE_INTEGER fr = {0, 0};
LARGE_INTEGER ti = {0, 0};
double time = 0.0;
QueryPerformanceCounter(&ti);
QueryPerformanceFrequency(&fr);
time = (double) ti.QuadPart / (double) fr.QuadPart;
return time;
}
int main(int argc, char* argv[])
{
std::string sFileName ;
size_t stSize, stTimes, stBytes ;
int retval = 0 ;
char *p = NULL ;
char *pPattern = NULL ;
char *pReadBuf = NULL ;
try {
stSize = 1<<30 ;
stTimes = 1000 ;
stBytes = 50 ;
bool bTruncate = false ;
bool bPre = false ;
bool bPreFast = false ;
bool bOrdered = false ;
bool bReverse = false ;
bool bWriteOnly = false ;
for(int index=1; index < argc; ++index)
{
if ( '-' != argv[index][0] ) throw ;
switch(argv[index][1])
{
case 'f': sFileName = argv[index]+2 ;
break ;
case 's': stSize = xw::str::strtol(argv[index]+2) ;
break ;
case 'n': stTimes = xw::str::strtol(argv[index]+2) ;
break ;
case 'b':stBytes = xw::str::strtol(argv[index]+2) ;
break ;
case 't': bTruncate = true ;
break ;
case 'p' : bPre = true, bPreFast = false ;
break ;
case 'v' : bPreFast = true, bPre = false ;
break ;
case 'o' : bOrdered = true, bReverse = false ;
break ;
case 'r' : bReverse = true, bOrdered = false ;
break ;
case 'w' : bWriteOnly = true ;
break ;
default: throw ;
break ;
}
}
if ( sFileName.empty() )
{
std::cout << "Usage: -f<File Name> -s<File Size> -n<Number of Reads and Writes> -b<Bytes per Read and Write> -t -p -v -o -r -w" << std::endl ;
std::cout << "-t truncates the file, -p pre load the file, -v pre load 'veloce', -o writes in order mode, -r write in reverse order mode, -w Write Only" << std::endl ;
std::cout << "Default: 1Gb, 1000 times, 50 bytes" << std::endl ;
throw ;
}
if ( !stSize || !stTimes || !stBytes )
{
std::cout << "Invalid Parameters" << std::endl ;
return -1 ;
}
size_t stBestSize = 0x00100000 ;
std::fstream fFile ;
fFile.open(sFileName.c_str(), std::ios_base::binary|std::ios_base::out|std::ios_base::in|(bTruncate?std::ios_base::trunc:0)) ;
p = new char[stBestSize] ;
pPattern = new char[stBytes] ;
pReadBuf = new char[stBytes] ;
memset(p, 0, stBestSize) ;
memset(pPattern, (int)(stBytes&0x000000ff), stBytes) ;
double dTime = RealTime_Microsecs() ;
size_t stCopySize, stSizeToCopy = stSize ;
if ( bPre )
{
do {
stCopySize = std::min(stSizeToCopy, stBestSize) ;
fFile.write(p, stCopySize) ;
stSizeToCopy -= stCopySize ;
} while (stSizeToCopy) ;
std::cout << "Creating time is: " << xw::str::itoa(RealTime_Microsecs()-dTime, 5, 'f') << std::endl ;
}
else if ( bPreFast )
{
fFile.seekp(stSize-1) ;
fFile.write(p, 1) ;
std::cout << "Creating Fast time is: " << xw::str::itoa(RealTime_Microsecs()-dTime, 5, 'f') << std::endl ;
}
size_t stPos ;
::srand((unsigned int)dTime) ;
double dReadTime, dWriteTime ;
stCopySize = stTimes ;
std::vector<size_t> inVect ;
std::vector<size_t> outVect ;
std::set<size_t> outSet ;
std::set<size_t> inSet ;
do {
stPos = (size_t)(::rand()<<16) % stSize ;
outVect.push_back(stPos) ;
outSet.insert(stPos) ;
stPos = (size_t)(::rand()<<16) % stSize ;
inVect.push_back(stPos) ;
inSet.insert(stPos) ;
} while (--stCopySize) ;
if ( !bReverse && !bOrdered )
{
std::vector<size_t>::iterator outI, inI ;
outI = outVect.begin() ;
inI = inVect.begin() ;
stCopySize = stTimes ;
dReadTime = 0.0 ;
dWriteTime = 0.0 ;
do {
dTime = RealTime_Microsecs() ;
fFile.seekp(*outI) ;
fFile.write(pPattern, stBytes) ;
dWriteTime += RealTime_Microsecs() - dTime ;
++outI ;
if ( !bWriteOnly )
{
dTime = RealTime_Microsecs() ;
fFile.seekg(*inI) ;
fFile.read(pReadBuf, stBytes) ;
dReadTime += RealTime_Microsecs() - dTime ;
++inI ;
}
} while (--stCopySize) ;
std::cout << "Write time is " << xw::str::itoa(dWriteTime, 5, 'f') << " (Ave: " << xw::str::itoa(dWriteTime/stTimes, 10, 'f') << ")" << std::endl ;
if ( !bWriteOnly )
{
std::cout << "Read time is " << xw::str::itoa(dReadTime, 5, 'f') << " (Ave: " << xw::str::itoa(dReadTime/stTimes, 10, 'f') << ")" << std::endl ;
}
}
if ( bOrdered )
{
std::set<size_t>::iterator i = outSet.begin() ;
dWriteTime = 0.0 ;
stCopySize = 0 ;
for(; i != outSet.end(); ++i)
{
stPos = *i ;
dTime = RealTime_Microsecs() ;
fFile.seekp(stPos) ;
fFile.write(pPattern, stBytes) ;
dWriteTime += RealTime_Microsecs() - dTime ;
++stCopySize ;
}
std::cout << "Ordered Write time is " << xw::str::itoa(dWriteTime, 5, 'f') << " in " << xw::str::itoa(stCopySize) << " (Ave: " << xw::str::itoa(dWriteTime/stCopySize, 10, 'f') << ")" << std::endl ;
if ( !bWriteOnly )
{
i = inSet.begin() ;
dReadTime = 0.0 ;
stCopySize = 0 ;
for(; i != inSet.end(); ++i)
{
stPos = *i ;
dTime = RealTime_Microsecs() ;
fFile.seekg(stPos) ;
fFile.read(pReadBuf, stBytes) ;
dReadTime += RealTime_Microsecs() - dTime ;
++stCopySize ;
}
std::cout << "Ordered Read time is " << xw::str::itoa(dReadTime, 5, 'f') << " in " << xw::str::itoa(stCopySize) << " (Ave: " << xw::str::itoa(dReadTime/stCopySize, 10, 'f') << ")" << std::endl ;
}
}
if ( bReverse )
{
std::set<size_t>::reverse_iterator i = outSet.rbegin() ;
dWriteTime = 0.0 ;
stCopySize = 0 ;
for(; i != outSet.rend(); ++i)
{
stPos = *i ;
dTime = RealTime_Microsecs() ;
fFile.seekp(stPos) ;
fFile.write(pPattern, stBytes) ;
dWriteTime += RealTime_Microsecs() - dTime ;
++stCopySize ;
}
std::cout << "Reverse ordered Write time is " << xw::str::itoa(dWriteTime, 5, 'f') << " in " << xw::str::itoa(stCopySize) << " (Ave: " << xw::str::itoa(dWriteTime/stCopySize, 10, 'f') << ")" << std::endl ;
if ( !bWriteOnly )
{
i = inSet.rbegin() ;
dReadTime = 0.0 ;
stCopySize = 0 ;
for(; i != inSet.rend(); ++i)
{
stPos = *i ;
dTime = RealTime_Microsecs() ;
fFile.seekg(stPos) ;
fFile.read(pReadBuf, stBytes) ;
dReadTime += RealTime_Microsecs() - dTime ;
++stCopySize ;
}
std::cout << "Reverse ordered Read time is " << xw::str::itoa(dReadTime, 5, 'f') << " in " << xw::str::itoa(stCopySize) << " (Ave: " << xw::str::itoa(dReadTime/stCopySize, 10, 'f') << ")" << std::endl ;
}
}
dTime = RealTime_Microsecs() ;
fFile.close() ;
std::cout << "Flush/Close Time is " << xw::str::itoa(RealTime_Microsecs()-dTime, 5, 'f') << std::endl ;
std::cout << "Program Terminated" << std::endl ;
}
catch(...)
{
std::cout << "Something wrong or wrong parameters" << std::endl ;
retval = -1 ;
}
if ( p ) delete []p ;
if ( pPattern ) delete []pPattern ;
if ( pReadBuf ) delete []pReadBuf ;
return retval ;
}