Nested STL Vector Using Too Much Memory

Question

I have an STL vector of My_Partition_Vectorobjects Partitiondefined as

struct Partition // the event log data structure
{
    int key;
    std::vector<std::vector<char> > partitions;
    float modularity;
};

The actual nested structure Partition.partitionschanges from object to object, but the total number of characters stored in the Partition.partitions section is always 16.

I suggested that the total size of the object should be greater than or less than 24 bytes (16 + 4 + 4). However, for every 100,000 items that I add in My_Partition_Vector, the memory consumption (found with ps -aux) increases by about 20 MB, indicating about 209 bytes for each section object.

Is that almost a 9x increase !? Where did all this extra memory use come from? Something like filling in an STL vector or in a structure? How can I solve this problem (and prevent its exchange)?

Answer 1

Firstly, it std::vectormodels a dynamic array, so if you know that you will always have 16 characters in partitions, using std::vectorit is overkill. Use the good old C. style array / matrix : boost :: array or raise :: multi_array .

, / - std::vector, upfront ( capacity() , ).

Answer 2

, -, DeadMG , , , .

( , - ) " , ", . , . , :

template <class T>
class vector { 
    T *data;
    size_t allocated;
    size_t valid;
public:
    // ...
};

32- . vector<vector<char> >, 12 , , . , , . , , - 32 64 . , , , , , (, , ).

, , 32- ( int) ( 36 ). , 204 - 209, , , .

, . - . . , , . , , , , - . , , Boost Pool Allocator Loki small object allocator.

( ) , vector<vector<char> > - :

char partitions[16];
struct parts { 
    int part0 : 4;
    int part1 : 4;
    int part2 : 4;
    int part3 : 4;
    int part4 : 4;
    int part5 : 4;
    int part6 : 4
    int part7 : 4;
};

8 - 16, parts. , , , ( ) . , 2D- , .

Answer 3

, .

. STL A LOT . debug.

Answer 4

sizeof (vector) 24. , , 3 8- : , . , , 1 16 ( ) 24 384 (sizeof (vector) * partition.capacity()).

...

   for ( int Y=1; Y<=16; Y++ )
      {

      const int X = 16/Y;
      if ( X*Y != 16 ) continue; // ignore imperfect geometries

      Partition a;
      a.partitions = vector< vector<char> >( Y, vector<char>(X) );

      int sum = sizeof(a); // main structure
      sum += sizeof(vector<char>) * a.partitions.capacity(); // outer vector
      for ( int i=0; i<(int)a.partitions.size(); i++ )
         sum += sizeof(char) * a.partitions[i].capacity(); // inner vector

      cerr <<"X="<<X<<", Y="<<Y<<", size = "<<sum<<"\n";

      }

, ( ) ...

X=16, Y=1, size = 80
X=8, Y=2, size = 104
X=4, Y=4, size = 152
X=2, Y=8, size = 248
X=1, Y=16, size = 440

, "", , .

64- . 32- , , , .

, std::vector < > . , .

---

, , , 16

std::tr1::array<char,16>

, 2D- .

, , . , - .

   template< typename T, int YSIZE, int XSIZE >
   class array_2D
      {
      std::tr1::array<char,YSIZE*XSIZE> data;
   public:
      T & operator () ( int y, int x ) { return data[y*XSIZE+x]; } // preferred accessor (avoid pointers)
      T * operator [] ( int index ) { return &data[index*XSIZE]; } // alternative accessor (mimics boost::multi_array syntax)
      };

Answer 5

... , boost:: multi_array OP . , multi_array OP.

Boost.MultiArray. , multi_array 10- , , , 16 .

,

( export CXXFLAGS="-Wall -DNDEBUG -O3" ; make main && ./main )

...

   #include <iostream>
   #include <vector>
   #include "boost/multi_array.hpp"
   #include <tr1/array>
   #include <cassert>

   #define USE_CUSTOM_ARRAY 0 // compare memory usage of my custom array vs. boost::multi_array

   using std::cerr;
   using std::vector;

  #ifdef USE_CUSTOM_ARRAY
   template< typename T, int YSIZE, int XSIZE >
   class array_2D
      {
      std::tr1::array<char,YSIZE*XSIZE> data;
   public:
      T & operator () ( int y, int x ) { return data[y*XSIZE+x]; } // preferred accessor (avoid pointers)
      T * operator [] ( int index ) { return &data[index*XSIZE]; } // alternative accessor (mimics boost::multi_array syntax)
      };
  #endif

int main ()
   {

   int COUNT = 1024*1024;

  #if USE_CUSTOM_ARRAY
   vector< array_2D<char,4,4> > A( COUNT );
   typedef int index;
  #else
   typedef boost::multi_array<char,2> array_type;
   typedef array_type::index index;
   vector<array_type> A( COUNT, array_type(boost::extents[4][4]) );
  #endif

  // Assign values to the elements
  int values = 0;
  for ( int n=0; n<COUNT; n++ )
     for(index i = 0; i != 4; ++i) 
       for(index j = 0; j != 4; ++j)
           A[n][i][j] = values++;

// Verify values
   int verify = 0;
    for ( int n=0; n<COUNT; n++ )
       for(index i = 0; i != 4; ++i) 
          for(index j = 0; j != 4; ++j)
             {
             assert( A[n][i][j] == (char)((verify++)&0xFF) );
            #if USE_CUSTOM_ARRAY
             assert( A[n][i][j] == A[n](i,j) ); // testing accessors
            #endif
             }

   cerr <<"spinning...\n";
   while ( 1 ) {} // wait here (so you can check memory usage in the system monitor)

   return 0;
   }

Answer 6

16 - . . - . sizeof (vector) - , . sizeof (vector) 20. , 4 + 4 + 16 + 20 + 20 * + , , .

16 , , , . - .