Stream multiple files continuously in C ++

Question

My question is similar to this one , but I did not find any C ++ references for this problem.

There is a list of large files for reading and processing. What is the best way to create an input stream that will receive data from files one by one, automatically opening the next file at the end of the previous file? This stream will be passed to a processing function that sequentially reads variable-sized blocks across file boundaries.

Answer 1

What you need to do is provide a type that inherits from std::basic_streambuf. There are many mysterious virtualmember functions, relevant ones are for you showmanyc(), underflow(), uflow()and xsgetn(). You will want to reload them by overflowing, automatically open the next file in your list (if any).

Here is an example implementation. We act std::filebufjust like we save the deque<string>following files that we need to read:

class multifilebuf : public std::filebuf
{
public:
    multifilebuf(std::initializer_list<std::string> filenames)
    : next_filenames(filenames.begin() + 1, filenames.end())
    {   
        open(*filenames.begin(), std::ios::in);
    }   

protected:
    std::streambuf::int_type underflow() override
    {   
        for (;;) {
            auto res = std::filebuf::underflow();
            if (res == traits_type::eof()) {
                // done with this file, move onto the next one
                if (next_filenames.empty()) {
                    // super done
                    return res;
                }
                else {
                    // onto the next file
                    close();
                    open(next_filenames.front(), std::ios::in);

                    next_filenames.pop_front();
                    continue;
                }
            }
            else {
                return res;
            }
        }
    }   

private:
    std::deque<std::string> next_filenames;
};

This way you can make everything transparent to your end user:

multifilebuf mfb{"file1", "file2", "file3"};

std::istream is(&mfb);
std::string word;
while (is >> word) {
    // transaparently read words from all the files
}

Answer 2

boost, istream . ++, , , TS Rangesv3.

: .

"" - , . .

- zip, , (zip- - , , for(:)).

, , ++ 14:

template<class It>
struct range_t {
  It b{};
  It e{};
  It begin() const { return b; }
  It end() const { return e; }
  bool empty() const { return begin()==end(); }
};

template<class It>
struct range_of_range_t {
  std::deque<range_t<It>> ranges;
  It cur;
  friend bool operator==(range_of_range_t const& lhs, range_of_range_t const& rhs) {
    return lhs.cur==rhs.cur;
  }
  friend bool operator!=(range_of_range_t const& lhs, range_of_range_t const& rhs) {
    return !(lhs==rhs);
  }
  void operator++(){
    ++cur;
    if (ranges.front().end() == cur) {
      next_range();
    }
  }
  void next_range() {
    while(ranges.size() > 1) {
      ranges.pop_front();
      if (ranges.front().empty()) continue;
      cur = ranges.front().begin();
      break;
    }
  }
  decltype(auto) operator*() const {
    return *cur;
  }
  range_of_range_t( std::deque<range_t<It>> in ):
    ranges(std::move(in)),
    cur{}
  {
    // easy way to find the starting cur:
    ranges.push_front({});
    next_range();
  }
};

, . - .

, .