Yesterday I asked if my asynchronous use of libpcap is supported so that I lose packets . Today I looked further, and it seems that the problem is not in asynchronous use of libpcap, but when using pcap_next_ex . Sometimes (10 ends out of 1000), pcap_next_ex will return before the pcap element has timed out, telling the program that there were no packets read (even if there are any).
The following proof of concept reproduces the problem. It depends on libpcap, pthread, boost and libcrafter (a good C ++ package development library). Essentially, it sends a set of TCP-SYN packets to the destination and tries to get responses using libpcap. The stream that calls pcap_loop starts in parallel - while the main program skips some responses (as described above), the stream captures all packets.
#include <boost/date_time/posix_time/posix_time.hpp> #include <boost/format.hpp> #include <boost/make_shared.hpp> #include <boost/shared_ptr.hpp> #include <crafter.h> #include <netinet/ip.h> #include <netinet/tcp.h> #include <pcap.h> #include <sys/types.h> #include <sys/socket.h> using namespace boost; using namespace Crafter; using namespace std; int captureThreadCount = 0; typedef vector<pair<shared_ptr<Packet>, posix_time::time_duration> > PacketTimestamp; PacketTimestamp capturedThreadPackets; void captureThreadCallback(u_char* user, const struct pcap_pkthdr* h, const u_char* bytes) { shared_ptr<Packet> packet = make_shared<Packet>(); packet->PacketFromIP(bytes + 16, h->caplen - 16); posix_time::time_duration timestamp = posix_time::seconds(h->ts.tv_sec) + posix_time::microseconds(h->ts.tv_usec); capturedThreadPackets.push_back(make_pair(packet, timestamp)); ++captureThreadCount; } void* captureThread(void* arg) { pcap_t* pcap = (pcap_t*) arg; pcap_loop(pcap, -1, captureThreadCallback, NULL); } int main(int argc, char* argv[]) { if (argc != 5) { cout << "Usage: " << argv[0] << " <source ip> <destination ip> <port> <# tries>" << endl; exit(1); } InitCrafter(); // Parameters. string sourceIp = argv[1]; string destinationIp = argv[2]; int port = atoi(argv[3]); int nTries = atoi(argv[4]); char errorBuffer[1024]; // Socket for sending, int sd = socket(PF_INET, SOCK_RAW, IPPROTO_RAW); // And sockaddr_in to send to. struct sockaddr_in sin; sin.sin_family = AF_INET; sin.sin_port = htons(port); sin.sin_addr.s_addr = inet_addr(destinationIp.c_str()); // One pcap for the main thread (calling pcap_next), pcap_t* pcapForNext = pcap_open_live(NULL, BUFSIZ, false, 1000, errorBuffer); // Another pcap for a capture thread (calling pcap_loop), pcap_t* pcapCapture = pcap_open_live(NULL, BUFSIZ, false, 1000, errorBuffer); // Both filtered for SYN+ACK or RST+ACK from destination:port to source. string filterExpression = (boost::format("ip src %s and dst %s and src port %d and ((tcp[tcpflags] & (tcp-syn|tcp-ack) != 0) or (tcp[tcpflags] & (tcp-rst|tcp-ack) != 0))") % destinationIp % sourceIp % port).str(); struct bpf_program filter; pcap_compile(pcapForNext, &filter, filterExpression.c_str(), false, 0); pcap_setfilter(pcapForNext, &filter); pcap_setfilter(pcapCapture, &filter); pcap_freecode(&filter); // Don't forget the capture thread! pthread_t thread; pthread_create(&thread, NULL, captureThread, pcapCapture); // Some statistics. int packetsSent = 0; int packetsReceived = 0; int packetsTimeout = 0; int packetsFailed = 0; // Let probe. for (int i = 0; i < nTries; ++i) { // Create packet, IP ipHeader; ipHeader.SetSourceIP(sourceIp); ipHeader.SetDestinationIP(destinationIp); TCP tcpHeader; tcpHeader.SetSrcPort(12345 + i); tcpHeader.SetDstPort(port); tcpHeader.SetFlags(TCP::SYN); shared_ptr<Packet> packet = make_shared<Packet>(ipHeader / tcpHeader); // Check the time, struct timeval tv; gettimeofday(&tv, NULL); posix_time::time_duration sentTime = posix_time::seconds(tv.tv_sec) + posix_time::microseconds(tv.tv_usec); // And send it. if (packet->SocketSend(sd) > 0) { cerr << "Sent packet " << i << " at " << sentTime << "." << endl; ++packetsSent; } else { cerr << "Sending packet " << i << " failed." << endl; continue; } struct pcap_pkthdr* pktHeader; const u_char* pktData; int r; // Wait for the response. if ((r = pcap_next_ex(pcapForNext, &pktHeader, &pktData)) > 0) { posix_time::time_duration timestamp = posix_time::seconds(pktHeader->ts.tv_sec) + posix_time::microseconds(pktHeader->ts.tv_usec); cerr << "Response " << i << " received at " << timestamp << "." << endl; ++packetsReceived; } else if (r == 0) { cerr << "Timeout receiving response for " << i << "." << endl; ++packetsTimeout; } else { cerr << "Failed receiving response for " << i << "." << endl; ++packetsFailed; } } // Wait (to ensure "fast packets" are captured by the capture thread), usleep(500000); // 500 ms. for (PacketTimestamp::iterator i = capturedThreadPackets.begin(); i != capturedThreadPackets.end(); ++i) { TCP* tcpLayer = GetTCP(*i->first); cout << "Captured packet " << (tcpLayer->GetDstPort() - 12345) << " at " << i->second << "." << endl; } cout << "SNT=" << packetsSent << ", RCV=" << packetsReceived << ", TIM=" << packetsTimeout << ", FLD=" << packetsFailed << ", CAP=" << captureThreadCount << "." << endl; CleanCrafter(); }