From 64d0c85f7eaf4634ccd97e8f859e6993bd239a04 Mon Sep 17 00:00:00 2001 From: Marek Majkowski Date: Wed, 21 May 2014 16:55:47 +0100 Subject: [PATCH 01/85] p0f-3.07b ae2c4fbcddf2a5ced33abd81992405b93207e7c8 /p0f-3.07b.tgz --- build.sh | 2 +- docs/ChangeLog | 13 ++++++++++++ docs/README | 1 + p0f.c | 51 +++++++++++++++++++++++----------------------- p0f.fp | 39 ++++++++++++++++++++++++++++------- process.c | 2 +- tools/README-TOOLS | 5 ++++- tools/p0f-client.c | 2 +- 8 files changed, 79 insertions(+), 36 deletions(-) diff --git a/build.sh b/build.sh index d48a969..b70896f 100755 --- a/build.sh +++ b/build.sh @@ -9,7 +9,7 @@ # PROGNAME="p0f" -VERSION="3.06b" +VERSION="3.07b" test "$CC" = "" && CC="gcc" diff --git a/docs/ChangeLog b/docs/ChangeLog index 03dc1c5..6b2bea9 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -1,3 +1,16 @@ +Version 3.07b: +-------------- + +Bug fixes: + + - Improvement to API handling to avoid FATAL() on short API reads & writes. + + - Minor bug fix to IP parsing in one of the companion utilities. + +Improvements: + + - New signatures. + Version 3.06b: -------------- diff --git a/docs/README b/docs/README index 4981c41..b1579f1 100644 --- a/docs/README +++ b/docs/README @@ -903,6 +903,7 @@ including: Jeff Weisberg Anthony Howe Tomoyuki Murakami + Michael Petch If you wish to help, the most immediate way to do so is to simply gather new signatures, especially from less popular or older platforms (servers, networking diff --git a/p0f.c b/p0f.c index fa746c3..c1418ba 100644 --- a/p0f.c +++ b/p0f.c @@ -481,7 +481,7 @@ static void prepare_pcap(void) { if (!use_iface) { /* See the earlier note on libpcap SEGV - same problem here. - Also, this retusns something stupid on Windows, but hey... */ + Also, this returns something stupid on Windows, but hey... */ if (!access("/sys/class/net", R_OK | X_OK) || errno == ENOENT) use_iface = (u8*)pcap_lookupdev(pcap_err); @@ -823,6 +823,30 @@ static void live_event_loop(void) { for (cur = 0; cur < pfd_count; cur++) { + if (pfds[cur].revents & (POLLERR | POLLHUP)) switch (cur) { + + case 0: + + FATAL("Packet capture interface is down."); + + case 1: + + FATAL("API socket is down."); + + default: + + /* Shut down API connection and free its state. */ + + DEBUG("[#] API connection on fd %d closed.\n", pfds[cur].fd); + + close(pfds[cur].fd); + ctable[cur]->fd = -1; + + pfd_count = regen_pfds(pfds, ctable); + goto poll_again; + + } + if (pfds[cur].revents & POLLOUT) switch (cur) { case 0: case 1: @@ -928,29 +952,6 @@ static void live_event_loop(void) { } - if (pfds[cur].revents & (POLLERR | POLLHUP)) switch (cur) { - - case 0: - - FATAL("Packet capture interface is down."); - - case 1: - - FATAL("API socket is down."); - - default: - - /* Shut down API connection and free its state. */ - - DEBUG("[#] API connection on fd %d closed.\n", pfds[cur].fd); - - close(pfds[cur].fd); - ctable[cur]->fd = -1; - - pfd_count = regen_pfds(pfds, ctable); - goto poll_again; - - } /* Processed all reported updates already? If so, bail out early. */ @@ -1099,7 +1100,7 @@ int main(int argc, char** argv) { case 'p': if (set_promisc) - FATAL("Even more promiscuous? People will call me slutty!"); + FATAL("Even more promiscuous? People will start talking!"); set_promisc = 1; break; diff --git a/p0f.fp b/p0f.fp index d02dc5f..33efda0 100644 --- a/p0f.fp +++ b/p0f.fp @@ -92,7 +92,11 @@ sig = 16436 ; Linux ; ----- -label = s:unix:Linux:3.x +label = s:unix:Linux:3.11 and newer +sig = *:64:0:*:mss*20,10:mss,sok,ts,nop,ws:df,id+:0 +sig = *:64:0:*:mss*20,7:mss,sok,ts,nop,ws:df,id+:0 + +label = s:unix:Linux:3.1-3.10 sig = *:64:0:*:mss*10,4:mss,sok,ts,nop,ws:df,id+:0 sig = *:64:0:*:mss*10,5:mss,sok,ts,nop,ws:df,id+:0 sig = *:64:0:*:mss*10,6:mss,sok,ts,nop,ws:df,id+:0 @@ -212,8 +216,10 @@ label = s:unix:Mac OS X:10.x sig = *:64:0:*:65535,1:mss,nop,ws,nop,nop,ts,sok,eol+1:df,id+:0 sig = *:64:0:*:65535,3:mss,nop,ws,nop,nop,ts,sok,eol+1:df,id+:0 -label = s:unix:iOS:iPhone or iPad +label = s:unix:MacOS X:10.9 or newer (sometimes iPhone or iPad) sig = *:64:0:*:65535,4:mss,nop,ws,nop,nop,ts,sok,eol+1:df,id+:0 + +label = s:unix:iOS:iPhone or iPad sig = *:64:0:*:65535,2:mss,nop,ws,nop,nop,ts,sok,eol+1:df,id+:0 ; Catch-all rules: @@ -225,7 +231,7 @@ sig = *:64:0:*:65535,*:mss,nop,ws,nop,nop,ts,sok,eol+1:df,id+:0 ; FreeBSD ; ------- -label = s:unix:FreeBSD:9.x +label = s:unix:FreeBSD:9.x or newer sig = *:64:0:*:65535,6:mss,nop,ws,sok,ts:df,id+:0 label = s:unix:FreeBSD:8.x @@ -585,9 +591,12 @@ sig = *:Host,User-Agent,Accept=[xml;q=0.9,*/*;q=0.8],Accept-Language,Accept-En ; MSIE ; ---- +; MSIE 11 no longer sends the 'MSIE' part in U-A, but we don't consider +; U-A to be a robust signal for fingerprinting, so no dice. + label = s:!:MSIE:8 or newer sys = Windows -sig = 1:Accept=[*/*],?Referer,?Accept-Language,User-Agent,Accept-Encoding=[gzip, deflate],Host,Connection=[Keep-Alive]:Keep-Alive,Accept-Charset,UA-CPU:(compatible; MSIE +sig = 1:Accept=[*/*],?Referer,?Accept-Language,User-Agent,Accept-Encoding=[gzip, deflate],Host,Connection=[Keep-Alive]:Keep-Alive,Accept-Charset,UA-CPU:Trident/ sig = 1:Accept=[*/*],?Referer,?Accept-Language,Accept-Encoding=[gzip, deflate],User-Agent,Host,Connection=[Keep-Alive]:Keep-Alive,Accept-Charset:(compatible; MSIE label = s:!:MSIE:7 @@ -605,17 +614,29 @@ sig = 1:Accept=[*/*],Connection=[Keep-Alive],Host,?Pragma=[no-cache],?Range,?R ; Chrome ; ------ -label = s:!:Chrome:11 or newer +label = s:!:Chrome:11.x to 26.x sys = Windows,@unix sig = 1:Host,Connection=[keep-alive],User-Agent,Accept=[*/*],?Referer,Accept-Encoding=[gzip,deflate,sdch],Accept-Language,Accept-Charset=[utf-8;q=0.7,*;q=0.3]:: Chrom sig = 1:Host,Connection=[keep-alive],User-Agent,Accept=[*/*],?Referer,Accept-Encoding=[gzip,deflate,sdch],Accept-Language,Accept-Charset=[UTF-8,*;q=0.5]:: Chrom sig = 1:Host,User-Agent,Accept=[*/*],?Referer,Accept-Encoding=[gzip,deflate,sdch],Accept-Language,Accept-Charset=[utf-8;q=0.7,*;q=0.3],Connection=[keep-alive]::Chrom +label = s:!:Chrome:27.x or newer +sys = Windows,@unix +sig = 1:Host,Connection=[keep-alive],Accept=[*/*],User-Agent,?Referer,Accept-Encoding=[gzip,deflate,sdch],Accept-Language:Accept-Charset,Keep-Alive: Chrom + ; ----- ; Opera ; ----- -label = s:!:Opera:11.x or newer +label = s:!:Opera:19.x or newer +sys = Windows,@unix +sig = 1:Host,Connection=[keep-alive],Accept=[*/*;q=0.8],User-Agent,Accept-Encoding=[gzip,deflate,lzma,sdch],Accept-Language=[;q=0.]:Accept-Charset,Keep-Alive:OPR/ + +label = s:!:Opera:15.x-18.x +sys = Windows,@unix +sig = 1:Host,Connection=[keep-alive],Accept=[*/*;q=0.8],User-Agent,Accept-Encoding=[gzip, deflate],Accept-Language=[;q=0.]:Accept-Charset,Keep-Alive:OPR/ + +label = s:!:Opera:11.x-14.x sys = Windows,@unix sig = 1:User-Agent,Host,Accept=[*/*;q=0.1],?Accept-Language=[;q=0.],Accept-Encoding=[gzip, deflate],Connection=[Keep-Alive]:Accept-Charset,X-OperaMini-Phone-UA:) Presto/ @@ -650,7 +671,11 @@ sig = 1:Host,Connection=[keep-alive],Accept=[,*/*;q=0.8],User-Agent,Accept-Enc ; Safari ; ------ -label = s:!:Safari:5.1 or newer +label = s:!:Safari:7 or newer +sys = @unix +sig = *:Host,Accept-Encoding=[gzip, deflate],Connection=[keep-alive],Accept=[*/*],User-Agent,Accept-Language,?Referer,?DNT:Accept-Charset,Keep-Alive:KHTML, like Gecko) + +label = s:!:Safari:5.1-6 sys = Windows,@unix sig = *:Host,User-Agent,Accept=[*/*],?Referer,Accept-Language,Accept-Encoding=[gzip, deflate],Connection=[keep-alive]:Accept-Charset:KHTML, like Gecko) sig = *:Host,User-Agent,Accept=[*/*],?Referer,Accept-Encoding=[gzip, deflate],Accept-Language,Connection=[keep-alive]:Accept-Charset:KHTML, like Gecko) diff --git a/process.c b/process.c index 3cee1da..ce142e9 100644 --- a/process.c +++ b/process.c @@ -748,7 +748,7 @@ static u32 get_flow_bucket(struct packet_data* pk) { bucket = hash32(pk->src, 16, hash_seed) ^ hash32(pk->dst, 16, hash_seed); } - bucket = hash32(&pk->sport, 2, hash_seed) ^ hash32(&pk->dport, 2, hash_seed); + bucket ^= hash32(&pk->sport, 2, hash_seed) ^ hash32(&pk->dport, 2, hash_seed); return bucket % FLOW_BUCKETS; diff --git a/tools/README-TOOLS b/tools/README-TOOLS index 5b7ae1a..fab3c6b 100644 --- a/tools/README-TOOLS +++ b/tools/README-TOOLS @@ -2,10 +2,13 @@ This directory contains several helper tools mentioned in ../README: p0f-sendsyn.c - a tool for gathering new SYN+ACK signatures - p0f-sendsyn6.c - the same, for IPv6 desstinations + p0f-sendsyn6.c - the same, for IPv6 destinations p0f-client.c - simple API client tool for p0f -s mode +Note that IPv6 addresses need to be passed to the utilities in a fully-expanded +form (i.e., no ::). + To build any of these programs, simply type 'make progname', e.g.: make p0f-sendsyn diff --git a/tools/p0f-client.c b/tools/p0f-client.c index ce5de3d..77db4ac 100644 --- a/tools/p0f-client.c +++ b/tools/p0f-client.c @@ -41,7 +41,7 @@ static void parse_addr4(char* str, u8* ret) { if (sscanf(str, "%u.%u.%u.%u", &a1, &a2, &a3, &a4) != 4) FATAL("Malformed IPv4 address."); - if (a1 > 255 || a2 > 255 || a3 > 255 || a3 > 255) + if (a1 > 255 || a2 > 255 || a3 > 255 || a4 > 255) FATAL("Malformed IPv4 address."); ret[0] = a1; From b8e94d4092173dd5bc61ec2fd70a5642c22e9d64 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:00:56 +1000 Subject: [PATCH 02/85] Should work with nflog sockets if BPF is disabled --- p0f.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index c1418ba..492e268 100644 --- a/p0f.c +++ b/p0f.c @@ -90,6 +90,8 @@ static FILE* lf; /* Log file stream */ static u8 stop_soon; /* Ctrl-C or so pressed? */ +bool disable_bpf; /* Dont compile and assign BPF */ + u8 daemon_mode; /* Running in daemon mode? */ static u8 set_promisc; /* Use promiscuous mode? */ @@ -1050,6 +1052,9 @@ int main(int argc, char** argv) { #endif /* ^__CYGWIN__ */ + case 'b': + disable_bpf = true; + break; case 'd': @@ -1202,7 +1207,12 @@ int main(int argc, char** argv) { read_config(fp_file ? fp_file : (u8*)FP_FILE); - prepare_pcap(); + if (disable_bpf) { + SAYF("[+] BPF Disabled\n"); + } + else { + prepare_pcap(); + } prepare_bpf(); if (log_file) open_log(); From a1523eb9b33bab9b1ad33f1f20b34c088f4bdd8f Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:04:07 +1000 Subject: [PATCH 03/85] add stdbool header --- p0f.c | 1 + 1 file changed, 1 insertion(+) diff --git a/p0f.c b/p0f.c index 492e268..cfd5890 100644 --- a/p0f.c +++ b/p0f.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include From db4a4e00972f6d170c28efc3248d4c86c6d14fb6 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:07:34 +1000 Subject: [PATCH 04/85] added -b to help output and getopt --- p0f.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index cfd5890..c60de5c 100644 --- a/p0f.c +++ b/p0f.c @@ -137,6 +137,7 @@ static void usage(void) { " -s name - answer to API queries at a named unix socket\n" #endif /* !__CYGWIN__ */ " -u user - switch to the specified unprivileged account and chroot\n" +" -b - dont compile and filter by BPF\n" " -d - fork into background (requires -o or -s)\n" "\n" "Performance-related options:\n" @@ -1026,7 +1027,7 @@ int main(int argc, char** argv) { if (getuid() != geteuid()) FATAL("Please don't make me setuid. See README for more.\n"); - while ((r = getopt(argc, argv, "+LS:df:i:m:o:pr:s:t:u:")) != -1) switch (r) { + while ((r = getopt(argc, argv, "+LS:df:i:m:o:pr:s:t:u:b:")) != -1) switch (r) { case 'L': From 7db3a92f6439ae48861bee68f0105f3d9aec522f Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:09:43 +1000 Subject: [PATCH 05/85] I have never used geopt before, one more attempt before man --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index c60de5c..39916ed 100644 --- a/p0f.c +++ b/p0f.c @@ -1027,7 +1027,7 @@ int main(int argc, char** argv) { if (getuid() != geteuid()) FATAL("Please don't make me setuid. See README for more.\n"); - while ((r = getopt(argc, argv, "+LS:df:i:m:o:pr:s:t:u:b:")) != -1) switch (r) { + while ((r = getopt(argc, argv, "+LS:dfb:i:m:o:pr:s:t:u:")) != -1) switch (r) { case 'L': From 2fe3b703629f7e29821fd162455a421f10f4652c Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:13:15 +1000 Subject: [PATCH 06/85] Check for multiple -b options --- p0f.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index 39916ed..36255b8 100644 --- a/p0f.c +++ b/p0f.c @@ -91,7 +91,7 @@ static FILE* lf; /* Log file stream */ static u8 stop_soon; /* Ctrl-C or so pressed? */ -bool disable_bpf; /* Dont compile and assign BPF */ +bool disable_bpf = false; /* Dont compile and assign BPF */ u8 daemon_mode; /* Running in daemon mode? */ @@ -1055,6 +1055,8 @@ int main(int argc, char** argv) { #endif /* ^__CYGWIN__ */ case 'b': + if (disable_bpf) + FATAL("Multiple -b options not supported."); disable_bpf = true; break; From b30fe4d3b324ffe658b148fb1b04e3136dc0c041 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:18:04 +1000 Subject: [PATCH 07/85] use u8 for consistency --- p0f.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/p0f.c b/p0f.c index 36255b8..580c5e9 100644 --- a/p0f.c +++ b/p0f.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include @@ -91,7 +90,7 @@ static FILE* lf; /* Log file stream */ static u8 stop_soon; /* Ctrl-C or so pressed? */ -bool disable_bpf = false; /* Dont compile and assign BPF */ +u8 disable_bpf; /* Dont compile and assign BPF */ u8 daemon_mode; /* Running in daemon mode? */ @@ -1057,7 +1056,7 @@ int main(int argc, char** argv) { case 'b': if (disable_bpf) FATAL("Multiple -b options not supported."); - disable_bpf = true; + disable_bpf = 1; break; case 'd': From 01b9d71d3ce0de241bcaf99e50a42aaf153c576d Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:21:56 +1000 Subject: [PATCH 08/85] attempt to make work --- p0f.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/p0f.c b/p0f.c index 580c5e9..2fd12e0 100644 --- a/p0f.c +++ b/p0f.c @@ -1165,9 +1165,12 @@ int main(int argc, char** argv) { } if (optind < argc) { - - if (optind + 1 == argc) orig_rule = (u8*)argv[optind]; - else FATAL("Filter rule must be a single parameter (use quotes)."); + if (disable_bpf) { + if (optind != argc) FATAL("BPF Disabled, no filter rule expected."); + } else { + if (optind + 1 == argc) orig_rule = (u8*)argv[optind]; + else FATAL("Filter rule must be a single parameter (use quotes)."); + } } @@ -1210,13 +1213,13 @@ int main(int argc, char** argv) { read_config(fp_file ? fp_file : (u8*)FP_FILE); + prepare_pcap(); if (disable_bpf) { SAYF("[+] BPF Disabled\n"); } else { - prepare_pcap(); + prepare_bpf(); } - prepare_bpf(); if (log_file) open_log(); if (api_sock) open_api(); From f3d6c71d53d349fffa5026c81e25ebf67138c313 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:28:44 +1000 Subject: [PATCH 09/85] . --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index 2fd12e0..d751137 100644 --- a/p0f.c +++ b/p0f.c @@ -1026,7 +1026,7 @@ int main(int argc, char** argv) { if (getuid() != geteuid()) FATAL("Please don't make me setuid. See README for more.\n"); - while ((r = getopt(argc, argv, "+LS:dfb:i:m:o:pr:s:t:u:")) != -1) switch (r) { + while ((r = getopt(argc, argv, "+LS:df:i:m:o:pr:s:t:u:b:")) != -1) switch (r) { case 'L': From cecb9aac89f6ecb257aa7284c1b2c833b3f91722 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 12:30:09 +1000 Subject: [PATCH 10/85] who cares about the check? --- p0f.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/p0f.c b/p0f.c index d751137..e4eb23d 100644 --- a/p0f.c +++ b/p0f.c @@ -1165,9 +1165,7 @@ int main(int argc, char** argv) { } if (optind < argc) { - if (disable_bpf) { - if (optind != argc) FATAL("BPF Disabled, no filter rule expected."); - } else { + if (!disable_bpf) { if (optind + 1 == argc) orig_rule = (u8*)argv[optind]; else FATAL("Filter rule must be a single parameter (use quotes)."); } From ada23377ef441a59e684629bb8b49d9b65fc3e88 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 14:13:27 +1000 Subject: [PATCH 11/85] read getopt manual, now I know what I was doing wrong! --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index e4eb23d..abee775 100644 --- a/p0f.c +++ b/p0f.c @@ -1026,7 +1026,7 @@ int main(int argc, char** argv) { if (getuid() != geteuid()) FATAL("Please don't make me setuid. See README for more.\n"); - while ((r = getopt(argc, argv, "+LS:df:i:m:o:pr:s:t:u:b:")) != -1) switch (r) { + while ((r = getopt(argc, argv, "+LS:df:i:m:o:pr:s:t:u:b")) != -1) switch (r) { case 'L': From 977fa70bb014f0557509b5f84e94ec7394dbe8e9 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 16:01:40 +1000 Subject: [PATCH 12/85] NFLOG packet parsing support --- process.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/process.c b/process.c index 5011632..1829fc3 100644 --- a/process.c +++ b/process.c @@ -90,6 +90,8 @@ static void find_offset(const u8* data, s32 total_len) { case DLT_RAW: link_off = 0; return; + case DLT_NFLOG: link_off = 4; return; //family, version, resource_id + case DLT_NULL: case DLT_PPP: link_off = 4; return; @@ -244,6 +246,27 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { } + /* NFLOG has multiple sections to the packet, with variable length */ + if (link_type == DLT_NFLOG){ + while (packet_len > MIN_TCP4){ + u16 nfsize = (*data & 0xFF); + if (nfsize % 4 != 0) + nfsize += 4 - nfsize % 4; + if (nfsize == 0) { + WARN("Invalid TLV length for NFLOG packet, aborting\n"); + return; + } + if ((*(data + 2) & 0xFF) == 9){ + data += 4; + packet_len -= 4; + DEBUG("Found TLV for packet payload payload\n"); + break; + } + data += nfsize; + packet_len -= nfsize; + } + } + /* If there is no way we could have received a complete TCP packet, bail out early. */ From 1456dd226f682f89dfa3fa533fdd253e8fa77e24 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 16:03:41 +1000 Subject: [PATCH 13/85] added [#] for debug message consistency --- process.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process.c b/process.c index 1829fc3..14e517c 100644 --- a/process.c +++ b/process.c @@ -253,13 +253,13 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { if (nfsize % 4 != 0) nfsize += 4 - nfsize % 4; if (nfsize == 0) { - WARN("Invalid TLV length for NFLOG packet, aborting\n"); + WARN("[#] Invalid TLV length for NFLOG packet, aborting\n"); return; } if ((*(data + 2) & 0xFF) == 9){ data += 4; packet_len -= 4; - DEBUG("Found TLV for packet payload payload\n"); + DEBUG("[#] Found TLV for packet payload payload\n"); break; } data += nfsize; From ee5591f2f84efec4d878eec524679f8ff699d972 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 16:09:12 +1000 Subject: [PATCH 14/85] nflog support tested as working. --- docs/README | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/README b/docs/README index 58b33fa..a160e68 100644 --- a/docs/README +++ b/docs/README @@ -125,7 +125,7 @@ verbose packet parsing and signature matching information will be written to stderr. This is useful when troubleshooting problems, but that's about it. The tool should compile cleanly under any reasonably new version of Linux, -FreeBSD, OpenBSD, MacOS X, and so forth. You can also builtdit on Windows using +FreeBSD, OpenBSD, MacOS X, and so forth. You can also build it on Windows using cygwin and winpcap. I have not tested it on all possible varieties of un*x, but if there are issues, they should be fairly superficial. @@ -181,6 +181,9 @@ command-line options: Only one instance of p0f can be listening on a particular socket at any given time. The mode is also incompatible with -r. + + -b - disables the compiling and filtering by BPF. This is required to + listen to a nflog:{X} interface -d - runs p0f in daemon mode: the program will fork into background and continue writing to the specified log file or API socket. It @@ -286,6 +289,11 @@ that fails, try this URL: Filters work both for online capture (-i) and for previously collected data produced by any other tool (-r). +It is also possible to listen to a NFLOG (LINKTYPE_NFLOG) interface. BPF can +not be used on this interface. To listen to nflog group 1: + +# ./p0f -i nflog:1 -b + ------------- 4. API access ------------- @@ -963,6 +971,7 @@ including: Tomoyuki Murakami Marek Majkowski Michael Petch + Mathew Heard If you wish to help, the most immediate way to do so is to simply gather new signatures, especially from less popular or older platforms (servers, networking From 367047cd06335ae3e1debd963abe8e0f2d65a529 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 16:34:58 +1000 Subject: [PATCH 15/85] added check for a packet where payload TLV isnt found --- process.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/process.c b/process.c index 14e517c..4472f0f 100644 --- a/process.c +++ b/process.c @@ -248,6 +248,7 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { /* NFLOG has multiple sections to the packet, with variable length */ if (link_type == DLT_NFLOG){ + u8 found_payload = false; while (packet_len > MIN_TCP4){ u16 nfsize = (*data & 0xFF); if (nfsize % 4 != 0) @@ -259,12 +260,17 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { if ((*(data + 2) & 0xFF) == 9){ data += 4; packet_len -= 4; + found_payload = true; DEBUG("[#] Found TLV for packet payload payload\n"); break; } data += nfsize; packet_len -= nfsize; } + if (!found_payload){ + WARN("Did not find payload TLV in NFLOG packet, skipping packet") + return; + } } /* If there is no way we could have received a complete TCP packet, bail From 893190aab858e9cf1b753e46de5b571f292feff1 Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 18:38:16 +1000 Subject: [PATCH 16/85] urgh no booleans in this C version --- p0f.c | 2 +- process.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/p0f.c b/p0f.c index abee775..b9d86f3 100644 --- a/p0f.c +++ b/p0f.c @@ -808,7 +808,7 @@ static void live_event_loop(void) { s32 pret, i; u32 cur; - /* We use a 250 ms timeout to keep Ctrl-C responsive without resortng to + /* We use a 250 ms timeout to keep Ctrl-C responsive without resorting to silly sigaction hackery or unsafe signal handler code. */ poll_again: diff --git a/process.c b/process.c index 4472f0f..fa92ba6 100644 --- a/process.c +++ b/process.c @@ -248,19 +248,19 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { /* NFLOG has multiple sections to the packet, with variable length */ if (link_type == DLT_NFLOG){ - u8 found_payload = false; + u8 found_payload = 0; while (packet_len > MIN_TCP4){ u16 nfsize = (*data & 0xFF); if (nfsize % 4 != 0) nfsize += 4 - nfsize % 4; if (nfsize == 0) { - WARN("[#] Invalid TLV length for NFLOG packet, aborting\n"); + WARN("Invalid TLV length for NFLOG packet, aborting\n"); return; } if ((*(data + 2) & 0xFF) == 9){ data += 4; packet_len -= 4; - found_payload = true; + found_payload = 1; DEBUG("[#] Found TLV for packet payload payload\n"); break; } From c2b547f4c5f4440b54e885116e5fd663d42c349b Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 18:39:13 +1000 Subject: [PATCH 17/85] missing semicolon --- process.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process.c b/process.c index fa92ba6..7ccd64d 100644 --- a/process.c +++ b/process.c @@ -268,7 +268,7 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { packet_len -= nfsize; } if (!found_payload){ - WARN("Did not find payload TLV in NFLOG packet, skipping packet") + WARN("Did not find payload TLV in NFLOG packet, skipping packet"); return; } } From e1edea42f3b9d84efecc433187a714c62de2a78e Mon Sep 17 00:00:00 2001 From: splitice Date: Sun, 15 Jun 2014 18:52:25 +1000 Subject: [PATCH 18/85] Allow for our traffic to be one-sided. --- process.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/process.c b/process.c index 7ccd64d..7fbb9c6 100644 --- a/process.c +++ b/process.c @@ -1327,8 +1327,9 @@ static void flow_dispatch(struct packet_data* pk) { if (!f->acked) { DEBUG("[#] Never received SYN+ACK to complete handshake, huh.\n"); - destroy_flow(f); - return; + //Allow one sided traffic. + //destroy_flow(f); + //return; } From 5e3bb94af9cc2103c6e2d8896827d34cf64d559c Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 16 Jun 2014 11:27:45 +1000 Subject: [PATCH 19/85] add p0f_open_live to increase the read buffer length to hopefully prevent peak issues. --- p0f.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index b9d86f3..127aaef 100644 --- a/p0f.c +++ b/p0f.c @@ -454,6 +454,58 @@ static u8* find_interface(int num) { #endif /* __CYGWIN__ */ +pcap_t * +p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *errbuf) +{ + pcap_t *p; + int status; + + p = pcap_create(source, errbuf); + if (p == NULL) + return (NULL); + status = pcap_set_snaplen(p, snaplen); + if (status < 0) + goto fail; + status = pcap_set_promisc(p, promisc); + if (status < 0) + goto fail; + status = pcap_set_timeout(p, to_ms); + if (status < 0) + goto fail; + status = pcap_set_buffer_size(p, 20971520); + if (status < 0) + goto fail; + /* + * Mark this as opened with pcap_open_live(), so that, for + * example, we show the full list of DLT_ values, rather + * than just the ones that are compatible with capturing + * when not in monitor mode. That allows existing applications + * to work the way they used to work, but allows new applications + * that know about the new open API to, for example, find out the + * DLT_ values that they can select without changing whether + * the adapter is in monitor mode or not. + */ + p->oldstyle = 1; + status = pcap_activate(p); + if (status < 0) + goto fail; + return (p); +fail: + if (status == PCAP_ERROR) + snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s", source, + p->errbuf); + else if (status == PCAP_ERROR_NO_SUCH_DEVICE || + status == PCAP_ERROR_PERM_DENIED || + status == PCAP_ERROR_PROMISC_PERM_DENIED) + snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s (%s)", source, + pcap_statustostr(status), p->errbuf); + else + snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s", source, + pcap_statustostr(status)); + pcap_close(p); + return (NULL); +} + /* Initialize PCAP capture */ @@ -509,14 +561,14 @@ static void prepare_pcap(void) { } - pt = pcap_open_live((char*)use_iface, SNAPLEN, set_promisc, 250, pcap_err); + pt = p0f_open_live((char*)use_iface, SNAPLEN, set_promisc, 250, pcap_err); #else /* PCAP timeouts tend to be broken, so we'll use a minimum value and rely on select() instead. */ - pt = pcap_open_live((char*)use_iface, SNAPLEN, set_promisc, 1, pcap_err); + pt = p0f_open_live((char*)use_iface, SNAPLEN, set_promisc, 1, pcap_err); #endif /* ^__CYGWIN__ */ From 011dee44d648b858994aeaffced63101f8a70cdc Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 16 Jun 2014 11:34:02 +1000 Subject: [PATCH 20/85] remove reliance on pcap internals --- p0f.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index 127aaef..7c928e2 100644 --- a/p0f.c +++ b/p0f.c @@ -493,12 +493,12 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err fail: if (status == PCAP_ERROR) snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s", source, - p->errbuf); + pcap_geterr(p)); else if (status == PCAP_ERROR_NO_SUCH_DEVICE || status == PCAP_ERROR_PERM_DENIED || status == PCAP_ERROR_PROMISC_PERM_DENIED) snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s (%s)", source, - pcap_statustostr(status), p->errbuf); + pcap_statustostr(status), pcap_geterr(p)); else snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s", source, pcap_statustostr(status)); From 85d85184bbf4bdaf0322dae5ee842515bd219950 Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 16 Jun 2014 11:35:11 +1000 Subject: [PATCH 21/85] Missed one place of reliance on internals. --- p0f.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/p0f.c b/p0f.c index 7c928e2..6de41bf 100644 --- a/p0f.c +++ b/p0f.c @@ -475,17 +475,7 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err status = pcap_set_buffer_size(p, 20971520); if (status < 0) goto fail; - /* - * Mark this as opened with pcap_open_live(), so that, for - * example, we show the full list of DLT_ values, rather - * than just the ones that are compatible with capturing - * when not in monitor mode. That allows existing applications - * to work the way they used to work, but allows new applications - * that know about the new open API to, for example, find out the - * DLT_ values that they can select without changing whether - * the adapter is in monitor mode or not. - */ - p->oldstyle = 1; + status = pcap_activate(p); if (status < 0) goto fail; From e6fd6824d6aa7fb113b22f51eaef3909c6af9167 Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 16 Jun 2014 21:41:28 +1000 Subject: [PATCH 22/85] readme updated with information on this fork. --- docs/README | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/docs/README b/docs/README index a160e68..b650354 100644 --- a/docs/README +++ b/docs/README @@ -39,7 +39,7 @@ information about the actors they are talking to. Common uses for p0f include reconnaissance during penetration tests; routine network monitoring; detection of unauthorized network interconnects in corporate -environments; providing signals for abuse-prevention tools; and miscellanous +environments; providing signals for abuse-prevention tools; and miscellaneous forensics. A snippet of typical p0f output may look like this: @@ -84,8 +84,24 @@ A live demonstration can be seen here: http://lcamtuf.coredump.cx/p0f3/ +------------------------------ +2. What does this fork change? +------------------------------ + + - Inclusion of SSL fingerprint support from majek/p0f + - Support for nflog capturing (capture after IPTables) + - One sided stream support (SYN+ACK not required) + - Capture buffer size increased to fix crash on heavily loaded systems. + + This fork is production ready, a version of it is in use at http://www.x4b.net/ + + Software is provided 'as-is'. No warranty is implied, we are not responsible for any damages or loss resulting + your use of this software. + + Contributions (via pull requests) welcome. + -------------------- -2. How does it work? +3. How does it work? -------------------- A vast majority of metrics used by p0f were invented specifically for this tool, @@ -113,7 +129,7 @@ just plain trickery. For example, a system where TCP timestamps jump back and forth, or where TTLs and MTUs change subtly, is probably a NAT device. ------------------------------- -3. How do I compile and use it? +4. How do I compile and use it? ------------------------------- To compile p0f, try running './build.sh'; if that fails, you will be probably @@ -295,7 +311,7 @@ not be used on this interface. To listen to nflog group 1: # ./p0f -i nflog:1 -b ------------- -4. API access +5. API access ------------- The API allows other applications running on the same system to get p0f's @@ -399,7 +415,7 @@ Developers using the API should be aware of several important constraints: go back hours or days, parse the logs instead of wasting RAM. ----------------------- -5. Fingerprint database +6. Fingerprint database ----------------------- Whenever p0f obtains a fingerprint from the observed traffic, it defers to @@ -788,7 +804,7 @@ bottom of the list. *** NOT IMPLEMENTED YET *** ---------------- -6. NAT detection +7. NAT detection ---------------- In addition to fairly straightforward measurements of intrinsic properties of @@ -845,7 +861,7 @@ even to very homogenous environments behind NAT. If you end up seeing false positives or other detection problems in your environment, please let me know! ----------- -7. Security +8. Security ----------- You should treat the output from this tool as advisory; the fingerprinting can @@ -897,7 +913,7 @@ able to compromise your system. The same goes for many other uses of sudo, by the way. -------------- -8. Limitations +9. Limitations -------------- Here are some of the known issues you may run into: @@ -946,9 +962,9 @@ Here are some of the known issues you may run into: Windows raw sockets (this should be relatively easy to fix if there are any users who care). ---------------------------- -9. Acknowledgments and more ---------------------------- +---------------------------- +10. Acknowledgments and more +---------------------------- P0f is made possible thanks to the contributions of several good souls, including: From 5289432601fbb419916ada8881e7928fc5d722b3 Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 16 Jun 2014 21:45:45 +1000 Subject: [PATCH 23/85] fix newline in README --- docs/README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README b/docs/README index b650354..ebe5d39 100644 --- a/docs/README +++ b/docs/README @@ -95,8 +95,8 @@ http://lcamtuf.coredump.cx/p0f3/ This fork is production ready, a version of it is in use at http://www.x4b.net/ - Software is provided 'as-is'. No warranty is implied, we are not responsible for any damages or loss resulting - your use of this software. + Software is provided 'as-is'. No warranty is implied, we are not responsible for any + damages or loss resulting your use of this software. Contributions (via pull requests) welcome. From 942e025275f4a886b5b341093c3c55d74fcc2951 Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 16 Jun 2014 22:40:40 +1000 Subject: [PATCH 24/85] NETLINK_NO_ENOBUFS added --- p0f.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/p0f.c b/p0f.c index 6de41bf..daf2882 100644 --- a/p0f.c +++ b/p0f.c @@ -34,6 +34,12 @@ #include #include +//TODO: compile option for linux options +#define NETLINK_NO_ENOBUFS 5 + +#ifndef SOL_NETLINK +#define SOL_NETLINK 270 +#endif #include #ifdef NET_BPF @@ -479,6 +485,11 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err status = pcap_activate(p); if (status < 0) goto fail; + + status = setsockopt(pcap_fileno(p), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); + if (status < 0) + goto fail; + return (p); fail: if (status == PCAP_ERROR) From 8aee80f8ca555326ef0ff1884c671f2a6aa8a3df Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 16 Jun 2014 22:57:54 +1000 Subject: [PATCH 25/85] re-factored NFLOG buffer setopt to only be applied to nflog sockets --- p0f.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/p0f.c b/p0f.c index daf2882..a29a292 100644 --- a/p0f.c +++ b/p0f.c @@ -486,10 +486,6 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err if (status < 0) goto fail; - status = setsockopt(pcap_fileno(p), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); - if (status < 0) - goto fail; - return (p); fail: if (status == PCAP_ERROR) @@ -584,6 +580,11 @@ static void prepare_pcap(void) { link_type = pcap_datalink(pt); + if (link_type == DLT_NFLOG){ + int status = setsockopt(pcap_fileno(p), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); + if (status < 0) + FATAL("setsockopt: %s", strerror(errno)); + } } From 7a3182080d8540f8c8f8a344026164b69a8320d0 Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 16 Jun 2014 22:59:02 +1000 Subject: [PATCH 26/85] fix incorrect variable name --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index a29a292..4d7a1ec 100644 --- a/p0f.c +++ b/p0f.c @@ -581,7 +581,7 @@ static void prepare_pcap(void) { link_type = pcap_datalink(pt); if (link_type == DLT_NFLOG){ - int status = setsockopt(pcap_fileno(p), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); + int status = setsockopt(pcap_fileno(pt), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); if (status < 0) FATAL("setsockopt: %s", strerror(errno)); } From dfcf540e72ee1024989dda7a4b17ad3b9c75f9f5 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 13:01:41 +1000 Subject: [PATCH 27/85] re-order NETLINK_NO_ENOBUFS --- p0f.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/p0f.c b/p0f.c index 4d7a1ec..69188bc 100644 --- a/p0f.c +++ b/p0f.c @@ -482,6 +482,15 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err if (status < 0) goto fail; + + link_type = pcap_datalink(pt); + + if (link_type == DLT_NFLOG){ + status = setsockopt(pcap_fileno(pt), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); + if (status < 0) + FATAL("setsockopt: %s", strerror(errno)); + } + status = pcap_activate(p); if (status < 0) goto fail; @@ -577,14 +586,6 @@ static void prepare_pcap(void) { if (!pt) FATAL("pcap_open_live: %s", pcap_err); } - - link_type = pcap_datalink(pt); - - if (link_type == DLT_NFLOG){ - int status = setsockopt(pcap_fileno(pt), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); - if (status < 0) - FATAL("setsockopt: %s", strerror(errno)); - } } From 6a83b67a1376166ce7b3248745933bcf4aecb09c Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 13:04:05 +1000 Subject: [PATCH 28/85] pt isnt initialized yet --- p0f.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/p0f.c b/p0f.c index 69188bc..75d17f0 100644 --- a/p0f.c +++ b/p0f.c @@ -482,11 +482,10 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err if (status < 0) goto fail; - - link_type = pcap_datalink(pt); + link_type = pcap_datalink(p); if (link_type == DLT_NFLOG){ - status = setsockopt(pcap_fileno(pt), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); + status = setsockopt(pcap_fileno(p), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); if (status < 0) FATAL("setsockopt: %s", strerror(errno)); } From 4c7f1f3a623e6781ff17d8b86511ef8625cca043 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 16:42:15 +1000 Subject: [PATCH 29/85] added ifdef around DLT_NFLOG and pcap promisc constants --- p0f.c | 9 +++++++-- process.c | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/p0f.c b/p0f.c index 75d17f0..d6c2199 100644 --- a/p0f.c +++ b/p0f.c @@ -484,11 +484,13 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err link_type = pcap_datalink(p); +#ifdef DLT_NFLOG if (link_type == DLT_NFLOG){ status = setsockopt(pcap_fileno(p), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); if (status < 0) FATAL("setsockopt: %s", strerror(errno)); } +#endif status = pcap_activate(p); if (status < 0) @@ -500,8 +502,11 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s", source, pcap_geterr(p)); else if (status == PCAP_ERROR_NO_SUCH_DEVICE || - status == PCAP_ERROR_PERM_DENIED || - status == PCAP_ERROR_PROMISC_PERM_DENIED) + status == PCAP_ERROR_PERM_DENIED +#ifdef PCAP_ERROR_PROMISC_PERM_DENIED + || status == PCAP_ERROR_PROMISC_PERM_DENIED +#endif + ) snprintf(errbuf, PCAP_ERRBUF_SIZE, "%s: %s (%s)", source, pcap_statustostr(status), pcap_geterr(p)); else diff --git a/process.c b/process.c index 7fbb9c6..f84de9b 100644 --- a/process.c +++ b/process.c @@ -89,9 +89,9 @@ static void find_offset(const u8* data, s32 total_len) { switch (link_type) { case DLT_RAW: link_off = 0; return; - +#ifdef DLT_NFLOG case DLT_NFLOG: link_off = 4; return; //family, version, resource_id - +#endif case DLT_NULL: case DLT_PPP: link_off = 4; return; @@ -246,6 +246,7 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { } +#ifdef DLT_NFLOG /* NFLOG has multiple sections to the packet, with variable length */ if (link_type == DLT_NFLOG){ u8 found_payload = 0; @@ -272,6 +273,7 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { return; } } +#endif /* If there is no way we could have received a complete TCP packet, bail out early. */ From 517425e5c031de24165fea18fd6190157797759f Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 19:18:10 +1000 Subject: [PATCH 30/85] added debug to pcap opening --- p0f.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/p0f.c b/p0f.c index d6c2199..2cad703 100644 --- a/p0f.c +++ b/p0f.c @@ -469,18 +469,27 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err p = pcap_create(source, errbuf); if (p == NULL) return (NULL); + DEBUG("PCAP created successfully"); + status = pcap_set_snaplen(p, snaplen); if (status < 0) goto fail; + DEBUG("PCAP snaplen set successfully"); + status = pcap_set_promisc(p, promisc); if (status < 0) goto fail; + DEBUG("PCAP promisc set successfully"); + status = pcap_set_timeout(p, to_ms); if (status < 0) goto fail; + DEBUG("PCAP timeout set successfully"); + status = pcap_set_buffer_size(p, 20971520); if (status < 0) goto fail; + DEBUG("PCAP buffer set successfully"); link_type = pcap_datalink(p); @@ -489,12 +498,14 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err status = setsockopt(pcap_fileno(p), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); if (status < 0) FATAL("setsockopt: %s", strerror(errno)); + DEBUG("PCAP overflow condition set successfully"); } #endif status = pcap_activate(p); if (status < 0) goto fail; + DEBUG("PCAP activated"); return (p); fail: From 4531e4519d506a55aa42d6cc538743c679c8b913 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 19:30:31 +1000 Subject: [PATCH 31/85] add DLT_LOOP, unsure of offset. --- p0f.c | 10 +++++----- process.c | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/p0f.c b/p0f.c index 2cad703..40bac38 100644 --- a/p0f.c +++ b/p0f.c @@ -469,27 +469,27 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err p = pcap_create(source, errbuf); if (p == NULL) return (NULL); - DEBUG("PCAP created successfully"); + DEBUG("PCAP created successfully\n"); status = pcap_set_snaplen(p, snaplen); if (status < 0) goto fail; - DEBUG("PCAP snaplen set successfully"); + DEBUG("PCAP snaplen set successfully\n"); status = pcap_set_promisc(p, promisc); if (status < 0) goto fail; - DEBUG("PCAP promisc set successfully"); + DEBUG("PCAP promisc set successfully\n"); status = pcap_set_timeout(p, to_ms); if (status < 0) goto fail; - DEBUG("PCAP timeout set successfully"); + DEBUG("PCAP timeout set successfully\n"); status = pcap_set_buffer_size(p, 20971520); if (status < 0) goto fail; - DEBUG("PCAP buffer set successfully"); + DEBUG("PCAP buffer set successfully\n"); link_type = pcap_datalink(p); diff --git a/process.c b/process.c index f84de9b..da1cc9b 100644 --- a/process.c +++ b/process.c @@ -89,6 +89,7 @@ static void find_offset(const u8* data, s32 total_len) { switch (link_type) { case DLT_RAW: link_off = 0; return; + case DLT_LOOP: link_off = 14; return; #ifdef DLT_NFLOG case DLT_NFLOG: link_off = 4; return; //family, version, resource_id #endif From bccd21a657e1f26c0bc4d6d0de3ea7358a9efa90 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 19:32:51 +1000 Subject: [PATCH 32/85] comment out old DLT_LOOP --- process.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/process.c b/process.c index da1cc9b..6a3816e 100644 --- a/process.c +++ b/process.c @@ -96,7 +96,7 @@ static void find_offset(const u8* data, s32 total_len) { case DLT_NULL: case DLT_PPP: link_off = 4; return; - case DLT_LOOP: + //case DLT_LOOP: #ifdef DLT_PPP_SERIAL case DLT_PPP_SERIAL: @@ -115,6 +115,8 @@ static void find_offset(const u8* data, s32 total_len) { case DLT_IEEE802_11: link_off = 32; return; } + DEBUG("[#] UNKNOWN DLT %d, attempting to find offset\n", link_type); + /* If this fails, try to auto-detect. There is a slight risk that if the first packet we see is maliciously crafted, and somehow gets past the configured BPF filter, we will configure the wrong offset. But that From bf9182ce40c219b838730cf971f198c61440792b Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 19:34:33 +1000 Subject: [PATCH 33/85] value appears to be wrong --- process.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process.c b/process.c index 6a3816e..32d1661 100644 --- a/process.c +++ b/process.c @@ -89,7 +89,7 @@ static void find_offset(const u8* data, s32 total_len) { switch (link_type) { case DLT_RAW: link_off = 0; return; - case DLT_LOOP: link_off = 14; return; + //case DLT_LOOP: link_off = 14; return; #ifdef DLT_NFLOG case DLT_NFLOG: link_off = 4; return; //family, version, resource_id #endif From ace481a0a754f5e68601ec4301a9dbf1dd2bd6dc Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 19:37:17 +1000 Subject: [PATCH 34/85] debug statement, what DLT? --- process.c | 1 + 1 file changed, 1 insertion(+) diff --git a/process.c b/process.c index 32d1661..a224b77 100644 --- a/process.c +++ b/process.c @@ -85,6 +85,7 @@ static void find_offset(const u8* data, s32 total_len) { u8 i; /* Check hardcoded values for some of the most common options. */ + DEBUG("[#] Looking for offset for link type: %d\n", link_type); switch (link_type) { From 652984025389f7750ae58be4629d56f370f37028 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 19:45:15 +1000 Subject: [PATCH 35/85] ok, this might work. --- p0f.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/p0f.c b/p0f.c index 40bac38..2ad8381 100644 --- a/p0f.c +++ b/p0f.c @@ -491,22 +491,22 @@ p0f_open_live(const char *source, int snaplen, int promisc, int to_ms, char *err goto fail; DEBUG("PCAP buffer set successfully\n"); - link_type = pcap_datalink(p); - -#ifdef DLT_NFLOG - if (link_type == DLT_NFLOG){ +/// if (link_type == DLT_NFLOG){ status = setsockopt(pcap_fileno(p), SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); - if (status < 0) - FATAL("setsockopt: %s", strerror(errno)); - DEBUG("PCAP overflow condition set successfully"); - } -#endif + ///If fails, probably not nfnetlink + // if (status < 0) + // FATAL("setsockopt: %s", strerror(errno)); + // DEBUG("PCAP overflow condition set successfully"); +// } status = pcap_activate(p); if (status < 0) goto fail; DEBUG("PCAP activated"); + link_type = pcap_datalink(p); + DEBUG("PCAP data link type: %d\n", link_type); + return (p); fail: if (status == PCAP_ERROR) From 38214f7660de79dc454f899571bd871c77fb4a43 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 20 Jun 2014 19:46:32 +1000 Subject: [PATCH 36/85] correct offset --- process.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/process.c b/process.c index a224b77..f2b1db4 100644 --- a/process.c +++ b/process.c @@ -90,14 +90,13 @@ static void find_offset(const u8* data, s32 total_len) { switch (link_type) { case DLT_RAW: link_off = 0; return; - //case DLT_LOOP: link_off = 14; return; #ifdef DLT_NFLOG case DLT_NFLOG: link_off = 4; return; //family, version, resource_id #endif case DLT_NULL: case DLT_PPP: link_off = 4; return; - //case DLT_LOOP: + case DLT_LOOP: #ifdef DLT_PPP_SERIAL case DLT_PPP_SERIAL: From 131a933cb2ea18ee5a221fc49486a741f93a440b Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:19:17 +1000 Subject: [PATCH 37/85] initial code to support epoll --- config.h | 3 + p0f.c | 323 +++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 212 insertions(+), 114 deletions(-) diff --git a/config.h b/config.h index 3408dd1..aafe4ee 100644 --- a/config.h +++ b/config.h @@ -17,6 +17,9 @@ * Things you may reasonably want to change * ********************************************/ +/* Use epoll, enable only if available */ +#define USE_EPOLL + /* Default location of p0f.fp: */ #ifndef FP_FILE diff --git a/p0f.c b/p0f.c index 2ad8381..12e4dc2 100644 --- a/p0f.c +++ b/p0f.c @@ -20,7 +20,11 @@ #include #include #include +#ifdef USE_EPOLL +#include +#else #include +#endif #include #include @@ -801,6 +805,87 @@ static void abort_handler(int sig) { #ifndef __CYGWIN__ +#ifdef USE_EPOLL + + +static void poll_event_loop(void){ + struct api_client* ctable; + + //Overallocate for now TODO: properly + ctable = ck_alloc(40 * sizeof(struct api_client)); + + struct epoll_event ev; + struct epoll_event events[5]; + int epfd = epoll_create(api_max_conn); + + for (;;){ + int nfds = epoll_wait(epfd, events, 5, -1); + for (int n = 0; n < nfds; ++n) { + int fd = events[n].data.fd; + if (fd == pcap_fileno(pt)){//TODO optimize + //Handle PCAP event + if (events[n].events & POLLIN){ + if (pcap_dispatch(pt, -1, (pcap_handler)parse_packet, 0) < 0) + FATAL("Packet capture interface is down."); + } + } + else if (fd == api_fd) { + //Accept api connection + + int client_sock = accept(api_fd, NULL, NULL); + ctable[fd].fd = client_sock; + + if (client_sock < 0) { + WARN("Unable to handle API connection: accept() fails."); + } + else { + if (fcntl(client_sock, F_SETFL, O_NONBLOCK)) + PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); + + ctable[fd].in_off = ctable[fd].out_off = 0; + + ev.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP; + ev.data.fd = client_sock; + int res = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sock, &ev); + + } + } + else{ + //Handle API query + if (events[n].events & POLLERR || events[n].events & POLLHUP){ + DEBUG("[#] API connection on fd %d closed.\n", events[n].data.fd); + + close(events[n].data.fd); + ctable[fd].fd = -1; + } + else if (events[n].events & POLLIN){ + /* Receive API query, dispatch when complete. */ + + if (ctable[fd].in_off >= sizeof(struct p0f_api_query)) + FATAL("Inconsistent p0f_api_query state.\n"); + + i = read(pfds[fd].fd, + ((char*)&ctable[fd].in_data) + ctable[fd].in_off, + sizeof(struct p0f_api_query) - ctable[fd].in_off); + + if (i < 0) PFATAL("read() on API socket fails despite POLLIN."); + + ctable[fd].in_off += i; + + /* Query in place? Compute response and prepare to send it back. */ + + if (ctable[fd].in_off == sizeof(struct p0f_api_query)) { + + handle_query(&ctable[fd]->in_data, &ctable[fd]->out_data); + //pfds[fd].events = (POLLOUT | POLLERR | POLLHUP); + + } + } + } + } + } +} +#else /* Regenerate pollfd data for poll() */ static u32 regen_pfds(struct pollfd* pfds, struct api_client** ctable) { @@ -838,204 +923,214 @@ static u32 regen_pfds(struct pollfd* pfds, struct api_client** ctable) { } -#endif /* !__CYGWIN__ */ +static void poll_event_loop(void){ + struct pollfd *pfds; + struct api_client** ctable; + u32 pfd_count; + /* We need room for pcap, and possibly api_fd + api_clients. */ -/* Event loop! Accepts and dispatches pcap data, API queries, etc. */ + pfds = ck_alloc((1 + (api_sock ? (1 + api_max_conn) : 0)) * + sizeof(struct pollfd)); -static void live_event_loop(void) { + ctable = ck_alloc((1 + (api_sock ? (1 + api_max_conn) : 0)) * + sizeof(struct api_client*)); -#ifndef __CYGWIN__ + pfd_count = regen_pfds(pfds, ctable); - /* The huge problem with winpcap on cygwin is that you can't get a file - descriptor suitable for poll() / select() out of it: + if (!daemon_mode) + SAYF("[+] Entered main event loop.\n\n"); - http://www.winpcap.org/pipermail/winpcap-users/2009-April/003179.html + while (!stop_soon) { - The only alternatives seem to be additional processes / threads, a - nasty busy loop, or a ton of Windows-specific code. If you need APi - queries on Windows, you are welcome to fix this :-) */ + s32 pret, i; + u32 cur; - struct pollfd *pfds; - struct api_client** ctable; - u32 pfd_count; + /* We use a 250 ms timeout to keep Ctrl-C responsive without resorting to + silly sigaction hackery or unsafe signal handler code. */ - /* We need room for pcap, and possibly api_fd + api_clients. */ + poll_again: - pfds = ck_alloc((1 + (api_sock ? (1 + api_max_conn) : 0)) * - sizeof(struct pollfd)); + pret = poll(pfds, pfd_count, 250); - ctable = ck_alloc((1 + (api_sock ? (1 + api_max_conn) : 0)) * - sizeof(struct api_client*)); + if (pret < 0) { + if (errno == EINTR) break; + PFATAL("poll() failed."); + } - pfd_count = regen_pfds(pfds, ctable); + if (!pret) { if (log_file) fflush(lf); continue; } - if (!daemon_mode) - SAYF("[+] Entered main event loop.\n\n"); + /* Examine pfds... */ - while (!stop_soon) { + for (cur = 0; cur < pfd_count; cur++) { - s32 pret, i; - u32 cur; + if (pfds[cur].revents & (POLLERR | POLLHUP)) switch (cur) { - /* We use a 250 ms timeout to keep Ctrl-C responsive without resorting to - silly sigaction hackery or unsafe signal handler code. */ + case 0: -poll_again: + FATAL("Packet capture interface is down."); - pret = poll(pfds, pfd_count, 250); + case 1: - if (pret < 0) { - if (errno == EINTR) break; - PFATAL("poll() failed."); - } + FATAL("API socket is down."); - if (!pret) { if (log_file) fflush(lf); continue; } + default: - /* Examine pfds... */ + /* Shut down API connection and free its state. */ - for (cur = 0; cur < pfd_count; cur++) { + DEBUG("[#] API connection on fd %d closed.\n", pfds[cur].fd); - if (pfds[cur].revents & (POLLERR | POLLHUP)) switch (cur) { + close(pfds[cur].fd); + ctable[cur]->fd = -1; - case 0: + pfd_count = regen_pfds(pfds, ctable); + goto poll_again; - FATAL("Packet capture interface is down."); + } - case 1: + if (pfds[cur].revents & POLLOUT) switch (cur) { - FATAL("API socket is down."); + case 0: case 1: - default: + FATAL("Unexpected POLLOUT on fd %d.\n", cur); - /* Shut down API connection and free its state. */ + default: - DEBUG("[#] API connection on fd %d closed.\n", pfds[cur].fd); + /* Write API response, restart state when complete. */ - close(pfds[cur].fd); - ctable[cur]->fd = -1; - - pfd_count = regen_pfds(pfds, ctable); - goto poll_again; + if (ctable[cur]->in_off < sizeof(struct p0f_api_query)) + FATAL("Inconsistent p0f_api_response state.\n"); - } + i = write(pfds[cur].fd, + ((char*)&ctable[cur]->out_data) + ctable[cur]->out_off, + sizeof(struct p0f_api_response) - ctable[cur]->out_off); - if (pfds[cur].revents & POLLOUT) switch (cur) { + if (i <= 0) PFATAL("write() on API socket fails despite POLLOUT."); - case 0: case 1: + ctable[cur]->out_off += i; - FATAL("Unexpected POLLOUT on fd %d.\n", cur); + /* All done? Back to square zero then! */ - default: + if (ctable[cur]->out_off == sizeof(struct p0f_api_response)) { - /* Write API response, restart state when complete. */ + ctable[cur]->in_off = ctable[cur]->out_off = 0; + pfds[cur].events = (POLLIN | POLLERR | POLLHUP); - if (ctable[cur]->in_off < sizeof(struct p0f_api_query)) - FATAL("Inconsistent p0f_api_response state.\n"); + } - i = write(pfds[cur].fd, - ((char*)&ctable[cur]->out_data) + ctable[cur]->out_off, - sizeof(struct p0f_api_response) - ctable[cur]->out_off); + } - if (i <= 0) PFATAL("write() on API socket fails despite POLLOUT."); + if (pfds[cur].revents & POLLIN) switch (cur) { - ctable[cur]->out_off += i; + case 0: - /* All done? Back to square zero then! */ + /* Process traffic on the capture interface. */ - if (ctable[cur]->out_off == sizeof(struct p0f_api_response)) { + if (pcap_dispatch(pt, -1, (pcap_handler)parse_packet, 0) < 0) + FATAL("Packet capture interface is down."); - ctable[cur]->in_off = ctable[cur]->out_off = 0; - pfds[cur].events = (POLLIN | POLLERR | POLLHUP); + break; - } + case 1: - } + /* Accept new API connection, limits permitting. */ - if (pfds[cur].revents & POLLIN) switch (cur) { - - case 0: + if (!api_sock) FATAL("Unexpected API connection."); - /* Process traffic on the capture interface. */ + if (pfd_count - 2 < api_max_conn) { - if (pcap_dispatch(pt, -1, (pcap_handler)parse_packet, 0) < 0) - FATAL("Packet capture interface is down."); + for (i = 0; i < api_max_conn && api_cl[i].fd >= 0; i++); - break; + if (i == api_max_conn) FATAL("Inconsistent API connection data."); - case 1: + api_cl[i].fd = accept(api_fd, NULL, NULL); - /* Accept new API connection, limits permitting. */ + if (api_cl[i].fd < 0) { - if (!api_sock) FATAL("Unexpected API connection."); + WARN("Unable to handle API connection: accept() fails."); - if (pfd_count - 2 < api_max_conn) { + } + else { - for (i = 0; i < api_max_conn && api_cl[i].fd >= 0; i++); + if (fcntl(api_cl[i].fd, F_SETFL, O_NONBLOCK)) + PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); - if (i == api_max_conn) FATAL("Inconsistent API connection data."); + api_cl[i].in_off = api_cl[i].out_off = 0; + pfd_count = regen_pfds(pfds, ctable); - api_cl[i].fd = accept(api_fd, NULL, NULL); + DEBUG("[#] Accepted new API connection, fd %d.\n", api_cl[i].fd); - if (api_cl[i].fd < 0) { + goto poll_again; - WARN("Unable to handle API connection: accept() fails."); + } - } else { + } + else WARN("Too many API connections (use -S to adjust).\n"); - if (fcntl(api_cl[i].fd, F_SETFL, O_NONBLOCK)) - PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); + break; - api_cl[i].in_off = api_cl[i].out_off = 0; - pfd_count = regen_pfds(pfds, ctable); + default: - DEBUG("[#] Accepted new API connection, fd %d.\n", api_cl[i].fd); + /* Receive API query, dispatch when complete. */ - goto poll_again; + if (ctable[cur]->in_off >= sizeof(struct p0f_api_query)) + FATAL("Inconsistent p0f_api_query state.\n"); - } + i = read(pfds[cur].fd, + ((char*)&ctable[cur]->in_data) + ctable[cur]->in_off, + sizeof(struct p0f_api_query) - ctable[cur]->in_off); - } else WARN("Too many API connections (use -S to adjust).\n"); + if (i < 0) PFATAL("read() on API socket fails despite POLLIN."); - break; + ctable[cur]->in_off += i; - default: + /* Query in place? Compute response and prepare to send it back. */ - /* Receive API query, dispatch when complete. */ + if (ctable[cur]->in_off == sizeof(struct p0f_api_query)) { - if (ctable[cur]->in_off >= sizeof(struct p0f_api_query)) - FATAL("Inconsistent p0f_api_query state.\n"); + handle_query(&ctable[cur]->in_data, &ctable[cur]->out_data); + pfds[cur].events = (POLLOUT | POLLERR | POLLHUP); - i = read(pfds[cur].fd, - ((char*)&ctable[cur]->in_data) + ctable[cur]->in_off, - sizeof(struct p0f_api_query) - ctable[cur]->in_off); + } - if (i < 0) PFATAL("read() on API socket fails despite POLLIN."); + } - ctable[cur]->in_off += i; - /* Query in place? Compute response and prepare to send it back. */ + /* Processed all reported updates already? If so, bail out early. */ - if (ctable[cur]->in_off == sizeof(struct p0f_api_query)) { + if (pfds[cur].revents && !--pret) break; - handle_query(&ctable[cur]->in_data, &ctable[cur]->out_data); - pfds[cur].events = (POLLOUT | POLLERR | POLLHUP); + } - } + } - } + ck_free(ctable); + ck_free(pfds); +} +#endif +#endif /* !__CYGWIN__ */ - /* Processed all reported updates already? If so, bail out early. */ +/* Event loop! Accepts and dispatches pcap data, API queries, etc. */ - if (pfds[cur].revents && !--pret) break; +static void live_event_loop(void) { - } +#ifndef __CYGWIN__ - } + /* The huge problem with winpcap on cygwin is that you can't get a file + descriptor suitable for poll() / select() out of it: + + http://www.winpcap.org/pipermail/winpcap-users/2009-April/003179.html - ck_free(ctable); - ck_free(pfds); + The only alternatives seem to be additional processes / threads, a + nasty busy loop, or a ton of Windows-specific code. If you need APi + queries on Windows, you are welcome to fix this :-) */ + +#ifdef USE_EPOLL + epoll_event_loop(); +#else + poll_event_loop(); +#endif #else From ed2fc9318c14f25f214255e80c09ca5ce0fc64e7 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:26:15 +1000 Subject: [PATCH 38/85] correct function name --- p0f.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index 12e4dc2..66c98ab 100644 --- a/p0f.c +++ b/p0f.c @@ -807,8 +807,7 @@ static void abort_handler(int sig) { #ifdef USE_EPOLL - -static void poll_event_loop(void){ +static void epoll_event_loop(void){ struct api_client* ctable; //Overallocate for now TODO: properly From 996f211827f598492c5f0bf008bae2719a82dd96 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:30:24 +1000 Subject: [PATCH 39/85] reorder includes so epoll is included --- p0f.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/p0f.c b/p0f.c index 66c98ab..00121c8 100644 --- a/p0f.c +++ b/p0f.c @@ -20,11 +20,6 @@ #include #include #include -#ifdef USE_EPOLL -#include -#else -#include -#endif #include #include @@ -62,6 +57,12 @@ #include "fp_http.h" #include "p0f.h" +#ifdef USE_EPOLL +#include +#else +#include +#endif + #ifndef PF_INET6 # define PF_INET6 10 #endif /* !PF_INET6 */ From c47cf5129880c056744f3ac81c283bcbbb322b63 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:32:10 +1000 Subject: [PATCH 40/85] fix some C errors --- p0f.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p0f.c b/p0f.c index 00121c8..d99c094 100644 --- a/p0f.c +++ b/p0f.c @@ -818,13 +818,13 @@ static void epoll_event_loop(void){ struct epoll_event events[5]; int epfd = epoll_create(api_max_conn); - for (;;){ + while (!stop_soon) { int nfds = epoll_wait(epfd, events, 5, -1); for (int n = 0; n < nfds; ++n) { int fd = events[n].data.fd; if (fd == pcap_fileno(pt)){//TODO optimize //Handle PCAP event - if (events[n].events & POLLIN){ + if (events[n].events & EPOLLIN){ if (pcap_dispatch(pt, -1, (pcap_handler)parse_packet, 0) < 0) FATAL("Packet capture interface is down."); } @@ -852,13 +852,13 @@ static void epoll_event_loop(void){ } else{ //Handle API query - if (events[n].events & POLLERR || events[n].events & POLLHUP){ + if (events[n].events & EPOLLERR || events[n].events & EPOLLHUP){ DEBUG("[#] API connection on fd %d closed.\n", events[n].data.fd); close(events[n].data.fd); ctable[fd].fd = -1; } - else if (events[n].events & POLLIN){ + else if (events[n].events & EPOLLIN){ /* Receive API query, dispatch when complete. */ if (ctable[fd].in_off >= sizeof(struct p0f_api_query)) From d4c930e779d62f5e09c01b9d35b0a79d37e6337d Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:35:57 +1000 Subject: [PATCH 41/85] more error fixes --- p0f.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/p0f.c b/p0f.c index d99c094..395ae02 100644 --- a/p0f.c +++ b/p0f.c @@ -820,7 +820,8 @@ static void epoll_event_loop(void){ while (!stop_soon) { int nfds = epoll_wait(epfd, events, 5, -1); - for (int n = 0; n < nfds; ++n) { + int n = 0; + for ( n < nfds ) { int fd = events[n].data.fd; if (fd == pcap_fileno(pt)){//TODO optimize //Handle PCAP event @@ -864,7 +865,7 @@ static void epoll_event_loop(void){ if (ctable[fd].in_off >= sizeof(struct p0f_api_query)) FATAL("Inconsistent p0f_api_query state.\n"); - i = read(pfds[fd].fd, + int i = read(pfds[fd].fd, ((char*)&ctable[fd].in_data) + ctable[fd].in_off, sizeof(struct p0f_api_query) - ctable[fd].in_off); @@ -876,12 +877,15 @@ static void epoll_event_loop(void){ if (ctable[fd].in_off == sizeof(struct p0f_api_query)) { - handle_query(&ctable[fd]->in_data, &ctable[fd]->out_data); + handle_query(&ctable[fd].in_data, &ctable[fd].out_data); //pfds[fd].events = (POLLOUT | POLLERR | POLLHUP); } } } + + //Increment n (like a for loop!) + ++n; } } } From 35f2f67a55944c47abfcfd1e33ce5206a20b9430 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:38:40 +1000 Subject: [PATCH 42/85] fixes, and epoll_ctl error handling --- p0f.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/p0f.c b/p0f.c index 395ae02..d2b68be 100644 --- a/p0f.c +++ b/p0f.c @@ -821,7 +821,7 @@ static void epoll_event_loop(void){ while (!stop_soon) { int nfds = epoll_wait(epfd, events, 5, -1); int n = 0; - for ( n < nfds ) { + while ( n < nfds ) { int fd = events[n].data.fd; if (fd == pcap_fileno(pt)){//TODO optimize //Handle PCAP event @@ -848,7 +848,9 @@ static void epoll_event_loop(void){ ev.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP; ev.data.fd = client_sock; int res = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sock, &ev); - + if (res != 0){ + PFATAL("epoll_ctl() failed."); + } } } else{ @@ -865,7 +867,7 @@ static void epoll_event_loop(void){ if (ctable[fd].in_off >= sizeof(struct p0f_api_query)) FATAL("Inconsistent p0f_api_query state.\n"); - int i = read(pfds[fd].fd, + int i = read(fd, ((char*)&ctable[fd].in_data) + ctable[fd].in_off, sizeof(struct p0f_api_query) - ctable[fd].in_off); From 291f1635434dc0d1036e378cb0e598f742115880 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:51:47 +1000 Subject: [PATCH 43/85] epoll - write out the response --- p0f.c | 63 ++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/p0f.c b/p0f.c index d2b68be..893892b 100644 --- a/p0f.c +++ b/p0f.c @@ -817,40 +817,47 @@ static void epoll_event_loop(void){ struct epoll_event ev; struct epoll_event events[5]; int epfd = epoll_create(api_max_conn); + int pcap_fd = pcap_fileno(pt); while (!stop_soon) { int nfds = epoll_wait(epfd, events, 5, -1); int n = 0; while ( n < nfds ) { int fd = events[n].data.fd; - if (fd == pcap_fileno(pt)){//TODO optimize + if (fd == pcap_fd){ //Handle PCAP event if (events[n].events & EPOLLIN){ if (pcap_dispatch(pt, -1, (pcap_handler)parse_packet, 0) < 0) FATAL("Packet capture interface is down."); } + else if(events[n].events & EPOLLERR || events[n].events & EPOLLHUP){ + FATAL("Packet capture interface is down."); + } } else if (fd == api_fd) { //Accept api connection + if (events[n].events & EPOLLIN){ + int client_sock = accept(api_fd, NULL, NULL); + ctable[fd].fd = client_sock; - int client_sock = accept(api_fd, NULL, NULL); - ctable[fd].fd = client_sock; - - if (client_sock < 0) { - WARN("Unable to handle API connection: accept() fails."); - } - else { - if (fcntl(client_sock, F_SETFL, O_NONBLOCK)) - PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); + if (client_sock < 0) { + WARN("Unable to handle API connection: accept() fails."); + } + else { + if (fcntl(client_sock, F_SETFL, O_NONBLOCK)) + PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); - ctable[fd].in_off = ctable[fd].out_off = 0; + ctable[fd].in_off = 0; - ev.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP; - ev.data.fd = client_sock; - int res = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sock, &ev); - if (res != 0){ - PFATAL("epoll_ctl() failed."); + ev.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP; + ev.data.fd = client_sock; + int res = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sock, &ev); + if (res != 0){ + PFATAL("epoll_ctl() failed."); + } } + else if (events[n].events & EPOLLERR || events[n].events & EPOLLHUP){ + FATAL("Packet capture interface is down."); } } else{ @@ -878,10 +885,28 @@ static void epoll_event_loop(void){ /* Query in place? Compute response and prepare to send it back. */ if (ctable[fd].in_off == sizeof(struct p0f_api_query)) { - handle_query(&ctable[fd].in_data, &ctable[fd].out_data); - //pfds[fd].events = (POLLOUT | POLLERR | POLLHUP); + //A unix socket shouldnt block here, unless we arent reading fast enough + //Lets assume we have reasonably good clients and that this is a non issue + //This reduces complexity and improves performance, providing it holds true + /* Write API response, restart state when complete. */ + + if (ctable[fd]->in_off < sizeof(struct p0f_api_query)) + FATAL("Inconsistent p0f_api_response state.\n"); + + //Disable non block for a minute + if (fcntl(fd, F_SETFL, ~O_NONBLOCK)) + PFATAL("fcntl() to set ~O_NONBLOCK on API connection fails."); + + i = write(fd + ((char*)&ctable[fd]->out_data), + sizeof(struct p0f_api_response)); + + if (fcntl(fd, F_SETFL, O_NONBLOCK)) + PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); + + if (i <= 0) PFATAL("write() on API socket fails despite POLLOUT."); } } } @@ -890,6 +915,8 @@ static void epoll_event_loop(void){ ++n; } } + + ck_free(ctable); } #else /* Regenerate pollfd data for poll() */ From 9c8607c380a98e1b0d4cf04e26b5120727df524c Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:53:04 +1000 Subject: [PATCH 44/85] simple fixes --- p0f.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/p0f.c b/p0f.c index 893892b..1b26078 100644 --- a/p0f.c +++ b/p0f.c @@ -892,15 +892,12 @@ static void epoll_event_loop(void){ //This reduces complexity and improves performance, providing it holds true /* Write API response, restart state when complete. */ - if (ctable[fd]->in_off < sizeof(struct p0f_api_query)) - FATAL("Inconsistent p0f_api_response state.\n"); - //Disable non block for a minute if (fcntl(fd, F_SETFL, ~O_NONBLOCK)) PFATAL("fcntl() to set ~O_NONBLOCK on API connection fails."); i = write(fd - ((char*)&ctable[fd]->out_data), + ((char*)&ctable[fd].out_data), sizeof(struct p0f_api_response)); if (fcntl(fd, F_SETFL, O_NONBLOCK)) From 10eee5cbebacdc8e57ae4a64140c46b5bdd076ba Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:54:35 +1000 Subject: [PATCH 45/85] missing brace and comma added --- p0f.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index 1b26078..fadb857 100644 --- a/p0f.c +++ b/p0f.c @@ -856,6 +856,7 @@ static void epoll_event_loop(void){ PFATAL("epoll_ctl() failed."); } } + } else if (events[n].events & EPOLLERR || events[n].events & EPOLLHUP){ FATAL("Packet capture interface is down."); } @@ -896,7 +897,7 @@ static void epoll_event_loop(void){ if (fcntl(fd, F_SETFL, ~O_NONBLOCK)) PFATAL("fcntl() to set ~O_NONBLOCK on API connection fails."); - i = write(fd + i = write(fd, ((char*)&ctable[fd].out_data), sizeof(struct p0f_api_response)); From e504dc0d5bcb4c6f39b0ae6d0d51c1e249823c56 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 27 Jun 2014 23:58:54 +1000 Subject: [PATCH 46/85] add initial api & pcap fds to epol --- p0f.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index fadb857..f2f4188 100644 --- a/p0f.c +++ b/p0f.c @@ -816,9 +816,28 @@ static void epoll_event_loop(void){ struct epoll_event ev; struct epoll_event events[5]; - int epfd = epoll_create(api_max_conn); int pcap_fd = pcap_fileno(pt); + //Initial epoll setup + int epfd = epoll_create(api_max_conn); + + //add PCAP fd + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; + ev.data.fd = pcap_fd; + int res = epoll_ctl(epfd, EPOLL_CTL_ADD, pcap_fd, &ev); + if (res != 0){ + PFATAL("epoll_ctl() failed."); + } + + //add api fd + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; + ev.data.fd = api_fd; + int res = epoll_ctl(epfd, EPOLL_CTL_ADD, api_fd, &ev); + if (res != 0){ + PFATAL("epoll_ctl() failed."); + } + + //Main loop while (!stop_soon) { int nfds = epoll_wait(epfd, events, 5, -1); int n = 0; @@ -849,7 +868,7 @@ static void epoll_event_loop(void){ ctable[fd].in_off = 0; - ev.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = client_sock; int res = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sock, &ev); if (res != 0){ From f767af09570f37d7944fb3f88b0cc60978d51d1d Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:00:28 +1000 Subject: [PATCH 47/85] only declare res once, and re-use it in more places --- p0f.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/p0f.c b/p0f.c index f2f4188..f692520 100644 --- a/p0f.c +++ b/p0f.c @@ -817,6 +817,7 @@ static void epoll_event_loop(void){ struct epoll_event ev; struct epoll_event events[5]; int pcap_fd = pcap_fileno(pt); + int res; //Initial epoll setup int epfd = epoll_create(api_max_conn); @@ -824,7 +825,7 @@ static void epoll_event_loop(void){ //add PCAP fd ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = pcap_fd; - int res = epoll_ctl(epfd, EPOLL_CTL_ADD, pcap_fd, &ev); + res = epoll_ctl(epfd, EPOLL_CTL_ADD, pcap_fd, &ev); if (res != 0){ PFATAL("epoll_ctl() failed."); } @@ -832,7 +833,7 @@ static void epoll_event_loop(void){ //add api fd ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = api_fd; - int res = epoll_ctl(epfd, EPOLL_CTL_ADD, api_fd, &ev); + res = epoll_ctl(epfd, EPOLL_CTL_ADD, api_fd, &ev); if (res != 0){ PFATAL("epoll_ctl() failed."); } @@ -870,7 +871,7 @@ static void epoll_event_loop(void){ ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = client_sock; - int res = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sock, &ev); + res = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sock, &ev); if (res != 0){ PFATAL("epoll_ctl() failed."); } @@ -894,13 +895,13 @@ static void epoll_event_loop(void){ if (ctable[fd].in_off >= sizeof(struct p0f_api_query)) FATAL("Inconsistent p0f_api_query state.\n"); - int i = read(fd, + res = read(fd, ((char*)&ctable[fd].in_data) + ctable[fd].in_off, sizeof(struct p0f_api_query) - ctable[fd].in_off); - if (i < 0) PFATAL("read() on API socket fails despite POLLIN."); + if (res < 0) PFATAL("read() on API socket fails despite POLLIN."); - ctable[fd].in_off += i; + ctable[fd].in_off += res; /* Query in place? Compute response and prepare to send it back. */ @@ -916,14 +917,14 @@ static void epoll_event_loop(void){ if (fcntl(fd, F_SETFL, ~O_NONBLOCK)) PFATAL("fcntl() to set ~O_NONBLOCK on API connection fails."); - i = write(fd, + res = write(fd, ((char*)&ctable[fd].out_data), sizeof(struct p0f_api_response)); if (fcntl(fd, F_SETFL, O_NONBLOCK)) PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); - if (i <= 0) PFATAL("write() on API socket fails despite POLLOUT."); + if (res <= 0) PFATAL("write() on API socket fails despite POLLOUT."); } } } From aed58b1e12ac706bfbfa82508c4a0585dce22117 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:02:21 +1000 Subject: [PATCH 48/85] only add api_fd if it is set (API enabled) --- p0f.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/p0f.c b/p0f.c index f692520..2e7a335 100644 --- a/p0f.c +++ b/p0f.c @@ -830,12 +830,14 @@ static void epoll_event_loop(void){ PFATAL("epoll_ctl() failed."); } - //add api fd - ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; - ev.data.fd = api_fd; - res = epoll_ctl(epfd, EPOLL_CTL_ADD, api_fd, &ev); - if (res != 0){ - PFATAL("epoll_ctl() failed."); + if (api_fd){ + //add api fd + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; + ev.data.fd = api_fd; + res = epoll_ctl(epfd, EPOLL_CTL_ADD, api_fd, &ev); + if (res != 0){ + PFATAL("epoll_ctl() failed."); + } } //Main loop From fa24ea5fb646c016886c578cb77d4e3779a98272 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:03:39 +1000 Subject: [PATCH 49/85] Check the flag api_sock not the unintialized varaible --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index 2e7a335..1554730 100644 --- a/p0f.c +++ b/p0f.c @@ -830,7 +830,7 @@ static void epoll_event_loop(void){ PFATAL("epoll_ctl() failed."); } - if (api_fd){ + if (api_sock){ //add api fd ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = api_fd; From 0ed548aea29c51163d1e78653c477b354cf30d16 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:06:56 +1000 Subject: [PATCH 50/85] added info message (same as poll) --- p0f.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/p0f.c b/p0f.c index 1554730..8820bd9 100644 --- a/p0f.c +++ b/p0f.c @@ -840,6 +840,9 @@ static void epoll_event_loop(void){ } } + if (!daemon_mode) + SAYF("[+] Entered main event loop.\n\n"); + //Main loop while (!stop_soon) { int nfds = epoll_wait(epfd, events, 5, -1); From 3c7a6cd2a76b1f3e052d4cff2f2edbf285445286 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:09:24 +1000 Subject: [PATCH 51/85] be a good function and close the epoll fd --- p0f.c | 1 + 1 file changed, 1 insertion(+) diff --git a/p0f.c b/p0f.c index 8820bd9..284071a 100644 --- a/p0f.c +++ b/p0f.c @@ -939,6 +939,7 @@ static void epoll_event_loop(void){ } } + close(epfd); ck_free(ctable); } #else From a8f02c2f60594d28e5ed6c1e2ede544067b2a643 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:14:00 +1000 Subject: [PATCH 52/85] improve api socket limiting, and increase limit to 250 inline with epoll scalability --- p0f.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/p0f.c b/p0f.c index 284071a..74d4d42 100644 --- a/p0f.c +++ b/p0f.c @@ -153,7 +153,11 @@ static void usage(void) { "Performance-related options:\n" "\n" #ifndef __CYGWIN__ +#ifdef USE_EPOLL +" -S limit - limit number of parallel API connections, not guarunteed (%u)\n" +#else " -S limit - limit number of parallel API connections (%u)\n" +#endif #endif /* !__CYGWIN__ */ " -t c,h - set connection / host cache age limits (%us,%um)\n" " -m c,h - cap the number of active connections / hosts (%u,%u)\n" @@ -811,8 +815,8 @@ static void abort_handler(int sig) { static void epoll_event_loop(void){ struct api_client* ctable; - //Overallocate for now TODO: properly - ctable = ck_alloc(40 * sizeof(struct api_client)); + int slots = 6 + api_max_conn; + ctable = ck_alloc(slots * sizeof(struct api_client)); struct epoll_event ev; struct epoll_event events[5]; @@ -868,6 +872,10 @@ static void epoll_event_loop(void){ if (client_sock < 0) { WARN("Unable to handle API connection: accept() fails."); } + else if (client_sock > slots){ + WARN("Unable to handle API connection: too many connection."); + close(client_sock); + } else { if (fcntl(client_sock, F_SETFL, O_NONBLOCK)) PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); @@ -1268,7 +1276,7 @@ int main(int argc, char** argv) { api_max_conn = atol(optarg); - if (!api_max_conn || api_max_conn > 100) + if (!api_max_conn || api_max_conn > 250) FATAL("Outlandish value specified for -S."); break; From 0599002eb6d2fef47feae107ef8473d88c551c14 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:15:21 +1000 Subject: [PATCH 53/85] prevent overflow of array --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index 74d4d42..c8f3885 100644 --- a/p0f.c +++ b/p0f.c @@ -872,7 +872,7 @@ static void epoll_event_loop(void){ if (client_sock < 0) { WARN("Unable to handle API connection: accept() fails."); } - else if (client_sock > slots){ + else if (client_sock >= slots){ WARN("Unable to handle API connection: too many connection."); close(client_sock); } From 5cb40b589247e897ae0eff02be20a253e2307c03 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:34:18 +1000 Subject: [PATCH 54/85] reset in offset so that further queries work :) --- p0f.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/p0f.c b/p0f.c index c8f3885..09786bc 100644 --- a/p0f.c +++ b/p0f.c @@ -921,6 +921,9 @@ static void epoll_event_loop(void){ if (ctable[fd].in_off == sizeof(struct p0f_api_query)) { handle_query(&ctable[fd].in_data, &ctable[fd].out_data); + //Reset in offset + ctable[fd].in_off = 0; + //A unix socket shouldnt block here, unless we arent reading fast enough //Lets assume we have reasonably good clients and that this is a non issue //This reduces complexity and improves performance, providing it holds true From 7738aca4f544fd3b11e52ef609e65f9f9db92024 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:36:56 +1000 Subject: [PATCH 55/85] set no flags --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index 09786bc..814b70d 100644 --- a/p0f.c +++ b/p0f.c @@ -930,7 +930,7 @@ static void epoll_event_loop(void){ /* Write API response, restart state when complete. */ //Disable non block for a minute - if (fcntl(fd, F_SETFL, ~O_NONBLOCK)) + if (fcntl(fd, F_SETFL, 0)) PFATAL("fcntl() to set ~O_NONBLOCK on API connection fails."); res = write(fd, From 5140da2dff5e1d71c717ee5789b838eaf270b306 Mon Sep 17 00:00:00 2001 From: splitice Date: Sat, 28 Jun 2014 00:57:52 +1000 Subject: [PATCH 56/85] extended readme for epoll --- docs/README | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/README b/docs/README index ebe5d39..ce24075 100644 --- a/docs/README +++ b/docs/README @@ -92,8 +92,9 @@ http://lcamtuf.coredump.cx/p0f3/ - Support for nflog capturing (capture after IPTables) - One sided stream support (SYN+ACK not required) - Capture buffer size increased to fix crash on heavily loaded systems. + - epoll support for increased performance and scalability (increased API connections) - This fork is production ready, a version of it is in use at http://www.x4b.net/ + This forks releases are possibly production ready, a version of it is in use at http://www.x4b.net/ Software is provided 'as-is'. No warranty is implied, we are not responsible for any damages or loss resulting your use of this software. From 42eac4de5fda2c44f2531ab745066aefc63f558d Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 3 Jul 2014 04:12:29 +1000 Subject: [PATCH 57/85] possible memory overrun --- p0f.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index 814b70d..7654cb9 100644 --- a/p0f.c +++ b/p0f.c @@ -803,6 +803,7 @@ static void fork_off(void) { /* Handler for Ctrl-C and related signals */ static void abort_handler(int sig) { + SAYF("Received signal: %d\n", sig); if (stop_soon) exit(1); stop_soon = 1; } @@ -867,7 +868,6 @@ static void epoll_event_loop(void){ //Accept api connection if (events[n].events & EPOLLIN){ int client_sock = accept(api_fd, NULL, NULL); - ctable[fd].fd = client_sock; if (client_sock < 0) { WARN("Unable to handle API connection: accept() fails."); @@ -877,6 +877,8 @@ static void epoll_event_loop(void){ close(client_sock); } else { + ctable[fd].fd = client_sock; + if (fcntl(client_sock, F_SETFL, O_NONBLOCK)) PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); @@ -899,7 +901,7 @@ static void epoll_event_loop(void){ if (events[n].events & EPOLLERR || events[n].events & EPOLLHUP){ DEBUG("[#] API connection on fd %d closed.\n", events[n].data.fd); - close(events[n].data.fd); + close(fd); ctable[fd].fd = -1; } else if (events[n].events & EPOLLIN){ From 785ce61a2fe8f084dc01a56e9a46ce3caf109bf9 Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 14 Jul 2014 13:52:49 +1000 Subject: [PATCH 58/85] first version supporting nfnetlink direct. untested. --- p0f.c | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++--- process.c | 83 +++++++++++++++++++++++++---------------- process.h | 8 ++++ 3 files changed, 164 insertions(+), 36 deletions(-) diff --git a/p0f.c b/p0f.c index 7654cb9..865d86a 100644 --- a/p0f.c +++ b/p0f.c @@ -33,6 +33,7 @@ #include #include +#ifdef USE_LIBPCAP //TODO: compile option for linux options #define NETLINK_NO_ENOBUFS 5 @@ -46,6 +47,14 @@ #else # include #endif /* !NET_BPF */ +#endif +#ifdef USE_LIBNML +#include + +#include +#include +#include +#endif #include "types.h" #include "debug.h" @@ -107,10 +116,16 @@ u8 daemon_mode; /* Running in daemon mode? */ static u8 set_promisc; /* Use promiscuous mode? */ +#ifdef USE_LIBPCAP static pcap_t *pt; /* PCAP capture thingy */ s32 link_type; /* PCAP link type */ +#elif USE_LIBMNL +struct mnl_socket mnlsock; /* Netlink socket */ + +#endif + u32 hash_seed; /* Hash seed */ static u8 obs_fields; /* No of pending observation fields */ @@ -135,8 +150,10 @@ static void usage(void) { "Network interface options:\n" "\n" " -i iface - listen on the specified network interface\n" +#ifdef USE_LIBPCAP " -r file - read offline pcap data from a given file\n" " -p - put the listening interface in promiscuous mode\n" +#endif " -L - list all available interfaces\n" "\n" "Operating mode and output settings:\n" @@ -389,6 +406,7 @@ void add_observation_field(char* key, u8* value) { } +#ifdef USE_LIBPCAP /* Show PCAP interface list */ static void list_interfaces(void) { @@ -440,7 +458,6 @@ static void list_interfaces(void) { } - #ifdef __CYGWIN__ /* List PCAP-recognized interfaces */ @@ -698,7 +715,50 @@ static void prepare_bpf(void) { } } +#elif USE_LIBMNL +static void prepare_netlink(void){ + char buf[MNL_SOCKET_BUFFER_SIZE]; + struct nlmsghdr *nlh; + unsigned int portid, qnum; + + + qnum = atoi(use_iface); + + nl = mnl_socket_open(NETLINK_NETFILTER); + if (nl == NULL) { + PFATAL("mnl_socket_open"); + } + + if (mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { + PFATAL("mnl_socket_bind"); + } + portid = mnl_socket_get_portid(nl); + + nlh = nflog_build_cfg_pf_request(buf, NFULNL_CFG_CMD_PF_UNBIND); + + if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { + PFATAL("mnl_socket_sendto"); + } + + nlh = nflog_build_cfg_pf_request(buf, NFULNL_CFG_CMD_PF_BIND); + + if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { + PFATAL("mnl_socket_sendto"); + } + + nlh = nflog_build_cfg_request(buf, NFULNL_CFG_CMD_BIND, qnum); + if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { + PFATAL("mnl_socket_sendto"); + } + + nlh = nflog_build_cfg_params(buf, NFULNL_COPY_PACKET, 0xFFFF, qnum); + + if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { + PFATAL("mnl_socket_sendto"); + } +} +#endif /* Drop privileges and chroot(), with some sanity checks */ @@ -821,7 +881,12 @@ static void epoll_event_loop(void){ struct epoll_event ev; struct epoll_event events[5]; +#ifdef USE_LIBPCAP int pcap_fd = pcap_fileno(pt); +#elif USE_LIBMNL + int pcap_fd = nlsock.fd; + char buf[MNL_SOCKET_BUFFER_SIZE]; +#endif int res; //Initial epoll setup @@ -857,8 +922,27 @@ static void epoll_event_loop(void){ if (fd == pcap_fd){ //Handle PCAP event if (events[n].events & EPOLLIN){ +#ifdef USE_LIBPCAP if (pcap_dispatch(pt, -1, (pcap_handler)parse_packet, 0) < 0) FATAL("Packet capture interface is down."); +#elif USE_LIBMNL + res = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + if (res == -1) { + PFATAL("mnl_socket_recvfrom"); + } + while (res > 0) { + res = mnl_cb_run(buf, res, 0, portid, parse_packet, NULL); + if (res < 0){ + PFATAL("mnl_cb_run"); + } + + res = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + if (ret == -1) { + PFATAL("mnl_socket_recvfrom"); + } + } +#endif + } else if(events[n].events & EPOLLERR || events[n].events & EPOLLHUP){ FATAL("Packet capture interface is down."); @@ -893,7 +977,7 @@ static void epoll_event_loop(void){ } } else if (events[n].events & EPOLLERR || events[n].events & EPOLLHUP){ - FATAL("Packet capture interface is down."); + FATAL("API socket is down."); } } else{ @@ -1212,7 +1296,9 @@ static void live_event_loop(void) { in pcap_dispatch() or pcap_loop() unless there's I/O. */ while (!stop_soon) { - +#ifndef USE_LIBPCAP +#error Only LIBPCAP supported in Cygwin +#endif s32 ret = pcap_dispatch(pt, -1, (pcap_handler)parse_packet, 0); if (ret < 0) return; @@ -1229,7 +1315,7 @@ static void live_event_loop(void) { } - +#ifdef USE_LIBPCAP /* Simple event loop for processing offline captures. */ static void offline_event_loop(void) { @@ -1246,6 +1332,7 @@ static void offline_event_loop(void) { WARN("User-initiated shutdown."); } +#endif /* Main entry point */ @@ -1446,6 +1533,7 @@ int main(int argc, char** argv) { read_config(fp_file ? fp_file : (u8*)FP_FILE); +#ifdef USE_LIBPCAP prepare_pcap(); if (disable_bpf) { SAYF("[+] BPF Disabled\n"); @@ -1453,6 +1541,9 @@ int main(int argc, char** argv) { else { prepare_bpf(); } +#elif USE_LIBMNL + prepare_netlink(); +#endif if (log_file) open_log(); if (api_sock) open_api(); @@ -1470,7 +1561,15 @@ int main(int argc, char** argv) { signal(SIGINT, abort_handler); signal(SIGTERM, abort_handler); - if (read_file) offline_event_loop(); else live_event_loop(); +#ifdef USE_LIBPCAP + if (read_file) + offline_event_loop(); + else + live_event_loop(); +#elif USE_LIBMNL + live_event_loop(); +#endif + if (!daemon_mode) SAYF("\nAll done. Processed %llu packets.\n", packet_cnt); diff --git a/process.c b/process.c index f2b1db4..8649571 100644 --- a/process.c +++ b/process.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -77,6 +76,35 @@ u32 get_unix_time(void) { return cur_time->tv_sec; } +/* Convert IPv4 or IPv6 address to a human-readable form. */ + +u8* addr_to_str(u8* data, u8 ip_ver) { + + static char tmp[128]; + + /* We could be using inet_ntop(), but on systems that have older libc + but still see passing IPv6 traffic, we would be in a pickle. */ + + if (ip_ver == IP_VER4) { + + sprintf(tmp, "%u.%u.%u.%u", data[0], data[1], data[2], data[3]); + + } + else { + + sprintf(tmp, "%x:%x:%x:%x:%x:%x:%x:%x", + (data[0] << 8) | data[1], (data[2] << 8) | data[3], + (data[4] << 8) | data[5], (data[6] << 8) | data[7], + (data[8] << 8) | data[9], (data[10] << 8) | data[11], + (data[12] << 8) | data[13], (data[14] << 8) | data[15]); + + } + + return (u8*)tmp; + +} + +#ifdef USE_LIBPCAP /* Find link-specific offset (pcap knows, but won't tell). */ @@ -182,35 +210,6 @@ static void find_offset(const u8* data, s32 total_len) { } - -/* Convert IPv4 or IPv6 address to a human-readable form. */ - -u8* addr_to_str(u8* data, u8 ip_ver) { - - static char tmp[128]; - - /* We could be using inet_ntop(), but on systems that have older libc - but still see passing IPv6 traffic, we would be in a pickle. */ - - if (ip_ver == IP_VER4) { - - sprintf(tmp, "%u.%u.%u.%u", data[0], data[1], data[2], data[3]); - - } else { - - sprintf(tmp, "%x:%x:%x:%x:%x:%x:%x:%x", - (data[0] << 8) | data[1], (data[2] << 8) | data[3], - (data[4] << 8) | data[5], (data[6] << 8) | data[7], - (data[8] << 8) | data[9], (data[10] << 8) | data[11], - (data[12] << 8) | data[13], (data[14] << 8) | data[15]); - - } - - return (u8*)tmp; - -} - - /* Parse PCAP input, with plenty of sanity checking. Store interesting details in a protocol-agnostic buffer that will be then examined upstream. */ @@ -768,7 +767,29 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { flow_dispatch(&pk); } - +#elif USE_LIBMNL +int parse_packet(const struct nlmsghdr *nlh, void *data) +{ + struct nlattr *tb[NFULA_MAX + 1] = {}; + struct nfulnl_msg_packet_hdr *ph = NULL; + const char *prefix = NULL; + uint32_t mark = 0; + + mnl_attr_parse(nlh, sizeof(struct nfgenmsg), parse_attr_cb, tb); + if (tb[NFULA_PACKET_HDR]) + ph = mnl_attr_get_payload(tb[NFULA_PACKET_HDR]); + if (tb[NFULA_PREFIX]) + prefix = mnl_attr_get_str(tb[NFULA_PREFIX]); + if (tb[NFULA_MARK]) + mark = ntohl(mnl_attr_get_u32(tb[NFULA_MARK])); + + printf("log received (prefix=\"%s\" hw=0x%04x hook=%u mark=%u)\n", + prefix ? prefix : "", ntohs(ph->hw_protocol), ph->hook, + mark); + + return MNL_CB_OK; +} +#endif /* Calculate hash bucket for packet_flow. Keep the hash symmetrical: switching source and dest should have no effect. */ diff --git a/process.h b/process.h index 2addf27..cfa809c 100644 --- a/process.h +++ b/process.h @@ -11,7 +11,11 @@ #ifndef _HAVE_PROCESS_H #define _HAVE_PROCESS_H +#ifdef USE_LIBPCAP #include +#elif USE_LIBMNL +#include +#endif #include "types.h" #include "fp_tcp.h" @@ -204,7 +208,11 @@ struct packet_flow { extern u64 packet_cnt; +#ifdef USE_LIBPCAP void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data); +#elif USE_LIBMNL +int parse_packet(const struct nlmsghdr *nlh, void *data) +#endif u8* addr_to_str(u8* data, u8 ip_ver); From 846b70f3eb3e7f4839dd0e80a20fb78db1dea1ee Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 14 Jul 2014 13:53:27 +1000 Subject: [PATCH 59/85] configuration to enable libmnfl by default --- config.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config.h b/config.h index aafe4ee..d6ee5b2 100644 --- a/config.h +++ b/config.h @@ -20,6 +20,10 @@ /* Use epoll, enable only if available */ #define USE_EPOLL +/* Use pcap or libmnl */ +#define USE_LIBNML +//#define USE_LIBPCAP + /* Default location of p0f.fp: */ #ifndef FP_FILE From 36de025a637359396f9dcc9e7c7e74fa0f46f484 Mon Sep 17 00:00:00 2001 From: splitice Date: Mon, 14 Jul 2014 23:13:02 +1000 Subject: [PATCH 60/85] work on compiler errors --- build.sh | 2 +- config.h | 4 ++-- p0f.c | 20 +++++++++++--------- process.c | 2 +- process.h | 6 +++--- types.h | 4 ++-- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/build.sh b/build.sh index 626de1e..742163f 100755 --- a/build.sh +++ b/build.sh @@ -28,7 +28,7 @@ if [ "$OSTYPE" = "cygwin" ]; then elif [ "$OSTYPE" = "solaris" ]; then USE_LIBS="-lsocket -lnsl $LIBS" else - USE_LIBS="-lpcap $LIBS" + USE_LIBS="-lpcap -lmnl $LIBS" fi OBJFILES="api.c process.c fp_tcp.c fp_mtu.c fp_http.c fp_ssl.c readfp.c" diff --git a/config.h b/config.h index d6ee5b2..a2c7882 100644 --- a/config.h +++ b/config.h @@ -21,8 +21,8 @@ #define USE_EPOLL /* Use pcap or libmnl */ -#define USE_LIBNML -//#define USE_LIBPCAP +#define USE_LIBMNL 1 +//#define USE_LIBPCAP 1 /* Default location of p0f.fp: */ diff --git a/p0f.c b/p0f.c index 865d86a..e47f1e5 100644 --- a/p0f.c +++ b/p0f.c @@ -11,6 +11,7 @@ #define _GNU_SOURCE #define _FROM_P0F +#include "config.h" #include #include #include @@ -47,13 +48,14 @@ #else # include #endif /* !NET_BPF */ -#endif -#ifdef USE_LIBNML +#elif defined(USE_LIBMNL) #include #include #include #include +#else +#error Neither LIBPCAP or LIBMNL selected #endif #include "types.h" @@ -121,8 +123,8 @@ static pcap_t *pt; /* PCAP capture thingy */ s32 link_type; /* PCAP link type */ -#elif USE_LIBMNL -struct mnl_socket mnlsock; /* Netlink socket */ +#elif defined(USE_LIBMNL) +struct mnl_socket nl; /* Netlink socket */ #endif @@ -715,7 +717,7 @@ static void prepare_bpf(void) { } } -#elif USE_LIBMNL +#elif defined(USE_LIBMNL) static void prepare_netlink(void){ char buf[MNL_SOCKET_BUFFER_SIZE]; struct nlmsghdr *nlh; @@ -883,7 +885,7 @@ static void epoll_event_loop(void){ struct epoll_event events[5]; #ifdef USE_LIBPCAP int pcap_fd = pcap_fileno(pt); -#elif USE_LIBMNL +#elif defined(USE_LIBMNL) int pcap_fd = nlsock.fd; char buf[MNL_SOCKET_BUFFER_SIZE]; #endif @@ -925,7 +927,7 @@ static void epoll_event_loop(void){ #ifdef USE_LIBPCAP if (pcap_dispatch(pt, -1, (pcap_handler)parse_packet, 0) < 0) FATAL("Packet capture interface is down."); -#elif USE_LIBMNL +#elif defined(USE_LIBMNL) res = mnl_socket_recvfrom(nl, buf, sizeof(buf)); if (res == -1) { PFATAL("mnl_socket_recvfrom"); @@ -1541,7 +1543,7 @@ int main(int argc, char** argv) { else { prepare_bpf(); } -#elif USE_LIBMNL +#elif defined(USE_LIBMNL) prepare_netlink(); #endif @@ -1566,7 +1568,7 @@ int main(int argc, char** argv) { offline_event_loop(); else live_event_loop(); -#elif USE_LIBMNL +#elif defined(USE_LIBMNL) live_event_loop(); #endif diff --git a/process.c b/process.c index 8649571..9edff7a 100644 --- a/process.c +++ b/process.c @@ -767,7 +767,7 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { flow_dispatch(&pk); } -#elif USE_LIBMNL +#elif defined(USE_LIBMNL) int parse_packet(const struct nlmsghdr *nlh, void *data) { struct nlattr *tb[NFULA_MAX + 1] = {}; diff --git a/process.h b/process.h index cfa809c..079954f 100644 --- a/process.h +++ b/process.h @@ -13,7 +13,7 @@ #ifdef USE_LIBPCAP #include -#elif USE_LIBMNL +#elif defined(USE_LIBMNL) #include #endif @@ -210,8 +210,8 @@ extern u64 packet_cnt; #ifdef USE_LIBPCAP void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data); -#elif USE_LIBMNL -int parse_packet(const struct nlmsghdr *nlh, void *data) +#elif defined(USE_LIBMNL) +int parse_packet(const struct nlmsghdr *nlh, void *data); #endif u8* addr_to_str(u8* data, u8 ip_ver); diff --git a/types.h b/types.h index bef7653..14f29b7 100644 --- a/types.h +++ b/types.h @@ -8,8 +8,8 @@ */ -#ifndef _HAVE_TYPES_H -#define _HAVE_TYPES_H +#ifndef _HAVE_P0F_TYPES_H +#define _HAVE_P0F_TYPES_H #include From b47e4d934f0ff29afc6980459496c98ff7791cd4 Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 00:14:16 +1000 Subject: [PATCH 61/85] compiles first build to compile after the introduction of libmnl --- p0f.c | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- process.c | 51 +++++++++++++++++ process.h | 3 + 3 files changed, 210 insertions(+), 6 deletions(-) diff --git a/p0f.c b/p0f.c index e47f1e5..daccdd1 100644 --- a/p0f.c +++ b/p0f.c @@ -54,6 +54,7 @@ #include #include #include +#include #else #error Neither LIBPCAP or LIBMNL selected #endif @@ -124,7 +125,9 @@ static pcap_t *pt; /* PCAP capture thingy */ s32 link_type; /* PCAP link type */ #elif defined(USE_LIBMNL) -struct mnl_socket nl; /* Netlink socket */ + +struct mnl_socket* nl; /* Netlink socket */ +unsigned int portid; #endif @@ -718,10 +721,157 @@ static void prepare_bpf(void) { } #elif defined(USE_LIBMNL) + +#ifdef NL_MMAP_STATUS_UNUSED //Has mmap support +static struct nlmsghdr *nflog_build_cfg_pf_request(struct mnl_socket *nl, uint8_t command) +{ + struct nl_mmap_hdr *hdr; + + hdr = mnl_socket_get_frame(nl, MNL_RING_TX); + if (hdr->nm_status != NL_MMAP_STATUS_UNUSED) + return NULL; + mnl_socket_advance_ring(nl, MNL_RING_TX); + + struct nlmsghdr *nlh = mnl_nlmsg_put_header((void *)hdr + NL_MMAP_HDRLEN); + nlh->nlmsg_type = (NFNL_SUBSYS_ULOG << 8) | NFULNL_MSG_CONFIG; + nlh->nlmsg_flags = NLM_F_REQUEST; + + struct nfgenmsg *nfg = mnl_nlmsg_put_extra_header(nlh, sizeof(*nfg)); + nfg->nfgen_family = AF_INET; + nfg->version = NFNETLINK_V0; + + struct nfulnl_msg_config_cmd cmd = { + .command = command, + }; + mnl_attr_put(nlh, NFULA_CFG_CMD, sizeof(cmd), &cmd); + + hdr->nm_len = nlh->nlmsg_len; + hdr->nm_status = NL_MMAP_STATUS_VALID; + return nlh; +} + +static struct nlmsghdr *nflog_build_cfg_request(struct mnl_socket *nl, uint8_t command, int nflognum) +{ + struct nl_mmap_hdr *hdr; + + hdr = mnl_socket_get_frame(nl, MNL_RING_TX); + if (hdr->nm_status != NL_MMAP_STATUS_UNUSED) + return NULL; + mnl_socket_advance_ring(nl, MNL_RING_TX); + + struct nlmsghdr *nlh = mnl_nlmsg_put_header((void *)hdr + NL_MMAP_HDRLEN); + nlh->nlmsg_type = (NFNL_SUBSYS_ULOG << 8) | NFULNL_MSG_CONFIG; + nlh->nlmsg_flags = NLM_F_REQUEST; + + struct nfgenmsg *nfg = mnl_nlmsg_put_extra_header(nlh, sizeof(*nfg)); + nfg->nfgen_family = AF_INET; + nfg->version = NFNETLINK_V0; + nfg->res_id = htons(nflognum); + + struct nfulnl_msg_config_cmd cmd = { + .command = command, + }; + mnl_attr_put(nlh, NFULA_CFG_CMD, sizeof(cmd), &cmd); + + hdr->nm_len = nlh->nlmsg_len; + hdr->nm_status = NL_MMAP_STATUS_VALID; + return nlh; +} + +static struct nlmsghdr *nflog_build_cfg_params(struct mnl_socket *nl, uint8_t mode, int range, int nflognum) +{ + struct nl_mmap_hdr *hdr; + + hdr = mnl_socket_get_frame(nl, MNL_RING_TX); + if (hdr->nm_status != NL_MMAP_STATUS_UNUSED) + return NULL; + mnl_socket_advance_ring(nl, MNL_RING_TX); + + struct nlmsghdr *nlh = mnl_nlmsg_put_header((void *)hdr + NL_MMAP_HDRLEN); + nlh->nlmsg_type = (NFNL_SUBSYS_ULOG << 8) | NFULNL_MSG_CONFIG; + nlh->nlmsg_flags = NLM_F_REQUEST; + + struct nfgenmsg *nfg = mnl_nlmsg_put_extra_header(nlh, sizeof(*nfg)); + nfg->nfgen_family = AF_UNSPEC; + nfg->version = NFNETLINK_V0; + nfg->res_id = htons(nflognum); + + struct nfulnl_msg_config_mode params = { + .copy_range = htonl(range), + .copy_mode = mode, + }; + mnl_attr_put(nlh, NFULA_CFG_MODE, sizeof(params), ¶ms); + + hdr->nm_len = nlh->nlmsg_len; + hdr->nm_status = NL_MMAP_STATUS_VALID; + return nlh; +} +#else +static struct nlmsghdr * +nflog_build_cfg_pf_request(char *buf, uint8_t command) +{ + struct nlmsghdr *nlh = mnl_nlmsg_put_header(buf); + nlh->nlmsg_type = (NFNL_SUBSYS_ULOG << 8) | NFULNL_MSG_CONFIG; + nlh->nlmsg_flags = NLM_F_REQUEST; + + struct nfgenmsg *nfg = mnl_nlmsg_put_extra_header(nlh, sizeof(*nfg)); + nfg->nfgen_family = AF_INET; + nfg->version = NFNETLINK_V0; + + struct nfulnl_msg_config_cmd cmd = { + .command = command, + }; + mnl_attr_put(nlh, NFULA_CFG_CMD, sizeof(cmd), &cmd); + + return nlh; +} + +static struct nlmsghdr * +nflog_build_cfg_request(char *buf, uint8_t command, int qnum) +{ + struct nlmsghdr *nlh = mnl_nlmsg_put_header(buf); + nlh->nlmsg_type = (NFNL_SUBSYS_ULOG << 8) | NFULNL_MSG_CONFIG; + nlh->nlmsg_flags = NLM_F_REQUEST; + + struct nfgenmsg *nfg = mnl_nlmsg_put_extra_header(nlh, sizeof(*nfg)); + nfg->nfgen_family = AF_INET; + nfg->version = NFNETLINK_V0; + nfg->res_id = htons(qnum); + + struct nfulnl_msg_config_cmd cmd = { + .command = command, + }; + mnl_attr_put(nlh, NFULA_CFG_CMD, sizeof(cmd), &cmd); + + return nlh; +} + +static struct nlmsghdr * +nflog_build_cfg_params(char *buf, uint8_t mode, int range, int qnum) +{ + struct nlmsghdr *nlh = mnl_nlmsg_put_header(buf); + nlh->nlmsg_type = (NFNL_SUBSYS_ULOG << 8) | NFULNL_MSG_CONFIG; + nlh->nlmsg_flags = NLM_F_REQUEST; + + struct nfgenmsg *nfg = mnl_nlmsg_put_extra_header(nlh, sizeof(*nfg)); + nfg->nfgen_family = AF_UNSPEC; + nfg->version = NFNETLINK_V0; + nfg->res_id = htons(qnum); + + struct nfulnl_msg_config_mode params = { + .copy_range = htonl(range), + .copy_mode = mode, + }; + mnl_attr_put(nlh, NFULA_CFG_MODE, sizeof(params), ¶ms); + + return nlh; +} +#endif + static void prepare_netlink(void){ char buf[MNL_SOCKET_BUFFER_SIZE]; struct nlmsghdr *nlh; - unsigned int portid, qnum; + unsigned int qnum; qnum = atoi(use_iface); @@ -886,7 +1036,7 @@ static void epoll_event_loop(void){ #ifdef USE_LIBPCAP int pcap_fd = pcap_fileno(pt); #elif defined(USE_LIBMNL) - int pcap_fd = nlsock.fd; + int pcap_fd = mnl_socket_get_fd(nl); char buf[MNL_SOCKET_BUFFER_SIZE]; #endif int res; @@ -939,7 +1089,7 @@ static void epoll_event_loop(void){ } res = mnl_socket_recvfrom(nl, buf, sizeof(buf)); - if (ret == -1) { + if (res == -1) { PFATAL("mnl_socket_recvfrom"); } } @@ -1351,12 +1501,12 @@ int main(int argc, char** argv) { FATAL("Please don't make me setuid. See README for more.\n"); while ((r = getopt(argc, argv, "+LS:df:i:m:o:pr:s:t:u:b")) != -1) switch (r) { - +#ifdef USE_PCAP case 'L': list_interfaces(); exit(0); - +#endif case 'S': #ifdef __CYGWIN__ diff --git a/process.c b/process.c index 9edff7a..8691538 100644 --- a/process.c +++ b/process.c @@ -768,6 +768,53 @@ void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { } #elif defined(USE_LIBMNL) +int parse_attr_cb(const struct nlattr *attr, void *data) +{ + const struct nlattr **tb = data; + int type = mnl_attr_get_type(attr); + + /* skip unsupported attribute in user-space */ + if (mnl_attr_type_valid(attr, NFULA_MAX) < 0) + return MNL_CB_OK; + + switch (type) { + case NFULA_MARK: + case NFULA_IFINDEX_INDEV: + case NFULA_IFINDEX_OUTDEV: + case NFULA_IFINDEX_PHYSINDEV: + case NFULA_IFINDEX_PHYSOUTDEV: + if (mnl_attr_validate(attr, MNL_TYPE_U32) < 0) { + perror("mnl_attr_validate"); + return MNL_CB_ERROR; + } + break; + case NFULA_TIMESTAMP: + if (mnl_attr_validate2(attr, MNL_TYPE_UNSPEC, + sizeof(struct nfulnl_msg_packet_timestamp)) < 0) { + perror("mnl_attr_validate"); + return MNL_CB_ERROR; + } + break; + case NFULA_HWADDR: + if (mnl_attr_validate2(attr, MNL_TYPE_UNSPEC, + sizeof(struct nfulnl_msg_packet_hw)) < 0) { + perror("mnl_attr_validate"); + return MNL_CB_ERROR; + } + break; + case NFULA_PREFIX: + if (mnl_attr_validate(attr, MNL_TYPE_NUL_STRING) < 0) { + perror("mnl_attr_validate"); + return MNL_CB_ERROR; + } + break; + case NFULA_PAYLOAD: + break; + } + tb[type] = attr; + return MNL_CB_OK; +} + int parse_packet(const struct nlmsghdr *nlh, void *data) { struct nlattr *tb[NFULA_MAX + 1] = {}; @@ -787,6 +834,10 @@ int parse_packet(const struct nlmsghdr *nlh, void *data) prefix ? prefix : "", ntohs(ph->hw_protocol), ph->hook, mark); + packet_cnt++; + + if (!(packet_cnt % EXPIRE_INTERVAL)) expire_cache(); + return MNL_CB_OK; } #endif diff --git a/process.h b/process.h index 079954f..2c586d6 100644 --- a/process.h +++ b/process.h @@ -11,10 +11,13 @@ #ifndef _HAVE_PROCESS_H #define _HAVE_PROCESS_H +#include "config.h" #ifdef USE_LIBPCAP #include #elif defined(USE_LIBMNL) #include +#include +#include #endif #include "types.h" From 52f2649f0e5ea27085376eb43763316b91d2cb0c Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 00:35:03 +1000 Subject: [PATCH 62/85] works, but segmentation faults after a few packets --- p0f.c | 2 + process.c | 904 ++++++++++++++++++++++++++++-------------------------- 2 files changed, 464 insertions(+), 442 deletions(-) diff --git a/p0f.c b/p0f.c index daccdd1..6eed585 100644 --- a/p0f.c +++ b/p0f.c @@ -1041,6 +1041,8 @@ static void epoll_event_loop(void){ #endif int res; + DEBUG("pcap fd: %d", pcap_fd); + //Initial epoll setup int epfd = epoll_create(api_max_conn); diff --git a/process.c b/process.c index 8691538..c1155e2 100644 --- a/process.c +++ b/process.c @@ -39,7 +39,9 @@ u64 packet_cnt; /* Total number of packets processed */ +#ifdef USE_LIBPCAP static s8 link_off = -1; /* Link-specific IP header offset */ +#endif static u8 bad_packets; /* Seen non-IP packets? */ static struct host_data *host_by_age, /* All host entries, by last mod */ @@ -104,668 +106,680 @@ u8* addr_to_str(u8* data, u8 ip_ver) { } -#ifdef USE_LIBPCAP +void process_packet(const u8* data, s32 packet_len){ + struct tcp_hdr* tcp; + struct packet_data pk; -/* Find link-specific offset (pcap knows, but won't tell). */ + u32 tcp_doff; -static void find_offset(const u8* data, s32 total_len) { + u8* opt_end; - u8 i; + /* If there is no way we could have received a complete TCP packet, bail + out early. */ - /* Check hardcoded values for some of the most common options. */ - DEBUG("[#] Looking for offset for link type: %d\n", link_type); + if (packet_len < MIN_TCP4) { + DEBUG("[#] Packet too short for any IPv4 + TCP headers, giving up!\n"); + return; + } - switch (link_type) { + pk.quirks = 0; - case DLT_RAW: link_off = 0; return; -#ifdef DLT_NFLOG - case DLT_NFLOG: link_off = 4; return; //family, version, resource_id -#endif - case DLT_NULL: - case DLT_PPP: link_off = 4; return; + if ((*data >> 4) == IP_VER4) { - case DLT_LOOP: + /************************ + * IPv4 header parsing. * + ************************/ -#ifdef DLT_PPP_SERIAL - case DLT_PPP_SERIAL: -#endif /* DLT_PPP_SERIAL */ + const struct ipv4_hdr* ip4 = (struct ipv4_hdr*)data; - case DLT_PPP_ETHER: link_off = 8; return; + u32 hdr_len = (ip4->ver_hlen & 0x0F) * 4; + u16 flags_off = ntohs(RD16(ip4->flags_off)); + u16 tot_len = ntohs(RD16(ip4->tot_len)); - case DLT_EN10MB: link_off = 14; return; + /* If the packet claims to be shorter than what we received off the wire, + honor this claim to account for etherleak-type bugs. */ -#ifdef DLT_LINUX_SLL - case DLT_LINUX_SLL: link_off = 16; return; -#endif /* DLT_LINUX_SLL */ + if (packet_len > tot_len) { + packet_len = tot_len; + // DEBUG("[#] ipv4.tot_len = %u, adjusted accordingly.\n", tot_len); + } - case DLT_PFLOG: link_off = 28; return; + /* Bail out if the result leaves no room for IPv4 + TCP headers. */ - case DLT_IEEE802_11: link_off = 32; return; - } + if (packet_len < MIN_TCP4) { + DEBUG("[#] packet_len = %u. Too short for IPv4 + TCP, giving up!\n", + packet_len); + return; + } - DEBUG("[#] UNKNOWN DLT %d, attempting to find offset\n", link_type); + /* Bail out if the declared length of IPv4 headers is nonsensical. */ - /* If this fails, try to auto-detect. There is a slight risk that if the - first packet we see is maliciously crafted, and somehow gets past the - configured BPF filter, we will configure the wrong offset. But that - seems fairly unlikely. */ + if (hdr_len < sizeof(struct ipv4_hdr)) { + DEBUG("[#] ipv4.hdr_len = %u. Too short for IPv4, giving up!\n", + hdr_len); + return; + } - for (i = 0; i < 40; i += 2, total_len -= 2) { + /* If the packet claims to be longer than the recv buffer, best to back + off - even though we could just ignore this and recover. */ - if (total_len < MIN_TCP4) break; + if (tot_len > packet_len) { + DEBUG("[#] ipv4.tot_len = %u but packet_len = %u, bailing out!\n", + tot_len, packet_len); + return; + } - /* Perhaps this is IPv6? We check three things: IP version (first 4 bits); - total length sufficient to accommodate IPv6 and TCP headers; and the - "next protocol" field equal to PROTO_TCP. */ + /* And finally, bail out if after skipping the IPv4 header as specified + (including options), there wouldn't be enough room for TCP. */ - if (total_len >= MIN_TCP6 && (data[i] >> 4) == IP_VER6) { + if (hdr_len + sizeof(struct tcp_hdr) > packet_len) { + DEBUG("[#] ipv4.hdr_len = %u, packet_len = %d, no room for TCP!\n", + hdr_len, packet_len); + return; + } - struct ipv6_hdr* hdr = (struct ipv6_hdr*)(data + i); + /* Bail out if the subsequent protocol is not TCP. */ - if (hdr->proto == PROTO_TCP) { + if (ip4->proto != PROTO_TCP) { + DEBUG("[#] Whoa, IPv4 packet with non-TCP payload (%u)?\n", ip4->proto); + return; + } - DEBUG("[#] Detected packet offset of %u via IPv6 (link type %u).\n", i, - link_type); - link_off = i; - break; + /* Ignore any traffic with MF or non-zero fragment offset specified. We + can do enough just fingerprinting the non-fragmented traffic. */ - } - - } + if (flags_off & ~(IP4_DF | IP4_MBZ)) { + DEBUG("[#] Packet fragment (0x%04x), letting it slide!\n", flags_off); + return; + } - /* Okay, let's try IPv4 then. The same approach, except the shortest packet - size must be just enough to accommodate IPv4 + TCP (already checked). */ + /* Store some relevant information about the packet. */ - if ((data[i] >> 4) == IP_VER4) { + pk.ip_ver = IP_VER4; - struct ipv4_hdr* hdr = (struct ipv4_hdr*)(data + i); + pk.ip_opt_len = hdr_len - 20; - if (hdr->proto == PROTO_TCP) { + memcpy(pk.src, ip4->src, 4); + memcpy(pk.dst, ip4->dst, 4); - DEBUG("[#] Detected packet offset of %u via IPv4 (link type %u).\n", i, - link_type); - link_off = i; - break; + pk.tos = ip4->tos_ecn >> 2; - } + pk.ttl = ip4->ttl; - } + if (ip4->tos_ecn & (IP_TOS_CE | IP_TOS_ECT)) pk.quirks |= QUIRK_ECN; - } + /* Tag some of the corner cases associated with implementation quirks. */ - /* If we found something, adjust for VLAN tags (ETH_P_8021Q == 0x8100). Else, - complain once and try again soon. */ + if (flags_off & IP4_MBZ) pk.quirks |= QUIRK_NZ_MBZ; - if (link_off >= 4 && data[i-4] == 0x81 && data[i-3] == 0x00) { + if (flags_off & IP4_DF) { - DEBUG("[#] Adjusting offset due to VLAN tagging.\n"); - link_off -= 4; + pk.quirks |= QUIRK_DF; + if (RD16(ip4->id)) pk.quirks |= QUIRK_NZ_ID; - } else if (link_off == -1) { + } + else { - link_off = -2; - WARN("Unable to find link-specific packet offset. This is bad."); + if (!RD16(ip4->id)) pk.quirks |= QUIRK_ZERO_ID; - } + } -} + pk.tot_hdr = hdr_len; -/* Parse PCAP input, with plenty of sanity checking. Store interesting details - in a protocol-agnostic buffer that will be then examined upstream. */ + tcp = (struct tcp_hdr*)(data + hdr_len); + packet_len -= hdr_len; -void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { + } + else if ((*data >> 4) == IP_VER6) { - struct tcp_hdr* tcp; - struct packet_data pk; + /************************ + * IPv6 header parsing. * + ************************/ - s32 packet_len; - u32 tcp_doff; + const struct ipv6_hdr* ip6 = (struct ipv6_hdr*)data; + u32 ver_tos = ntohl(RD32(ip6->ver_tos)); + u32 tot_len = ntohs(RD16(ip6->pay_len)) + sizeof(struct ipv6_hdr); - u8* opt_end; + /* If the packet claims to be shorter than what we received off the wire, + honor this claim to account for etherleak-type bugs. */ - packet_cnt++; + if (packet_len > tot_len) { + packet_len = tot_len; + // DEBUG("[#] ipv6.tot_len = %u, adjusted accordingly.\n", tot_len); + } - cur_time = (struct timeval*)&hdr->ts; + /* Bail out if the result leaves no room for IPv6 + TCP headers. */ - if (!(packet_cnt % EXPIRE_INTERVAL)) expire_cache(); + if (packet_len < MIN_TCP6) { + DEBUG("[#] packet_len = %u. Too short for IPv6 + TCP, giving up!\n", + packet_len); + return; + } - /* Be paranoid about how much data we actually have off the wire. */ + /* If the packet claims to be longer than the data we have, best to back + off - even though we could just ignore this and recover. */ - packet_len = MIN(hdr->len, hdr->caplen); - if (packet_len > SNAPLEN) packet_len = SNAPLEN; + if (tot_len > packet_len) { + DEBUG("[#] ipv6.tot_len = %u but packet_len = %u, bailing out!\n", + tot_len, packet_len); + return; + } - // DEBUG("[#] Received packet: len = %d, caplen = %d, limit = %d\n", - // hdr->len, hdr->caplen, SNAPLEN); + /* Bail out if the subsequent protocol is not TCP. One day, we may try + to parse and skip IPv6 extensions, but there seems to be no point in + it today. */ - /* Account for link-level headers. */ + if (ip6->proto != PROTO_TCP) { + DEBUG("[#] IPv6 packet with non-TCP payload (%u).\n", ip6->proto); + return; + } - if (link_off < 0) find_offset(data, packet_len); + /* Store some relevant information about the packet. */ - if (link_off > 0) { + pk.ip_ver = IP_VER6; - data += link_off; - packet_len -= link_off; + pk.ip_opt_len = 0; - } + memcpy(pk.src, ip6->src, 16); + memcpy(pk.dst, ip6->dst, 16); -#ifdef DLT_NFLOG - /* NFLOG has multiple sections to the packet, with variable length */ - if (link_type == DLT_NFLOG){ - u8 found_payload = 0; - while (packet_len > MIN_TCP4){ - u16 nfsize = (*data & 0xFF); - if (nfsize % 4 != 0) - nfsize += 4 - nfsize % 4; - if (nfsize == 0) { - WARN("Invalid TLV length for NFLOG packet, aborting\n"); - return; - } - if ((*(data + 2) & 0xFF) == 9){ - data += 4; - packet_len -= 4; - found_payload = 1; - DEBUG("[#] Found TLV for packet payload payload\n"); - break; - } - data += nfsize; - packet_len -= nfsize; - } - if (!found_payload){ - WARN("Did not find payload TLV in NFLOG packet, skipping packet"); - return; - } - } -#endif + pk.tos = (ver_tos >> 22) & 0x3F; - /* If there is no way we could have received a complete TCP packet, bail - out early. */ + pk.ttl = ip6->ttl; - if (packet_len < MIN_TCP4) { - DEBUG("[#] Packet too short for any IPv4 + TCP headers, giving up!\n"); - return; - } + if (ver_tos & 0xFFFFF) pk.quirks |= QUIRK_FLOW; - pk.quirks = 0; + if ((ver_tos >> 20) & (IP_TOS_CE | IP_TOS_ECT)) pk.quirks |= QUIRK_ECN; - if ((*data >> 4) == IP_VER4) { + pk.tot_hdr = sizeof(struct ipv6_hdr); - /************************ - * IPv4 header parsing. * - ************************/ - - const struct ipv4_hdr* ip4 = (struct ipv4_hdr*)data; + tcp = (struct tcp_hdr*)(ip6 + 1); + packet_len -= sizeof(struct ipv6_hdr); - u32 hdr_len = (ip4->ver_hlen & 0x0F) * 4; - u16 flags_off = ntohs(RD16(ip4->flags_off)); - u16 tot_len = ntohs(RD16(ip4->tot_len)); + } + else { - /* If the packet claims to be shorter than what we received off the wire, - honor this claim to account for etherleak-type bugs. */ + if (!bad_packets) { + WARN("Unknown packet type %u, link detection issue?", *data >> 4); + bad_packets = 1; + } - if (packet_len > tot_len) { - packet_len = tot_len; - // DEBUG("[#] ipv4.tot_len = %u, adjusted accordingly.\n", tot_len); - } + return; - /* Bail out if the result leaves no room for IPv4 + TCP headers. */ + } - if (packet_len < MIN_TCP4) { - DEBUG("[#] packet_len = %u. Too short for IPv4 + TCP, giving up!\n", - packet_len); - return; - } + /*************** + * TCP parsing * + ***************/ - /* Bail out if the declared length of IPv4 headers is nonsensical. */ + data = (u8*)tcp; - if (hdr_len < sizeof(struct ipv4_hdr)) { - DEBUG("[#] ipv4.hdr_len = %u. Too short for IPv4, giving up!\n", - hdr_len); - return; - } + tcp_doff = (tcp->doff_rsvd >> 4) * 4; - /* If the packet claims to be longer than the recv buffer, best to back - off - even though we could just ignore this and recover. */ + /* As usual, let's start with sanity checks. */ - if (tot_len > packet_len) { - DEBUG("[#] ipv4.tot_len = %u but packet_len = %u, bailing out!\n", - tot_len, packet_len); - return; - } + if (tcp_doff < sizeof(struct tcp_hdr)) { + DEBUG("[#] tcp.hdr_len = %u, not enough for TCP!\n", tcp_doff); + return; + } - /* And finally, bail out if after skipping the IPv4 header as specified - (including options), there wouldn't be enough room for TCP. */ + if (tcp_doff > packet_len) { + DEBUG("[#] tcp.hdr_len = %u, past end of packet!\n", tcp_doff); + return; + } - if (hdr_len + sizeof(struct tcp_hdr) > packet_len) { - DEBUG("[#] ipv4.hdr_len = %u, packet_len = %d, no room for TCP!\n", - hdr_len, packet_len); - return; - } + pk.tot_hdr += tcp_doff; - /* Bail out if the subsequent protocol is not TCP. */ + pk.sport = ntohs(RD16(tcp->sport)); + pk.dport = ntohs(RD16(tcp->dport)); - if (ip4->proto != PROTO_TCP) { - DEBUG("[#] Whoa, IPv4 packet with non-TCP payload (%u)?\n", ip4->proto); - return; - } + pk.tcp_type = tcp->flags & (TCP_SYN | TCP_ACK | TCP_FIN | TCP_RST); - /* Ignore any traffic with MF or non-zero fragment offset specified. We - can do enough just fingerprinting the non-fragmented traffic. */ + /* NUL, SYN+FIN, SYN+RST, FIN+RST, etc, should go to /dev/null. */ - if (flags_off & ~(IP4_DF | IP4_MBZ)) { - DEBUG("[#] Packet fragment (0x%04x), letting it slide!\n", flags_off); - return; - } + if (((tcp->flags & TCP_SYN) && (tcp->flags & (TCP_FIN | TCP_RST))) || + ((tcp->flags & TCP_FIN) && (tcp->flags & TCP_RST)) || + !pk.tcp_type) { - /* Store some relevant information about the packet. */ + DEBUG("[#] Silly combination of TCP flags: 0x%02x.\n", tcp->flags); + return; - pk.ip_ver = IP_VER4; + } - pk.ip_opt_len = hdr_len - 20; + pk.win = ntohs(RD16(tcp->win)); - memcpy(pk.src, ip4->src, 4); - memcpy(pk.dst, ip4->dst, 4); + pk.seq = ntohl(RD32(tcp->seq)); - pk.tos = ip4->tos_ecn >> 2; + /* Take note of miscellanous features and quirks. */ - pk.ttl = ip4->ttl; + if ((tcp->flags & (TCP_ECE | TCP_CWR)) || + (tcp->doff_rsvd & TCP_NS_RES)) pk.quirks |= QUIRK_ECN; - if (ip4->tos_ecn & (IP_TOS_CE | IP_TOS_ECT)) pk.quirks |= QUIRK_ECN; + if (!pk.seq) pk.quirks |= QUIRK_ZERO_SEQ; - /* Tag some of the corner cases associated with implementation quirks. */ - - if (flags_off & IP4_MBZ) pk.quirks |= QUIRK_NZ_MBZ; + if (tcp->flags & TCP_ACK) { - if (flags_off & IP4_DF) { + if (!RD32(tcp->ack)) pk.quirks |= QUIRK_ZERO_ACK; - pk.quirks |= QUIRK_DF; - if (RD16(ip4->id)) pk.quirks |= QUIRK_NZ_ID; + } + else { - } else { + /* A good proportion of RSTs tend to have "illegal" ACK numbers, so + ignore these. */ - if (!RD16(ip4->id)) pk.quirks |= QUIRK_ZERO_ID; + if (RD32(tcp->ack) & !(tcp->flags & TCP_RST)) { - } + DEBUG("[#] Non-zero ACK on a non-ACK packet: 0x%08x.\n", + ntohl(RD32(tcp->ack))); - pk.tot_hdr = hdr_len; + pk.quirks |= QUIRK_NZ_ACK; - tcp = (struct tcp_hdr*)(data + hdr_len); - packet_len -= hdr_len; - - } else if ((*data >> 4) == IP_VER6) { + } - /************************ - * IPv6 header parsing. * - ************************/ - - const struct ipv6_hdr* ip6 = (struct ipv6_hdr*)data; - u32 ver_tos = ntohl(RD32(ip6->ver_tos)); - u32 tot_len = ntohs(RD16(ip6->pay_len)) + sizeof(struct ipv6_hdr); + } - /* If the packet claims to be shorter than what we received off the wire, - honor this claim to account for etherleak-type bugs. */ + if (tcp->flags & TCP_URG) { - if (packet_len > tot_len) { - packet_len = tot_len; - // DEBUG("[#] ipv6.tot_len = %u, adjusted accordingly.\n", tot_len); - } + pk.quirks |= QUIRK_URG; - /* Bail out if the result leaves no room for IPv6 + TCP headers. */ + } + else { - if (packet_len < MIN_TCP6) { - DEBUG("[#] packet_len = %u. Too short for IPv6 + TCP, giving up!\n", - packet_len); - return; - } + if (RD16(tcp->urg)) { - /* If the packet claims to be longer than the data we have, best to back - off - even though we could just ignore this and recover. */ + DEBUG("[#] Non-zero UPtr on a non-URG packet: 0x%08x.\n", + ntohl(RD16(tcp->urg))); - if (tot_len > packet_len) { - DEBUG("[#] ipv6.tot_len = %u but packet_len = %u, bailing out!\n", - tot_len, packet_len); - return; - } + pk.quirks |= QUIRK_NZ_URG; - /* Bail out if the subsequent protocol is not TCP. One day, we may try - to parse and skip IPv6 extensions, but there seems to be no point in - it today. */ + } - if (ip6->proto != PROTO_TCP) { - DEBUG("[#] IPv6 packet with non-TCP payload (%u).\n", ip6->proto); - return; - } + } - /* Store some relevant information about the packet. */ + if (tcp->flags & TCP_PUSH) pk.quirks |= QUIRK_PUSH; - pk.ip_ver = IP_VER6; + /* Handle payload data. */ - pk.ip_opt_len = 0; + if (tcp_doff == packet_len) { - memcpy(pk.src, ip6->src, 16); - memcpy(pk.dst, ip6->dst, 16); + pk.payload = NULL; + pk.pay_len = 0; - pk.tos = (ver_tos >> 22) & 0x3F; + } + else { - pk.ttl = ip6->ttl; + pk.payload = (u8*)data + tcp_doff; + pk.pay_len = packet_len - tcp_doff; - if (ver_tos & 0xFFFFF) pk.quirks |= QUIRK_FLOW; + } - if ((ver_tos >> 20) & (IP_TOS_CE | IP_TOS_ECT)) pk.quirks |= QUIRK_ECN; + /********************** + * TCP option parsing * + **********************/ - pk.tot_hdr = sizeof(struct ipv6_hdr); + opt_end = (u8*)data + tcp_doff; /* First byte of non-option data */ + data = (u8*)(tcp + 1); - tcp = (struct tcp_hdr*)(ip6 + 1); - packet_len -= sizeof(struct ipv6_hdr); + pk.opt_cnt = 0; + pk.opt_eol_pad = 0; + pk.mss = 0; + pk.wscale = 0; + pk.ts1 = 0; - } else { + /* Option parsing problems are non-fatal, but we want to keep track of + them to spot buggy TCP stacks. */ - if (!bad_packets) { - WARN("Unknown packet type %u, link detection issue?", *data >> 4); - bad_packets = 1; - } + while (data < opt_end && pk.opt_cnt < MAX_TCP_OPT) { - return; + pk.opt_layout[pk.opt_cnt++] = *data; - } + switch (*data++) { - /*************** - * TCP parsing * - ***************/ + case TCPOPT_EOL: - data = (u8*)tcp; + /* EOL is a single-byte option that aborts further option parsing. + Take note of how many bytes of option data are left, and if any of + them are non-zero. */ - tcp_doff = (tcp->doff_rsvd >> 4) * 4; + pk.opt_eol_pad = opt_end - data; - /* As usual, let's start with sanity checks. */ + while (data < opt_end && !*data++); - if (tcp_doff < sizeof(struct tcp_hdr)) { - DEBUG("[#] tcp.hdr_len = %u, not enough for TCP!\n", tcp_doff); - return; - } + if (data != opt_end) { + pk.quirks |= QUIRK_OPT_EOL_NZ; + data = opt_end; + } - if (tcp_doff > packet_len) { - DEBUG("[#] tcp.hdr_len = %u, past end of packet!\n", tcp_doff); - return; - } + break; - pk.tot_hdr += tcp_doff; + case TCPOPT_NOP: - pk.sport = ntohs(RD16(tcp->sport)); - pk.dport = ntohs(RD16(tcp->dport)); + /* NOP is a single-byte option that does nothing. */ - pk.tcp_type = tcp->flags & (TCP_SYN | TCP_ACK | TCP_FIN | TCP_RST); + break; - /* NUL, SYN+FIN, SYN+RST, FIN+RST, etc, should go to /dev/null. */ + case TCPOPT_MAXSEG: - if (((tcp->flags & TCP_SYN) && (tcp->flags & (TCP_FIN | TCP_RST))) || - ((tcp->flags & TCP_FIN) && (tcp->flags & TCP_RST)) || - !pk.tcp_type) { + /* MSS is a four-byte option with specified size. */ - DEBUG("[#] Silly combination of TCP flags: 0x%02x.\n", tcp->flags); - return; + if (*data != 4) { + DEBUG("[#] MSS option expected to have 4 bytes, not %u.\n", *data); + pk.quirks |= QUIRK_OPT_BAD; + } - } + if (data + 3 > opt_end) { + DEBUG("[#] MSS option would end past end of header (%u left).\n", + opt_end - data); + goto abort_options; + } - pk.win = ntohs(RD16(tcp->win)); + pk.mss = ntohs(RD16p(data + 1)); - pk.seq = ntohl(RD32(tcp->seq)); + data += 3; - /* Take note of miscellanous features and quirks. */ + break; - if ((tcp->flags & (TCP_ECE | TCP_CWR)) || - (tcp->doff_rsvd & TCP_NS_RES)) pk.quirks |= QUIRK_ECN; + case TCPOPT_WSCALE: - if (!pk.seq) pk.quirks |= QUIRK_ZERO_SEQ; + /* WS is a three-byte option with specified size. */ - if (tcp->flags & TCP_ACK) { + if (*data != 3) { + DEBUG("[#] MSS option expected to have 3 bytes, not %u.\n", *data); + pk.quirks |= QUIRK_OPT_BAD; + } - if (!RD32(tcp->ack)) pk.quirks |= QUIRK_ZERO_ACK; + if (data + 2 > opt_end) { + DEBUG("[#] WS option would end past end of header (%u left).\n", + opt_end - data); + goto abort_options; + } - } else { + pk.wscale = data[1]; - /* A good proportion of RSTs tend to have "illegal" ACK numbers, so - ignore these. */ + if (pk.wscale > 14) pk.quirks |= QUIRK_OPT_EXWS; - if (RD32(tcp->ack) & !(tcp->flags & TCP_RST)) { + data += 2; - DEBUG("[#] Non-zero ACK on a non-ACK packet: 0x%08x.\n", - ntohl(RD32(tcp->ack))); + break; - pk.quirks |= QUIRK_NZ_ACK; + case TCPOPT_SACKOK: - } + /* SACKOK is a two-byte option with specified size. */ - } + if (*data != 2) { + DEBUG("[#] SACKOK option expected to have 2 bytes, not %u.\n", *data); + pk.quirks |= QUIRK_OPT_BAD; + } - if (tcp->flags & TCP_URG) { + if (data + 1 > opt_end) { + DEBUG("[#] SACKOK option would end past end of header (%u left).\n", + opt_end - data); + goto abort_options; + } - pk.quirks |= QUIRK_URG; + data++; - } else { + break; - if (RD16(tcp->urg)) { + case TCPOPT_SACK: - DEBUG("[#] Non-zero UPtr on a non-URG packet: 0x%08x.\n", - ntohl(RD16(tcp->urg))); + /* SACK is a variable-length option of 10 to 34 bytes. Because we don't + know the size any better, we need to bail out if it looks wonky. */ - pk.quirks |= QUIRK_NZ_URG; + if (*data < 10 || *data > 34) { + DEBUG("[#] SACK length out of range (%u), bailing out.\n", *data); + goto abort_options; + } - } + if (data - 1 + *data > opt_end) { + DEBUG("[#] SACK option (len %u) is too long (%u left).\n", + *data, opt_end - data); + goto abort_options; + } - } + data += *data - 1; - if (tcp->flags & TCP_PUSH) pk.quirks |= QUIRK_PUSH; + break; - /* Handle payload data. */ + case TCPOPT_TSTAMP: - if (tcp_doff == packet_len) { + /* Timestamp is a ten-byte option with specified size. */ - pk.payload = NULL; - pk.pay_len = 0; + if (*data != 10) { + DEBUG("[#] TStamp option expected to have 10 bytes, not %u.\n", + *data); + pk.quirks |= QUIRK_OPT_BAD; + } - } else { + if (data + 9 > opt_end) { + DEBUG("[#] TStamp option would end past end of header (%u left).\n", + opt_end - data); + goto abort_options; + } - pk.payload = (u8*)data + tcp_doff; - pk.pay_len = packet_len - tcp_doff; + pk.ts1 = ntohl(RD32p(data + 1)); - } + if (!pk.ts1) pk.quirks |= QUIRK_OPT_ZERO_TS1; - /********************** - * TCP option parsing * - **********************/ + if (pk.tcp_type == TCP_SYN && RD32p(data + 5)) { - opt_end = (u8*)data + tcp_doff; /* First byte of non-option data */ - data = (u8*)(tcp + 1); + DEBUG("[#] Non-zero second timestamp: 0x%08x.\n", + ntohl(*(u32*)(data + 5))); - pk.opt_cnt = 0; - pk.opt_eol_pad = 0; - pk.mss = 0; - pk.wscale = 0; - pk.ts1 = 0; + pk.quirks |= QUIRK_OPT_NZ_TS2; - /* Option parsing problems are non-fatal, but we want to keep track of - them to spot buggy TCP stacks. */ + } - while (data < opt_end && pk.opt_cnt < MAX_TCP_OPT) { + data += 9; - pk.opt_layout[pk.opt_cnt++] = *data; + break; - switch (*data++) { + default: - case TCPOPT_EOL: + /* Unknown option, presumably with specified size. */ - /* EOL is a single-byte option that aborts further option parsing. - Take note of how many bytes of option data are left, and if any of - them are non-zero. */ + if (*data < 2 || *data > 40) { + DEBUG("[#] Unknown option 0x%02x has invalid length %u.\n", + data[-1], *data); + goto abort_options; + } - pk.opt_eol_pad = opt_end - data; - - while (data < opt_end && !*data++); + if (data - 1 + *data > opt_end) { + DEBUG("[#] Unknown option 0x%02x (len %u) is too long (%u left).\n", + data[-1], *data, opt_end - data); + goto abort_options; + } - if (data != opt_end) { - pk.quirks |= QUIRK_OPT_EOL_NZ; - data = opt_end; - } + data += *data - 1; - break; + } - case TCPOPT_NOP: + } - /* NOP is a single-byte option that does nothing. */ + if (data != opt_end) { - break; - - case TCPOPT_MAXSEG: + abort_options: - /* MSS is a four-byte option with specified size. */ + DEBUG("[#] Option parsing aborted (cnt = %u, remainder = %u).\n", + pk.opt_cnt, opt_end - data); - if (*data != 4) { - DEBUG("[#] MSS option expected to have 4 bytes, not %u.\n", *data); - pk.quirks |= QUIRK_OPT_BAD; - } + pk.quirks |= QUIRK_OPT_BAD; - if (data + 3 > opt_end) { - DEBUG("[#] MSS option would end past end of header (%u left).\n", - opt_end - data); - goto abort_options; - } + } - pk.mss = ntohs(RD16p(data+1)); + flow_dispatch(&pk); +} - data += 3; +#ifdef USE_LIBPCAP - break; +/* Find link-specific offset (pcap knows, but won't tell). */ + +static void find_offset(const u8* data, s32 total_len) { - case TCPOPT_WSCALE: + u8 i; - /* WS is a three-byte option with specified size. */ + /* Check hardcoded values for some of the most common options. */ + DEBUG("[#] Looking for offset for link type: %d\n", link_type); - if (*data != 3) { - DEBUG("[#] MSS option expected to have 3 bytes, not %u.\n", *data); - pk.quirks |= QUIRK_OPT_BAD; - } + switch (link_type) { - if (data + 2 > opt_end) { - DEBUG("[#] WS option would end past end of header (%u left).\n", - opt_end - data); - goto abort_options; - } + case DLT_RAW: link_off = 0; return; +#ifdef DLT_NFLOG + case DLT_NFLOG: link_off = 4; return; //family, version, resource_id +#endif + case DLT_NULL: + case DLT_PPP: link_off = 4; return; - pk.wscale = data[1]; + case DLT_LOOP: - if (pk.wscale > 14) pk.quirks |= QUIRK_OPT_EXWS; +#ifdef DLT_PPP_SERIAL + case DLT_PPP_SERIAL: +#endif /* DLT_PPP_SERIAL */ - data += 2; + case DLT_PPP_ETHER: link_off = 8; return; - break; + case DLT_EN10MB: link_off = 14; return; - case TCPOPT_SACKOK: +#ifdef DLT_LINUX_SLL + case DLT_LINUX_SLL: link_off = 16; return; +#endif /* DLT_LINUX_SLL */ - /* SACKOK is a two-byte option with specified size. */ + case DLT_PFLOG: link_off = 28; return; - if (*data != 2) { - DEBUG("[#] SACKOK option expected to have 2 bytes, not %u.\n", *data); - pk.quirks |= QUIRK_OPT_BAD; - } + case DLT_IEEE802_11: link_off = 32; return; + } - if (data + 1 > opt_end) { - DEBUG("[#] SACKOK option would end past end of header (%u left).\n", - opt_end - data); - goto abort_options; - } + DEBUG("[#] UNKNOWN DLT %d, attempting to find offset\n", link_type); - data++; + /* If this fails, try to auto-detect. There is a slight risk that if the + first packet we see is maliciously crafted, and somehow gets past the + configured BPF filter, we will configure the wrong offset. But that + seems fairly unlikely. */ + + for (i = 0; i < 40; i += 2, total_len -= 2) { + + if (total_len < MIN_TCP4) break; + + /* Perhaps this is IPv6? We check three things: IP version (first 4 bits); + total length sufficient to accommodate IPv6 and TCP headers; and the + "next protocol" field equal to PROTO_TCP. */ + if (total_len >= MIN_TCP6 && (data[i] >> 4) == IP_VER6) { + + struct ipv6_hdr* hdr = (struct ipv6_hdr*)(data + i); + + if (hdr->proto == PROTO_TCP) { + + DEBUG("[#] Detected packet offset of %u via IPv6 (link type %u).\n", i, + link_type); + link_off = i; break; - case TCPOPT_SACK: + } + + } - /* SACK is a variable-length option of 10 to 34 bytes. Because we don't - know the size any better, we need to bail out if it looks wonky. */ + /* Okay, let's try IPv4 then. The same approach, except the shortest packet + size must be just enough to accommodate IPv4 + TCP (already checked). */ - if (*data < 10 || *data > 34) { - DEBUG("[#] SACK length out of range (%u), bailing out.\n", *data); - goto abort_options; - } + if ((data[i] >> 4) == IP_VER4) { - if (data - 1 + *data > opt_end) { - DEBUG("[#] SACK option (len %u) is too long (%u left).\n", - *data, opt_end - data); - goto abort_options; - } + struct ipv4_hdr* hdr = (struct ipv4_hdr*)(data + i); - data += *data - 1; + if (hdr->proto == PROTO_TCP) { + DEBUG("[#] Detected packet offset of %u via IPv4 (link type %u).\n", i, + link_type); + link_off = i; break; - case TCPOPT_TSTAMP: + } - /* Timestamp is a ten-byte option with specified size. */ + } - if (*data != 10) { - DEBUG("[#] TStamp option expected to have 10 bytes, not %u.\n", - *data); - pk.quirks |= QUIRK_OPT_BAD; - } + } - if (data + 9 > opt_end) { - DEBUG("[#] TStamp option would end past end of header (%u left).\n", - opt_end - data); - goto abort_options; - } + /* If we found something, adjust for VLAN tags (ETH_P_8021Q == 0x8100). Else, + complain once and try again soon. */ - pk.ts1 = ntohl(RD32p(data + 1)); + if (link_off >= 4 && data[i-4] == 0x81 && data[i-3] == 0x00) { - if (!pk.ts1) pk.quirks |= QUIRK_OPT_ZERO_TS1; + DEBUG("[#] Adjusting offset due to VLAN tagging.\n"); + link_off -= 4; - if (pk.tcp_type == TCP_SYN && RD32p(data + 5)) { + } else if (link_off == -1) { - DEBUG("[#] Non-zero second timestamp: 0x%08x.\n", - ntohl(*(u32*)(data + 5))); + link_off = -2; + WARN("Unable to find link-specific packet offset. This is bad."); - pk.quirks |= QUIRK_OPT_NZ_TS2; + } - } +} - data += 9; +/* Parse PCAP input, with plenty of sanity checking. Store interesting details + in a protocol-agnostic buffer that will be then examined upstream. */ - break; +void parse_packet(void* junk, const struct pcap_pkthdr* hdr, const u8* data) { + struct tcp_hdr* tcp; + struct packet_data pk; - default: + s32 packet_len; - /* Unknown option, presumably with specified size. */ + packet_cnt++; - if (*data < 2 || *data > 40) { - DEBUG("[#] Unknown option 0x%02x has invalid length %u.\n", - data[-1], *data); - goto abort_options; - } + cur_time = (struct timeval*)&hdr->ts; - if (data - 1 + *data > opt_end) { - DEBUG("[#] Unknown option 0x%02x (len %u) is too long (%u left).\n", - data[-1], *data, opt_end - data); - goto abort_options; - } + if (!(packet_cnt % EXPIRE_INTERVAL)) expire_cache(); - data += *data - 1; + /* Be paranoid about how much data we actually have off the wire. */ - } + packet_len = MIN(hdr->len, hdr->caplen); + if (packet_len > SNAPLEN) packet_len = SNAPLEN; - } + // DEBUG("[#] Received packet: len = %d, caplen = %d, limit = %d\n", + // hdr->len, hdr->caplen, SNAPLEN); - if (data != opt_end) { + /* Account for link-level headers. */ -abort_options: + if (link_off < 0) find_offset(data, packet_len); - DEBUG("[#] Option parsing aborted (cnt = %u, remainder = %u).\n", - pk.opt_cnt, opt_end - data); + if (link_off > 0) { - pk.quirks |= QUIRK_OPT_BAD; + data += link_off; + packet_len -= link_off; } - flow_dispatch(&pk); +#ifdef DLT_NFLOG + /* NFLOG has multiple sections to the packet, with variable length */ + if (link_type == DLT_NFLOG){ + u8 found_payload = 0; + while (packet_len > MIN_TCP4){ + u16 nfsize = (*data & 0xFF); + if (nfsize % 4 != 0) + nfsize += 4 - nfsize % 4; + if (nfsize == 0) { + WARN("Invalid TLV length for NFLOG packet, aborting\n"); + return; + } + if ((*(data + 2) & 0xFF) == 9){ + data += 4; + packet_len -= 4; + found_payload = 1; + DEBUG("[#] Found TLV for packet payload payload\n"); + break; + } + data += nfsize; + packet_len -= nfsize; + } + if (!found_payload){ + WARN("Did not find payload TLV in NFLOG packet, skipping packet"); + return; + } + } +#endif + process_packet(data, packet_len); } #elif defined(USE_LIBMNL) int parse_attr_cb(const struct nlattr *attr, void *data) @@ -820,6 +834,7 @@ int parse_packet(const struct nlmsghdr *nlh, void *data) struct nlattr *tb[NFULA_MAX + 1] = {}; struct nfulnl_msg_packet_hdr *ph = NULL; const char *prefix = NULL; + const u8* payload = NULL; uint32_t mark = 0; mnl_attr_parse(nlh, sizeof(struct nfgenmsg), parse_attr_cb, tb); @@ -829,14 +844,19 @@ int parse_packet(const struct nlmsghdr *nlh, void *data) prefix = mnl_attr_get_str(tb[NFULA_PREFIX]); if (tb[NFULA_MARK]) mark = ntohl(mnl_attr_get_u32(tb[NFULA_MARK])); + if (tb[NFULA_PAYLOAD]){ + payload = mnl_attr_get_payload(tb[NFULA_PAYLOAD]); - printf("log received (prefix=\"%s\" hw=0x%04x hook=%u mark=%u)\n", - prefix ? prefix : "", ntohs(ph->hw_protocol), ph->hook, - mark); + process_packet(payload, mnl_attr_get_len(tb[NFULA_PAYLOAD])); - packet_cnt++; + printf("log received (prefix=\"%s\" hw=0x%04x hook=%u mark=%u)\n", + prefix ? prefix : "", ntohs(ph->hw_protocol), ph->hook, + mark); - if (!(packet_cnt % EXPIRE_INTERVAL)) expire_cache(); + packet_cnt++; + + if (!(packet_cnt % EXPIRE_INTERVAL)) expire_cache(); + } return MNL_CB_OK; } From a70459f34f9be3b6c1300deafd39993047000b17 Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 01:17:57 +1000 Subject: [PATCH 63/85] build appears to work --- p0f.c | 2 +- process.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/p0f.c b/p0f.c index 6eed585..46d7a84 100644 --- a/p0f.c +++ b/p0f.c @@ -1041,7 +1041,7 @@ static void epoll_event_loop(void){ #endif int res; - DEBUG("pcap fd: %d", pcap_fd); + DEBUG("[#] pcap fd: %d\n", pcap_fd); //Initial epoll setup int epfd = epoll_create(api_max_conn); diff --git a/process.c b/process.c index c1155e2..83ba24c 100644 --- a/process.c +++ b/process.c @@ -50,8 +50,44 @@ static struct host_data *host_by_age, /* All host entries, by last mod */ static struct packet_flow *flow_by_age, /* All flows, by creation time */ *newest_flow; /* Tail of the list */ +#ifdef USE_LIBPCAP static struct timeval* cur_time; /* Current time, courtesy of pcap */ +#define CURTIME_SEC cur_time->tv_sec +#define CURTIME_USEC cur_time->tv_usec +#else +static struct timeval cur_time; /* Current time, courtesy of libmnl */ + +#define CURTIME_SEC cur_time.tv_sec +#define CURTIME_USEC cur_time.tv_usec + +/* Structure used to swap the bytes in a 64-bit unsigned long long. */ +union byteswap_64_u { + unsigned long long a; + uint32_t b[2]; +}; + +/* Function to byteswap big endian 64bit unsigned integers +* back to little endian host order on little endian machines. +* As above, on big endian machines this will be a null macro. +* The macro ntohll() is defined in byteorder64.h, and if needed, +* refers to _ntohll() here. +* +* Source: http://www.opensource.apple.com/source/CyrusIMAP/CyrusIMAP-187/cyrus_imap/lib/byteorder64.c?txt +*/ +unsigned long long ntohll(unsigned long long x) +{ + union byteswap_64_u u1; + union byteswap_64_u u2; + + u1.a = x; + + u2.b[1] = ntohl(u1.b[0]); + u2.b[0] = ntohl(u1.b[1]); + + return u2.a; +} +#endif /* Bucketed hosts and flows: */ static struct host_data *host_b[HOST_BUCKETS]; @@ -68,14 +104,14 @@ static void expire_cache(void); u64 get_unix_time_ms(void) { - return ((u64)cur_time->tv_sec) * 1000 + (cur_time->tv_usec / 1000); + return ((u64)CURTIME_SEC) * 1000 + (CURTIME_USEC / 1000); } /* Get unix time in seconds. */ u32 get_unix_time(void) { - return cur_time->tv_sec; + return CURTIME_SEC; } /* Convert IPv4 or IPv6 address to a human-readable form. */ @@ -847,6 +883,15 @@ int parse_packet(const struct nlmsghdr *nlh, void *data) if (tb[NFULA_PAYLOAD]){ payload = mnl_attr_get_payload(tb[NFULA_PAYLOAD]); + if (tb[NFULA_TIMESTAMP]){ + struct nfulnl_msg_packet_timestamp *timestamp = mnl_attr_get_str(tb[NFULA_TIMESTAMP]); + CURTIME_SEC = ntohll(timestamp->sec); + CURTIME_USEC = ntohll(timestamp->usec); + } + else{ + gettimeofday(&cur_time, NULL); + } + process_packet(payload, mnl_attr_get_len(tb[NFULA_PAYLOAD])); printf("log received (prefix=\"%s\" hw=0x%04x hook=%u mark=%u)\n", From 022e5208d292680e3145e38ed9e91eae576001a5 Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 01:19:56 +1000 Subject: [PATCH 64/85] made log received debug only --- process.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process.c b/process.c index 83ba24c..a713c3c 100644 --- a/process.c +++ b/process.c @@ -894,7 +894,7 @@ int parse_packet(const struct nlmsghdr *nlh, void *data) process_packet(payload, mnl_attr_get_len(tb[NFULA_PAYLOAD])); - printf("log received (prefix=\"%s\" hw=0x%04x hook=%u mark=%u)\n", + DEBUG("[#] log received (prefix=\"%s\" hw=0x%04x hook=%u mark=%u)\n", prefix ? prefix : "", ntohs(ph->hw_protocol), ph->hook, mark); From 26f623db6606034891e611daf27a8c7ab7ffbfdd Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 01:26:19 +1000 Subject: [PATCH 65/85] fixed compiler warnings --- p0f.c | 6 ++++-- process.c | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/p0f.c b/p0f.c index 46d7a84..18a6f76 100644 --- a/p0f.c +++ b/p0f.c @@ -873,14 +873,16 @@ static void prepare_netlink(void){ struct nlmsghdr *nlh; unsigned int qnum; + //Get queue + qnum = atoi((const char*)use_iface); - qnum = atoi(use_iface); - + //open netfilter socket nl = mnl_socket_open(NETLINK_NETFILTER); if (nl == NULL) { PFATAL("mnl_socket_open"); } + //setup if (mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { PFATAL("mnl_socket_bind"); } diff --git a/process.c b/process.c index a713c3c..e9052e8 100644 --- a/process.c +++ b/process.c @@ -868,23 +868,25 @@ int parse_attr_cb(const struct nlattr *attr, void *data) int parse_packet(const struct nlmsghdr *nlh, void *data) { struct nlattr *tb[NFULA_MAX + 1] = {}; + const u8* payload = NULL; + mnl_attr_parse(nlh, sizeof(struct nfgenmsg), parse_attr_cb, tb); + +#ifdef DEBUG_BUILD struct nfulnl_msg_packet_hdr *ph = NULL; const char *prefix = NULL; - const u8* payload = NULL; uint32_t mark = 0; - - mnl_attr_parse(nlh, sizeof(struct nfgenmsg), parse_attr_cb, tb); if (tb[NFULA_PACKET_HDR]) ph = mnl_attr_get_payload(tb[NFULA_PACKET_HDR]); if (tb[NFULA_PREFIX]) prefix = mnl_attr_get_str(tb[NFULA_PREFIX]); if (tb[NFULA_MARK]) mark = ntohl(mnl_attr_get_u32(tb[NFULA_MARK])); +#endif if (tb[NFULA_PAYLOAD]){ payload = mnl_attr_get_payload(tb[NFULA_PAYLOAD]); if (tb[NFULA_TIMESTAMP]){ - struct nfulnl_msg_packet_timestamp *timestamp = mnl_attr_get_str(tb[NFULA_TIMESTAMP]); + struct nfulnl_msg_packet_timestamp *timestamp = (struct nfulnl_msg_packet_timestamp *)mnl_attr_get_str(tb[NFULA_TIMESTAMP]); CURTIME_SEC = ntohll(timestamp->sec); CURTIME_USEC = ntohll(timestamp->usec); } From f26ea1c45ec078b474797e5db69a52926bc1a49b Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 01:27:36 +1000 Subject: [PATCH 66/85] updated readme to reflect recent features --- docs/README | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/README b/docs/README index ce24075..ba92279 100644 --- a/docs/README +++ b/docs/README @@ -93,6 +93,7 @@ http://lcamtuf.coredump.cx/p0f3/ - One sided stream support (SYN+ACK not required) - Capture buffer size increased to fix crash on heavily loaded systems. - epoll support for increased performance and scalability (increased API connections) + - optional support for using libmnl instead of libpcap This forks releases are possibly production ready, a version of it is in use at http://www.x4b.net/ From cbd992e2f911523c2d4def0551dff1824a636e1e Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 02:19:38 +1000 Subject: [PATCH 67/85] set ENOBUFS on nflog socket --- p0f.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/p0f.c b/p0f.c index 18a6f76..c29c889 100644 --- a/p0f.c +++ b/p0f.c @@ -900,6 +900,13 @@ static void prepare_netlink(void){ PFATAL("mnl_socket_sendto"); } + int status = mnl_socket_setsockopt(nl, SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); + ///If fails, probably not nfnetlink + if (status < 0){ + FATAL("setsockopt: %s", strerror(errno)); + // DEBUG("PCAP overflow condition set successfully"); + } + nlh = nflog_build_cfg_request(buf, NFULNL_CFG_CMD_BIND, qnum); if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { From 35b6411e7f07806ea6ab59f20eeb2f97e243b219 Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 02:26:11 +1000 Subject: [PATCH 68/85] increase buffer size to hopefully reduce dropped packets --- p0f.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index c29c889..1f249b5 100644 --- a/p0f.c +++ b/p0f.c @@ -900,13 +900,17 @@ static void prepare_netlink(void){ PFATAL("mnl_socket_sendto"); } - int status = mnl_socket_setsockopt(nl, SOL_NETLINK, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)); ///If fails, probably not nfnetlink - if (status < 0){ + if (mnl_socket_setsockopt(nl, NETLINK_NO_ENOBUFS, &(int){1}, sizeof(int)) < 0){ FATAL("setsockopt: %s", strerror(errno)); // DEBUG("PCAP overflow condition set successfully"); } + int a = 655350; + if (setsockopt(mnl_socket_get_fd(nl), SOL_SOCKET, SO_RCVBUF, &a, sizeof(int)) < 0) { + FATAL("Error setting socket opts: %s\n", strerror(errno)); + } + nlh = nflog_build_cfg_request(buf, NFULNL_CFG_CMD_BIND, qnum); if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { From d70738fae2438c4f395982d1b6c5f8c048615ac8 Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 15 Jul 2014 02:28:12 +1000 Subject: [PATCH 69/85] added reasoning for libmnl --- docs/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README b/docs/README index ba92279..9d54683 100644 --- a/docs/README +++ b/docs/README @@ -93,7 +93,7 @@ http://lcamtuf.coredump.cx/p0f3/ - One sided stream support (SYN+ACK not required) - Capture buffer size increased to fix crash on heavily loaded systems. - epoll support for increased performance and scalability (increased API connections) - - optional support for using libmnl instead of libpcap + - optional support for using libmnl instead of libpcap (tiny cpu usage, no ENOBUFS issues) This forks releases are possibly production ready, a version of it is in use at http://www.x4b.net/ From 3c6ed6ef28c2428b13778e48b26ce1c3ea217529 Mon Sep 17 00:00:00 2001 From: splitice Date: Wed, 6 Aug 2014 20:30:38 +1000 Subject: [PATCH 70/85] fix netlink socket taking all processing from API sockets --- p0f.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/p0f.c b/p0f.c index 1f249b5..98d5c8c 100644 --- a/p0f.c +++ b/p0f.c @@ -1097,16 +1097,9 @@ static void epoll_event_loop(void){ if (res == -1) { PFATAL("mnl_socket_recvfrom"); } - while (res > 0) { - res = mnl_cb_run(buf, res, 0, portid, parse_packet, NULL); - if (res < 0){ - PFATAL("mnl_cb_run"); - } - - res = mnl_socket_recvfrom(nl, buf, sizeof(buf)); - if (res == -1) { - PFATAL("mnl_socket_recvfrom"); - } + res = mnl_cb_run(buf, res, 0, portid, parse_packet, NULL); + if (res < 0){ + PFATAL("mnl_cb_run"); } #endif From f768082702b7e03730492a8b00e4357f1ff73b41 Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 6 Nov 2014 10:38:57 +1100 Subject: [PATCH 71/85] Remove max API connection limitation, scale as needed --- p0f.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/p0f.c b/p0f.c index 98d5c8c..e5d60c2 100644 --- a/p0f.c +++ b/p0f.c @@ -176,7 +176,7 @@ static void usage(void) { "\n" #ifndef __CYGWIN__ #ifdef USE_EPOLL -" -S limit - limit number of parallel API connections, not guarunteed (%u)\n" +" -S limit - default storage capacity for parallel API connections, not guarunteed (%u)\n" #else " -S limit - limit number of parallel API connections (%u)\n" #endif @@ -1044,6 +1044,7 @@ static void epoll_event_loop(void){ int slots = 6 + api_max_conn; ctable = ck_alloc(slots * sizeof(struct api_client)); + struct epoll_event ev; struct epoll_event events[5]; #ifdef USE_LIBPCAP @@ -1116,11 +1117,13 @@ static void epoll_event_loop(void){ if (client_sock < 0) { WARN("Unable to handle API connection: accept() fails."); } - else if (client_sock >= slots){ - WARN("Unable to handle API connection: too many connection."); - close(client_sock); - } else { + if (client_sock >= slots){ + WARN("Too many connections, enlarging connection table"); + slots *= 2; + ctable = ck_realloc(ctable, slots * sizeof(struct api_client)); + } + ctable[fd].fd = client_sock; if (fcntl(client_sock, F_SETFL, O_NONBLOCK)) @@ -1528,9 +1531,6 @@ int main(int argc, char** argv) { api_max_conn = atol(optarg); - if (!api_max_conn || api_max_conn > 250) - FATAL("Outlandish value specified for -S."); - break; #endif /* ^__CYGWIN__ */ From 044c70ca84d286a2125f3c397201d5fcc7fc71ee Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 13 Nov 2014 11:52:51 +1100 Subject: [PATCH 72/85] Handle broken API connection --- debug.h | 9 +++++++++ p0f.c | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/debug.h b/debug.h index 0b7da14..62877a0 100644 --- a/debug.h +++ b/debug.h @@ -23,6 +23,15 @@ #define ERRORF(x...) fprintf(stderr, x) #define SAYF(x...) printf(x) +#define PWARN(x...) do { \ + ERRORF("[!] SYSTEM WARNING: " x); \ + ERRORF("\n Location : %s(), %s:%u\n", \ + __FUNCTION__, __FILE__, __LINE__); \ + perror(" OS message "); \ + ERRORF("\n"); \ + exit(1); \ + } + #define WARN(x...) do { \ ERRORF("[!] WARNING: " x); \ ERRORF("\n"); \ diff --git a/p0f.c b/p0f.c index e5d60c2..f878875 100644 --- a/p0f.c +++ b/p0f.c @@ -1186,10 +1186,10 @@ static void epoll_event_loop(void){ ((char*)&ctable[fd].out_data), sizeof(struct p0f_api_response)); - if (fcntl(fd, F_SETFL, O_NONBLOCK)) - PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); + if (fcntl(fd, F_SETFL, O_NONBLOCK)){ + PWARN("fcntl() to set O_NONBLOCK on API connection fails."); - if (res <= 0) PFATAL("write() on API socket fails despite POLLOUT."); + if (res <= 0) PWARN("write() on API socket fails despite POLLOUT."); } } } From 419b35c3608f681fb99a6bc16345df92ec7ed29f Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 13 Nov 2014 11:53:58 +1100 Subject: [PATCH 73/85] syntax error fix --- debug.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug.h b/debug.h index 62877a0..4b17a42 100644 --- a/debug.h +++ b/debug.h @@ -30,7 +30,8 @@ perror(" OS message "); \ ERRORF("\n"); \ exit(1); \ - } + } while (0) + #define WARN(x...) do { \ ERRORF("[!] WARNING: " x); \ From 7a7653fdfe8073b4b33d775416b5c21f2785a5ae Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 13 Nov 2014 11:54:53 +1100 Subject: [PATCH 74/85] mistaken start brace removed --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index f878875..5a94d6e 100644 --- a/p0f.c +++ b/p0f.c @@ -1186,7 +1186,7 @@ static void epoll_event_loop(void){ ((char*)&ctable[fd].out_data), sizeof(struct p0f_api_response)); - if (fcntl(fd, F_SETFL, O_NONBLOCK)){ + if (fcntl(fd, F_SETFL, O_NONBLOCK)) PWARN("fcntl() to set O_NONBLOCK on API connection fails."); if (res <= 0) PWARN("write() on API socket fails despite POLLOUT."); From 5fd7c3535b51d2419cc183f96486f8a859b3d019 Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 9 Dec 2014 13:40:47 +1100 Subject: [PATCH 75/85] add signal handler to ignore sigpipe --- p0f.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p0f.c b/p0f.c index 5a94d6e..6b75b63 100644 --- a/p0f.c +++ b/p0f.c @@ -1715,6 +1715,8 @@ int main(int argc, char** argv) { if (switch_user) drop_privs(); + signal(SIGPIPE, SIG_IGN); + if (daemon_mode) fork_off(); signal(SIGHUP, daemon_mode ? SIG_IGN : abort_handler); From 92679ad092cdfa1bc659219702ffe09c9a09e87e Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 16 Jan 2015 14:53:04 +1100 Subject: [PATCH 76/85] Start of better handling for EPIPE & non blocking epoll write --- p0f.c | 57 ++++++++++++++++++++++++++++++++++++--------------------- p0f.h | 5 +++-- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/p0f.c b/p0f.c index 6b75b63..7ed1e5e 100644 --- a/p0f.c +++ b/p0f.c @@ -1041,7 +1041,7 @@ static void abort_handler(int sig) { static void epoll_event_loop(void){ struct api_client* ctable; - int slots = 6 + api_max_conn; + int slots = 6 + api_max_conn;//Start with 128 slots ctable = ck_alloc(slots * sizeof(struct api_client)); @@ -1168,28 +1168,37 @@ static void epoll_event_loop(void){ /* Query in place? Compute response and prepare to send it back. */ if (ctable[fd].in_off == sizeof(struct p0f_api_query)) { - handle_query(&ctable[fd].in_data, &ctable[fd].out_data); - - //Reset in offset - ctable[fd].in_off = 0; + ev.events = EPOLLOUT | EPOLLERR | EPOLLHUP; + ev.data.fd = fd; + res = epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &ev); + } + } + else if (events[n].events & EPOLLOUT){ + if (ctable[fd]->in_off < sizeof(struct p0f_api_query)){ + WARN("Inconsistent p0f_api_response state.\n"); + } - //A unix socket shouldnt block here, unless we arent reading fast enough - //Lets assume we have reasonably good clients and that this is a non issue - //This reduces complexity and improves performance, providing it holds true - /* Write API response, restart state when complete. */ + i = write(pfds[fd].fd, + ((char*)&ctable[cur]->out_data) + ctable[fd]->out_off, + sizeof(struct p0f_api_response) - ctable[fd]->out_off); - //Disable non block for a minute - if (fcntl(fd, F_SETFL, 0)) - PFATAL("fcntl() to set ~O_NONBLOCK on API connection fails."); + if (i <= 0) { + PWARN("write() on API socket fails despite POLLOUT."); + close(pfds[fd].fd); + ctable[fd]->fd = -1; + continue; + } - res = write(fd, - ((char*)&ctable[fd].out_data), - sizeof(struct p0f_api_response)); + ctable[cur]->out_off += i; - if (fcntl(fd, F_SETFL, O_NONBLOCK)) - PWARN("fcntl() to set O_NONBLOCK on API connection fails."); + /* All done? Back to square zero then! */ - if (res <= 0) PWARN("write() on API socket fails despite POLLOUT."); + if (ctable[cur]->out_off == sizeof(struct p0f_api_response)) { + ctable[cur]->in_off = ctable[cur]->out_off = 0; + + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; + ev.data.fd = fd; + res = epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &ev); } } } @@ -1315,14 +1324,20 @@ static void poll_event_loop(void){ /* Write API response, restart state when complete. */ - if (ctable[cur]->in_off < sizeof(struct p0f_api_query)) - FATAL("Inconsistent p0f_api_response state.\n"); + if (ctable[cur]->in_off < sizeof(struct p0f_api_query)){ + WARN("Inconsistent p0f_api_response state.\n"); + } i = write(pfds[cur].fd, ((char*)&ctable[cur]->out_data) + ctable[cur]->out_off, sizeof(struct p0f_api_response) - ctable[cur]->out_off); - if (i <= 0) PFATAL("write() on API socket fails despite POLLOUT."); + if (i <= 0) { + PWARN("write() on API socket fails despite POLLOUT."); + close(pfds[cur].fd); + ctable[cur]->fd = -1; + continue; + } ctable[cur]->out_off += i; diff --git a/p0f.h b/p0f.h index c9290a6..4ca599a 100644 --- a/p0f.h +++ b/p0f.h @@ -37,10 +37,11 @@ struct api_client { s32 fd; /* -1 if slot free */ struct p0f_api_query in_data; /* Query recv buffer */ - u32 in_off; /* Query buffer offset */ struct p0f_api_response out_data; /* Response transmit buffer */ - u32 out_off; /* Response buffer offset */ + + u8 in_off; /* Query buffer offset */ + u8 out_off; /* Response buffer offset */ }; From bb3de4a36de8f8f117fd4f78be6b31eefc6feef9 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 16 Jan 2015 14:55:49 +1100 Subject: [PATCH 77/85] Bug fixes --- p0f.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/p0f.c b/p0f.c index 7ed1e5e..473f3b8 100644 --- a/p0f.c +++ b/p0f.c @@ -1174,18 +1174,18 @@ static void epoll_event_loop(void){ } } else if (events[n].events & EPOLLOUT){ - if (ctable[fd]->in_off < sizeof(struct p0f_api_query)){ + if (ctable[fd].in_off < sizeof(struct p0f_api_query)){ WARN("Inconsistent p0f_api_response state.\n"); } - i = write(pfds[fd].fd, - ((char*)&ctable[cur]->out_data) + ctable[fd]->out_off, - sizeof(struct p0f_api_response) - ctable[fd]->out_off); + res = write(pfds[fd].fd, + ((char*)&ctable[fd].out_data) + ctable[fd].out_off, + sizeof(struct p0f_api_response) - ctable[fd].out_off); - if (i <= 0) { + if (res <= 0) { PWARN("write() on API socket fails despite POLLOUT."); - close(pfds[fd].fd); - ctable[fd]->fd = -1; + close(fd); + ctable[fd].fd = -1; continue; } @@ -1193,8 +1193,8 @@ static void epoll_event_loop(void){ /* All done? Back to square zero then! */ - if (ctable[cur]->out_off == sizeof(struct p0f_api_response)) { - ctable[cur]->in_off = ctable[cur]->out_off = 0; + if (ctable[cur].out_off == sizeof(struct p0f_api_response)) { + ctable[cur].in_off = ctable[cur].out_off = 0; ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = fd; From 43283d2d479655065f0127d444ecc8fd8401da6c Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 16 Jan 2015 14:57:01 +1100 Subject: [PATCH 78/85] ... --- p0f.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p0f.c b/p0f.c index 473f3b8..94d2e17 100644 --- a/p0f.c +++ b/p0f.c @@ -1178,7 +1178,7 @@ static void epoll_event_loop(void){ WARN("Inconsistent p0f_api_response state.\n"); } - res = write(pfds[fd].fd, + res = write(fd, ((char*)&ctable[fd].out_data) + ctable[fd].out_off, sizeof(struct p0f_api_response) - ctable[fd].out_off); @@ -1189,12 +1189,12 @@ static void epoll_event_loop(void){ continue; } - ctable[cur]->out_off += i; + ctable[fd]->out_off += res; /* All done? Back to square zero then! */ - if (ctable[cur].out_off == sizeof(struct p0f_api_response)) { - ctable[cur].in_off = ctable[cur].out_off = 0; + if (ctable[fd].out_off == sizeof(struct p0f_api_response)) { + ctable[fd].in_off = ctable[fd].out_off = 0; ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = fd; From 7951b1e54576e1b71176e8f018b73bb2f343cf73 Mon Sep 17 00:00:00 2001 From: splitice Date: Fri, 16 Jan 2015 14:57:46 +1100 Subject: [PATCH 79/85] mistake, not pointer access --- p0f.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index 94d2e17..da6d367 100644 --- a/p0f.c +++ b/p0f.c @@ -1189,7 +1189,7 @@ static void epoll_event_loop(void){ continue; } - ctable[fd]->out_off += res; + ctable[fd].out_off += res; /* All done? Back to square zero then! */ From 27dd301e259d9273e9a0456873a7b3be074f85e6 Mon Sep 17 00:00:00 2001 From: splitice Date: Tue, 20 Jan 2015 18:36:35 +1100 Subject: [PATCH 80/85] zero epoll event before using, valgrind warning --- p0f.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/p0f.c b/p0f.c index da6d367..096a292 100644 --- a/p0f.c +++ b/p0f.c @@ -1060,6 +1060,9 @@ static void epoll_event_loop(void){ //Initial epoll setup int epfd = epoll_create(api_max_conn); + //Zero epoll event + memset(&ev, 0, sizeof ev); + //add PCAP fd ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = pcap_fd; From c81e4af8ab68104174db6b40780c3a89baaacb11 Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 12 Feb 2015 09:31:24 +1100 Subject: [PATCH 81/85] sigpipe fix --- p0f.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index 096a292..d8c43bd 100644 --- a/p0f.c +++ b/p0f.c @@ -1733,10 +1733,9 @@ int main(int argc, char** argv) { if (switch_user) drop_privs(); - signal(SIGPIPE, SIG_IGN); - if (daemon_mode) fork_off(); + signal(SIGPIPE, SIG_IGN); signal(SIGHUP, daemon_mode ? SIG_IGN : abort_handler); signal(SIGINT, abort_handler); signal(SIGTERM, abort_handler); From 6904638e76c9cca0d0129d1b3ae338fbd6148036 Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 12 Feb 2015 09:57:34 +1100 Subject: [PATCH 82/85] change EPIPE handling on write() --- p0f.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/p0f.c b/p0f.c index d8c43bd..8913ed4 100644 --- a/p0f.c +++ b/p0f.c @@ -1186,22 +1186,25 @@ static void epoll_event_loop(void){ sizeof(struct p0f_api_response) - ctable[fd].out_off); if (res <= 0) { - PWARN("write() on API socket fails despite POLLOUT."); - close(fd); - ctable[fd].fd = -1; - continue; + if (errno != EPIPE){ + PWARN("write() on API socket fails despite POLLOUT."); + close(fd); + ctable[fd].fd = -1; + } } + else{ - ctable[fd].out_off += res; + ctable[fd].out_off += res; - /* All done? Back to square zero then! */ + /* All done? Back to square zero then! */ - if (ctable[fd].out_off == sizeof(struct p0f_api_response)) { - ctable[fd].in_off = ctable[fd].out_off = 0; - - ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; - ev.data.fd = fd; - res = epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &ev); + if (ctable[fd].out_off == sizeof(struct p0f_api_response)) { + ctable[fd].in_off = ctable[fd].out_off = 0; + + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; + ev.data.fd = fd; + res = epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &ev); + } } } } From 216a0cd11d79db934d51082c0ddb0bc52867bf16 Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 12 Feb 2015 10:40:35 +1100 Subject: [PATCH 83/85] zero api sock to make more resiliant --- p0f.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index 8913ed4..ecacf08 100644 --- a/p0f.c +++ b/p0f.c @@ -1127,13 +1127,13 @@ static void epoll_event_loop(void){ ctable = ck_realloc(ctable, slots * sizeof(struct api_client)); } + memset(&ctable[fd], 0, sizeof(struct api_client)); + ctable[fd].fd = client_sock; if (fcntl(client_sock, F_SETFL, O_NONBLOCK)) PFATAL("fcntl() to set O_NONBLOCK on API connection fails."); - ctable[fd].in_off = 0; - ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = client_sock; res = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sock, &ev); From 8c68ccc0d3e9470dec06987808060b2b7b7c4569 Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 12 Feb 2015 10:58:58 +1100 Subject: [PATCH 84/85] . --- p0f.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p0f.c b/p0f.c index ecacf08..93b857b 100644 --- a/p0f.c +++ b/p0f.c @@ -1188,9 +1188,9 @@ static void epoll_event_loop(void){ if (res <= 0) { if (errno != EPIPE){ PWARN("write() on API socket fails despite POLLOUT."); - close(fd); - ctable[fd].fd = -1; } + close(fd); + ctable[fd].fd = -1; } else{ From e12dea09cfc4a4b8763eb2c8c6652b53294631c5 Mon Sep 17 00:00:00 2001 From: splitice Date: Thu, 12 Feb 2015 21:31:11 +1100 Subject: [PATCH 85/85] reset in_off for a more consistent query state --- p0f.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p0f.c b/p0f.c index 93b857b..875a859 100644 --- a/p0f.c +++ b/p0f.c @@ -1173,6 +1173,7 @@ static void epoll_event_loop(void){ if (ctable[fd].in_off == sizeof(struct p0f_api_query)) { ev.events = EPOLLOUT | EPOLLERR | EPOLLHUP; ev.data.fd = fd; + ctable[fd].in_off = 0; res = epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &ev); } } @@ -1199,7 +1200,7 @@ static void epoll_event_loop(void){ /* All done? Back to square zero then! */ if (ctable[fd].out_off == sizeof(struct p0f_api_response)) { - ctable[fd].in_off = ctable[fd].out_off = 0; + ctable[fd].out_off = 0; ev.events = EPOLLIN | EPOLLERR | EPOLLHUP; ev.data.fd = fd;