new software on the server, ..., almost done

Carter Bullard carter at qosient.com
Mon Sep 10 13:08:48 EDT 2007


So it looks quickly like your trying to build an Arp flow key,  for  
either an
illegal op code, or you didn't get enough data in the snaplen to get
all the arp information.  So, if you still have the info, lets dump  
the packet
contents, and I'll try to figure it out:

    (gdb) print model->ArgusThisSnapLen
    (gdb) print model->ArgusThisSnapEnd
    (gdb) print model->ArgusThisEpHdr
    (gdb) print model->ArgusThisNetworkHdr

    (gdb) print *((struct arphdr *)model->ArgusThisUpHdr)
    (gdb) x/32x model->ArgusThisUpHdr

Carter

On Sep 10, 2007, at 11:45 AM, Michael Hornung wrote:

> (gdb) print *flow
> $1 = {hdr = {type = 2 '\002', subtype = 6 '\006', dsr_un = {fl =  
> {data =
> 0},
>       vl8 = {qual = 0 '\0', len = 0 '\0'}, vl16 = {len = 0}}},  
> flow_un = {
>     ipv6 = {ip_src = {0, 0, 0, 0}, ip_dst = {0, 0, 0, 0}, flow = 0,  
> resv =
> 0,
>       ip_p = 0, sport = 0, dport = 0}, ip = {ip_src = 0, ip_dst = 0,
>       ip_p = 0 '\0', tp_p = 0 '\0', sport = 0, dport = 0, pad = 0},  
> mac =
> {
>       ehdr = {ether_dhost = "\000\000\000\000\000",
>         ether_shost = "\000\000\000\000\000", ether_type = 0}, dsap  
> = 0
> '\0',
>       ssap = 0 '\0'}, icmpv6 = {ip_src = {0, 0, 0, 0}, ip_dst = {0,  
> 0, 0,
> 0},
>       flow = 0, resv = 0, ip_p = 0, type = 0 '\0', code = 0 '\0',  
> id = 0},
>     icmp = {ip_src = 0, ip_dst = 0, ip_p = 0 '\0', tp_p = 0 '\0',
>       type = 0 '\0', code = 0 '\0', id = 0, ip_id = 0}, igmpv6 =  
> {ip_src =
> {0,
>         0, 0, 0}, ip_dst = {0, 0, 0, 0}, flow = 0, resv = 0, ip_p = 0,
>       type = 0 '\0', code = 0 '\0', pad = 0}, igmp = {ip_src = 0,  
> ip_dst =
> 0,
>       ip_p = 0 '\0', tp_p = 0 '\0', type = 0 '\0', code = 0 '\0',  
> pad = 0,
>       ip_id = 0}, espv6 = {ip_src = {0, 0, 0, 0}, ip_dst = {0, 0,  
> 0, 0},
>       flow = 0, resv = 0, ip_p = 0, spi = 0}, esp = {ip_src = 0,  
> ip_dst =
> 0,
>       ip_p = 0 '\0', tp_p = 0 '\0', pad = 0, spi = 0}, arp = {hrd = 0,
>       pro = 0, hln = 0 '\0', pln = 0 '\0', op = 0, arp_spa = 0,  
> arp_tpa =
> 0,
>       haddr = {{ethernet = "\000\000\000\000\000",
>           ib = '\0' <repeats 31 times>, ieee1394 = '\0' <repeats 15
> times>,
>           framerelay = "\000\000\000", tokenring = "\000\000\000\000 
> \000",
>           arcnet = "", fiberchannel = '\0' <repeats 11 times>,
>           atm = '\0' <repeats 19 times>}}}, rarp = {hrd = 0, pro = 0,
>       hln = 0 '\0', pln = 0 '\0', op = 0, arp_tpa = 0, shaddr = {{
>           ethernet = "\000\000\000\000\000", ib = '\0' <repeats 31  
> times>,
>           ieee1394 = '\0' <repeats 15 times>, framerelay = "\000\000 
> \000",
>           tokenring = "\000\000\000\000\000", arcnet = "",
>           fiberchannel = '\0' <repeats 11 times>,
>           atm = '\0' <repeats 19 times>}}, dhaddr = {{
>           ethernet = "\000\000\000\000\000", ib = '\0' <repeats 31  
> times>,
>           ieee1394 = '\0' <repeats 15 times>, framerelay = "\000\000 
> \000",
>           tokenring = "\000\000\000\000\000", arcnet = "",
>           fiberchannel = '\0' <repeats 11 times>,
>           atm = '\0' <repeats 19 times>}}}, fragv6 = {ip_src = {0,  
> 0, 0,
> 0},
>       ip_dst = {0, 0, 0, 0}, flow = 0, resv = 0, ip_p = 0, ip_id = 0},
> frag = {
>       ip_src = 0, ip_dst = 0, ip_p = 0 '\0', tp_p = 0 '\0', pad =  
> {0, 0},
>       ip_id = 0}}}
>
>
> (gdb) print *hstruct
> $3 = {len = 0, hash = 0, key = {0 <repeats 24 times>}}
>
>
> -Mike
>
> On Sun, 9 Sep 2007 at 16:52, Carter Bullard wrote:
>
> |OK, well like I said earlier, we need to know what kind of packet  
> this is,
> |and the flow struct that we created as the key to the flow should  
> tell us.
> |
> |So, in gdb:
> |
> |(gdb) print *flow
> |
> |in ArgusNewFlow() will give us most of the info we need.
> |But in addition, we should also get the contents of the hstruct:
> |
> |(gdb) print *hstruct
> |
> |that should tell us enough in this situation.
> |
> |Carter
> |
> |On Sep 7, 2007, at 6:27 PM, Michael Hornung wrote:
> |
> |> Here's where it gets me:
> |>
> |> (gdb) run
> |> Starting program: /usr/local/sbin/argus
> |>  ArgusWarning: argus[29876]: 07 Sep 07 15:19:15.920346 started
> |>  ArgusWarning: argus[29876]: 07 Sep 07 15:19:15.920527
> |> ArgusGetInterfaceStatus: interface eth2 is up
> |>     ArgusInfo: argus[29876]: 07 Sep 07 15:19:18.541138 connect from
> |> marathon.cac.washington.edu
> |>
> |> Program received signal SIGSEGV, Segmentation fault.
> |> 0x4c13e663 in bcopy () from /lib/libc.so.6
> |>
> |> (gdb) bt full
> |> #0  0x4c13e663 in bcopy () from /lib/libc.so.6
> |> No symbol table info available.
> |> #1  0x0804ff55 in ArgusNewFlow (model=0x8fab008, flow=0x8fab3a8,
> |>    hstruct=0x8fab310, queue=0x8fab380) at ArgusModeler.c:1469
> |>        retn = (struct ArgusFlowStruct *) 0x9281420
> |>        timeout = 5
> |>        userlen = 0
> |> #2  0x0804ef30 in ArgusProcessPacket (model=0x8fab008,  
> p=0x8fac1ea "",
> |>    length=90, tvp=0xbfd68c70, type=0) at ArgusModeler.c:1072
> |>        retn = 0
> |>        tflow = (struct ArgusSystemFlow *) 0x8fab3a8
> |>        flow = (struct ArgusFlowStruct *) 0x0
> |>        nflow = (struct ArgusFlowStruct *) 0x9eb7b80
> |>        ptr = 0x8fac1ea ""
> |>        value = 0
> |> #3  0x0805655b in ArgusEtherPacket (user=0xb7ed9008 "",  
> h=0xbfd68c70,
> |>    p=0x8fac1ea "") at ArgusSource.c:683
> |>        ep = (struct ether_header *) 0x8fac1ea
> |>        ind = 0
> |>        src = (struct ArgusSourceStruct *) 0xb7ed9008
> |>        tvp = (struct timeval *) 0xbfd68c70
> |>        caplen = 90
> |>        length = 90
> |>        statbuf = {st_dev = 578110229122026696, __pad1 = 45768,
> |>  __st_ino = 3218508768, st_mode = 3218508904, st_nlink =  
> 1275960740,
> |>  st_uid = 3086401536, st_gid = 1935745139, st_rdev =  
> 5480000866624733183,
> |>  __pad2 = 41952, st_size = -4623353967097284856, st_blksize =  
> 1275861536,
> |>  st_blocks = -5190746013132413544, st_atim = {tv_sec = 1,  
> tv_nsec = 1},
> |>  st_mtim = {tv_sec = 0, tv_nsec = 4589194}, st_ctim = {tv_sec =  
> 4583424,
> |>    tv_nsec = 164972}, st_ino = 20393674228473252}
> |> #4  0x00464517 in pcap_open_live () from /usr/lib/libpcap.so.0.9.4
> |> No symbol table info available.
> |> #5  0x00464987 in pcap_dispatch () from /usr/lib/libpcap.so.0.9.4
> |> No symbol table info available.
> |> #6  0x080585c1 in ArgusGetPackets (src=0xb7ed9008) at  
> ArgusSource.c:1730
> |>        ArgusReadMask = {__fds_bits = {128, 0 <repeats 31 times>}}
> |>        ArgusWriteMask = {__fds_bits = {0 <repeats 32 times>}}
> |>        ArgusExceptMask = {__fds_bits = {0 <repeats 32 times>}}
> |>        tmp = 1
> |>        i = 0
> |>        width = 7
> |>        noerror = 1
> |>        fd = 7
> |>        found = 1
> |>        up = 1
> |>        wait = {tv_sec = 0, tv_usec = 20000}
> |> #7  0x0804b687 in main (argc=1, argv=0xbfd69084) at argus.c:567
> |>        commandlinew = 0
> |>        doconf = 0
> |>        dodebug = 0
> |>        i = 1
> |>        pid = 0
> |>        tmparg = 0x0
> |>        filter = 0x0
> |>        statbuf = {st_dev = 64768, __pad1 = 0, __st_ino = 36308349,
> |>  st_mode = 33188, st_nlink = 1, st_uid = 0, st_gid = 0, st_rdev  
> = 0,
> |>  __pad2 = 0, st_size = 11781, st_blksize = 4096, st_blocks = 32,  
> st_atim
> |> = {
> |>    tv_sec = 1189199340, tv_nsec = 0}, st_mtim = {tv_sec =  
> 1189199328,
> |>    tv_nsec = 0}, st_ctim = {tv_sec = 1189199328, tv_nsec = 0},
> |>  st_ino = 36308349}
> |>        op = -1
> |>        commandlinei = 0
> |>        path = "/etc/argus.conf\000argus", '\0' <repeats 8170 times>
> |>
> |> (gdb) up
> |> #1  0x0804ff55 in ArgusNewFlow (model=0x8fab008, flow=0x8fab3a8,
> |>    hstruct=0x8fab310, queue=0x8fab380) at ArgusModeler.c:1469
> |> 1469          bcopy ((char *)&flow->flow_un, (char
> |> *)&retn->canon.flow.flow_un, (flow->hdr.argus_dsrvl8.len - 1) * 4);
> |>
> |> (gdb) print (char *)&flow->flow_un
> |> $1 = 0x8fab3ac ""
> |>
> |> (gdb) print (char *)&retn->canon.flow.flow_un
> |> $3 = 0x92815b4 ""
> |>
> |> (gdb) print flow->hdr.argus_dsrvl8.len
> |> There is no member named argus_dsrvl8.
> |>
> |> Hrmm.
> |>
> |> -Mike
> |>
> |> On Fri, 7 Sep 2007 at 17:04, Carter Bullard wrote:
> |>
> |> |I think I found something that could be the cause of your problem,
> |> |although its a medium shot (as opposed to a long shot ;o)
> |> |
> |> |I'll have a new argus and clients up on the server, and it  
> could fix
> |> |Michael's problem as well, only because there are multiple changes
> |> |in this update.
> |> |
> |> |So for Peter, I have a few more checks, and we shouldn't die if we
> |> |have your problem, now (should not die).  For Michael, I put in  
> some
> |> |checks for zero length hash structs, and we should survive them
> |> |much better.
> |> |
> |> |Carter
> |> |
> |> |On Sep 7, 2007, at 2:20 PM, Peter Van Epp wrote:
> |> |
> |> |> On Fri, Sep 07, 2007 at 12:36:38PM -0400, Carter Bullard wrote:
> |> |> > Hey Peter,
> |> |> > Well that is good news!!!
> |> |> >
> |> |> > So there is another update, to support ARP functions for IP  
> over
> |> |> > Inifiniband and
> |> |> > ATM, which is a pretty big change, as the physical  
> addresses (mac
> |> |> > addresses)
> |> |> > can be rather large (physical address for infiniband is  
> what 32 bytes
> |> |> > long.
> |> |> > So the flow model for ARP had to change to accommodate that.
> |> |> >
> |> |> > I'll put it and new matching clients up later today, or on  
> Sunday,
> |> |> > depending on
> |> |> > how far I get on documentation, etc....,   If someone is  
> interested
> |> |> > in monitoring
> |> |> > IP over infinifband on there OpenIB adapter this weekend,  
> yell and
> |> |> > I'll put it up
> |> |> > sooner.
> |> |> >
> |> |> > Carter
> |> |>
> |> |> 	Unfortunatly it didn't hold (must be traffic of some kind).  
> It looks
> |> |> like the problem is that retn->dsrs[i] is NULL which makes  
> copying in to
> |> |> it difficult :-).
> |> |>
> |> |> test4:/var/log/argus vanepp$ ra3 -r
> |> |>
> |> /archive/argus3/com_argus.archive/2007/09/07/com_argus. 
> 2007.09.07.09.00.00.0.gz
> |> |> -n >t
> |> |> ra3(10073,0xa000ed88) malloc: *** vm_allocate(size=8421376)  
> failed (error
> |> |> code=3)
> |> |> ra3(10073,0xa000ed88) malloc: *** error: can't allocate region
> |> |> ra3(10073,0xa000ed88) malloc: *** set a breakpoint in  
> szone_error to debug
> |> |> Bus error (core dumped)
> |> |> test4:/var/log/argus vanepp$ ls /cores
> |> |> core.10073      core.5848
> |> |> test4:/var/log/argus vanepp$ ls -l /cores
> |> |> total 8574696
> |> |> -r--------   1 vanepp  admin  2191327232 Sep  7 11:15 core.10073
> |> |> -r--------   1 vanepp  admin  2198917120 Sep  6 19:03 core.5848
> |> |> test4:/var/log/argus vanepp$ gdb ra3 /cores/core.10073
> |> |> GNU gdb 6.3.50-20050815 (Apple version gdb-563) (Wed Jul 19  
> 05:17:43 GMT
> |> |> 2006)
> |> |> Copyright 2004 Free Software Foundation, Inc.
> |> |> GDB is free software, covered by the GNU General Public  
> License, and you
> |> are
> |> |> welcome to change it and/or distribute copies of it under  
> certain
> |> conditions.
> |> |> Type "show copying" to see the conditions.
> |> |> There is absolutely no warranty for GDB.  Type "show  
> warranty" for
> |> details.
> |> |> This GDB was configured as "powerpc-apple-darwin"...Reading  
> symbols for
> |> |> shared libraries .. done
> |> |>
> |> |> Core was generated by `/usr/local/bin/ra3'.
> |> |> #0  0xffff8a74 in ___memcpy () at
> |> |>
> |> /System/Library/Frameworks/System.framework/PrivateHeaders/ppc/ 
> cpu_capabilities.h:189
> |> |> 189
> |> |>
> |> /System/Library/Frameworks/System.framework/PrivateHeaders/ppc/ 
> cpu_capabilities.h:
> |> |> No such file or directory.
> |> |>        in
> |> |>
> |> /System/Library/Frameworks/System.framework/PrivateHeaders/ppc/ 
> cpu_capabilities.h
> |> |> (gdb) where
> |> |> #0  0xffff8a74 in ___memcpy () at
> |> |>
> |> /System/Library/Frameworks/System.framework/PrivateHeaders/ppc/ 
> cpu_capabilities.h:189
> |> |> #1  0x0005c804 in ArgusCopyRecordStruct (rec=0x405234) at
> |> |> ./argus_client.c:3359
> |> |> #2  0x0000979c in RaScheduleRecord (parser=0x288000,  
> argus=0x405234) at
> |> |> ./argus_util.c:840
> |> |> #3  0x00009c2c in ArgusHandleDatum (parser=0x288000,  
> input=0x405000,
> |> |> ptr=0x632bc4, filter=0x299f40) at ./argus_util.c:919
> |> |> #4  0x00056e34 in ArgusReadStreamSocket (parser=0x288000,  
> input=0x405000)
> |> at
> |> |> ./argus_client.c:1638
> |> |> #5  0x0005713c in ArgusReadFileStream (parser=0x288000,  
> input=0x405000) at
> |> |> ./argus_client.c:1700
> |> |> #6  0x00003b44 in main (argc=4, argv=0xbffffc18) at ./ 
> argus_main.c:238
> |> |> (gdb) up
> |> |> #1  0x0005c804 in ArgusCopyRecordStruct (rec=0x405234) at
> |> |> ./argus_client.c:3359
> |> |> 3359                            bcopy (rec->dsrs[i], retn- 
> >dsrs[i], size +
> |> |> 8);
> |> |> (gdb) print rec->dsrs[i]
> |> |> $1 = (struct ArgusDSRHeader *) 0x45550c
> |> |> (gdb) print *rec->dsrs[i]
> |> |> $2 = {
> |> |>  type = 80 'P',
> |> |>  subtype = 160 '?',
> |> |>  dsr_un = {
> |> |>    fl = {
> |> |>      data = 130
> |> |>    },
> |> |>    vl8 = {
> |> |>      qual = 0 '\0',
> |> |>      len = 130 '?'
> |> |>    },
> |> |>    vl16 = {
> |> |>      len = 130
> |> |>    }
> |> |>  }
> |> |> }
> |> |> (gdb) print retn->dsrs[i]
> |> |> $3 = (struct ArgusDSRHeader *) 0x0
> |> |> (gdb) print *retn->dsrs[i]
> |> |> $4 = {
> |> |>  type = 0 '\0',
> |> |>  subtype = 0 '\0',
> |> |>  dsr_un = {
> |> |>    fl = {
> |> |>      data = 0
> |> |>    },
> |> |>    vl8 = {
> |> |>      qual = 0 '\0',
> |> |>      len = 0 '\0'
> |> |>    },
> |> |>    vl16 = {
> |> |>      len = 0
> |> |>    }
> |> |>  }
> |> |> }
> |> |> (gdb) print *user
> |> |> $5 = {
> |> |>  hdr = {
> |> |>    type = 80 'P',
> |> |>    subtype = 160 '?',
> |> |>    dsr_un = {
> |> |>      fl = {
> |> |>        data = 130
> |> |>      },
> |> |>      vl8 = {
> |> |>        qual = 0 '\0',
> |> |>        len = 130 '?'
> |> |>      },
> |> |>      vl16 = {
> |> |>        len = 130
> |> |>      }
> |> |>    }
> |> |>  },
> |> |>  size = 512,
> |> |>  count = 512,
> |> |>  array = "=en-us\">"
> |> |> }
> |> |> (gdb) print i
> |> |> $6 = 12
> |> |>
> |> |> Peter Van Epp / Operations and Technical Support
> |> |> Simon Fraser University, Burnaby, B.C. Canada
> |> |>
> |> |
> |>
> |
>



More information about the argus mailing list