new argus-clients-3.0.7.29 on the server
David Edelman
dedelman at iname.com
Thu May 22 22:39:32 EDT 2014
Carter,
There is still a problem with file processing in rasqlinsert but I can
reproduce it at will and might be able to explain it.
If my MySQL table is set to contain one day of flow data, and if my source
file contains records that span more than one MySQL table and the size of
the data from the source file is small (I think that this means small enough
that it will all fit in a single buffer) then only one table will be
populated. It will be populated with the correct day's data but the other
tables are only created, not populated.
If I attempt to populate the database with two full days of data, even if
the days are not consecutive, it seems to work correctly. If I create a
source file with only a very small amount of data for each day I get this:
ra -N 2 -r 10/argus.2014.05.10.00.gz -w /tmp/small.arg
ra -N 2 -r 11/argus.2014.05.11.00.gz -w /tmp/small.arg
ra -r /tmp/small.arg
StartTime Flgs Proto SrcAddr Sport Dir
DstAddr Dport SrcPkts DstPkts SrcBytes DstBytes State
2014-05-10-00:00:00.000 * tcp 10.1.1.60 34064 ->
10.1.1.45 monit* 2 2 140 1725
CON
2014-05-10-00:00:00.000 * tcp 10.1.1.60 59181 ->
216.17.8.7 https 3 2 258 140
CON
2014-05-11-00:00:00.000 * i tcp 10.1.1.101 49157 <?>
216.17.8.231 https 545 261 822577 18276
CON
2014-05-11-00:00:00.000 * tcp 10.1.1.60 34064 ->
10.1.1.45 monit* 5 5 350 3390
CON
[root at monolith 05]# cd /tmp
[root at monolith tmp]# rasqlinsert -D 3 -r small.arg -M time 1d -w
mysql://argus:XXX@localhost/argus/small_%Y_%m_%d -m srcid saddr daddr proto
-s ltime dur srcid saddr daddr proto bytes sco dco
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:24.426
ArgusAddFileList (0x470a8010, small.arg, 1, -1, -1) returning 1
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:24.426
RaCursesNewProcess(0x470a8010) returns 0x422d430
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:24.426
RaCursesNewProcess(0x470a8010) returns 0x422bbb0
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:24.426
RaCursesNewProcess(0x470a8010) returns 0x4230370
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:24.431 RaMySQLInit:
connect localhost argus 0
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:24.786 RaMySQLInit ()
RaSource (null) RaArchive (null) RaFormat (null)
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:24.796
ArgusInitAddrtoname (0x7f30470a8010, 0x0, 0x0)
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:24.796
ArgusParseInit(0x7f30470a8010, NULL)
rasqlinsert[32240.00f76345307f0000]: 2014-05-23-02:16:24.796
ArgusMySQLInsertProcess() starting
rasqlinsert[32240.00f7ff3f307f0000]: 2014-05-23-02:16:24.797
ArgusMySQLSelectProcess() starting
rasqlinsert[32240.00e77f3f307f0000]: 2014-05-23-02:16:24.797
ArgusMySQLUpdateProcess() starting
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:24.797
ArgusProcessData() starting
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:24.797
ArgusReadConnection() read 16 bytes
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:24.797
ArgusReadConnection() read 112 bytes
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:24.797
ArgusParseInit(0x7f30470a8010 0x7f3046fb6010
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:24.797
ArgusReadConnection(0x46fb6010, 1) returning 1
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:24.797
RaProcessSplitOptions(small_2014_05_10, 4096, 0x46fb6630): returns 0
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:24.812
ArgusCreateSQLSaveTable: CREATE table argus.small_2014_05_10 (ltime
double(18,6) unsigned not null,dur double(18,6) not null,srcid
varchar(64),saddr varchar(64) not null,daddr varchar(64) not null,proto
varchar(16) not null,bytes bigint,sco varchar(2),dco varchar(2), primary key
(srcid,proto,saddr,daddr), record blob) ENGINE=MyISAM
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:26.747
ArgusCreateSQLSaveTable (small_2014_05_10) returning
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:26.747
RaProcessSplitOptions(small_2014_05_11, 4096, 0x46fb6630): returns 0
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:26.763
ArgusCreateSQLSaveTable: CREATE table argus.small_2014_05_11 (ltime
double(18,6) unsigned not null,dur double(18,6) not null,srcid
varchar(64),saddr varchar(64) not null,daddr varchar(64) not null,proto
varchar(16) not null,bytes bigint,sco varchar(2),dco varchar(2), primary key
(srcid,proto,saddr,daddr), record blob) ENGINE=MyISAM
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:27.556
ArgusCreateSQLSaveTable (small_2014_05_11) returning
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:27.556
ArgusCloseInput(0x46fb6010) closing
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:27.556
ArgusCloseInput(0x46fb6010) done
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:27.556
ArgusProcessData: flushing sql queues
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:27.556
ArgusScheduleSQLQuery (0x470a8010, 0x422c330, 0x380027c0, INSERT INTO
argus.small_2014_05_11
(ltime,dur,srcid,saddr,daddr,proto,bytes,sco,dco,record) VALUES
("1399766400.827","0.827","108.50.164.35","10.1.1.101","216.17.8.231","tcp",
"840853","ZZ","US",...), 32) done
rasqlinsert[32240.00f76345307f0000]: 2014-05-23-02:16:27.556 ArgusSQLQuery
(INSERT INTO argus.small_2014_05_11
(ltime,dur,srcid,saddr,daddr,proto,bytes,sco,dco,record) VALUES
("1399766400.827","0.827","108.50.164.35","10.1.1.101","216.17.8.231","tcp",
"840853","ZZ","US",...))
rasqlinsert[32240.00f76345307f0000]: 2014-05-23-02:16:27.557
ArgusMySQLInsertProcess: residual buffer Count 1 SQL Query len 1991
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:27.557
ArgusScheduleSQLQuery (0x470a8010, 0x422c330, 0x38001670, INSERT INTO
argus.small_2014_05_11
(ltime,dur,srcid,saddr,daddr,proto,bytes,sco,dco,record) VALUES
("1399766402.090","2.091","108.50.164.35","10.1.1.60","10.1.1.45","tcp","374
0","ZZ","ZZ",...), 32) done
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:27.557
ArgusProcessData: flushed 2 records
rasqlinsert[32240.00c77f3e307f0000]: 2014-05-23-02:16:27.557
RaParseComplete(caught signal 0)
rasqlinsert[32240.00f7ff3f307f0000]: 2014-05-23-02:16:27.565
ArgusMySQLSelectProcess() done!
rasqlinsert[32240.00e77f3f307f0000]: 2014-05-23-02:16:27.565
ArgusMySQLUpdateProcess() done!
rasqlinsert[32240.00f76345307f0000]: 2014-05-23-02:16:28.048 ArgusSQLQuery
(INSERT INTO argus.small_2014_05_11
(ltime,dur,srcid,saddr,daddr,proto,bytes,sco,dco,record) VALUES
("1399766402.090","2.091","108.50.164.35","10.1.1.60","10.1.1.45","tcp","374
0","ZZ","ZZ",...))
rasqlinsert[32240.00f76345307f0000]: 2014-05-23-02:16:28.048
ArgusMySQLInsertProcess: residual buffer Count 1 SQL Query len 2187
rasqlinsert[32240.00f76345307f0000]: 2014-05-23-02:16:28.532
ArgusMySQLInsertProcess() done!
rasqlinsert[32240.80d81447307f0000]: 2014-05-23-02:16:28.532
ArgusWindowClose () returning
[root at monolith tmp]# mysql -p argus
mysql> show tables like 'small%';
+--------------------------+
| Tables_in_argus (small%) |
+--------------------------+
| small_2014_05_10 |
| small_2014_05_11 |
+--------------------------+
2 rows in set (0.01 sec)
mysql> select count(*) from small_2014_05_10;
+----------+
| count(*) |
+----------+
| 0 |
+----------+
1 row in set (0.00 sec)
mysql> select count(*) from small_2014_05_11;
+----------+
| count(*) |
+----------+
| 2 |
+----------+
1 row in set (0.00 sec)
One additional rasqlinsert() observation - If you build it with debug, you
don't see the -D option when you invoke rasqlinsert -h - not a big deal but
the other clients do it
One nice to have but not for this release if there is a -N value for an
input count and more than a single -r|R value the count should be applied on
a source file basis eg: -N i5 would mean take the first five records of
each file specified.
To my thinking this is counterintuitive: ra -N i2 -r
10/argus.2014.05.10.00.gz -r 11/argus.2014.05.11.00
StartTime Flgs Proto SrcAddr Sport Dir
DstAddr Dport SrcPkts DstPkts SrcBytes DstBytes State
2014-05-10-00:00:00.000 * tcp 10.1.1.60 34064 ->
10.1.1.45 monit* 2 2 140 1725
CON
2014-05-10-00:00:00.000 * tcp 10.1.1.60 59181 ->
216.17.8.7 https 3 2 258 140
CON
--Dave
From: Carter Bullard [mailto:carter at qosient.com]
Sent: Thursday, May 22, 2014 3:16 PM
To: David Edelman
Cc: Argus
Subject: new argus-clients-3.0.7.29 on the server
Hey Dave, et. al,
I've uploaded client-3.0.7.29 which should fix all the issues
that have come up on the list, and a few others.
rasqlinsert - complete overhaul of thread completion and scheduling.
this should solve incomplete flushing of records, and
deal with the new problems Dave reported with file vs
pipe processing, and zero metrics being stuffed into the db.
sasl - fixes for struct typing and compiling issues.
rarc.5 - updated for new rarc variables for color and flow direction
hints.
MYSQL_ENGINE - fixes for default engine when using -X option.
cco + matrix - should be fixed but historically aggregated data
will be affected, need to run historical data with
-M dsrs="-cocode" to remove any mislabeled flow data.
Hoping that this is close to release. I'll put up the release
candidate tonight, so we can start testing that, the numbers will
become argus[-clients]-3.0.8 !!!
Thanks !!!
Carter
On May 19, 2014, at 11:43 PM, David Edelman <dedelman at iname.com
<mailto:dedelman at iname.com> > wrote:
I added a debug statement to rasqlinsert.c in ArgusOutputProcessClose at the
end of the loop that calls ArgusScheduleSQLQuery. It looks like both the
ArgusMySQLUpdateProcess andArgusMySQLSelectProcess threads were already
stopped before the items are scheduled. This is with -D 2
RaProcessSplitOptions(xtyst_2013_09_23, 4096, 0x9beec630): returns 0
ArgusCreateSQLSaveTable (xtyst_2013_09_23) returning
RaProcessSplitOptions(xtyst_2013_09_24, 4096, 0x9beec630): returns 0
ArgusCreateSQLSaveTable (xtyst_2013_09_24) returning
RaProcessSplitOptions(xtyst_2013_09_27, 4096, 0x9beec630): returns 0
ArgusCreateSQLSaveTable (xtyst_2013_09_27) returning
RaProcessSplitOptions(xtyst_2013_09_30, 4096, 0x9beec630): returns 0
ArgusCreateSQLSaveTable (xtyst_2013_09_30) returning
RaProcessSplitOptions(xtyst_2013_10_01, 4096, 0x9beec630): returns 0
ArgusCreateSQLSaveTable (xtyst_2013_10_01) returning
ArgusScheduleSQLQuery (0x9bfde010, 0x3e06330, 0x8c0039f0, INSERT INTO
argus.xtyst_2013_10_01
(ltime,dur,srcid,saddr,daddr,proto,bytes,sco,dco,record) VALUES
("0.000","0.000","10.25.236.7","5.161.164.145","169.173.35.180","udp","0","I
R","US",...), 32) done
ArgusScheduleSQLQuery (0x9bfde010, 0x3e06330, 0x8c004190, INSERT INTO
argus.xtyst_2013_10_01
(ltime,dur,srcid,saddr,daddr,proto,bytes,sco,dco,record) VALUES
("0.000","0.000","169.185.96.6","5.161.164.145","169.185.208.76","tcp","0","
IR","US",...), 32) done
ArgusMySQLUpdateProcess() done!
ArgusMySQLSelectProcess() done!
ArgusOutputProcessClose: ArgusParser->RaParseDone set after 53 items were
sent toArgusScheduleSQLQuery
ArgusMySQLInsertProcess() done!
ArgusWindowClose () returning
RaParseComplete(caught signal 0)
ArgusShutDown (0)
ArgusWindowClose () returning
RaParseComplete(caught signal 0)
ArgusDeleteModeList () returning
ArgusDeleteFileList () returning
ArgusDeleteLabeler (0x7f7d9bfde010, 0x3e05d10) returning
ArgusDeleteAggregator(0x7f7d9bfde010, 0x3e06330) returned
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://pairlist1.pair.net/pipermail/argus/attachments/20140522/d1814a03/attachment.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: smime.p7s
Type: application/pkcs7-signature
Size: 6283 bytes
Desc: not available
URL: <https://pairlist1.pair.net/pipermail/argus/attachments/20140522/d1814a03/attachment.bin>
More information about the argus
mailing list