diff --git a/docker-compose.yml b/docker-compose.yml
index 0ac931260..e8f20ffdd 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -94,6 +94,31 @@ services:
- .:/srv/ixpmanager
- mrtg-data:/srv/mrtg
+ sflow-scripts:
+ build:
+ context: tools/docker/containers/sflow-scripts
+ depends_on:
+ - "mysql"
+ ports:
+ - 6343:6343/udp # sFlow
+ networks:
+ ixpmanager:
+ ipv4_address: 172.30.201.21
+ ipv6_address: fd99::21
+ aliases:
+ - ixpmanager-sflow-scripts
+ extra_hosts:
+ - "rs1-ipv4:172.30.201.30"
+ - "rs1-ipv4.ixpmanager:172.30.201.30"
+ - "rs1-ipv6:172.30.201.30"
+ - "rs1-ipv6.ixpmanager:172.30.201.30"
+ - "switch1:172.30.201.60"
+ - "switch2:172.30.201.61"
+ volumes:
+ - .:/srv/ixpmanager
+ stdin_open: true
+ tty: true
+
routinator:
build:
context: tools/docker/containers/routinator
diff --git a/ipfix.clab.yml b/ipfix.clab.yml
new file mode 100644
index 000000000..bb70229e4
--- /dev/null
+++ b/ipfix.clab.yml
@@ -0,0 +1,40 @@
+name: ipfix-sros
+
+mgmt:
+ network: ixpm_ixpmanager # Connect to IXP manager network
+ # bridge: mgmt-bridge
+ ipv4-subnet: 172.30.201.0/24
+
+topology:
+ kinds:
+ linux:
+ image: ghcr.io/hellt/network-multitool
+
+ nodes:
+ sros:
+ kind: vr-nokia_sros
+ image: vrnetlab/vr-sros:23.10.R3
+ license: /Projects/SR_OS_VSR-SIM_license.txt
+ startup-config: |
+ /configure port 1/1/c1 admin-state enable connector breakout c1-100g
+ /configure port 1/1/c1/1 admin-state enable ethernet mode access encap-type dot1q
+ /configure cflowd collector 172.30.201.1 port 2056 version 10 router-instance "management" admin-state enable template-set l2-ip
+ /configure cflowd active-flow-timeout 30 sample-profile 1 sample-rate 1
+ /configure service vpls "1" admin-state enable service-id 1 customer "1" sap 1/1/c1/1:100 { cflowd true }
+ /configure service vpls "1" interface "i" { ipv4 primary address 192.168.0.0 prefix-length 31 }
+
+ ### CLIENTS ###
+ client1:
+ kind: linux
+ exec:
+ - ip link add link eth1 name eth1.100 type vlan id 100
+ - ip address add 192.168.0.1/31 dev eth1.100
+ - ip link set dev eth1.100 up
+ - ip route replace default via 192.168.0.0
+ - ip -6 address add 2002::192:168:0:1/64 dev eth1.100
+ # - iperf3 -s -p 5201 -D > iperf3_1.log
+ # - iperf3 -s -p 5202 -D > iperf3_2.log
+ group: server
+
+ links:
+ - endpoints: ["sros:eth1","client1:eth1"]
diff --git a/tools/docker/containers/sflow-scripts/Dockerfile b/tools/docker/containers/sflow-scripts/Dockerfile
new file mode 100644
index 000000000..1b847103f
--- /dev/null
+++ b/tools/docker/containers/sflow-scripts/Dockerfile
@@ -0,0 +1,42 @@
+FROM ubuntu:23.04
+
+# RUN apt-get -y update && \
+# apt-get -y upgrade && \
+# apt-get -y autoremove --purge && \
+# apt-get -y clean && \
+# rm -rf /var/lib/apt/lists/* && \
+# rm -rf /tmp/*
+
+# JvB added for sflow testing
+RUN apt-get -y update && apt-get install -y perl-base bash make gcc cpanminus libmysqlclient-dev
+
+RUN cpanm install NetAddr::IP Config::General NetPacket::TCP
+
+# Tests fail, force install
+RUN cpanm install DBD::mysql --force
+
+COPY ixpmanager.conf /usr/local/etc/ixpmanager.conf
+
+# sflowtool for traditional workflow
+RUN apt-get install -y git autoconf
+RUN git clone https://github.com/sflow/sflowtool.git && \
+ cd sflowtool && \
+ ./boot.sh && \
+ ./configure --prefix=/usr && \
+ make && \
+ make install
+
+# goflow2 script dependencies - does not support IPFIX v10 format
+RUN cpanm install JSON REST::Client
+RUN apt-get install -y wget librrds-perl
+RUN wget https://github.com/netsampler/goflow2/releases/download/v1.3.6/goflow2_1.3.6_amd64.deb
+RUN apt install -y ./goflow2_1.3.6_amd64.deb
+
+# nfdump does support v10 with MAC addresses
+RUN apt-get install -y nfdump
+# Usage: sudo nfcapd -M /tmp -b 172.30.201.1 -E
+
+WORKDIR /
+
+ENTRYPOINT "/bin/bash"
+CMD "/bin/bash"
diff --git a/tools/docker/containers/sflow-scripts/README.md b/tools/docker/containers/sflow-scripts/README.md
new file mode 100644
index 000000000..049db0b0c
--- /dev/null
+++ b/tools/docker/containers/sflow-scripts/README.md
@@ -0,0 +1,50 @@
+# Notes
+
+* Need to use a secure connection or restore the default mysql password plugin:
+```
+mysql --protocol=TCP --port 33060 -u root
+ALTER USER 'ixpmanager' IDENTIFIED WITH mysql_native_password BY 'ixpmanager';
+```
+
+## nfcapd
+
+```nfcapd -p 9995 -w /tmp -E```
+
+This will listen for IPFIX UDP packets on port 9995, and print them out to the console for debugging
+
+## nfdump
+
+Only accept from from a single exporter and only extension 6,7,8,10,11 are accepted. Run a given command when files are rotated
+and automatically expire flows:
+```nfcapd -w -D -T 6,7,8,10,11 -n upstream,192.168.1.1,/netflow/spool/upstream -p 23456 -B 128000 -s 100 -x '/path/command -r %d/%f' -P /var/run/nfcapd/nfcapd.pid -e```
+
+From ```man nfcapd```:
+Extensions:
+ v5/v7/v9/IPFIX extensions:
+ 1 input/output interface SNMP numbers.
+ 2 src/dst AS numbers.
+ 3 src/dst mask, (dst)TOS, direction.
+ 4 line Next hop IP addr line
+ 5 line BGP next hop IP addr line
+-> 6 src/dst vlan id labels
+-> 7 counter output packets
+-> 8 counter output bytes
+ 9 counter aggregated flows
+-> 10 in_src/out_dst MAC address
+-> 11 in_dst/out_src MAC address
+ 12 MPLS labels 1-10
+ 13 Exporting router IPv4/IPv6 address
+ 14 Exporting router ID
+ 15 BGP adjacent prev/next AS
+ 16 time stamp flow received by the collector
+ NSEL/ASA/NAT extensions
+ 26 NSEL ASA event, xtended event, ICMP type/code
+ 27 NSEL/NAT xlate ports
+ 28 NSEL/NAT xlate IPv4/IPv6 addr
+ 29 NSEL ASA ACL ingress/egress acl ID
+ 30 NSEL ASA username
+ NEL/NAT extensions
+ 31 NAT event, ingress egress vrfid
+ 32 NAT Block port allocation - block start, end step and size
+ latency extension
+ 64 nfpcapd/nprobe client/server/application latency"},
diff --git a/tools/docker/containers/sflow-scripts/ixpmanager.conf b/tools/docker/containers/sflow-scripts/ixpmanager.conf
new file mode 100644
index 000000000..78954e5b2
--- /dev/null
+++ b/tools/docker/containers/sflow-scripts/ixpmanager.conf
@@ -0,0 +1,20 @@
+
+ dbase_type = mysql
+ dbase_database = ixpmanager
+ dbase_username = ixpmanager
+ dbase_password = ixpmanager
+ dbase_hostname = ixpmanager-mysql
+
+
+
+ # sflowtool = /usr/bin/sflowtool
+ # sflowtool_opts = -4 -p 6343 -l
+ sflowtool = /usr/bin/goflow2
+ debug = 1
+ sflow_rrdcached = 1
+ sflow_rrddir = /data/ixpmatrix
+
+ apikey = APIKeyFromIXPManager
+ apibaseurl = http://ixpmanager-www/ixp/api/v4
+ macdbtype = configured
+
\ No newline at end of file
diff --git a/tools/runtime/sflow/goflow2-detect-ixp-bgp-sessions b/tools/runtime/sflow/goflow2-detect-ixp-bgp-sessions
new file mode 100755
index 000000000..7df56fe64
--- /dev/null
+++ b/tools/runtime/sflow/goflow2-detect-ixp-bgp-sessions
@@ -0,0 +1,247 @@
+#!/usr/bin/env perl
+#
+# goflow2-detect-ixp-bgp-sessions
+#
+# Copyright (C) 2009 - 2019 Internet Neutral Exchange Association Company Limited By Guarantee.
+# All Rights Reserved.
+#
+# This file is part of IXP Manager.
+#
+# IXP Manager is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, version v2.0 of the License.
+#
+# IXP Manager is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License v2.0
+# along with IXP Manager. If not, see:
+#
+# http://www.gnu.org/licenses/gpl-2.0.html
+#
+# Description:
+#
+# This script take the JSON output from goflow2, traps active BGP/tcp sessions
+# and populates the IXP Manager peering database with this data. This
+# allows the IXP operator to create a live peering matrix.
+#
+# goflow2 supports both sFlow and IPFix
+
+use v5.10.1;
+use warnings;
+use strict;
+use Getopt::Long;
+use Data::Dumper;
+use Socket;
+use NetAddr::IP qw (:lower);
+use Time::HiRes qw(ualarm gettimeofday tv_interval);
+
+use JSON qw(decode_json);
+
+use FindBin qw($Bin);
+use File::Spec;
+use lib File::Spec->catdir( $Bin, File::Spec->updir(), File::Spec->updir(), 'perl-lib', 'IXPManager', 'lib' );
+
+use IXPManager::Config;
+
+my $ixp = new IXPManager::Config; # (configfile => $configfile);
+my $dbh = $ixp->{db};
+
+my $debug = defined($ixp->{ixp}->{debug}) ? $ixp->{ixp}->{debug} : 0;
+my $insanedebug = 0;
+my $sflowtool = defined($ixp->{ixp}->{sflowtool}) ? $ixp->{ixp}->{sflowtool} : '/usr/bin/goflow2';
+my $sflowtool_opts = defined($ixp->{ixp}->{sflowtool_bgp_opts}) ? $ixp->{ixp}->{sflowtool_bgp_opts} : '';
+my $timer_period = 600;
+my $bgpretaindays = 14;
+my $daemon = 1;
+
+GetOptions(
+ 'bgpretaindays=i' => \$bgpretaindays,
+ 'debug!' => \$debug,
+ 'insanedebug!' => \$insanedebug,
+ 'daemon!' => \$daemon,
+ 'sflowtool=s' => \$sflowtool,
+ 'sflowtool_bgp_opts=s' => \$sflowtool_opts,
+ 'periodictimer=i' => \$timer_period
+);
+
+if ($insanedebug) {
+ $debug = 1;
+}
+
+my $ipmappings = reload_ipmappings($dbh);
+
+my $execute_periodic = 0;
+my $quit_after_periodic = 0;
+
+# handle signals gracefully
+$SIG{TERM} = sub { $execute_periodic = 1; $quit_after_periodic = 1; };
+$SIG{QUIT} = sub { $execute_periodic = 1; $quit_after_periodic = 1; };
+$SIG{HUP} = sub { $execute_periodic = 1; };
+
+# set up a periodic timer to signal that stuff should be flushed out. The
+# flush isn't performed in the SIGALRM callback function because perl has
+# historical problems doing complicated stuff in signal handler functions.
+# Much more sensible to raise a flag and have the main code body handle this
+# during normal code execution.
+$SIG{ALRM} = sub { $execute_periodic = 1 };
+ualarm ( $timer_period*1000000, $timer_period*1000000);
+
+my $tv = [gettimeofday()];
+my $lastdailyrun = 0;
+
+# FIXME - spaces embedded *within* args will be split too
+my $sflowpid = open (SFLOWTOOL, '-|', $sflowtool, split(' ', $sflowtool_opts));
+
+my $sth = $dbh->prepare('INSERT INTO bgpsessiondata (srcipaddressid, dstipaddressid, protocol, vlan, packetcount, source, timestamp) VALUES (?, ?, ?, ?, 1, ?, NOW())');
+my $expiresth = $dbh->prepare('DELETE FROM bgpsessiondata WHERE timestampdiff(DAY, timestamp, NOW()) > ?');
+
+# methodology is to throw away as much crap as possible before parsing
+while () {
+ my ($ipprotocol);
+
+ chomp;
+
+ $insanedebug && print STDERR "DEBUG: $_\n";
+
+ # parse and split out all the data. most of this is unused at the
+ # moment, but it's useful to collect it anyway
+ # FLOW,193.242.111.152,2,21,0013136f2fc0,0010a52f261f,0x0800,10,10,94.1.115.114,80.1.2.222,6,0x00,124,1863,750,0x18,179,165,1024
+ my $sample = decode_json( $_ );
+ # my (undef, $agent, $srcswport, $dstswport, $srcmac, $dstmac, $ethertype, $vlan, undef,
+ # $srcip, $dstip, $protocol, $tos, $ttl,
+ # $srcport, $dstport, $tcpflags, $pktsize, $payloadsize, $samplerate) = @sample;
+
+ # SrcAddr,DstAddr,Etype,Proto,SrcPort,DstPort,TCPFlags,SamplerAddress,VlanId
+ # see https://github.com/netsampler/goflow2/blob/main/docs/protocols.md for more fields
+ my $ethertype = $sample->{ Etype };
+ my $srcip = $sample->{ SrcAddr };
+ my $dstip = $sample->{ DstAddr };
+
+ if ($ethertype == 2048) {
+ $ipprotocol = 4;
+ } elsif ($ethertype == 34525) { # 0x86dd
+ $ipprotocol = 6;
+ $srcip = NetAddr::IP->new($srcip)->short();
+ $dstip = NetAddr::IP->new($dstip)->short();
+ } else {
+ next;
+ }
+
+ my $protocol = $sample->{ Proto };
+ my $srcport = $sample->{ SrcPort };
+ my $dstport = $sample->{ DstPort };
+
+ # BGP data is protocol 6 (tcp) and one port == 179
+ if ($protocol == 6 && ($srcport == 179 || $dstport == 179)) {
+ use NetPacket::TCP;
+
+ my $tcpflags = hex($sample->{TcpFlags});
+
+ # we're only interested in established sessions
+ if (($tcpflags & ACK) && !(($tcpflags & SYN) || ($tcpflags & RST) ||($tcpflags & FIN))) {
+ if ($debug) {
+ print STDERR "DEBUG: [$srcip]:$srcport - [$dstip]:$dstport ".debug_tcpflags($tcpflags).".";
+ }
+
+ # we're also only interested in ip addresses that have a database match
+ my $agent = $sample->{ SamplerAddress };
+ my $vlan = $sample->{ VlanId };
+ if ($ipmappings->{$ipprotocol}->{$srcip} && $ipmappings->{$ipprotocol}->{$dstip}) {
+ print STDERR " database updated" if ($debug);
+ if (!$sth->execute($ipmappings->{$ipprotocol}->{$srcip}, $ipmappings->{$ipprotocol}->{$dstip}, $ipprotocol, $vlan, $agent)) {
+ print STDERR " unsuccessfully" if ($debug);
+ }
+ } else {
+ print STDERR " ignored - no address match in database" if ($debug);
+ }
+ print STDERR ".\n" if ($debug);
+ } else {
+ if ($debug) {
+ print STDERR "DEBUG: ignoring [$srcip]:$srcport - [$dstip]:$dstport ".debug_tcpflags($tcpflags).".";
+ }
+ }
+ }
+
+ if ($execute_periodic) {
+ if ($quit_after_periodic) {
+ # sometimes sflowtool doesn't die properly. Need to prioritise kill.
+ kill 9, $sflowpid;
+ }
+ my $newtv = [gettimeofday()];
+ my $interval = tv_interval($tv, $newtv);
+ $tv = $newtv;
+ $debug && print STDERR "DEBUG: periodic reload at time interval: $interval, time: ".time()."\n";
+ if ($quit_after_periodic) {
+ $debug && print STDERR "DEBUG: orderly quit at ".time()."\n";
+ exit 0;
+ }
+ $execute_periodic = 0;
+ $ipmappings = reload_ipmappings($dbh);
+ $debug && print STDERR "DEBUG: periodic reload completed at ".time()."\n";
+ $debug && print Dumper ($ipmappings);
+
+ if (time() - $lastdailyrun > 86400) {
+ $lastdailyrun = time();
+ $debug && print STDERR "DEBUG: started daily run at ".time()."\n";
+ $debug && print STDERR "DEBUG: clearing out stale bgp session data older than $bgpretaindays days\n";
+ if (!$expiresth->execute($bgpretaindays)) {
+ $debug && print STDERR "WARNING: could not expire session data";
+ }
+ $debug && print STDERR "DEBUG: completed daily run at ".time()."\n";
+ }
+ }
+}
+
+close (SFLOWTOOL);
+
+# try to kill off sflowtool if it's not already dead
+kill 9, $sflowpid;
+
+# oops, we should never exit
+die "Oops, input pipe died. Aborting.\n";
+
+sub debug_tcpflags
+{
+ my ($tcpflags) = @_;
+
+ use NetPacket::TCP;
+
+ my $ret = sprintf ("tcpflags %09b:", $tcpflags);
+
+ $ret .= " cwr" if ($tcpflags & CWR);
+ $ret .= " ece" if ($tcpflags & ECE);
+ $ret .= " urg" if ($tcpflags & URG);
+ $ret .= " ack" if ($tcpflags & ACK);
+ $ret .= " psh" if ($tcpflags & PSH);
+ $ret .= " rst" if ($tcpflags & RST);
+ $ret .= " syn" if ($tcpflags & SYN);
+ $ret .= " fin" if ($tcpflags & FIN);
+
+ return $ret;
+}
+
+#
+# Create a mapping from $ipmappings->{address}
+#
+sub reload_ipmappings
+{
+ my ($d) = @_;
+ my ($rec, $s, $mapping);
+
+ $s = $d->prepare('SELECT ipv4address.id AS id, ipv4address.address AS address FROM ipv4address LEFT JOIN vlan ON vlan.id = ipv4address.vlanid WHERE vlan.peering_matrix = 1');
+ $s->execute();
+ while (my $rec = $s->fetchrow_hashref) {
+ $mapping->{4}->{$rec->{address}} = $rec->{id};
+ }
+
+ $s = $d->prepare('SELECT ipv6address.id AS id, ipv6address.address AS address FROM ipv6address LEFT JOIN vlan ON vlan.id = ipv6address.vlanid WHERE vlan.peering_matrix = 1');
+ $s->execute();
+ while (my $rec = $s->fetchrow_hashref) {
+ $mapping->{6}->{NetAddr::IP->new($rec->{address})->short()} = $rec->{id};
+ }
+
+ return $mapping;
+}
diff --git a/tools/runtime/sflow/goflow2-to-rrd-handler b/tools/runtime/sflow/goflow2-to-rrd-handler
new file mode 100755
index 000000000..a68ef0041
--- /dev/null
+++ b/tools/runtime/sflow/goflow2-to-rrd-handler
@@ -0,0 +1,454 @@
+#!/usr/bin/env perl
+#
+# goflow2-to-rrd-handler
+#
+# Copyright (C) 2009 - 2019 Internet Neutral Exchange Association Company Limited By Guarantee.
+# All Rights Reserved.
+#
+# This file is part of IXP Manager.
+#
+# IXP Manager is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, version v2.0 of the License.
+#
+# IXP Manager is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License v2.0
+# along with IXP Manager. If not, see:
+#
+# http://www.gnu.org/licenses/gpl-2.0.html
+#
+# Description:
+#
+# This script take the JSON output from goflow2, builds up a peer-to-peer
+# traffic matrix + aggregate stats and periodically writes the results out
+# to a stash of RRD files.
+
+use warnings;
+use strict;
+use Getopt::Long;
+use Data::Dumper;
+use RRDs;
+use Time::HiRes qw(ualarm gettimeofday tv_interval);
+use REST::Client;
+
+use JSON qw(decode_json);
+
+use FindBin qw($Bin);
+use File::Spec;
+use lib File::Spec->catdir( $Bin, File::Spec->updir(), File::Spec->updir(), 'perl-lib', 'IXPManager', 'lib' );
+
+use IXPManager::Config;
+
+my $ixp = new IXPManager::Config (dbase_disable => 1); # (configfile => $configfile);
+
+my $debug = defined($ixp->{ixp}->{debug}) ? $ixp->{ixp}->{debug} : 0;
+my $insanedebug = 0;
+my $rrdcached = defined($ixp->{ixp}->{sflow_rrdcached}) ? $ixp->{ixp}->{sflow_rrdcached} : 1;
+my $sflowtool = defined($ixp->{ixp}->{sflowtool}) ? $ixp->{ixp}->{sflowtool} : '/usr/bin/sflowtool';
+my $sflowtool_opts = defined($ixp->{ixp}->{sflowtool_opts}) ? $ixp->{ixp}->{sflowtool_opts} : '-l';
+my $basedir = defined($ixp->{ixp}->{sflow_rrddir}) ? $ixp->{ixp}->{sflow_rrddir} : '/data/ixpmatrix';
+my $apikey = defined($ixp->{ixp}->{apikey}) ? $ixp->{ixp}->{apikey} : undef;
+my $apibaseurl = defined($ixp->{ixp}->{apibaseurl}) ? $ixp->{ixp}->{apibaseurl} : undef;
+
+my $timer_period = 60;
+my $mactabletimeout = 86400;
+my $daemon = 1;
+my $infraid = undef;
+my $macdbtype = defined($ixp->{ixp}->{macdbtype}) ? $ixp->{ixp}->{macdbtype} : '';
+my $macdbrest;
+
+# conundrum: do we run GetOptions() before creating a new IXPManager::Config
+# object, which would allow us to set the configfile location on the command
+# line? Or do we do it after, which allows us to override the config file
+# arguments on the command line. first world problems.
+
+GetOptions(
+ 'debug!' => \$debug,
+ 'insanedebug!' => \$insanedebug,
+ 'daemon!' => \$daemon,
+ 'sflowtool=s' => \$sflowtool,
+ 'sflowtool_opts=s' => \$sflowtool_opts,
+ 'sflow_rrddir=s' => \$basedir,
+ 'flushtimer=i' => \$timer_period,
+ 'infraid=i' => \$infraid,
+ 'macdbtype=s' => \$macdbtype,
+ 'apikey=s' => \$apikey,
+ 'apibaseurl=s' => \$apibaseurl,
+);
+
+if ($macdbtype eq 'configured') {
+ $macdbrest = '/sflow-db-mapper/configured-macs';
+} else {
+ $macdbrest = '/sflow-db-mapper/learned-macs';
+}
+
+if ($insanedebug) {
+ $debug = 1;
+}
+
+if (!defined ($apikey)) {
+ die "FATAL: must set 'apikey' parameter in section of ixpmanager.conf\n";
+}
+
+if (!defined ($apibaseurl)) {
+ die "FATAL: must set 'apibaseurl' parameter in section of ixpmanager.conf\n";
+}
+
+my $client = REST::Client->new();
+$client->setTimeout(5);
+$client->setHost($apibaseurl);
+$client->addHeader('X-IXP-Manager-API-Key', $apikey);
+
+my $mactable = reload_mactable($client, $macdbrest);
+$mactable || die "FATAL: could not read IXP Manager API call on $apibaseurl$macdbrest\n";
+my $matrix = matrix_init($mactable, $infraid);
+
+my $execute_periodic = 0;
+my $quit_after_periodic = 0;
+my $mactablereloadfails = 0;
+
+# handle signals gracefully
+$SIG{TERM} = sub { $execute_periodic = 1; $quit_after_periodic = 1; };
+$SIG{QUIT} = sub { $execute_periodic = 1; $quit_after_periodic = 1; };
+$SIG{HUP} = sub { $execute_periodic = 1; };
+
+# set up a periodic timer to signal that stuff should be flushed out. The
+# flush isn't performed in the SIGALRM callback function because perl has
+# historical problems doing complicated stuff in signal handler functions.
+# Much more sensible to raise a flag and have the main code body handle this
+# during normal code execution.
+$SIG{ALRM} = sub { $execute_periodic = 1 };
+ualarm ( $timer_period*1000000, $timer_period*1000000);
+
+my $tv = [gettimeofday()];
+
+# FIXME - spaces embedded *within* sflowtool args will be split too
+# Should only ever matter for the "-r" option if the filename has spaces in it...
+my $sflowpid = open (SFLOWTOOL, '-|', $sflowtool, split(' ', $sflowtool_opts));
+
+# methodology is to throw away as much crap as possible before parsing
+while () {
+ # next unless (substr($_, 0, 4) eq 'FLOW'); # don't use regexp here for performance reasons
+ my ($ipprotocol);
+
+ chomp;
+
+ $insanedebug && print STDERR "DEBUG: $_\n";
+
+ # FLOW,193.242.111.152,2,21,0013136faaaa,0010a52fbbbb,0x0800,10,10,192.168.1.1,172.16.12.255,6,0x00,124,1863,750,0x18,179,165,1024
+ # don't use regexp here for performance reasons
+ # my ($sampletype, $agent, $srcmac, $dstmac, $ethertype, $vlan, $pktsize, $samplerate) = (split (/,/))[0,1,4,5,6,7,17,19];
+ my $sample = decode_json( $_ );
+ # next unless ($sampletype eq 'FLOW');
+
+ my $ethertype = $sample->{ Etype };
+
+ # Note: MAC addresses require template-set 'l2-ip' which is only supported with IPFIX version 10
+ # (which is not yet supported by goflow2)
+ my $srcmac = $sample->{ SrcMac };
+ my $dstmac = $sample->{ DstMac };
+ my $vlan = $sample->{ VlanId }; # or 'SrcVlan' and 'DstVlan'
+ my $pktsize = int( $sample->{ Bytes } / $sample->{ Packets });
+ my $samplerate = $sample->{ SamplingRate };
+
+ if ($ethertype == 2048) {
+ $ipprotocol = 4;
+ } elsif ($ethertype == 34525) { # 0x86dd
+ $ipprotocol = 6;
+ } else {
+ $insanedebug && print STDERR "DEBUG: unknown Ethertype $ethertype";
+ next;
+ }
+
+ my $srcvli = getvlifrommactable ($mactable, $infraid, $vlan, $srcmac);
+ my $dstvli = getvlifrommactable ($mactable, $infraid, $vlan, $dstmac);
+
+ # the sflow accounting perimeter on the switches will ensure that
+ # each packet will be counted exactly once. because of this, the
+ # packet sample needs to be added to both the source vlaninterface
+ # for bytes in and the destination vlaninterface for bytes out for
+ # the individual counts.
+
+ if ($ipprotocol && $srcvli && $dstvli && ($srcvli != $dstvli) ) {
+ $insanedebug && print STDERR "DEBUG: accepted update for: ".
+ "protocol: $ipprotocol ".
+ "vlan: $vlan ".
+ "srcvli: $srcvli ".
+ "dstvli: $dstvli ".
+ "pktsize: $pktsize ".
+ "samplerate: $samplerate ".
+ "\n";
+ $matrix->{p2p}->{$ipprotocol}->{bytes}->{$srcvli}->{$dstvli} += $pktsize * $samplerate;
+ $matrix->{p2p}->{$ipprotocol}->{pkts}->{$srcvli}->{$dstvli} += $samplerate;
+ $matrix->{individual}->{$ipprotocol}->{bytes}->{$srcvli}->{in} += $pktsize * $samplerate;
+ $matrix->{individual}->{$ipprotocol}->{bytes}->{$dstvli}->{out} += $pktsize * $samplerate;
+ $matrix->{individual}->{$ipprotocol}->{pkts}->{$srcvli}->{in} += $samplerate;
+ $matrix->{individual}->{$ipprotocol}->{pkts}->{$dstvli}->{out} += $samplerate;
+ } else {
+ $debug && print STDERR "DEBUG: dropped update for: ".
+ "protocol: $ipprotocol ".
+ "vlan: $vlan ".
+ "srcvli: $srcvli ".
+ "dstvli: $dstvli ".
+ "pktsize: $pktsize ".
+ "samplerate: $samplerate ".
+ "\n";
+ $debug && print STDERR "DEBUG: rejected: ".$_."\n";
+ }
+
+ if ($execute_periodic) {
+ if ($quit_after_periodic) {
+ # sometimes sflowtool doesn't die properly. Need to prioritise kill.
+ kill 9, $sflowpid;
+ }
+ my $newtv = [gettimeofday()];
+ my $interval = tv_interval($tv, $newtv);
+ $tv = $newtv;
+ $debug && print STDERR "DEBUG: starting rrd flush at time interval: $interval, time: ".time()."\n";
+ process_rrd($interval, $matrix, $rrdcached);
+ if ($quit_after_periodic) {
+ $debug && print STDERR "DEBUG: orderly quit at ".time()."\n";
+ exit 0;
+ }
+ $execute_periodic = 0;
+ my $newmactable = reload_mactable($client, $macdbrest);
+ if ($newmactable) {
+ $mactable = $newmactable;
+ $matrix = matrix_init($mactable, $infraid);
+ $mactablereloadfails = 0;
+ } else {
+ $mactablereloadfails++;
+ $debug && print STDERR "DEBUG: mactable reload failed: $mactablereloadfails\n";
+ # quit after specified period of unavailability
+ if ($mactablereloadfails > ($mactabletimeout / $timer_period)) {
+ kill 9, $sflowpid;
+ die "FATAL: could not reload mactable after $mactabletimeout seconds. Aborting.\n";
+ }
+ }
+ $debug && print STDERR "DEBUG: flush completed at ".time()."\n";
+ }
+}
+
+close (SFLOWTOOL);
+
+# try to kill off sflowtool if it's not already dead
+kill 9, $sflowpid;
+
+# oops, we should never exit
+die "Oops, input pipe died. Aborting.\n";
+
+#
+# write traffic matrix out to RRD file while calculating totals
+#
+sub process_rrd {
+ my ($interval, $matrix, $rrdcached) = @_;
+ my ($aggregate, $rrdfile);
+
+ foreach my $ipprotocol (qw(4 6)) {
+ foreach my $rrdtype (qw(bytes pkts)) {
+ foreach my $vlan (keys %{$matrix->{vlilist}}) {
+ foreach my $srcvli (keys %{$matrix->{vlilist}->{$vlan}}) {
+ foreach my $dstvli (keys %{$matrix->{vlilist}->{$vlan}}) {
+ next if ($srcvli == $dstvli);
+
+ $rrdfile = sprintf("$basedir/ipv$ipprotocol/$rrdtype/p2p/src-%05d/p2p.ipv$ipprotocol.$rrdtype.src-%05d.dst-%05d.rrd", $srcvli, $srcvli, $dstvli);
+
+ # look up peer-to-peer traffic
+ my $in = $matrix->{p2p}->{$ipprotocol}->{$rrdtype}->{$dstvli}->{$srcvli};
+ my $out = $matrix->{p2p}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{$dstvli};
+
+ # this is too noisy for normal debugging.
+ $insanedebug && print STDERR "DEBUG: p2p: building update for ".
+ "protocol: $ipprotocol ".
+ "type: $rrdtype ".
+ "srcvli: $srcvli ".
+ "dstvli: $dstvli ".
+ "in: $in out: $out ".
+ "\n";
+ build_update_rrd ($rrdfile, $rrdtype, $ipprotocol, $in, $out, $interval, $rrdcached);
+ }
+
+ # Handle aggregate per-vli traffic
+ $rrdfile = sprintf("$basedir/ipv$ipprotocol/$rrdtype/individual/individual.ipv$ipprotocol.$rrdtype.src-%05d.rrd", $srcvli);
+
+ # these hashrefs are guaranteed to be defined due to matrix_init
+ my $in = $matrix->{individual}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{in};
+ my $out = $matrix->{individual}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{out};
+
+ # build a running total for the per-vlan traffic. these
+ # hashrefs are guaranteed to be defined due to matrix_init
+ $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{in} += $in;
+ $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{out} += $out;
+
+ $debug && print STDERR "DEBUG: individual: building update for ".
+ "vlan: $vlan ".
+ "type: $rrdtype ".
+ "protocol: $ipprotocol ".
+ "srcvli: $srcvli ".
+ "\n";
+ build_update_rrd ($rrdfile, $rrdtype, $ipprotocol, $in, $out, $interval, $rrdcached);
+ }
+
+ # write per-vlan aggregates out to rrd
+ $rrdfile = sprintf ("$basedir/ipv$ipprotocol/$rrdtype/aggregate/aggregate.ipv$ipprotocol.$rrdtype.vlan%05d.rrd", $vlan);
+ $debug && print STDERR "DEBUG: aggregate: building update for vlan: $vlan type: $rrdtype protocol: $ipprotocol file: $rrdfile\n";
+
+ my $in = $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{in};
+ my $out = $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{out};
+
+ build_update_rrd ($rrdfile, $rrdtype, $ipprotocol, $in, $out, $interval, $rrdcached);
+ }
+ }
+ }
+}
+
+sub build_update_rrd
+{
+ use File::Path qw(make_path);
+ use File::Basename;
+
+ my ($rrdfile, $rrdtype, $ipprotocol, $in, $out, $interval, $rrdcached) = @_;
+ my @rrds_options = ();
+ my $rrd_err;
+
+ $in = 0 if (!defined($in));
+ $out = 0 if (!defined($out));
+
+
+ if (!-s $rrdfile) {
+ my $dir = dirname($rrdfile);
+ if (!-d $dir) {
+ make_path($dir) or die "Could not make directory: $dir: $!\n";
+ }
+ my @rrds_create_options = (
+ 'DS:traffic_in:GAUGE:600:U:U',
+ 'DS:traffic_out:GAUGE:600:U:U',
+ 'RRA:AVERAGE:0.5:1:600', 'RRA:MAX:0.5:1:600',
+ 'RRA:AVERAGE:0.5:6:700', 'RRA:MAX:0.5:6:700',
+ 'RRA:AVERAGE:0.5:24:750', 'RRA:MAX:0.5:24:750',
+ 'RRA:AVERAGE:0.5:288:3650', 'RRA:MAX:0.5:288:3650',
+ );
+
+ RRDs::create ($rrdfile, @rrds_create_options);
+ $rrd_err = RRDs::error;
+ print STDERR "WARNING: while updating $rrdfile: $rrd_err\n" if $rrd_err;
+ }
+
+ if ($rrdcached) {
+ push @rrds_options, '--daemon', 'unix:/var/run/rrdcached.sock';
+ }
+
+ my $rrdvalues = "N:".int($in/$interval).":".int($out/$interval);
+ RRDs::update ($rrdfile, @rrds_options, $rrdvalues);
+
+ $rrd_err = RRDs::error;
+ print STDERR "WARNING: while updating $rrdfile: $rrd_err\n" if $rrd_err;
+}
+
+#
+# extract vli from mac table, given input from sflow
+#
+
+sub getvlifrommactable {
+ my ($mactable, $infraid, $vlan, $mac) = @_;
+
+ my @infras;
+
+ if (defined ($infraid)) {
+ @infras = qw ( $infraid );
+ } else {
+ @infras = keys %{$mactable};
+ }
+
+ foreach my $infra (@infras) {
+ # we need some sanitisation here because sflowtool could pass in anything
+ next unless (
+ defined ($mactable->{$infra}) &&
+ defined ($vlan) && defined ($mactable->{$infra}->{$vlan}) &&
+ defined ($mac) && defined ($mactable->{$infra}->{$vlan}->{$mac})
+ );
+
+ return $mactable->{$infra}->{$vlan}->{$mac};
+ }
+
+ return 0;
+}
+
+#
+# build up complete up-to-date blank matrix of all relevant traffic data
+#
+
+sub matrix_init
+{
+ my ($mactable, $infraid) = @_;
+ my (@infras, $matrix);
+
+ if (defined ($infraid)) {
+ @infras = qw ( $infraid );
+ } else {
+ @infras = keys %{$mactable};
+ }
+
+ # create list of all vlan interface IDs per vlan
+ foreach my $infra (@infras) {
+ next unless (defined ($mactable->{$infra}));
+ foreach my $vlan (keys %{$mactable->{$infra}}) {
+ foreach my $mac (keys %{$mactable->{$infra}->{$vlan}}) {
+ $matrix->{vlilist}->{$vlan}->{$mactable->{$infra}->{$vlan}->{$mac}} = 1;
+ }
+ }
+ }
+
+ # Not all srcvli/dstvli combinations are valid. We only consider
+ # those which are on the same vlan.
+
+ foreach my $ipprotocol (qw(4 6)) {
+ foreach my $rrdtype (qw(bytes pkts)) {
+ foreach my $vlan (keys %{$matrix->{vlilist}}) {
+ foreach my $srcvli (keys %{$matrix->{vlilist}->{$vlan}}) {
+ foreach my $dstvli (keys %{$matrix->{vlilist}->{$vlan}}) {
+ next if ($srcvli == $dstvli);
+ $matrix->{p2p}->{$ipprotocol}->{$rrdtype}->{$dstvli}->{$srcvli} = 0;
+ }
+ $matrix->{individual}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{in} = 0;
+ $matrix->{individual}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{out} = 0;
+ }
+ $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{in} = 0;
+ $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{out} = 0;
+ }
+ }
+ }
+
+ return $matrix;
+}
+
+
+#
+# Create a mapping from macaddress->virtualinterfaceid
+#
+
+sub reload_mactable
+{
+ my ($client, $uri) = @_;
+
+ $client->GET($uri);
+
+ my $content = $client->responseContent();
+ my $code = $client->responseCode();
+ if ($code != 200) {
+ $debug && print STDERR "WARNING: HTTP response $code: $content\n";
+ return undef;
+ }
+
+ my $json = eval { from_json($content) };
+ if (!$json) {
+ $debug && print STDERR "WARNING: HTTP response was not legitimate json: '$content'\n";
+ return undef;
+ }
+
+ return $json;
+}
diff --git a/tools/runtime/sflow/nfcap-to-rrd-handler b/tools/runtime/sflow/nfcap-to-rrd-handler
new file mode 100755
index 000000000..b3b7ca803
--- /dev/null
+++ b/tools/runtime/sflow/nfcap-to-rrd-handler
@@ -0,0 +1,383 @@
+#!/usr/bin/env perl
+#
+# nfcap-to-rrd-handler
+#
+# Copyright (C) 2009 - 2019 Internet Neutral Exchange Association Company Limited By Guarantee.
+# All Rights Reserved.
+#
+# This file is part of IXP Manager.
+#
+# IXP Manager is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, version v2.0 of the License.
+#
+# IXP Manager is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License v2.0
+# along with IXP Manager. If not, see:
+#
+# http://www.gnu.org/licenses/gpl-2.0.html
+#
+# Description:
+#
+# This script take the JSON output from nfdump, builds up a peer-to-peer
+# traffic matrix + aggregate stats and periodically writes the results out
+# to a stash of RRD files.
+#
+# TODO: Could be invoked as script by 'nfcapd' upon rotating capture file
+
+use warnings;
+use strict;
+use Getopt::Long;
+use Data::Dumper;
+use RRDs;
+use Time::HiRes qw(ualarm gettimeofday tv_interval);
+use REST::Client;
+
+use JSON qw(decode_json);
+
+use FindBin qw($Bin);
+use File::Spec;
+use lib File::Spec->catdir( $Bin, File::Spec->updir(), File::Spec->updir(), 'perl-lib', 'IXPManager', 'lib' );
+
+use IXPManager::Config;
+
+my $ixp = new IXPManager::Config (dbase_disable => 1); # (configfile => $configfile);
+
+my $debug = defined($ixp->{ixp}->{debug}) ? $ixp->{ixp}->{debug} : 0;
+my $insanedebug = 0;
+my $rrdcached = defined($ixp->{ixp}->{sflow_rrdcached}) ? $ixp->{ixp}->{sflow_rrdcached} : 1;
+my $sflowtool = defined($ixp->{ixp}->{sflowtool}) ? $ixp->{ixp}->{sflowtool} : '/usr/bin/sflowtool';
+my $sflowtool_opts = defined($ixp->{ixp}->{sflowtool_opts}) ? $ixp->{ixp}->{sflowtool_opts} : '-l';
+my $basedir = defined($ixp->{ixp}->{sflow_rrddir}) ? $ixp->{ixp}->{sflow_rrddir} : '/data/ixpmatrix';
+my $apikey = defined($ixp->{ixp}->{apikey}) ? $ixp->{ixp}->{apikey} : undef;
+my $apibaseurl = defined($ixp->{ixp}->{apibaseurl}) ? $ixp->{ixp}->{apibaseurl} : undef;
+
+my $interval = 60; # Sampling interval in seconds
+my $mactabletimeout = 86400;
+my $daemon = 1;
+my $infraid = undef;
+my $macdbtype = defined($ixp->{ixp}->{macdbtype}) ? $ixp->{ixp}->{macdbtype} : '';
+my $macdbrest;
+
+# conundrum: do we run GetOptions() before creating a new IXPManager::Config
+# object, which would allow us to set the configfile location on the command
+# line? Or do we do it after, which allows us to override the config file
+# arguments on the command line. first world problems.
+
+GetOptions(
+ 'debug!' => \$debug,
+ 'insanedebug!' => \$insanedebug,
+ 'daemon!' => \$daemon,
+ 'sflowtool=s' => \$sflowtool,
+ 'sflowtool_opts=s' => \$sflowtool_opts,
+ 'sflow_rrddir=s' => \$basedir,
+ 'interval=i' => \$interval,
+ 'infraid=i' => \$infraid,
+ 'macdbtype=s' => \$macdbtype,
+ 'apikey=s' => \$apikey,
+ 'apibaseurl=s' => \$apibaseurl,
+);
+
+if ($macdbtype eq 'configured') {
+ $macdbrest = '/sflow-db-mapper/configured-macs';
+} else {
+ $macdbrest = '/sflow-db-mapper/learned-macs';
+}
+
+if ($insanedebug) {
+ $debug = 1;
+}
+
+if (!defined ($apikey)) {
+ die "FATAL: must set 'apikey' parameter in section of ixpmanager.conf\n";
+}
+
+if (!defined ($apibaseurl)) {
+ die "FATAL: must set 'apibaseurl' parameter in section of ixpmanager.conf\n";
+}
+
+my $client = REST::Client->new();
+$client->setTimeout(5);
+$client->setHost($apibaseurl);
+$client->addHeader('X-IXP-Manager-API-Key', $apikey);
+
+my $mactable = reload_mactable($client, $macdbrest);
+$mactable || die "FATAL: could not read IXP Manager API call on $apibaseurl$macdbrest\n";
+my $matrix = matrix_init($mactable, $infraid);
+
+# Read JSON input from stdin
+my $json_input = do { local $/; };
+
+# Attempt to decode the JSON input
+my $flowrecords;
+eval {
+ $flowrecords = decode_json($json_input);
+};
+
+if ($@) {
+ die "Error decoding JSON: $@";
+}
+
+if (ref $flowrecords eq 'ARRAY') {
+ foreach my $flow (@{ $flowrecords }) {
+ my $srcmac = $flow->{ in_src_mac }; $srcmac =~ tr/://d; # Remove colons
+ my $dstmac = $flow->{ in_dst_mac }; $dstmac =~ tr/://d;
+ my $vlan = $flow->{ vlanID }; # or 'cust_vlanID' when using QinQ
+ my $pktsize = int( $flow->{ in_bytes } / $flow->{ in_packets });
+ my $samplerate = $flow->{ sampled }; # Sampling Packet Interval, i.e. 1 out of every X packets sampled
+ my $ipprotocol = int( $flow->{ ip_version } );
+
+ my $srcvli = getvlifrommactable ($mactable, $infraid, $vlan, $srcmac);
+ my $dstvli = getvlifrommactable ($mactable, $infraid, $vlan, $dstmac);
+
+ if ($srcvli && $dstvli && ($srcvli != $dstvli) ) { # if $ipprotocol && ...
+ $insanedebug && print STDERR "DEBUG: accepted update for: ".
+ "protocol: $ipprotocol ".
+ "vlan: $vlan ".
+ "srcvli: $srcvli ".
+ "dstvli: $dstvli ".
+ "pktsize: $pktsize ".
+ "samplerate: $samplerate ".
+ "\n";
+ $matrix->{p2p}->{$ipprotocol}->{bytes}->{$srcvli}->{$dstvli} += $pktsize * $samplerate;
+ $matrix->{p2p}->{$ipprotocol}->{pkts}->{$srcvli}->{$dstvli} += $samplerate;
+ $matrix->{individual}->{$ipprotocol}->{bytes}->{$srcvli}->{in} += $pktsize * $samplerate;
+ $matrix->{individual}->{$ipprotocol}->{bytes}->{$dstvli}->{out} += $pktsize * $samplerate;
+ $matrix->{individual}->{$ipprotocol}->{pkts}->{$srcvli}->{in} += $samplerate;
+ $matrix->{individual}->{$ipprotocol}->{pkts}->{$dstvli}->{out} += $samplerate;
+ } else {
+ $debug && print STDERR "DEBUG: VLAN(s) not found, dropped update for: ".
+ "protocol: $ipprotocol ".
+ "vlan: $vlan ".
+ "srcvli: $srcvli ".
+ "dstvli: $dstvli ".
+ "pktsize: $pktsize ".
+ "samplerate: $samplerate ".
+ "\n";
+ $debug && print STDERR "DEBUG: rejected: ".$_."\n";
+ }
+ }
+
+ $debug && print STDERR "DEBUG: starting rrd flush at time interval: $interval, time: ".time()."\n";
+ process_rrd($interval, $matrix, $rrdcached);
+} else {
+ print "No flows found in the JSON data.\n";
+}
+
+exit 0;
+
+#
+# write traffic matrix out to RRD file while calculating totals
+#
+sub process_rrd {
+ my ($interval, $matrix, $rrdcached) = @_;
+ my ($aggregate, $rrdfile);
+
+ foreach my $ipprotocol (qw(4 6)) {
+ foreach my $rrdtype (qw(bytes pkts)) {
+ foreach my $vlan (keys %{$matrix->{vlilist}}) {
+ foreach my $srcvli (keys %{$matrix->{vlilist}->{$vlan}}) {
+ foreach my $dstvli (keys %{$matrix->{vlilist}->{$vlan}}) {
+ next if ($srcvli == $dstvli);
+
+ $rrdfile = sprintf("$basedir/ipv$ipprotocol/$rrdtype/p2p/src-%05d/p2p.ipv$ipprotocol.$rrdtype.src-%05d.dst-%05d.rrd", $srcvli, $srcvli, $dstvli);
+
+ # look up peer-to-peer traffic
+ my $in = $matrix->{p2p}->{$ipprotocol}->{$rrdtype}->{$dstvli}->{$srcvli};
+ my $out = $matrix->{p2p}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{$dstvli};
+
+ # this is too noisy for normal debugging.
+ $insanedebug && print STDERR "DEBUG: p2p: building update for ".
+ "protocol: $ipprotocol ".
+ "type: $rrdtype ".
+ "srcvli: $srcvli ".
+ "dstvli: $dstvli ".
+ "in: $in out: $out ".
+ "\n";
+ build_update_rrd ($rrdfile, $rrdtype, $ipprotocol, $in, $out, $interval, $rrdcached);
+ }
+
+ # Handle aggregate per-vli traffic
+ $rrdfile = sprintf("$basedir/ipv$ipprotocol/$rrdtype/individual/individual.ipv$ipprotocol.$rrdtype.src-%05d.rrd", $srcvli);
+
+ # these hashrefs are guaranteed to be defined due to matrix_init
+ my $in = $matrix->{individual}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{in};
+ my $out = $matrix->{individual}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{out};
+
+ # build a running total for the per-vlan traffic. these
+ # hashrefs are guaranteed to be defined due to matrix_init
+ $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{in} += $in;
+ $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{out} += $out;
+
+ $debug && print STDERR "DEBUG: individual: building update for ".
+ "vlan: $vlan ".
+ "type: $rrdtype ".
+ "protocol: $ipprotocol ".
+ "srcvli: $srcvli ".
+ "\n";
+ build_update_rrd ($rrdfile, $rrdtype, $ipprotocol, $in, $out, $interval, $rrdcached);
+ }
+
+ # write per-vlan aggregates out to rrd
+ $rrdfile = sprintf ("$basedir/ipv$ipprotocol/$rrdtype/aggregate/aggregate.ipv$ipprotocol.$rrdtype.vlan%05d.rrd", $vlan);
+ $debug && print STDERR "DEBUG: aggregate: building update for vlan: $vlan type: $rrdtype protocol: $ipprotocol file: $rrdfile\n";
+
+ my $in = $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{in};
+ my $out = $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{out};
+
+ build_update_rrd ($rrdfile, $rrdtype, $ipprotocol, $in, $out, $interval, $rrdcached);
+ }
+ }
+ }
+}
+
+sub build_update_rrd
+{
+ use File::Path qw(make_path);
+ use File::Basename;
+
+ my ($rrdfile, $rrdtype, $ipprotocol, $in, $out, $interval, $rrdcached) = @_;
+ my @rrds_options = ();
+ my $rrd_err;
+
+ $in = 0 if (!defined($in));
+ $out = 0 if (!defined($out));
+
+
+ if (!-s $rrdfile) {
+ my $dir = dirname($rrdfile);
+ if (!-d $dir) {
+ make_path($dir) or die "Could not make directory: $dir: $!\n";
+ }
+ my @rrds_create_options = (
+ 'DS:traffic_in:GAUGE:600:U:U',
+ 'DS:traffic_out:GAUGE:600:U:U',
+ 'RRA:AVERAGE:0.5:1:600', 'RRA:MAX:0.5:1:600',
+ 'RRA:AVERAGE:0.5:6:700', 'RRA:MAX:0.5:6:700',
+ 'RRA:AVERAGE:0.5:24:750', 'RRA:MAX:0.5:24:750',
+ 'RRA:AVERAGE:0.5:288:3650', 'RRA:MAX:0.5:288:3650',
+ );
+
+ RRDs::create ($rrdfile, @rrds_create_options);
+ $rrd_err = RRDs::error;
+ print STDERR "WARNING: while updating $rrdfile: $rrd_err\n" if $rrd_err;
+ }
+
+ if ($rrdcached) {
+ push @rrds_options, '--daemon', 'unix:/var/run/rrdcached.sock';
+ }
+
+ my $rrdvalues = "N:".int($in/$interval).":".int($out/$interval);
+ RRDs::update ($rrdfile, @rrds_options, $rrdvalues);
+
+ $rrd_err = RRDs::error;
+ print STDERR "WARNING: while updating $rrdfile: $rrd_err\n" if $rrd_err;
+}
+
+#
+# extract vli from mac table, given input from sflow
+#
+
+sub getvlifrommactable {
+ my ($mactable, $infraid, $vlan, $mac) = @_;
+
+ my @infras;
+
+ if (defined ($infraid)) {
+ @infras = qw ( $infraid );
+ } else {
+ @infras = keys %{$mactable};
+ }
+
+ foreach my $infra (@infras) {
+ # we need some sanitisation here because sflowtool could pass in anything
+ next unless (
+ defined ($mactable->{$infra}) &&
+ defined ($vlan) && defined ($mactable->{$infra}->{$vlan}) &&
+ defined ($mac) && defined ($mactable->{$infra}->{$vlan}->{$mac})
+ );
+
+ return $mactable->{$infra}->{$vlan}->{$mac};
+ }
+
+ return 0;
+}
+
+#
+# build up complete up-to-date blank matrix of all relevant traffic data
+#
+
+sub matrix_init
+{
+ my ($mactable, $infraid) = @_;
+ my (@infras, $matrix);
+
+ if (defined ($infraid)) {
+ @infras = qw ( $infraid );
+ } else {
+ @infras = keys %{$mactable};
+ }
+
+ # create list of all vlan interface IDs per vlan
+ foreach my $infra (@infras) {
+ next unless (defined ($mactable->{$infra}));
+ foreach my $vlan (keys %{$mactable->{$infra}}) {
+ foreach my $mac (keys %{$mactable->{$infra}->{$vlan}}) {
+ $matrix->{vlilist}->{$vlan}->{$mactable->{$infra}->{$vlan}->{$mac}} = 1;
+ }
+ }
+ }
+
+ # Not all srcvli/dstvli combinations are valid. We only consider
+ # those which are on the same vlan.
+
+ foreach my $ipprotocol (qw(4 6)) {
+ foreach my $rrdtype (qw(bytes pkts)) {
+ foreach my $vlan (keys %{$matrix->{vlilist}}) {
+ foreach my $srcvli (keys %{$matrix->{vlilist}->{$vlan}}) {
+ foreach my $dstvli (keys %{$matrix->{vlilist}->{$vlan}}) {
+ next if ($srcvli == $dstvli);
+ $matrix->{p2p}->{$ipprotocol}->{$rrdtype}->{$dstvli}->{$srcvli} = 0;
+ }
+ $matrix->{individual}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{in} = 0;
+ $matrix->{individual}->{$ipprotocol}->{$rrdtype}->{$srcvli}->{out} = 0;
+ }
+ $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{in} = 0;
+ $matrix->{aggregate}->{$ipprotocol}->{$rrdtype}->{$vlan}->{out} = 0;
+ }
+ }
+ }
+
+ return $matrix;
+}
+
+
+#
+# Create a mapping from macaddress->virtualinterfaceid
+#
+
+sub reload_mactable
+{
+ my ($client, $uri) = @_;
+
+ $client->GET($uri);
+
+ my $content = $client->responseContent();
+ my $code = $client->responseCode();
+ if ($code != 200) {
+ $debug && print STDERR "WARNING: HTTP response $code: $content\n";
+ return undef;
+ }
+
+ my $json = eval { decode_json($content) };
+ if (!$json) {
+ $debug && print STDERR "WARNING: HTTP response was not legitimate json: '$content'\n";
+ return undef;
+ }
+
+ return $json;
+}
diff --git a/tools/runtime/sflow/sflow_bgp_handler b/tools/runtime/sflow/sflow_bgp_handler
index 793d03ea4..28fec0eeb 100755
--- a/tools/runtime/sflow/sflow_bgp_handler
+++ b/tools/runtime/sflow/sflow_bgp_handler
@@ -25,19 +25,22 @@
# Add the following line to /etc/rc.conf to enable sflow_bgp_handler:
# sflow_bgp_handler_enable="YES"
# sflow_bgp_handler_flags=""
+#
+# To use goflow2 instead of sflowtool:
+# sflow_bgp_handler_command="/usr/local/bin/goflow2-detect-ixp-bgp-sessions"
. /etc/rc.subr
name="sflow_bgp_handler"
rcvar=sflow_bgp_handler_enable
-command="/usr/local/bin/sflow-detect-ixp-bgp-sessions"
load_rc_config $name
# Set defaults
: ${sflow_bgp_handler_enable="NO"}
+: ${sflow_bgp_handler_command="/usr/local/bin/sflow-detect-ixp-bgp-sessions"}
: ${sflow_bgp_handler_pidfile="/var/run/${name}.pid"}
-start_cmd="/usr/sbin/daemon -f -p ${sflow_bgp_handler_pidfile} ${command} ${sflow_bgp_handler_flags}"
+start_cmd="/usr/sbin/daemon -f -p ${sflow_bgp_handler_pidfile} ${sflow_bgp_handler_command} ${sflow_bgp_handler_flags}"
start_postcmd="echo Starting ${name}."
run_rc_command "$1"
diff --git a/tools/runtime/sflow/sflow_rrd_handler b/tools/runtime/sflow/sflow_rrd_handler
index 6ca3b737c..b7b196da6 100755
--- a/tools/runtime/sflow/sflow_rrd_handler
+++ b/tools/runtime/sflow/sflow_rrd_handler
@@ -26,18 +26,21 @@
# sflow_rrd_handler_enable="YES"
# sflow_rrd_handler_flags=""
+# To use goflow2 instead of sflowtool:
+# sflow_rrd_handler_command="/usr/local/bin/goflow2-to-rrd-handler"
+
. /etc/rc.subr
name="sflow_rrd_handler"
rcvar=sflow_rrd_handler_enable
-command="/usr/local/bin/sflow-to-rrd-handler"
load_rc_config $name
# Set defaults
: ${sflow_rrd_handler_enable="NO"}
: ${sflow_rrd_handler_pidfile="/var/run/${name}.pid"}
+: ${sflow_rrd_handler_command="/usr/local/bin/sflow-to-rrd-handler"}
-start_cmd="/usr/sbin/daemon -f -p ${sflow_rrd_handler_pidfile} ${command} ${sflow_rrd_handler_flags}"
+start_cmd="/usr/sbin/daemon -f -p ${sflow_rrd_handler_pidfile} ${sflow_rrd_handler_command} ${sflow_rrd_handler_flags}"
start_postcmd="echo Starting ${name}."
run_rc_command "$1"