Skip to content

Commit 7ddd9bd

Browse files
brymon68brymon68
committed
Merge branch 'master' of baltig.sandia.gov:scot/SCOT
2 parents be52307 + 48a428e commit 7ddd9bd

File tree

21 files changed

+523
-187283
lines changed

21 files changed

+523
-187283
lines changed

‎bin/david.pl‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/usr/bin/env perl
2+
3+
use strict;
4+
use warnings;
5+
use v5.18;
6+
7+
use lib '../lib';
8+
use Scot::Env;
9+
use Scot::App::Report;
10+
use Data::Dumper;
11+
12+
my $config = $ENV{'scot_monthly_report_config_file'} //
13+
'/opt/scot/etc/david.cfg.pl';
14+
15+
my $env = Scot::Env->new( config_file => $config );
16+
my $app = Scot::App::Report->new( env => $env );
17+
18+
my $href = $app->alertgroup_counts;
19+
20+
say "Alertgroups";
21+
say "date\tcount";
22+
foreach my $date (sort keys %$href) {
23+
say "$date\t".$href->{$date};
24+
}
25+
26+
$href = $app->event_counts;
27+
say "\nEvents";
28+
say "date\tcount";
29+
foreach my $date (sort keys %$href) {
30+
say "$date\t".$href->{$date};
31+
}
32+
33+
$href = $app->incident_counts;
34+
say "\nIncidents";
35+
say "date\tcount";
36+
foreach my $date (sort keys %$href) {
37+
say "$date\t".$href->{$date};
38+
}
39+
40+
$href = $app->response_times;
41+
say "\nResponse Times";
42+
printf "%8s\t%8s\t%9s\t%20s\n","date","category","avg","human readable avg";
43+
foreach my $date (sort keys %$href) {
44+
foreach my $category (sort keys %{$href->{$date}}) {
45+
printf "%8s\t%8s\t%6.2f\t%20s\n", $date, $category, $href->{$date}->{$category}->{avg}, $href->{$date}->{$category}->{humanavg} // '';
46+
}
47+
print "\n";
48+
}
49+

‎bin/reports.pl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ =head1 PROGRAM ARGUMENTS
7171
7272
=cut
7373

74-
my $config_file = $ENV{'scot_reports_config_file'} // '/opt/scot/etc/reports.cfg.pl';
74+
my $config_file = $ENV{'scot_reports_config_file'} // '/opt/scot/etc/scot.cfg.pl';
7575

7676
my $env = Scot::Env->new(
7777
config_file => $config_file,
@@ -414,7 +414,7 @@ sub event_report {
414414
print "Events By Month\n";
415415
print "Date, Events, Views, Counts, Score\n";
416416
foreach my $y (sort keys %results) {
417-
foreach my $m (sort keys $results{$y}) {
417+
foreach my $m (sort keys %{$results{$y}}) {
418418

419419
printf "%d-%d, %d, %d, %d, %d\n",
420420
$y,

‎install/install_mongodb.sh‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,18 +219,39 @@ function configure_for_scot {
219219
initialize_database
220220
}
221221

222+
function install_36_mongo {
223+
MONGO_KEY_SERVER="--keyserver hkp://keyserver.ubuntu.com:80"
224+
MONGO_RECV="--recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5"
225+
MONGO_PROXY=""
226+
227+
if [[ -v $http_proxy ]]; then
228+
MONGO_PROXY="--keyserver-options http-proxy=$http_proxy"
229+
fi
230+
231+
apt-key adv $MONGO_PROXY $MONGO_KEY_SERVER $MONGO_RECV
232+
233+
if [[ $OSVERSION == "16" ]]; then
234+
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list
235+
else
236+
echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list
237+
fi
238+
239+
apt-get update
240+
apt-get install -y mongodb-org
241+
}
242+
243+
222244
function install_mongodb {
223245

224246
echo "---"
225247
echo "--- Installing Mongodb "
226248
echo "---"
227249

228-
ensure_mongo_repo
229250

230251
if [[ $OS == "Ubuntu" ]]; then
231-
apt-get-update
232-
apt-get install -y mongodb-org
252+
install_36_mongo
233253
else
254+
ensure_mongo_repo
234255
yum install mongodb-org -y
235256
fi
236257

‎install/src/mongodb/mongod.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ storage:
1010
dbPath: /var/lib/mongodb
1111
journal:
1212
enabled: true
13-
engine: "wiredTiger"
13+
engine: wiredTiger
1414

1515
# where to write logging data.
1616
systemLog:

‎install/src/scot/scot.cfg.pl‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,11 @@
8484
disable_filters => [],
8585
},
8686

87+
# this file helps scot determine valid domain "entities"
88+
# keep up to date, by creating a root cron job that does the following:
89+
# @daily (cd /opt/scot/etc; export https_proxy=yourproxy.com; wget -q -N https://publicsuffix.org/list/public_suffix_list.dat)
90+
mozilla_public_suffix_file => '/opt/scot/etc/public_suffix_list.dat',
91+
8792
# modules to instantiate at Env.pm startup. will be done in
8893
# order of the array
8994
modules => [

‎lib/Scot/App/Report.pm‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
package Scot::App::Report;
2+
3+
use lib '../../../lib';
4+
use strict;
5+
use warnings;
6+
use v5.18;
7+
8+
use DateTime;
9+
use Time::Duration;
10+
use Data::Dumper;
11+
12+
use Moose;
13+
extends 'Scot::App';
14+
15+
has months_ago => (
16+
is => 'ro',
17+
isa => 'Int',
18+
required => 1,
19+
default => 6,
20+
);
21+
22+
has nowdt => (
23+
is => 'rw',
24+
isa => 'DateTime',
25+
lazy => 1,
26+
required => 1,
27+
default => sub { DateTime->now; },
28+
);
29+
30+
has thendt => (
31+
is => 'rw',
32+
isa => 'DateTime',
33+
required => 1,
34+
lazy => 1,
35+
builder => '_build_thendt',
36+
);
37+
38+
sub _build_thendt {
39+
my $self = shift;
40+
my $nowdt = $self->nowdt;
41+
my $thendt = $nowdt->clone();
42+
$thendt->subtract( months => $self->months_ago );
43+
return $thendt;
44+
}
45+
46+
sub alertgroup_counts {
47+
my $self = shift;
48+
my $env = $self->env;
49+
my $mongo = $env->mongo;
50+
my $log = $env->log;
51+
my $nowdt = $self->nowdt;
52+
my $thendt = $self->thendt;
53+
54+
$log->debug("counting alertgroups");
55+
56+
my $query = {
57+
metric => 'alertgroups created',
58+
epoch => {
59+
'$gte' => $thendt->epoch,
60+
'$lte' => $nowdt->epoch,
61+
},
62+
};
63+
64+
$log->debug("query is ",{filter=>\&Dumper, value=>$query});
65+
66+
my $cursor = $mongo->collection('Stat')->find($query);
67+
my %result;
68+
69+
while ( my $stat = $cursor->next ) {
70+
my $key = $stat->year . '/' . $stat->month;
71+
$result{$key} += $stat->value;
72+
}
73+
return wantarray ? %result : \%result;
74+
}
75+
76+
sub event_counts {
77+
my $self = shift;
78+
my $env = $self->env;
79+
my $mongo = $env->mongo;
80+
my $log = $env->log;
81+
my $nowdt = $self->nowdt;
82+
my $thendt = $self->thendt;
83+
84+
$log->debug("counting events");
85+
86+
my $query = {
87+
metric => 'event created',
88+
epoch => {
89+
'$gte' => $thendt->epoch,
90+
'$lte' => $nowdt->epoch,
91+
},
92+
};
93+
94+
$log->debug("query is ",{filter=>\&Dumper, value=>$query});
95+
96+
my $cursor = $mongo->collection('Stat')->find($query);
97+
my %result;
98+
99+
while ( my $stat = $cursor->next ) {
100+
my $key = $stat->year . '/' . $stat->month;
101+
$result{$key} += $stat->value;
102+
}
103+
return wantarray ? %result : \%result;
104+
}
105+
106+
sub incident_counts {
107+
my $self = shift;
108+
my $env = $self->env;
109+
my $mongo = $env->mongo;
110+
my $log = $env->log;
111+
my $nowdt = $self->nowdt;
112+
my $thendt = $self->thendt;
113+
114+
$log->debug("counting incidents");
115+
116+
my $query = {
117+
metric => 'incident created',
118+
epoch => {
119+
'$gte' => $thendt->epoch,
120+
'$lte' => $nowdt->epoch,
121+
},
122+
};
123+
124+
$log->debug("query is ",{filter=>\&Dumper, value=>$query});
125+
126+
my $cursor = $mongo->collection('Stat')->find($query);
127+
my %result;
128+
129+
while ( my $stat = $cursor->next ) {
130+
my $key = $stat->year . '/' . $stat->month;
131+
$result{$key} += $stat->value;
132+
}
133+
return wantarray ? %result : \%result;
134+
}
135+
136+
sub response_times {
137+
my $self = shift;
138+
my $env = $self->env;;
139+
my $mongo = $env->mongo;
140+
my $log = $env->log;
141+
my $nowdt = $self->nowdt;
142+
my $thendt = $self->thendt;
143+
144+
my $query = {
145+
metric => qr/Alertgroup Response Times/i,
146+
epoch => {
147+
'$gte' => $thendt->epoch,
148+
'$lte' => $nowdt->epoch,
149+
},
150+
};
151+
152+
my $cursor = $mongo->collection('Stat')->find($query);
153+
my %result;
154+
155+
while ( my $stat = $cursor->next ) {
156+
my $key = $stat->year . '/' . $stat->month;
157+
my $metric = $stat->metric;
158+
my @words = split(/ /,$metric);
159+
my $type = lc($words[0]);
160+
my $category = lc($words[2]);
161+
my $value = $stat->value;
162+
$result{$key}{$category}{$type} += $value;
163+
}
164+
165+
foreach my $key (keys %result) {
166+
$log->debug("$key");
167+
foreach my $category (keys %{$result{$key}}) {
168+
$log->debug(" $category");
169+
my $seconds = $result{$key}{$category}{sum};
170+
my $count = $result{$key}{$category}{count};
171+
$log->debug({filter=>\&Dumper, value=> $result{$key}{$category}});
172+
if ( !defined $count or $count == 0 ) {
173+
$log->error("count zero or undef!");
174+
if ($seconds == 0) {
175+
$result{$key}{$category}{avg} = 0;
176+
}
177+
else {
178+
$result{$key}{$category}{avg} = 'na';
179+
}
180+
next;
181+
}
182+
183+
$result{$key}{$category}{avg} = $seconds / $count;
184+
$result{$key}{$category}{humanavg} = duration($seconds / $count);
185+
}
186+
}
187+
188+
return wantarray ? %result : \%result;
189+
}
190+
191+
192+
193+
1;

‎lib/Scot/Collection.pm‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -625,8 +625,8 @@ sub api_find {
625625
if ( $href->{collection} eq "entity" ) {
626626
$log->debug("finding an entity");
627627
if ( $href->{id} eq "byname" ) {
628-
$log->debug("byname, please");
629628
my $name = $href->{request}->{params}->{name};
629+
$log->debug("byname $name, please");
630630
my $obj = $self->find_one({ value => $name });
631631
return $obj;
632632
}

‎lib/Scot/Collection/Entity.pm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ sub api_subthing {
185185
if ( $subthing eq "alert" or
186186
$subthing eq "event" or
187187
$subthing eq "intel" or
188+
$subthing eq "guide" or
188189
$subthing eq "signature" or
189190
$subthing eq "incident" ) {
190191
return $mongo->collection('Link')

‎lib/Scot/Controller/Api.pm‎

Lines changed: 8 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -958,31 +958,14 @@ sub promote {
958958
}
959959
});
960960

961-
# disabling until we can test
962-
# my $lbmem = $env->get_config_item('lb_memorialization');
963-
# if ( defined $lbmem ) {
964-
# XXX
965-
# my ($send_url,
966-
# $memuser,
967-
# $mempass ) = $lbmem->($object);
968-
# my $ua = Mojo::UserAgent->new();
969-
# # authenticate to the webservice
970-
# $ua->on(start => sub {
971-
# my $ua = shift;
972-
# my $tx = shift;
973-
# $tx->req->headers->header(
974-
# 'Authorization' => $memuser . ":" . $mempass
975-
# );
976-
# });
977-
# my $tx = $ua->post($send_url);
978-
# my $response = $tx->success;
979-
# if ( defined $response ) {
980-
# $log->debug("lb_memorialization: ",{filter=>&Dumper, value=> $response->json});
981-
# }
982-
# else {
983-
## $log->error("error in lb_memorialization: ".$tx->result->message);
984-
# }
985-
# }
961+
## use this to memorialize the likaboss object
962+
if ( $env->meta->has_attribute('lbwebservice') ) {
963+
foreach my $rootuid (@{$object->data->{rootUID}}) {
964+
$log->debug("memorializing $rootuid");
965+
$self->env->lbwebservice->memorialize($rootuid);
966+
}
967+
}
968+
986969
}
987970

988971
# update the promotee

‎lib/Scot/Env.pm‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ sub BUILD {
122122
my $log = $self->build_logger($self->log_config);
123123
$meta->add_attribute( log => ( is => 'rw', isa => 'Log::Log4perl::Logger'));
124124
$self->log($log);
125+
126+
$log->debug("\@INC = ",{filter=>\&Dumper, value => \@INC});
125127

126128
# now build the modules
127129
foreach my $href (@{ $modules_aref }) {

0 commit comments

Comments
 (0)