Better tables, search fixes, host lists, etc.

monitor/job.py

@@ -284,37 +284,39 @@ def interpret_amd64_pmc_cpu(self):
         ['UserCycles', 'DCacheSysFills', 'HTlink1Use', 'SSEFLOPS'],
         ['UserCycles', 'DCacheSysFills', 'SSEFLOPS', 'HTlink2Use']]
 
-	cpus = self.stats['amd64_pmc']
+        cpus = self.stats['amd64_pmc']
 
         CNTRS = dict()
 
         for core, stats in cpus.iteritems():
-             core_str = "core" + core
-	     core = int(core)
-
-             SSEFLOPS_CNTR = EVENTS[core % 4].index('SSEFLOPS') + 4
-             UserCycles_CNTR = EVENTS[core % 4].index('UserCycles') + 4
-             DCacheSysFills_CNTR = EVENTS[core % 4].index('DCacheSysFills') + 4             
-             core_cntrs = dict()
-             core_cntrs['SSEFLOPS'] = stats[ : , SSEFLOPS_CNTR]
-             core_cntrs['UserCycles'] = stats[ :, UserCycles_CNTR]
-             core_cntrs['DCacheSysFills'] = stats[ :, DCacheSysFills_CNTR]
-
-             CNTRS[core_str] = core_cntrs
-
-             #soc_index returns the index in the EVENTS array of the element
-             # which has not been accessed
-             soc_index = 4*3 + 6 - SSEFLOPS_CNTR - UserCycles_CNTR - DCacheSysFills_CNTR
-             soc_field = EVENTS[core % 4][soc_index]
-             soc_str = "soc" + repr(core / 4)
-
-             soc_cntrs = dict()
-	     if CNTRS.has_key(soc_str):
-                 soc_cntrs = CNTRS[soc_str]
-
-             soc_cntrs[soc_field] = stats[ :, soc_index + 4]
-
-             CNTRS[soc_str] = soc_cntrs
+            core_str = "core" + core
+            core = int(core)
+
+            SSEFLOPS_CNTR = EVENTS[core % 4].index('SSEFLOPS') + 4
+            UserCycles_CNTR = EVENTS[core % 4].index('UserCycles') + 4
+            DCacheSysFills_CNTR = EVENTS[core % 4].index('DCacheSysFills') + 4             
+            core_cntrs = dict()
+            core_cntrs['SSEFLOPS'] = stats[ : , SSEFLOPS_CNTR]
+            core_cntrs['UserCycles'] = stats[ :, UserCycles_CNTR]
+            core_cntrs['DCacheSysFills'] = stats[ :, DCacheSysFills_CNTR]
+
+            CNTRS[core_str] = core_cntrs
+
+            #soc_index returns the index in the EVENTS array of the element
+            # which has not been accessed
+            soc_index = 4*3 + 6 - SSEFLOPS_CNTR - UserCycles_CNTR - DCacheSysFills_CNTR
+            soc_field = EVENTS[core % 4][soc_index]
+            soc_str = "soc" + repr(core / 4)
+
+            soc_cntrs = dict()
+
+
+            if CNTRS.has_key(soc_str):
+                soc_cntrs = CNTRS[soc_str]
+
+            soc_cntrs[soc_field] = stats[ :, soc_index + 4]
+
+            CNTRS[soc_str] = soc_cntrs
 
 
         return CNTRS

tacc_stats_web/apps/tacc_stats/build_database.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+"""build_database.py
+
+Synopsis
+--------
+    build_database.py [system_name] [archive_path]
+
+Description
+-----------
+    Builds the database from archive directory
+
+Environment
+-----------
+    To access the database, django needs to know the settings module. Use the
+    environment variable DJANGO_SETTINGS_MODULE to name the module. The module
+    needs to be in the python path, for example assuming this file is in
+    /var/www/django_sites.
+
+    $ export PYTHONPATH=<path_to_tacc_stats_web>:<path_to_tacc_stats_web/apps>:$PYTHONPATH
+    $ export DJANGO_SETTINGS_MODULE='tacc_stats_web.settings'
+
+Commands
+--------
+    build <system_name> <archive_path> :
+        Builds the database from job archive
+        $ ./build_database.py ranger.tacc.utexas.edu /home/tacc_stats/sample-jobs
+
+    flush:
+       Clears the database to be cleaned for next usage
+
+"""
+from django.conf import settings
+from django.core.management import call_command
+import os
+import shelve
+import sys
+
+sys.path.insert(0, '/home/aterrel/workspace/tacc_stats/monitor')
+
+from tacc_stats.models import Node, System, User
+from tacc_stats.models import Job as tsm_Job
+
+def get_job_shelf(archive_path):
+    """Returns the on-disk python job monitor database"""
+    return shelve.open(os.path.join(archive_path, 'jobs'))
+
+def get_system(system_name):
+    """Returns system, adding it if system if not in db."""
+    systems = System.objects.filter(name=system_name)
+    if len(systems) == 0:
+        print "Adding system: %s" % system_name
+        system = System(name=system_name)
+        system.save()
+    else:
+        system = systems[0]
+    return system
+
+def get_node(system, node_name):
+    """Returns node, if it doesn't exist it is created"""
+    node_name = strip_system_name(system.name, node_name)
+    nodes = system.node_set.filter(name=node_name)
+    if len(nodes) == 0:
+        print "Adding node: %s" % node_name
+        node = system.node_set.create(name=node_name)
+    else:
+        node = nodes[0]
+    return node
+
+def get_user(user_name, system):
+    """Returns the user cooresponding to the user_name, if it doesn't exist it is created"""
+    users = User.objects.filter(user_name=user_name)
+    if len(users) == 0:
+        print "Adding user:", user_name
+        user = User(user_name = user_name)
+        user.save()
+    else:
+        user = users[0]
+    user.systems.add(system)
+    return user
+
+def strip_system_name(system_name, node):
+    """Removes the system name from a node"""
+    if system_name in node:
+        end = node.find(system_name)
+        end = end - 1 if node[end-1] == '.' else end
+        node = node[:end]
+    return node
+
+def add_Job(system, a_job):
+    if system.job_set.filter(acct_id=int(a_job.id)):
+        print "Job %s already exists" % a_job.id
+        return
+    owner = get_user(a_job.acct['account'], system)
+
+    job_dict = {
+        'system': system,
+        'acct_id': a_job.id,
+        'owner': owner,
+        'begin': a_job.start_time,
+        'end': a_job.end_time,
+        'nr_slots': a_job.acct['slots'],
+        'pe': a_job.acct['granted_pe'],
+        'failed': a_job.acct['failed'],
+        'exit_status': a_job.acct['exit_status'],
+    }
+    #newJob.nr_hosts = len(a_job.hosts)
+    newJob = tsm_Job(**job_dict)
+    newJob.save()
+    hosts = map(lambda node: get_node(system, node), a_job.hosts.keys())
+    newJob.hosts = hosts
+
+    print "Added job:", a_job.id
+
+def process_job_archive(system_name, archive_path):
+    """Given a system name and the job archive, builds the django database"""
+    system = get_system(system_name)
+    job_shelf = get_job_shelf(archive_path)
+    for a_job in job_shelf.values():
+        add_Job(system, a_job)
+
+def build_database(system_name, archive_dir, clean=False):
+    if clean:
+        clean_database()
+    call_command('syncdb')
+    process_job_archive(system_name, archive_dir)
+
+def clean_database():
+    print "cleaning db"
+    call_command('flush')
+
+def print_usage_and_exit():
+    print "Invalid usage:"
+    print __doc__
+    sys.exit(1)
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print_usage_and_exit()
+    elif sys.argv[1] == 'flush':
+        clean_database()
+    elif sys.argv[1] == 'build':
+        if len(sys.argv) != 4:
+            print_usage_and_exit()
+        build_database(*sys.argv[2:4])
+    else:
+        print_usage_and_exit()

tacc_stats_web/apps/tacc_stats/forms.py

@@ -1,14 +1,40 @@
 from django import forms
 from django.forms import ModelForm
 from models import Job
+from django.core.validators import RegexValidator
+
+CHOICES = (
+    ('acct_id', 'ID'),
+    ('owner', 'Owner'),
+    ('nr_slots', '# of Hosts'),
+    ('begin', 'Start Time'),
+    ('end', 'End Time'),
+    ('mem_MemUsed', 'Memory Used'),
+    ('llite_open_work', 'Open Work'),
+    ('cpu_irq', 'IRQ'),
+   # ('timespent', 'Timespent'),
+)
+
+def validate_date(value):
+    if 0:
+        raise ValidationError('poop')
+
+class CalendarWidget(forms.TextInput):
+    class Media:
+        css = {
+            'all': ('anytime.css')
+        }
+        js = ('anytime.js', 'jquery.min.js')
 
 class SearchForm(ModelForm):
-    begin = forms.DateTimeField()
-    end = forms.DateTimeField()
+    begin = forms.DateTimeField(widget=CalendarWidget)
+    end = forms.DateTimeField(widget=CalendarWidget)
+#    sort = forms.ChoiceField(choices=CHOICES)
+    host = forms.CharField(max_length=100)
 
     class Meta:
         model = Job
-        fields = ('owner', 'begin', 'end', 'acct_id')
+        fields = ('owner', 'acct_id')
 #        widgets = {
 #            'begin': forms.DateTimeField(),
 #        }
@@ -18,4 +44,3 @@ def __init__(self, *args, **kwargs):
 
         for key in self.fields:
             self.fields[key].required = False
-

tacc_stats_web/apps/tacc_stats/models.py

@@ -3,13 +3,23 @@
 from django.db import models
 import time
 import math
+import datetime
+import dateutil
 
 COLORS = { 
-    'Normal' : "background-color: rgba(0%, 0%, 100%, .2);",
-    'High Files Open' : "background-color: rgba(100%, 0%, 0%, .2);",
-    'High Memory Used' : "background-color: rgba(80%, 30%, 0%, .2)",
-    'High Runtime' : "background-color: rgba(0%, 100%, 0%, .2)",
-    'High Idle' : "background-color: rgba(50%, 0%, 50%, .2)"
+    'Normal' : "rgba(0%, 0%, 100%, .2)",
+    'High Files Open' : "rgba(100%, 0%, 0%, .2)",
+    'High Memory Used' : "rgba(80%, 30%, 0%, .2)",
+    'High Runtime' : "rgba(0%, 100%, 0%, .2)",
+    'High Idle' : "rgba(50%, 0%, 50%, .2)"
+}
+
+COLOR_LEGEND = {
+    'Normal' : "Default Job Color",
+    'High Files Open' : "Greater than 30000 files opened",
+    'High Memory Used' : "Greater than 30 GB Used",
+    'High Runtime' : "Greater than 10 Hour",
+    'High Idle' : "Greater than 10% Idle"
 }
 
 class System(models.Model):
@@ -57,6 +67,15 @@ class Job(models.Model):
     pe = models.CharField(max_length=8, null=True)
     failed = models.BooleanField()
     exit_status = models.IntegerField(null=True)
+
+    avg_flops = models.FloatField(null=True)
+    flops_25 = models.IntegerField(null=True)
+    flops_50 = models.IntegerField(null=True)
+    flops_75 = models.IntegerField(null=True)
+
+    mem_25 = models.IntegerField(null=True)
+    mem_50 = models.IntegerField(null=True)
+    mem_75 = models.IntegerField(null=True)
 
     amd64_pmc_CTL0 = models.BigIntegerField(null=True)
     amd64_pmc_CTL1 = models.BigIntegerField(null=True)
@@ -269,17 +288,22 @@ def color(self):
         Returns the color of the job row as a css style field
         """
         ret_val = COLORS['Normal']
-        if self.llite_open_work > 3000:
+        if self.llite_open_work > 30000:
             ret_val = COLORS['High Files Open']
         elif self.mem_MemUsed > 30*2**30:
             ret_val = COLORS['High Memory Used']
-        elif self.runtime > 3000:
+        elif self.runtime > 36000:
             ret_val = COLORS['High Runtime']
+        elif self.cpu_idle * 10 > self.cpu_user:
+            ret_val = COLORS['High Idle']
         return ret_val
 
     def timespent(self):
         """ Returns the runtime of the job in a readable format """
-        return time.strftime('%H:%M:%S', time.gmtime(self.runtime))
+        d1 = datetime.datetime.fromtimestamp(self.begin)
+        d2 = datetime.datetime.fromtimestamp(self.end)
+        rd = dateutil.relativedelta.relativedelta(d2, d1)
+        return "%d days, %d hours, %d min %d sec" % (rd.days, rd.hours, rd.minutes, rd.seconds)
 
     def start_time(self):
         """ Returns the start time of a the job in a readable format """
@@ -295,6 +319,10 @@ def host_list(self):
         hosts.append(host.name for host in self.hosts.all())
         return hosts
 
+    def attrs(self):
+        for field in self._meta.fields:
+            yield field.name, getattr(self, field.name)
+
 class Monitor(models.Model):
     kind = models.CharField(max_length=32)
     system = models.ForeignKey(System)

tacc_stats_web/apps/tacc_stats/static/Autocompleter.Local.js

@@ -0,0 +1,31 @@
+/**
+ * Autocompleter.Local
+ *
+ * http://digitarald.de/project/autocompleter/
+ *
+ * @version		1.1.2
+ *
+ * @license		MIT-style license
+ * @author		Harald Kirschner <mail [at] digitarald.de>
+ * @copyright	Author
+ */
+
+Autocompleter.Local = new Class({
+
+	Extends: Autocompleter,
+
+	options: {
+		minLength: 0,
+		delay: 200
+	},
+
+	initialize: function(element, tokens, options) {
+		this.parent(element, options);
+		this.tokens = tokens;
+	},
+
+	query: function() {
+		this.update(this.filter());
+	}
+
+});