From acd58b00e850a29522999c27573eb55496a65d00 Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Tue, 28 Apr 2026 14:54:06 -0600 Subject: [PATCH 1/9] Add install script; config via json --- usage-monitoring/config.json | 10 ++++ usage-monitoring/install.sh | 11 ++++ usage-monitoring/usage_monitoring.py | 64 +++++++++++++-------- usage-monitoring/usage_monitoring_config.py | 18 ------ 4 files changed, 60 insertions(+), 43 deletions(-) create mode 100644 usage-monitoring/config.json create mode 100644 usage-monitoring/install.sh delete mode 100644 usage-monitoring/usage_monitoring_config.py diff --git a/usage-monitoring/config.json b/usage-monitoring/config.json new file mode 100644 index 0000000..9f7bcd4 --- /dev/null +++ b/usage-monitoring/config.json @@ -0,0 +1,10 @@ +{ + "token_file": "~/.config/usage-monitoring/os-token.json", + "allocation_resources": [ + "jetstream2.indiana.xsede.org", + "jetstream2-gpu.indiana.xsede.org", + "jetstream2-lm.indiana.xsede.org" + ], + "data_file": "~/usage_monitoring.csv", + "test_csv_file": "/tmp/usage_monitoring_test.csv" +} diff --git a/usage-monitoring/install.sh b/usage-monitoring/install.sh new file mode 100644 index 0000000..8343176 --- /dev/null +++ b/usage-monitoring/install.sh @@ -0,0 +1,11 @@ +# /usr/bin/env /usr/bin/bash + +set -x + +INSTALL_PATH=$HOME/.local/bin/ +CONFIG_PATH=$HOME/.config/usage-monitoring/ + +mkdir -p $INSTALL_PATH $CONFIG_PATH + +ln -s ./usage_monitoring.py $INSTALL_PATH +ln -s ./usage_monitoring_config.py $CONFIG_PATH diff --git a/usage-monitoring/usage_monitoring.py b/usage-monitoring/usage_monitoring.py index dcbb163..b5dd40b 100755 --- a/usage-monitoring/usage_monitoring.py +++ b/usage-monitoring/usage_monitoring.py @@ -1,6 +1,6 @@ from os import system from subprocess import run -from os.path import isfile +from os.path import isfile, expanduser import argparse @@ -11,7 +11,13 @@ import requests from matplotlib import pyplot as plt -from usage_monitoring_config import * +def load_config(config_path): + try: + with open(expanduser(config_path)) as c: + return json.load(c) + except FileNotFoundError: + print(f'File {config_path} not found! Exiting ...') + exit(1) def create_os_token(token_file): token = run( @@ -19,32 +25,36 @@ def create_os_token(token_file): capture_output=True, check=True ) - with open(token_file, 'w') as f: + with open(expanduser(token_file), 'w') as f: f.write(token.stdout.decode()) def token_expired(token_file): - with open(token_file, 'r') as f: + with open(expanduser(token_file), 'r') as f: expires_str = json.load(f)['expires'] date_format = '%Y-%m-%dT%H:%M:%S+0000' expire = datetime.strptime(expires_str, date_format).timestamp() now = datetime.now(UTC).timestamp() expire < now -def get_os_token(token_file='/tmp/os-token.json', force_new_token=False): - if not isfile(token_file) or force_new_token or token_expired(token_file): +def get_os_token(token_file, force_new_token=False): + if not isfile(expanduser(token_file)) or force_new_token or token_expired(token_file): create_os_token(token_file) - with open(token_file, 'r') as f: - json.load(f)['id'] + with open(expanduser(token_file), 'r') as f: + return json.load(f)['id'] def query_accounting_api(token): url = 'https://js2.jetstream-cloud.org:9001' headers = { 'X-Auth-Token': f'{token}' } response = requests.get(url, headers=headers) - response.raise_for_status() + try: + response.raise_for_status() + except Exception as ex: + print(ex) + exit(1) query = json.loads(response.text) return query -def get_js2_resources(query): +def get_js2_resources(query,allocation_resources): now = datetime.now() date_format = '%Y-%m-%d' all_resources = [ @@ -72,6 +82,7 @@ def write_resource_csv(resources, data_file): ] now = datetime.now(UTC).timestamp() + data_file = expanduser(data_file) # Create file and write headers if it doesn't exist if not isfile(data_file): with open(data_file, 'w') as f: @@ -244,7 +255,7 @@ def usage_analysis(data,days_prior): def generate_usage_plot(resources, analyses): fig, ax = plt.subplots() - for resource_type in allocation_resources: + for resource_type in c['allocation_resources']: data = get_data_by_resource(resources, resource_type) timestamps = pd.array(data['timestamp']) @@ -258,43 +269,46 @@ def generate_usage_plot(resources, analyses): plt.show() return 0 -def main(data_file): +def main(): parser = argparse.ArgumentParser() parser.add_argument('-n', '--force-new-token', help='Force the creation of a new openstack token before query', action='store_true') - parser.add_argument('-w', '--write', help=f'Query Jetstream2 for new allocation data and write to data file: {data_file}', action='store_true') - parser.add_argument('-c', '--dump-csv', help=f'Dump the data from {data_file} in csv format', action='store_true') - parser.add_argument('-j', '--dump-json', help=f'Dump the data from {data_file} in json format', action='store_true') + parser.add_argument('-w', '--write', help='Query Jetstream2 for new allocation data and write to data file', action='store_true') + parser.add_argument('-c', '--dump-csv', help='Dump the data from data_file in csv format', action='store_true') + parser.add_argument('-j', '--dump-json', help='Dump the data from data_file in json format', action='store_true') parser.add_argument('-p', '--plot', help='Generate an interactive plot of SU usage data', action='store_true') parser.add_argument('-a', '--analysis-days', help='Days prior for which to perform an analysis', action='extend', nargs='+', type=int) parser.add_argument('-d', '--devel', help='Use test_csv_file for development work', action='store_true') + parser.add_argument('--config', help='Configuration file path', type=str, default="~/.config/usage-monitoring/config.json") args = vars(parser.parse_args()) + c = load_config(args['config']) + if not any([ args[key] for key in args.keys() ]): parser.parse_args(['--help']) if args['devel']: - data_file = test_csv_file + c['data_file'] = c['test_csv_file'] if args['write']: - token = get_os_token(token_file,force_new_token=args['force_new_token']) + token = get_os_token(c['token_file'],force_new_token=args['force_new_token']) query = query_accounting_api(token) - resources = get_js2_resources(query) - write_resource_csv(resources, data_file) + resources = get_js2_resources(query,c['allocation_resources']) + write_resource_csv(resources, c['data_file']) if args['dump_csv']: - system(f'cat {data_file}') + system(f'cat {c['data_file']}') if args['dump_json']: - resources = read_resource_csv(data_file) + resources = read_resource_csv(c['data_file']) print(json.dumps(resources, indent=2)) if args['analysis_days']: # Get resources - resources = read_resource_csv(data_file) + resources = read_resource_csv(c['data_file']) analyses = [] # Loop over resources to get each type of data found in allocation_resources - for resource_type in allocation_resources: + for resource_type in c['allocation_resources']: data = get_data_by_resource(resources, resource_type) # Perform analysis (usage rates, "forecast", ) analyses.append(usage_analysis(data,args['analysis_days'])) @@ -304,8 +318,8 @@ def main(data_file): if args['plot']: if 'analyses' not in locals(): analyses = None - resources = read_resource_csv(data_file) + resources = read_resource_csv(c['data_file']) generate_usage_plot(resources, analyses) if __name__ == "__main__": - main(data_file) + main() diff --git a/usage-monitoring/usage_monitoring_config.py b/usage-monitoring/usage_monitoring_config.py deleted file mode 100644 index 22f856d..0000000 --- a/usage-monitoring/usage_monitoring_config.py +++ /dev/null @@ -1,18 +0,0 @@ -from os.path import expanduser -home = expanduser('~') - -# String: Path to file used for the openstack token -token_file = '/path/to/os-token.json' - -# List of strings: Used to filter out the Jetstream2 resources that are irrelevant, i.e. -# storage, which doesn't use service units -allocation_resources = [ 'jetstream2.indiana.xsede.org', - 'jetstream2-gpu.indiana.xsede.org', - #'jetstream2-lm.indiana.xsede.org', - ] - -# String: Path to data file which stores persistent data -data_file = f'{home}/usage_monitoring.csv' - -# String: Path to a test data file used for development -test_csv_file = '/tmp/usage_monitoring_test.csv' From c9aa2b2db7b6b76f82d10183c6a18320edb19d28 Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Tue, 28 Apr 2026 15:06:33 -0600 Subject: [PATCH 2/9] Fix dump json option --- usage-monitoring/usage_monitoring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usage-monitoring/usage_monitoring.py b/usage-monitoring/usage_monitoring.py index b5dd40b..b6d67d6 100755 --- a/usage-monitoring/usage_monitoring.py +++ b/usage-monitoring/usage_monitoring.py @@ -300,7 +300,7 @@ def main(): if args['dump_json']: resources = read_resource_csv(c['data_file']) - print(json.dumps(resources, indent=2)) + print(resources.to_json(orient='records', indent=2)) if args['analysis_days']: # Get resources From 2587bca5ea38c8ed9dcaba913c9cea457a9729a2 Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Tue, 28 Apr 2026 15:59:56 -0600 Subject: [PATCH 3/9] Fix symbolic linking --- usage-monitoring/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usage-monitoring/install.sh b/usage-monitoring/install.sh index 8343176..a3908e2 100644 --- a/usage-monitoring/install.sh +++ b/usage-monitoring/install.sh @@ -7,5 +7,5 @@ CONFIG_PATH=$HOME/.config/usage-monitoring/ mkdir -p $INSTALL_PATH $CONFIG_PATH -ln -s ./usage_monitoring.py $INSTALL_PATH -ln -s ./usage_monitoring_config.py $CONFIG_PATH +ln -s $PWD/usage_monitoring.py $INSTALL_PATH/usage_monitoring.py +ln -s $PWD/config.json $CONFIG_PATH/config.json From 70a72627832a040ee148b530872f71ae3cb78285 Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Tue, 28 Apr 2026 16:03:01 -0600 Subject: [PATCH 4/9] Add 'conda run' hashbang --- usage-monitoring/usage_monitoring.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/usage-monitoring/usage_monitoring.py b/usage-monitoring/usage_monitoring.py index b6d67d6..9ae3334 100755 --- a/usage-monitoring/usage_monitoring.py +++ b/usage-monitoring/usage_monitoring.py @@ -1,3 +1,5 @@ +#! /usr/bin/env conda run -n usage-monitoring python + from os import system from subprocess import run from os.path import isfile, expanduser From b23f2094b43c4a59e3718c314d0362a480df8107 Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Tue, 28 Apr 2026 16:53:35 -0600 Subject: [PATCH 5/9] Handle the case of missing resource types --- usage-monitoring/usage_monitoring.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/usage-monitoring/usage_monitoring.py b/usage-monitoring/usage_monitoring.py index 9ae3334..48972bc 100755 --- a/usage-monitoring/usage_monitoring.py +++ b/usage-monitoring/usage_monitoring.py @@ -255,10 +255,13 @@ def usage_analysis(data,days_prior): return analysis -def generate_usage_plot(resources, analyses): +def generate_usage_plot(resources, analyses, allocation_resources): fig, ax = plt.subplots() - for resource_type in c['allocation_resources']: + for resource_type in allocation_resources: data = get_data_by_resource(resources, resource_type) + if data.empty: + print(f'No available data for {resource_type}') + continue timestamps = pd.array(data['timestamp']) dates = [ datetime.fromtimestamp(ts) for ts in timestamps ] @@ -312,6 +315,9 @@ def main(): # Loop over resources to get each type of data found in allocation_resources for resource_type in c['allocation_resources']: data = get_data_by_resource(resources, resource_type) + if data.empty: + print(f'No available data for {resource_type}') + continue # Perform analysis (usage rates, "forecast", ) analyses.append(usage_analysis(data,args['analysis_days'])) @@ -321,7 +327,7 @@ def main(): if 'analyses' not in locals(): analyses = None resources = read_resource_csv(c['data_file']) - generate_usage_plot(resources, analyses) + generate_usage_plot(resources, analyses, c['allocation_resources']) if __name__ == "__main__": main() From 6d52cb3d5b08ae290aa83a41cd97b4b57bcb1b7f Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Wed, 29 Apr 2026 09:16:07 -0600 Subject: [PATCH 6/9] Handle a rate of 0 SU/s The usage analysis will predict an exhausted_date based on a basic linear model given a SU usage rate. In some cases, this rate is calculated to be 0 SU/s, such as: 1) You're not using any SUs for the given resource type 2) Jetstream2 had not updated your SU usage between different queries In these cases, report the 0 SU usage rate, but a None/null exhausted_date. --- usage-monitoring/usage_monitoring.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/usage-monitoring/usage_monitoring.py b/usage-monitoring/usage_monitoring.py index 48972bc..1191768 100755 --- a/usage-monitoring/usage_monitoring.py +++ b/usage-monitoring/usage_monitoring.py @@ -219,7 +219,10 @@ def usage_analysis(data,days_prior): # tot_sus - s1 = remaining_sus = r*(t2 - t1) --> t2 = remaining_sus/r + t1 exhausted_ts = remaining_sus/r + cur_ts - exhausted_date = datetime.fromtimestamp(exhausted_ts) + try: + exhausted_date = datetime.fromtimestamp(exhausted_ts) + except OverflowError: + exhausted_date = None # s2 - s1 = r*(t2 - t1) --> s2 = r*(t2 - t1) + s1 date_format = '%Y-%m-%d' From 80b38d019754e3bc2b797fdf82a4321d5f9d1549 Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Wed, 29 Apr 2026 09:47:45 -0600 Subject: [PATCH 7/9] Handle too small data sets --- usage-monitoring/usage_monitoring.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usage-monitoring/usage_monitoring.py b/usage-monitoring/usage_monitoring.py index 1191768..2781a4a 100755 --- a/usage-monitoring/usage_monitoring.py +++ b/usage-monitoring/usage_monitoring.py @@ -319,8 +319,10 @@ def main(): for resource_type in c['allocation_resources']: data = get_data_by_resource(resources, resource_type) if data.empty: - print(f'No available data for {resource_type}') + print(f'No available data for {resource_type}. Skipping ...') continue + if len(data) < 2: + print(f'Not enough data for {resource_type}: len(data) = {len(data)}. Skipping ...') # Perform analysis (usage rates, "forecast", ) analyses.append(usage_analysis(data,args['analysis_days'])) From cd7b9c10cbf2ad4315d877e1decd1accdf0c6fb8 Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Wed, 29 Apr 2026 10:02:42 -0600 Subject: [PATCH 8/9] Drop 'system' dependency --- usage-monitoring/usage_monitoring.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/usage-monitoring/usage_monitoring.py b/usage-monitoring/usage_monitoring.py index 2781a4a..17857b3 100755 --- a/usage-monitoring/usage_monitoring.py +++ b/usage-monitoring/usage_monitoring.py @@ -1,6 +1,5 @@ #! /usr/bin/env conda run -n usage-monitoring python -from os import system from subprocess import run from os.path import isfile, expanduser @@ -304,7 +303,10 @@ def main(): write_resource_csv(resources, c['data_file']) if args['dump_csv']: - system(f'cat {c['data_file']}') + dump = run( + ['cat', f'{expanduser(c['data_file'])}'], + check=True + ) if args['dump_json']: resources = read_resource_csv(c['data_file']) From 64ce1ea1f2c02619f2638e2100a93fd93d6ec47b Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Wed, 29 Apr 2026 10:09:10 -0600 Subject: [PATCH 9/9] Clean: expanduser() on paths when loading config --- usage-monitoring/usage_monitoring.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/usage-monitoring/usage_monitoring.py b/usage-monitoring/usage_monitoring.py index 17857b3..4ff485a 100755 --- a/usage-monitoring/usage_monitoring.py +++ b/usage-monitoring/usage_monitoring.py @@ -14,8 +14,12 @@ def load_config(config_path): try: - with open(expanduser(config_path)) as c: - return json.load(c) + with open(expanduser(config_path)) as config: + c = json.load(config) + c['token_file'] = expanduser(c['token_file']) + c['data_file'] = expanduser(c['data_file']) + c['test_csv_file'] = expanduser(c['test_csv_file']) + return c except FileNotFoundError: print(f'File {config_path} not found! Exiting ...') exit(1) @@ -26,11 +30,11 @@ def create_os_token(token_file): capture_output=True, check=True ) - with open(expanduser(token_file), 'w') as f: + with open(token_file, 'w') as f: f.write(token.stdout.decode()) def token_expired(token_file): - with open(expanduser(token_file), 'r') as f: + with open(token_file, 'r') as f: expires_str = json.load(f)['expires'] date_format = '%Y-%m-%dT%H:%M:%S+0000' expire = datetime.strptime(expires_str, date_format).timestamp() @@ -38,9 +42,9 @@ def token_expired(token_file): expire < now def get_os_token(token_file, force_new_token=False): - if not isfile(expanduser(token_file)) or force_new_token or token_expired(token_file): + if not isfile(token_file) or force_new_token or token_expired(token_file): create_os_token(token_file) - with open(expanduser(token_file), 'r') as f: + with open(token_file, 'r') as f: return json.load(f)['id'] def query_accounting_api(token): @@ -83,7 +87,6 @@ def write_resource_csv(resources, data_file): ] now = datetime.now(UTC).timestamp() - data_file = expanduser(data_file) # Create file and write headers if it doesn't exist if not isfile(data_file): with open(data_file, 'w') as f: @@ -304,7 +307,7 @@ def main(): if args['dump_csv']: dump = run( - ['cat', f'{expanduser(c['data_file'])}'], + ['cat', f'{c['data_file']}'], check=True )