Merge pull request #89 from cesar-avalos3/update-to-python3

Update get-accel-sim-traces.py to support python 3

Author: JRPan

README.md

@@ -104,6 +104,7 @@ AccelWattch microbenchmarks and AccelWattch validation set benchmarks are also i
 
 2. **Accel-Sim SASS Frontend and Simulation Engine**: A simulator frontend that consumes SASS traces and feeds them into a performance model. The intial release of Accel-Sim coincides with the release of [GPGPU-Sim 4.0](https://github.com/accel-sim/accel-sim-framework/blob/dev/gpu-simulator/gpgpu-sim4.md), which acts as the detailed performance model. To build the Accel-Sim simulator that uses the traces, do the following:
     ```bash
+    pip3 install -r requirements.txt
     source ./gpu-simulator/setup_environment.sh
     make -j -C ./gpu-simulator/
     ```
@@ -278,4 +279,4 @@ Note that *AccelWattch HW* and *AccelWattch HYBRID* configurations require hardw
 
 5. **Microbenchmarks and Quadratic Optimization Solver**: The source code for the microbenchmarks used for AccelWattch dynamic power modeling are located [here](https://github.com/accel-sim/gpu-app-collection/tree/release-accelwattch/src/cuda/accelwattch-ubench) and can be compiled by following the README [here](https://github.com/accel-sim/gpu-app-collection/tree/release-accelwattch). The Quadratic Optimization Solver MATLAB script is located at `./util/accelwattch/quadprog_solver.m`.
 
-6. **SASS to Power Component Mapping**: The header file `gpu-simulator/ISA_Def/accelwattch_component_mapping.h` contains the Accel-Sim instruction opcode to AccelWattch power component mapping and can be extended to support new SASS instructions for future architectures. Please look at the *opcode.h* files for respective GPU Architectures in the same directory `gpu-simulator/ISA_Def/` for SASS instruction to Accel-Sim opcode mapping.
+6. **SASS to Power Component Mapping**: The header file `gpu-simulator/ISA_Def/accelwattch_component_mapping.h` contains the Accel-Sim instruction opcode to AccelWattch power component mapping and can be extended to support new SASS instructions for future architectures. Please look at the *opcode.h* files for respective GPU Architectures in the same directory `gpu-simulator/ISA_Def/` for SASS instruction to Accel-Sim opcode mapping.

get-accel-sim-traces.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from optparse import OptionParser
 import re
@@ -49,7 +49,7 @@ def __init__(self, name):
 
     def getTotalCompressed(self):
         total = 0.0
-        for name,suite in self.suites.iteritems():
+        for name,suite in self.suites.items():
             try:
                 total += suite.compressedSize
             except TypeError:
@@ -58,13 +58,13 @@ def getTotalCompressed(self):
 
     def getTotalUncompressed(self):
         total = 0.0
-        for name,suite in self.suites.iteritems():
+        for name,suite in self.suites.items():
             total += suite.uncompressedSize
         return total
 
 def downloadTrace(cardName, suiteName):
     webFile = os.path.join(WEB_DIRECTORY, cardName, VERSION + ".latest", suiteName + ".tgz")
-    print "\n\nDownloading {0}".format(webFile)
+    print("\n\nDownloading {0}".format(webFile))
     wget = Popen(["wget " + webFile], stderr=STDOUT, shell=True)
     wget.communicate()
     if wget.returncode != 0:
@@ -121,13 +121,13 @@ def main():
                 suite.uncompressedSize = getNumRaw(size)
 
     # Infor the user what is available - ask them what they want to do
-    print "\n\nCurrently Available Traces:"
-    for cardName, card in sizeDict.iteritems():
-        print "GPU Name: {0}. All Apps Compressed (Download size): {1}, All Apps Uncompressed (Size on disk when used): {2}"\
-            .format(card.name, millify(card.getTotalCompressed()), millify(card.getTotalUncompressed()))
-        for name, suite in card.suites.iteritems():
-            print "\t{0}: Compressed = {1}, Uncompressed = {2}".format(suite.name,\
-                millify(suite.compressedSize), millify(suite.uncompressedSize))
+    print("\n\nCurrently Available Traces:")
+    for cardName, card in sizeDict.items():
+        print("GPU Name: {0}. All Apps Compressed (Download size): {1}, All Apps Uncompressed (Size on disk when used): {2}"\
+            .format(card.name, millify(card.getTotalCompressed()), millify(card.getTotalUncompressed())))
+        for name, suite in card.suites.items():
+            print("\t{0}: Compressed = {1}, Uncompressed = {2}".format(suite.name,\
+                millify(suite.compressedSize), millify(suite.uncompressedSize)))
 
     selectionValid = False
     while not selectionValid:
@@ -144,29 +144,29 @@ def main():
                 cardName = item.split(r"/")[0]
                 suiteName = item.split(r"/")[1]
                 if cardName == "all":
-                    for cardName, card in sizeDict.iteritems():
+                    for cardName, card in sizeDict.items():
                         if suiteName == "all":
-                            for suiteName, suite in card.suites.iteritems():
+                            for suiteName, suite in card.suites.items():
                                 downloadTrace(cardName, suiteName)
                         else:
                             downloadTrace(cardName, suiteName)
                 else:
                     card = sizeDict[cardName]
                     if suiteName == "all":
-                        for suiteName, suite in card.suites.iteritems():
+                        for suiteName, suite in card.suites.items():
                             downloadTrace(cardName, suiteName)
                     else:
                         downloadTrace(cardName, suiteName)
 
             selectionValid = True
         except Exception as e:
             selectionValid = False
-            print "Invalid Input: {0}".format(e)
+            print("Invalid Input: {0}".format(e))
             if options.apps != None:
                 sys.exit(1)
 
-    print "\n\nDownload successful to {0}.\nFiles must be uncompressed with tar -xzvf <filename> to be usable by accel-sim"\
-        .format(hw_run_dir)
+    print("\n\nDownload successful to {0}.\nFiles must be uncompressed with tar -xzvf <filename> to be usable by accel-sim"\
+        .format(hw_run_dir))
 
 if __name__ == '__main__':
     main()

requirements.txt

@@ -0,0 +1,5 @@
+pyyaml
+pandas
+plotly
+kaleido
+psutil

util/hw_stats/run_hw.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright (c) 2018-2021, Mahmoud Khairy, Vijay Kandiah, Timothy Rogers, Tor M. Aamodt, Nikos Hardavellas
 # Northwestern University, Purdue University, The University of British Columbia
@@ -177,8 +177,8 @@
         if not options.norun:
             saved_dir = os.getcwd()
             os.chdir(this_run_dir)
-            print "Running {0}".format(exe)
+            print("Running {0}".format(exe))
 
             if subprocess.call(["bash", "run.sh"]) != 0:
-                print "Error invoking profiler on {0}".format(this_run_dir)
+                print("Error invoking profiler on {0}".format(this_run_dir))
             os.chdir(saved_dir)

util/job_launching/monitor_func_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from optparse import OptionParser
 import re
@@ -37,21 +37,21 @@ def kill_all_running_jobs(jobstatus_out_filename):
                     app = tokens[appCol].strip()
                     appArgs = tokens[appArgsCol].strip()
                     nodeName = tokens[nodeCol].strip()
-                    print "Calling qdel {0}: {1}/{2} ({3})".format(torqueID, app, appArgs, nodeName)
+                    print("Calling qdel {0}: {1}/{2} ({3})".format(torqueID, app, appArgs, nodeName))
                     if subprocess.call( ["qdel" , torqueID] ) != 0:
-                        print "WARNING error calling qdel"
-    print "Sleeping 30s to wait for the file system to calm down"
+                        print("WARNING error calling qdel")
+    print("Sleeping 30s to wait for the file system to calm down")
     time.sleep(30)
     jobstatus_out_file.close()
 
 def print_statsfile(options, this_directory):
     get_stats_out_file = open(options.statsfile, 'w+')
-    print "Calling get_stats.py"
+    print("Calling get_stats.py")
     if subprocess.call([os.path.join(this_directory, "get_stats.py") ,"-R" ,"-l", options.logfile, "-N", options.sim_name, "-A"],
         stdout=get_stats_out_file, stderr=get_stats_out_file) != 0:
-        print "Error Launching get_stats.py"
+        print("Error Launching get_stats.py")
     get_stats_out_file.seek(0)
-    print get_stats_out_file.read()
+    print(get_stats_out_file.read())
     get_stats_out_file.close()
 
 def handle_exit(jobstatus_out_filename):
@@ -116,14 +116,14 @@ def handle_exit(jobstatus_out_filename):
 while True:
     jobstatus_out_file = open(jobstatus_out_filename, 'w+')
     if options.verbose:
-        print "Calling job_status.py"
+        print("Calling job_status.py")
     if subprocess.call([os.path.join(this_directory, "job_status.py"),\
             "-l", options.logfile, \
             "-N", options.sim_name, \
             "-j", job_manager],\
             stdout=jobstatus_out_file, stderr=jobstatus_out_file) != 0:
         jobstatus_out_file.seek(0)
-        print jobstatus_out_file.read()
+        print(jobstatus_out_file.read())
         exit("Error Launching job_status.py")
     else:
         jobstatus_out_file.seek(0)
@@ -135,7 +135,7 @@ def handle_exit(jobstatus_out_filename):
         num_no_err = 0
         for line in jobstatus_out_file.readlines():
             if options.verbose:
-                print line.strip()
+                print(line.strip())
             if jobStatusCol == None:
                 jobStatusCol = getColId("(.*)JobStatus.*", line)
             else:
@@ -159,27 +159,27 @@ def handle_exit(jobstatus_out_filename):
         jobstatus_out_file.close()
 
     total = num_passed + num_running + num_waiting + num_error + num_no_err
-    print "Passed:{0}/{1}, No error:{2}/{1}, Failed/Error:{3}/{1}, Running:{4}/{1}, Waiting:{5}/{1}"\
-        .format(num_passed, total, num_no_err, num_error, num_running, num_waiting)
+    print("Passed:{0}/{1}, No error:{2}/{1}, Failed/Error:{3}/{1}, Running:{4}/{1}, Waiting:{5}/{1}"\
+        .format(num_passed, total, num_no_err, num_error, num_running, num_waiting))
     if num_error > 0:
-        print "Contents {0}:".format(failed_job_file)
+        print("Contents {0}:".format(failed_job_file))
         if options.verbose and os.path.exists(failed_job_file):
-            print open(failed_job_file).read()
+            print(open(failed_job_file).read())
 
     if num_running + num_waiting == 0:
-        print "All {0} Tests Done.".format(total)
+        print("All {0} Tests Done.".format(total))
         if num_error == 0:
-            print "Congratulations! All Tests Pass!"
+            print("Congratulations! All Tests Pass!")
         else:
-            print "Something did not pass."
+            print("Something did not pass.")
 
         handle_exit(jobstatus_out_filename)
     else:
-        print "Sleeping for {0}s".format(options.sleep_time)
+        print("Sleeping for {0}s".format(options.sleep_time))
         time.sleep(int(options.sleep_time))
         options.timeout -= float(options.sleep_time)
         if options.timeout <= 0:
-            print "Monitor has timed-out"
+            print("Monitor has timed-out")
             if options.killwhentimedout:
                 kill_all_running_jobs(jobstatus_out_filename)
             handle_exit(jobstatus_out_filename)

util/plotting/merge-stats.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 from optparse import OptionParser

util/plotting/plot-correlation.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 from optparse import OptionParser
@@ -58,12 +58,12 @@ def getAppData(kernels, x, y, xaxis_title, correlmap):
     # For rates, take the average across all the kernels in the app
     if correlmap.stattype == "rate":
         new_map = {}
-        for k,v in app_map.iteritems():
+        for k,v in app_map.items():
             x1,y1,numk = v
             new_map[k] = (x1 / numk, y1 / numk, numk)
         app_map = new_map
 
-    for k,v in app_map.iteritems():
+    for k,v in app_map.items():
         apps.append(k)
         x1,y1,numk = v
         newx.append(x1)
@@ -480,7 +480,7 @@ def get_sim_csv_data(filepath, logger):
                     count += 1
     for stat in stats_missing:
         appargs, num, current_stat = stat
-        for cfg in all_kerns.iterkeys():
+        for cfg in all_kerns.keys():
             del all_kerns[cfg][appargs][num][current_stat]
     return all_kerns
 
@@ -614,18 +614,18 @@ def parse_hw_csv(csv_file, hw_data, appargs, kdata, logger):
 
 def summarize_hw_data(hw_data, logger):
     print("-----------------------------------------------------------------")
-    for device,appargslist in hw_data.iteritems():
+    for device,appargslist in hw_data.items():
         print("All Card Summary:")
         print("HW Summary for {0} [Contains {1} Apps]:"
             .format(device,len(appargslist)))
     print("----------------------------------------------------------------\n\n")
 
     # Print HW data summary
-    for device,appargslist in hw_data.iteritems():
+    for device,appargslist in hw_data.items():
         logger.logchan("-----------------------------------------------------------------","hwsummary")
         logger.logchan("HW Summary for {0} [Contains {1} Apps]:"
             .format(device,len(appargslist)), "hwsummary")
-        for appargs,kdata in appargslist.iteritems():
+        for appargs,kdata in appargslist.items():
             logger.logchan("\t{0}:".format(appargs), "hwsummary")
             logger.logchan("\t\tContatins {0} kernels:".format(len(kdata)), "hwsummary")
 
@@ -763,13 +763,13 @@ def summarize_hw_data(hw_data, logger):
 exec(open(options.data_mappings,'r').read())
 
 fig_data = {} # map of HW config to a list of scatters
-for cfg,sim_for_cfg in sim_data.iteritems():
+for cfg,sim_for_cfg in sim_data.items():
     if cfg.split('-')[0] not in config_maps:
         logger.log("cfg {0} not in config_maps:{1}.".format(cfg, config_maps))
         continue
 
     hw_cfg = None
-    for device in hw_data.iterkeys():
+    for device in hw_data.keys():
         logger.log("Testing hw_cfg={0}".format(device))
         logger.log("\tcfg={0}, config_maps={1}".format(cfg,config_maps))
 
@@ -804,14 +804,14 @@ def summarize_hw_data(hw_data, logger):
         num_under = 0
         num_over = 0
         errs = []
-        sim_appargs_leftover = set(copy.deepcopy(sim_for_cfg.keys()))
-        hw_appargs_leftover = set(copy.deepcopy(hw_data[hw_cfg].keys()))
+        sim_appargs_leftover = set(copy.deepcopy(list(sim_for_cfg.keys())))
+        hw_appargs_leftover = set(copy.deepcopy(list(hw_data[hw_cfg].keys())))
         max_axis_val = 0.0
         min_axis_val = 99999999999999999999999999999.9
         err_dropped_stats = 0
         hw_low_drop_stats = 0
         apps_included = {}
-        for appargs,sim_klist in sim_for_cfg.iteritems():
+        for appargs,sim_klist in sim_for_cfg.items():
             if appargs in hw_data[hw_cfg]:
                 if (isAppBanned( appargs, blacklist )):
                     continue
@@ -979,6 +979,6 @@ def summarize_hw_data(hw_data, logger):
 
 
 correl_outdir = os.path.join(this_directory, "correl-html")
-for (plotfile,hw_cfg), traces in fig_data.iteritems():
+for (plotfile,hw_cfg), traces in fig_data.items():
     make_submission_quality_image(options.image_type, traces, hw_cfg)
 print("Output Available at: file://{0}".format(correl_outdir))

util/tracer_nvbit/run_hw_trace.py

@@ -105,7 +105,7 @@
         sh_contents += "\nexport CUDA_VERSION=\"" + cuda_version + "\"; export CUDA_VISIBLE_DEVICES=\"" + options.device_num + "\" ; " +\
             "export TRACES_FOLDER="+ this_trace_folder + "; CUDA_INJECTION64_PATH=" + os.path.join(nvbit_tracer_path, "tracer_tool.so") + " " +\
             exec_path + " " + str(args) + " ; " + os.path.join(nvbit_tracer_path,"traces-processing", "post-traces-processing") + " " +\
-            os.path.join(this_trace_folder, "kernelslist") #+ " ; rm -f " + this_trace_folder + "/*.trace ; rm -f " + this_trace_folder + "/kernelslist "
+            os.path.join(this_trace_folder, "kernelslist") + " ; rm -f " + this_trace_folder + "/*.trace ; rm -f " + this_trace_folder + "/kernelslist "
 
         open(os.path.join(this_run_dir,"run.sh"), "w").write(sh_contents)
         if subprocess.call(['chmod', 'u+x', os.path.join(this_run_dir,"run.sh")]) != 0: