From afeb8afbf7ccde89bbf8cc377fe2c4bd76849b33 Mon Sep 17 00:00:00 2001 From: Emmett Gaines Date: Wed, 27 Sep 2023 20:22:07 -0400 Subject: [PATCH 1/3] Specify python 3.6 in case multiple python installations exist --- Update.bat | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Update.bat b/Update.bat index 0257bd2..531c376 100644 --- a/Update.bat +++ b/Update.bat @@ -1,12 +1,15 @@ -#!/bin/sh - -@echo off -pushd "%~dp0" -cd root -echo Grabbing Logs -py RawLogsScraper.py -echo Averaging CSV files -py MaptickAverager.py -echo Plotting Graphs +@ECHO OFF + +PUSHD "%~dp0" +CD root + +ECHO Grabbing Logs +py -3.6 RawLogsScraper.py + +ECHO Averaging CSV files +py -3.6 MaptickAverager.py + +ECHO Plotting Graphs gnuplot maptick_plot.gnuplot -popd "%~dp0" \ No newline at end of file + +POPD "%~dp0" \ No newline at end of file From 52c42df09b0b2e0038b19fd3551a04faf44e4d78 Mon Sep 17 00:00:00 2001 From: Emmett Gaines Date: Wed, 27 Sep 2023 20:23:48 -0400 Subject: [PATCH 2/3] Adds a setup batch file for one time operations Just creating the default config and installing pip modules for now --- .gitignore | 3 ++- Setup.bat | 17 +++++++++++++++++ root/README.md | 2 +- root/RawLogsScraper.py | 2 +- root/config/_template.json | 10 ++++++++++ 5 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 Setup.bat create mode 100644 root/config/_template.json diff --git a/.gitignore b/.gitignore index 4803dc0..4d11683 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ root/mood.json settings.json round.csv.gz round.csv -parseRoundInfoTable.py \ No newline at end of file +parseRoundInfoTable.py +root/config/header_config.json diff --git a/Setup.bat b/Setup.bat new file mode 100644 index 0000000..21b0656 --- /dev/null +++ b/Setup.bat @@ -0,0 +1,17 @@ +@ECHO OFF + +SET CONFIGFOLDER=root\config\ +SET TEMPLATEPATH=%CONFIGFOLDER%_template.json +SET CONFIGPATH=%CONFIGFOLDER%header_config.json + +IF EXIST %CONFIGPATH% ( + ECHO Config already exists, skipping. +) ELSE ( + ECHO No config found, copying default... + COPY %TEMPLATEPATH% %CONFIGPATH% +) + +ECHO Installing pip dependencies... +py -3.6 -m pip install requests + +PAUSE \ No newline at end of file diff --git a/root/README.md b/root/README.md index 86179eb..724fe89 100644 --- a/root/README.md +++ b/root/README.md @@ -14,7 +14,7 @@ In the event of an error, this info can be found in [data/scraping.json](data/sc Also has some internal procs you can use to look at how many holes you have. No way to self heal if they happen before buffering was introduced though. -Oh right uh, mood.json exists. Create one and edit it to match your server creds if for some reason you want to pull from raw logs (Don't forget to change the url) +Oh right uh, `header_config.json` exists. One should be created when `Setup.bat` was run in the [config](config) folder. Edit it to match your server creds if for some reason you want to pull from raw logs (Don't forget to change the url) ### [MaptickAverager](MaptickAverager.py) diff --git a/root/RawLogsScraper.py b/root/RawLogsScraper.py index 92bf41e..8b2c023 100644 --- a/root/RawLogsScraper.py +++ b/root/RawLogsScraper.py @@ -163,7 +163,7 @@ def clearDataFile(): #owning mso with facts and logic (you need beautiful soup to parse raw logs for reasons) #from bs4 import BeautifulSoup -#cookie_file = "mood.json" +#cookie_file = "config/header_config.json" #do_not_post_this_4head = open(cookie_file) #fakingIdentity = json.load(do_not_post_this_4head) #Loads a .json file containing the cookie and other params to send to mso #do_not_post_this_4head.close() diff --git a/root/config/_template.json b/root/config/_template.json new file mode 100644 index 0000000..28aaf33 --- /dev/null +++ b/root/config/_template.json @@ -0,0 +1,10 @@ +{ + "Host": "tgstation13.download", + "User-Agent": "Default Graphing Scraper (Lemon Scented)", + "Accept": "Remember to not accept zip files", + "Accept-Language": "hhhhhhhhhhh", + "Referer": "Pick Your Poison", + "Connection": "keep-alive", + "Cookie": "Do not post this publically anywhere 4head", + "Upgrade-Insecure-Requests": "1" +} \ No newline at end of file From 411db2f3142da4afe6c40c4a1427f4e88ea03f4d Mon Sep 17 00:00:00 2001 From: Emmett Gaines Date: Wed, 27 Sep 2023 20:25:29 -0400 Subject: [PATCH 3/3] Brings up the setup info in the readme earlier I missed it myself and went digging around to figure out dependencies before realizing too late it's at the bottom of the readme. --- README.MD | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/README.MD b/README.MD index e8dae14..0bae560 100644 --- a/README.MD +++ b/README.MD @@ -4,31 +4,37 @@ This is a set of tools developed to aid in the tracking of maptick, td, and pop It scrapes all available performance logs, processes them into csv files (One for each map, one for each server, and a few global ones), and then uses `gnuplot` to plot them in a few different ways I've judged as useful. -## Scraping +## Setup -We don't scrape all performance logs, since some of them from before 2020/11/11 don't scrape well and don't match the current system. +### Install Dependencies -## Parsing +Download and install the following: +- [Python 3.6](https://www.python.org/downloads/release/python-360/) +- [gnuplot 5+](http://www.gnuplot.info/) -A brief note on parsing, we compile each round into one data-point, attached to some other info about it. +Make sure both are in your path, then restart your computer. You should then run `Setup.bat`, this will setup the default header config in [root/config](root/config) and download any python modules needed through pip. -## Updating your data +### Update data Running `Update.bat` will pull all the logs you don't have (Outside some edgecases I don't have the brainpower to fix, read the python file) parse them into csv files, and generate a new plot from them. -## Single Round Parsing +## Usage Notes -If you'd like to graph a single round you've already pulled, use `GraphOne.bat`. It'll dump images for that round into [root/single_round](root/single_round) +### Scraping -## More Information +We don't scrape all performance logs, since some of them from before 2020/11/11 don't scrape well and don't match the current system. -If you'd like to learn more about any of this go read [root/README.md](root/README.md), it goes into more detail about how things work and how to interact with them +### Parsing -## Dependencies +A brief note on parsing, we compile each round into one data-point, attached to some other info about it. + +### Single Round Parsing + +If you'd like to graph a single round you've already pulled, use `GraphOne.bat`. It'll dump images for that round into [root/single_round](root/single_round) -This project was created with `python 3.6`, you'll need at least `python 3` for the logs parser to run properly. +## More Information -Additionally you will need gnuplot installed, with a version above 5. This is to support graphing several files onto one graph. Needs to be in your path too, because I'm a moron. [Have another Link](http://www.gnuplot.info/) +If you'd like to learn more about any of this go read [root/README.md](root/README.md), it goes into more detail about how things work and how to interact with them ## License