Code cleanup

Author: source-nerd

.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

README.md

@@ -0,0 +1,41 @@
+# YFinance Data Downloader - Docker and Kubernetes Cron Job
+
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
+YFinance minute and daily data downloader, with Dockerfile and kubernetes pod yaml declaration. The Pod has restart policy on failure and can be run as a CROn job on daily basis. The downloaded data is written to elasticsearch - which is running in the same K8 environment and is connected via service.
+
+> This project makes use of Github Actions to build and publish the docker image.
+
+## Publishing image to GitHub Packages Manually
+docker login docker.pkg.github.com -u USER_NAME -p PERSONAL_ACCESS_TOKEN
+
+docker tag BUILD_IMAGE_ID docker.pkg.github.com/source-nerd/yahoo-data-downloader-docker-elastic/yahoo-data-downloader-docker-elastic:VERSION
+
+docker build -t docker.pkg.github.com/source-nerd/yahoo-data-downloader-docker-elastic/yahoo-data-downloader-docker-elastic:VERSION .
+
+docker push docker.pkg.github.com/source-nerd/yahoo-data-downloader-docker-elastic/yahoo-data-downloader-docker-elastic:VERSION
+
+
+## Deploying to Kubernetes
+1. Initialize the namespace
+2. Install the creds - [REF HERE](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/)
+```shell
+# Example:
+kubectl create secret docker-registry regcred \
+  --docker-server=$DOCKER_REGISTRY_SERVER \
+  --docker-username=$DOCKER_USER \
+  --docker-password=$DOCKER_PASSWORD \
+  --docker-email=$DOCKER_EMAIL
+
+# Actual
+kubectl create secret docker-registry regcred --docker-server=docker.pkg.github.com --docker-username=source-nerd --docker-password=PERSONAL_ACCESS_TOKEN --docker-email=EMAIL
+```
+
+kubectl --kubeconfig KUBE_CONF_PATH -n NAMESPACE apply -f pod.yaml
+
+
+## Feedback & Final Thoughts
+The code may not be very optimized, so if you tend to find any bug fixes, feature requests, pull requests, feedback, etc., are welcome... If you like this project, please do give it a star.
+
+[![ForTheBadge built-with-love](http://ForTheBadge.com/images/badges/built-with-love.svg)](https://GitHub.com/Naereen/)
+

__init__.py

@@ -1,9 +1,9 @@
-ES_HOST=elasticsearch.equitian-elk.svc.cluster.local
+ES_HOST=elasticsearch.NAMESPACE.svc.cluster.local
 ES_PORT=9200
-ES_USERNAME=elastic
-ES_PASSWORD=5qq17MySsmBhN4DFvsJW
+ES_USERNAME=ES_USERNAME
+ES_PASSWORD=ES_PASSWORD
 
-INDEX_1_MIN=us_tickers_1m
-INDEX_1_DAY=us_tickers_1d
+INDEX_1_MIN=INDEX_1M
+INDEX_1_DAY=INDEX_1D
 
-STOCKS=AAPL,TSLA,MSFT
+STOCKS=AAPL,AMZN,AMD,FB,FISV,GS,GOOGL,INTC,MCD,MSFT,NVDA,PG,TSLA,V

build.sh

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: data-downloader-pod
+  namespace: equitian-app
+spec:
+  imagePullSecrets:
+    - name: regcred
+  containers:
+  - name: data-downloader-container
+    image: docker.pkg.github.com/source-nerd/yahoo-data-downloader-docker-elastic/yahoo-data-downloader-docker-elastic:0.1
+    env:
+    - name: ES_HOST
+      value: ES_HOST
+    - name: ES_PORT
+      value: "9200"
+    - name: ES_USERNAME
+      value: "elastic"
+    - name: ES_PASSWORD
+      value: ES_PASSWORD
+    - name: INDEX_1_MIN
+      value: INDEX_1_MIN
+    - name: INDEX_1_DAY
+      value: INDEX_1_DAY
+    - name: STOCKS
+      value: "AAPL,TSLA,MSFT"
+  restartPolicy: OnFailure

config

@@ -14,29 +14,29 @@
 STOCKS_LIST = str(os.getenv("STOCKS", "AAPL,TSLA,MSFT")).split(",")
 INDEX_1_MIN = os.getenv("INDEX_1_MIN", "us_tickers_1m")
 INDEX_1_DAY = os.getenv("INDEX_1_DAY", "us_tickers_1d")
-ES_HOST = os.getenv('ES_HOST', "localhost")
-ES_PORT = os.getenv('ES_PORT', "9200")
-ES_USERNAME = os.getenv('ES_USERNAME', "elastic")
-ES_PASSWORD = os.getenv('ES_PASSWORD', "SOME_RANDOM_PASS")
+ES_HOST = os.getenv("ES_HOST", "localhost")
+ES_PORT = os.getenv("ES_PORT", "9200")
+ES_USERNAME = os.getenv("ES_USERNAME", "elastic")
+ES_PASSWORD = os.getenv("ES_PASSWORD", "SOME_RANDOM_PASS")
 
 
 def random_id_generator():
-    id_size = randrange(5,10)
-    return ''.join(random.choices(string.ascii_uppercase + string.digits, k=id_size))
+    id_size = randrange(5, 10)
+    return "".join(random.choices(string.ascii_uppercase + string.digits, k=id_size))
 
 
 def doc_generator(df, ticker_name, date_time_field, es_index):
     df_iter = df.iterrows()
     for df_idx, document in df_iter:
         doc_dict = document.to_dict()
         doc_dict["ticker"] = ticker_name
-        doc_id= str(document[date_time_field].value) + "-" + str(random_id_generator())
+        doc_id = str(document[date_time_field].value) + "-" + str(random_id_generator())
         final_doc = {
-                "_index": es_index,
-                "_type": "_doc",
-                "_id" : f"{doc_id}",
-                "_source": doc_dict,
-            }
+            "_index": es_index,
+            "_type": "_doc",
+            "_id": f"{doc_id}",
+            "_source": doc_dict,
+        }
 
         yield final_doc
 
@@ -50,17 +50,18 @@ def download_1d_data(ticker, s_date, e_data):
     :return:
     """
     data_interval = "1d"
-    data_1_day = yf.download(tickers=ticker,
-                   start=s_date,
-                   end=e_data,
-                   interval=data_interval)
+    data_1_day = yf.download(
+        tickers=ticker, start=s_date, end=e_data, interval=data_interval
+    )
     data_1_day.reset_index(level=0, inplace=True)
     return data_1_day
 
 
 def download_min_data(ticker, s_date, e_date):
     data_interval = "1m"
-    LOGGER.info(f"Downloading {ticker} from Yahoo finance - s_data={s_date}, e_date={e_date}")
+    LOGGER.info(
+        f"Downloading {ticker} from Yahoo finance - s_data={s_date}, e_date={e_date}"
+    )
     data_1_min = yf.download(
         tickers=ticker,
         start=s_date,
@@ -74,38 +75,49 @@ def download_min_data(ticker, s_date, e_date):
 
 
 def get_es_client():
-    es_client = Elasticsearch([{'host': ES_HOST, 'port': ES_PORT}],
-                              http_auth=('elastic', '5qq17MySsmBhN4DFvsJW'),
-                              http_compress=True)
+    es_client = Elasticsearch(
+        [{"host": ES_HOST, "port": ES_PORT}],
+        http_auth=("elastic", "5qq17MySsmBhN4DFvsJW"),
+        http_compress=True
+    )
     # es_client = Elasticsearch(['localhost'], port=9200, http_compress=True)
     return es_client
 
 
 def day_downloader_main():
-    today_date = datetime.today().strftime('%Y-%m-%d')
+    today_date = datetime.today().strftime("%Y-%m-%d")
     LOGGER.info(f"Downloading day's worth of data for date - [{today_date}]")
+    es_client = get_es_client()
+
     for item in STOCKS_LIST:
         res_data = download_1d_data(item, today_date, today_date)
-        helpers.bulk(get_es_client(), doc_generator(res_data, item, "Date", INDEX_1_DAY))
+        helpers.bulk(
+            es_client, doc_generator(res_data, item, "Date", INDEX_1_DAY)
+        )
     LOGGER.info(f"Done downloading data for - [{today_date}]")
 
 
 def min_downloader_main():
-    start_date = (datetime.today() + timedelta(days=-2)).strftime('%Y-%m-%d')
-    end_date = (datetime.today() + timedelta(days=1)).strftime('%Y-%m-%d')
+    start_date = datetime.today().strftime("%Y-%m-%d")
+    end_date = (datetime.today() + timedelta(days=1)).strftime("%Y-%m-%d")
+    es_client = get_es_client()
 
     LOGGER.info(f"Downloading day's worth of data for date - [{start_date}]")
     for item in STOCKS_LIST:
         res_data = download_min_data(item, start_date, end_date)
         LOGGER.info(res_data)
         if res_data.shape[0] > 0:
-            helpers.bulk(get_es_client(), doc_generator(res_data, item, "Datetime", INDEX_1_MIN))
+            helpers.bulk(
+                es_client, doc_generator(res_data, item, "Datetime", INDEX_1_MIN)
+            )
     LOGGER.info(f"Done downloading data for - [{start_date}]")
 
 
 if __name__ == "__main__":
     LOGGER.info("Inside main function")
-    LOGGER.info(f"Global variables are:: {STOCKS_LIST}, {ES_HOST}, {ES_PORT}, {ES_USERNAME}, {ES_PASSWORD}")
+    LOGGER.info(
+        f"Global variables are:: {STOCKS_LIST}, {ES_HOST}, {ES_PORT}, {ES_USERNAME}, {ES_PASSWORD}"
+    )
 
     min_downloader_main()
     day_downloader_main()