init commit of extract_wind_data

Dockerfile

@@ -45,7 +45,7 @@ RUN apt-get update && \
 RUN apt-get install -y python3 python3-setuptools python3-pip git curl unzip python3-click python3-fiona python3-gdal python3-lxml python3-parse python3-pyproj python3-rasterio python3-requests python3-slugify python3-sqlalchemy python3-xarray python3-yaml
 
 # install pygeoapi
-RUN git clone $PYGEOAPI_GITREPO -b 0.19.0 && \
+RUN git clone $PYGEOAPI_GITREPO -b 0.20.0 && \
     cd pygeoapi && \
     pip3 install -r requirements.txt && \
     pip3 install flask_cors gunicorn gevent greenlet && \

debian/postinst

@@ -31,7 +31,7 @@
 export MSC_PYGEOAPI_ES_TIMEOUT=90
 export MSC_PYGEOAPI_ES_URL=http://localhost:9200
 export MSC_PYGEOAPI_CACHEDIR=/tmp
-export MSC_PYGEOAPI_OGC_API_URL=https://api.wxod-dev.cmc.ec.gc.ca
+export MSC_PYGEOAPI_OGC_API_URL=https://api.wxod-dev.edc-mtl.ec.gc.ca
 export MSC_PYGEOAPI_OGC_API_URL_BASEPATH=/
 export MSC_PYGEOAPI_METPX_EVENT_FILE_PY=msc_pygeoapi.event.EventAfterWork
 export MSC_PYGEOAPI_METPX_EVENT_MESSAGE_PY=msc_pygeoapi.event.EventAfterAccept

deploy/default/msc-pygeoapi.wsgi

@@ -48,7 +48,7 @@ import os
 os.environ['PYGEOAPI_CONFIG'] = '/opt/msc-pygeoapi/conf/msc-pygeoapi-config.yml'
 os.environ['PYGEOAPI_OPENAPI'] = '/opt/msc-pygeoapi/conf/msc-pygeoapi-openapi.yml'
 os.environ['MSC_PYGEOAPI_ES_URL'] = 'http://localhost:9200'
-os.environ['MSC_PYGEOAPI_OGC_API_URL'] = 'https://api.wxod-dev.cmc.ec.gc.ca'
+os.environ['MSC_PYGEOAPI_OGC_API_URL'] = 'https://api.wxod-dev.edc-mtl.ec.gc.ca'
 os.environ['MSC_PYGEOAPI_OGC_API_URL_BASEPATH'] = '/'
 os.environ['MSC_PYGEOAPI_LOCALE'] = '/opt/msc-pygeoapi/locale'
 os.environ['MSC_PYGEOAPI_TEMPLATES'] = '/opt/msc-pygeoapi/theme/templates'

deploy/default/sarracenia/bulletins-realtime.conf

@@ -6,7 +6,8 @@ instances 4
 subtopic *.WXO-DD.bulletins.alphanumeric.#
 
 mirror True
-directory /tmp/bulletins
+download off
+directory /dev/null
 callback ${MSC_PYGEOAPI_METPX_EVENT_MESSAGE_PY}
 logLevel ${MSC_PYGEOAPI_LOGGING_LOGLEVEL}
 report False

docker/default.env

@@ -14,6 +14,7 @@ MSC_PYGEOAPI_LOCALE=${BASEDIR}/msc-pygeoapi/locale
 
 GEOMET_HPFX_BASEPATH=/data/geomet/feeds/hpfx
 GEOMET_SCIENCE_BASEPATH=/data/geomet/feeds/cmoi-science
+GEOMET_LOCAL_BASEPATH=/data/geomet/local
 
 XDG_CACHE_HOME=/tmp/msc-pygeoapi-sarra-logs
 

msc-pygeoapi.env

@@ -6,7 +6,7 @@ export MSC_PYGEOAPI_ES_TIMEOUT=90
 #export MSC_PYGEOAPI_ES_PASSWORD=bar
 export MSC_PYGEOAPI_ES_URL=http://${MSC_PYGEOAPI_ES_USERNAME}:${MSC_PYGEOAPI_ES_PASSWORD}@localhost:9200
 export MSC_PYGEOAPI_CACHEDIR=/tmp
-export MSC_PYGEOAPI_OGC_API_URL=https://api.wxod-dev.cmc.ec.gc.ca
+export MSC_PYGEOAPI_OGC_API_URL=https://api.wxod-dev.edc-mtl.ec.gc.ca
 export MSC_PYGEOAPI_OGC_API_URL_BASEPATH=/
 export MSC_PYGEOAPI_METPX_EVENT_FILE_PY=msc_pygeoapi.event.EventAfterWork
 export MSC_PYGEOAPI_METPX_EVENT_MESSAGE_PY=msc_pygeoapi.event.EventAfterAccept
@@ -17,6 +17,7 @@ export MSC_PYGEOAPI_DMS_API_URL=http://host/search/v2.0
 
 export GEOMET_HPFX_BASEPATH=/data/geomet/feeds/hpfx
 export GEOMET_SCIENCE_BASEPATH=/data/geomet/feeds/cmoi-science
+export GEOMET_LOCAL_BASEPATH=/data/geomet/local
 
 export XDG_CACHE_HOME=/tmp/msc-pygeoapi-sarra-logs
 

msc_pygeoapi/env.py

@@ -63,3 +63,4 @@
 
 GEOMET_HPFX_BASEPATH = os.getenv('GEOMET_HPFX_BASEPATH', None)
 GEOMET_SCIENCE_BASEPATH = os.getenv('GEOMET_SCIENCE_BASEPATH', None)
+GEOMET_LOCAL_BASEPATH = os.getenv('GEOMET_LOCAL_BASEPATH', None)

msc_pygeoapi/loader/__init__.py

@@ -64,6 +64,7 @@ def metadata():
     ('msc_pygeoapi.loader.swob_realtime', 'swob_realtime'),
     ('msc_pygeoapi.loader.aqhi_realtime', 'aqhi_realtime'),
     ('msc_pygeoapi.loader.aqhi_stations', 'aqhi_stations'),
+    ('msc_pygeoapi.loader.swob_stations', 'swob_stations'),
     ('msc_pygeoapi.loader.ltce', 'ltce'),
     ('msc_pygeoapi.loader.climate_archive', 'climate_archive'),
     ('msc_pygeoapi.loader.metnotes', 'metnotes'),

msc_pygeoapi/loader/bulletins_realtime.py

@@ -43,7 +43,7 @@
 LOGGER = logging.getLogger(__name__)
 
 # cleanup settings
-DAYS_TO_KEEP = 140
+DAYS_TO_KEEP = 30
 
 # index settings
 INDEX_BASENAME = 'bulletins.'
@@ -122,7 +122,8 @@ def __init__(self, filepath, conn_config={}):
 
         BaseLoader.__init__(self)
 
-        self.DD_URL = 'https://dd.weather.gc.ca/bulletins/alphanumeric'
+        today_date = datetime.today().strftime('%Y%m%d')
+        self.DD_URL = f'https://dd.weather.gc.ca/{today_date}/WXO-DD/bulletins/alphanumeric' # noqa
         self.conn = ElasticsearchConnector(conn_config)
         self.conn.create_template(INDEX_BASENAME, SETTINGS)
 

msc_pygeoapi/loader/hydat.py

@@ -306,6 +306,10 @@ def create_index(self, index):
                                     "type": "text",
                                     "fields": {"raw": {"type": "keyword"}},
                                 },
+                                "REAL_TIME": {"type": "byte"},
+                                "RHBN": {"type": "byte"},
+                                "DRAINAGE_AREA_GROSS": {"type": "float"},
+                                "DRAINAGE_AREA_EFFECT": {"type": "float"},
                             }
                         },
                         "geometry": {"type": "geo_shape"},
@@ -682,6 +686,16 @@ def generate_stations(self, station_table, annual_peaks_table,
                 float(station_metadata[station_keys.index('LONGITUDE')]),
                 float(station_metadata[station_keys.index('LATITUDE')]),
             ]
+            station_rhbn = station_metadata[station_keys.index('RHBN')]
+            station_real_time = station_metadata[
+                station_keys.index('REAL_TIME')
+            ]
+            station_drainage_area_gross = station_metadata[
+                station_keys.index('DRAINAGE_AREA_GROSS')
+            ]
+            station_draininage_area_effect = station_metadata[
+                station_keys.index('DRAINAGE_AREA_EFFECT')
+            ]
             agency_id = station_metadata[station_keys.index('CONTRIBUTOR_ID')]
             datum_id = station_metadata[station_keys.index('DATUM_ID')]
             if agency_id is not None:
@@ -745,7 +759,11 @@ def generate_stations(self, station_table, annual_peaks_table,
                     'STATUS_FR': status_fr,
                     'CONTRIBUTOR_EN': agency_en,
                     'CONTRIBUTOR_FR': agency_fr,
-                    'VERTICAL_DATUM': datum_en
+                    'VERTICAL_DATUM': datum_en,
+                    'REAL_TIME': station_real_time,
+                    'RHBN': station_rhbn,
+                    'DRAINAGE_AREA_GROSS': station_drainage_area_gross,
+                    'DRAINAGE_AREA_EFFECT': station_draininage_area_effect,
                 },
                 'links': [
                     {

msc_pygeoapi/loader/hydrometric_realtime.py

@@ -48,7 +48,8 @@
 LOGGER = logging.getLogger(__name__)
 
 STATIONS_LIST_NAME = 'hydrometric_StationList.csv'
-STATIONS_LIST_URL = f'https://dd.weather.gc.ca/hydrometric/doc/{STATIONS_LIST_NAME}'  # noqa
+TODAY_DATE = datetime.today().strftime('%Y%m%d')
+STATIONS_LIST_URL = f'https://dd.weather.gc.ca/{TODAY_DATE}/WXO-DD/hydrometric/doc/{STATIONS_LIST_NAME}'  # noqa
 
 STATIONS_CACHE = os.path.join(MSC_PYGEOAPI_CACHEDIR, STATIONS_LIST_NAME)