diff --git a/.travis.yml b/.travis.yml index eaaecdb..f8238b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ install: - pip install -r dev-requirements.txt - pip install -e . - git clone -b 2.4.x https://github.com/GeoNode/geonode.git - - cp local_settings.py geonode/geonode/. + - cp local_settings.py.template geonode/geonode/local_settings.py - pip install -e geonode script: diff --git a/dataqs/airnow/airnow.py b/dataqs/airnow/airnow.py index 60a0be5..2efabcb 100755 --- a/dataqs/airnow/airnow.py +++ b/dataqs/airnow/airnow.py @@ -50,31 +50,31 @@ class AirNowGRIB2HourlyProcessor(GeoDataMosaicProcessor): "airnow_aqi_combined"] img_patterns = ["", "_pm25", "_combined"] layer_titles = ["Ozone", "PM25", "Combined Ozone & PM25"] - description = """ -U.S. Environmental Protection Agency’s (EPA) nationwide, voluntary program, -AirNow(www.airnow.gov), provides real-time air quality data and forecasts to -protect public health across the United States, Canada, and parts of Mexico. -AirNowreceives real-time ozone and PM2.5data from over 2,000 monitors and -collects air quality forecasts for more than 300 cities. - -As part of the Global Earth Observation System of Systems (GEOSS) -(www.epa.gov/geoss) program, the AirNow API system broadens access to AirNowdata - and data products. AirNow API produces data products in several standard data -formats and makes them available via FTP and web services. This document -describes the GRIB2 file formats. - -All data provided by AirNow API are made possible by the efforts of more than -120 local, state, tribal, provincial, and federal government agencies -(www.airnow.gov/index.cfm?action=airnow.partnerslist). These data are not fully -verified or validated and should be considered preliminary and subject to -change. Data and information reported to AirNow from federal, state, local, and -tribal agencies are for the express purpose of reporting and forecasting the -Air Quality Index (AQI). As such, they should not be used to formulate or -support regulation, trends, guidance, or any other government or public -decision making. Official regulatory air quality data must be obtained from -EPA’s Air Quality System (AQS) (www.epa.gov/ttn/airs/airsaqs). See the AirNow -Data Exchange Guidelines at http://airnowapi.org/docs/DataUseGuidelines.pdf. -""" + description = ( + u"U.S. Environmental Protection Agency's (EPA) nationwide, voluntary " + u"program, AirNow(www.airnow.gov), provides real-time air quality data " + u"and forecasts to protect public health across the United States, " + u" Canada, and parts of Mexico. AirNow receives real-time ozone and " + u"PM2.5 data from over 2000 monitors and collects air quality forecasts" + u" for more than 300 cities.\n\nAs part of the Global Earth Observation" + u" System of Systems (GEOSS)(www.epa.gov/geoss) program, the AirNow API" + u" system broadens access to AirNowdata and data products. AirNow API " + u"produces data products in several standard data formats and makes " + u"them available via FTP and web services. This documen describes the " + u"GRIB2 file formats. All data provided by AirNow API are made " + u"possible by the efforts of more than 120 local, state, tribal, " + u"provincial, and federal government agencies (www.airnow.gov/index.cfm" + u"?action=airnow.partnerslist). These data are not fully verified or " + u"validated and should be considered preliminary and subject to change." + u" Data and information reported to AirNow from federal, state, local, " + u"and tribal agencies are for the express purpose of reporting and" + u" forecasting the Air Quality Index (AQI). As such, they should not be" + u" used to formulate or support regulation, trends, guidance, or any " + u"other government or public decision making. Official regulatory air " + u"quality data must be obtained from EPA’s Air Quality System (AQS) " + u"(www.epa.gov/ttn/airs/airsaqs). See the AirNow Data Exchange " + u"Guidelines at http://airnowapi.org/docs/DataUseGuidelines.pdf." + ) def download(self, auth_account=AIRNOW_ACCOUNT, days=1): """ diff --git a/dataqs/cmap/cmap.py b/dataqs/cmap/cmap.py index b47b611..6a00aa5 100644 --- a/dataqs/cmap/cmap.py +++ b/dataqs/cmap/cmap.py @@ -5,10 +5,10 @@ import shutil from datetime import date from ftplib import FTP - +from time import sleep from dateutil.relativedelta import relativedelta from dataqs.processor_base import GeoDataMosaicProcessor, GS_DATA_DIR, \ - GS_TMP_DIR + GS_TMP_DIR, RSYNC_WAIT_TIME from dataqs.helpers import get_band_count, gdal_translate, \ nc_convert, style_exists, cdo_fixlng @@ -30,57 +30,54 @@ class CMAPProcessor(GeoDataMosaicProcessor): layer_name = 'cmap' bounds = '-178.75,178.75,-88.75,88.75' title = 'CPC Merged Analysis of Precipitation, 1979/01 - {}' - abstract = """The CPC Merged Analysis of Precipitation ("CMAP") is a -technique which produces pentad and monthly analyses of global precipitation - in which observations from raingauges are merged with precipitation estimates -from several satellite-based algorithms (infrared and microwave). The analyses -are are on a 2.5 x 2.5 degree latitude/longitude grid and extend back to 1979. -These data are comparable (but should not be confused with) similarly combined -analyses by the Global Precipitation Climatology Project which are described in -Huffman et al (1997). - -It is important to note that the input data sources to make these analyses are -not constant throughout the period of record. For example, SSM/I (passive -microwave - scattering and emission) data became available in July of 1987; -prior to that the only microwave-derived estimates available are from the MSU -algorithm (Spencer 1993) which is emission-based thus precipitation estimates -are avaialble only over oceanic areas. Furthermore, high temporal resolution IR -data from geostationary satellites (every 3-hr) became available during 1986; -prior to that, estimates from the OPI technique (Xie and Arkin 1997) are used -based on OLR from polar orbiting satellites. - -The merging technique is thoroughly described in Xie and Arkin (1997). Briefly, -the methodology is a two-step process. First, the random error is reduced by -linearly combining the satellite estimates using the maximum likelihood method, -in which case the linear combination coefficients are inversely proportional to -the square of the local random error of the individual data sources. Over global - land areas the random error is defined for each time period and grid location -by comparing the data source with the raingauge analysis over the surrounding -area. Over oceans, the random error is defined by comparing the data sources -with the raingauge observations over the Pacific atolls. Bias is reduced when -the data sources are blended in the second step using the blending technique of -Reynolds (1988). Here the data output from step 1 is used to define the "shape" -of the precipitation field and the rain gauge data are used to constrain the -amplitude. - -Monthly and pentad CMAP estimates back to the 1979 are available from CPC ftp -server. - -References: - -Huffman, G. J. and co-authors, 1997: The Global Precipitation Climatology -Project (GPCP) combined data set. Bull. Amer. Meteor. Soc., 78, 5-20. - -Reynolds, R. W., 1988: A real-time global sea surface temperature analysis. J. -Climate, 1, 75-86. - -Spencer, R. W., 1993: Global oceanic precipitation from the MSU during 1979-91 -and comparisons to other climatologies. J. Climate, 6, 1301-1326. - -Xie P., and P. A. Arkin, 1996: Global precipitation: a 17-year monthly analysis -based on gauge observations, satellite estimates, and numerical model outputs. -Bull. Amer. Meteor. Soc., 78, 2539-2558. -""" + abstract = ( + "The CPC Merged Analysis of Precipitation ('CMAP') is a technique which" + " produces pentad and monthly analyses of global precipitation in which" + " observations from raingauges are merged with precipitation estimates " + "from several satellite-based algorithms (infrared and microwave). The " + "analyses are on a 2.5 x 2.5 degree latitude/longitude grid and extend " + "back to 1979.\n\nThese data are comparable (but should not be confused" + " with) similarly combined analyses by the Global Precipitation " + "Climatology Project which are described in Huffman et al (1997).\n\n" + "It is important to note that the input data sources to make these " + "analyses are not constant throughout the period of record. For example" + ", SSM/I (passive microwave - scattering and emission) data became " + "available in July of 1987; prior to that the only microwave-derived " + "estimates available are from the MSU algorithm (Spencer 1993) which is" + " emission-based thus precipitation estimates are avaialble only over " + " oceanic areas. Furthermore, high temporal resolution IR data from " + "geostationary satellites (every 3-hr) became available during 1986;" + " prior to that, estimates from the OPI technique (Xie and Arkin 1997) " + "are used based on OLR from polar orbiting satellites.\n\nThe merging " + "technique is thoroughly described in Xie and Arkin (1997). Briefly, " + "the methodology is a two-step process. First, the random error is " + "reduced by linearly combining the satellite estimates using the " + "maximum likelihood method, in which case the linear combination " + "coefficients are inversely proportional to the square of the local " + "random error of the individual data sources. Over global land areas " + "the random error is defined for each time period and grid location by " + "comparing the data source with the raingauge analysis over the " + "surrounding area. Over oceans, the random error is defined by " + "comparing the data sources with the raingauge observations over the " + "Pacific atolls. Bias is reduced when the data sources are blended in " + "the second step using the blending technique of Reynolds (1988). Here " + "the data output from step 1 is used to define the \"shape\" of the " + "precipitation field and the rain gauge data are used to constrain the " + "amplitude.\n\nMonthly and pentad CMAP estimates back to the 1979 are " + "available from CPC ftp server.\n\nSource: " + "http://www.esrl.noaa.gov/psd/data/gridded/data.cmap.html\n\nRaw data " + "file: ftp://ftp.cdc.noaa.gov/Datasets/cmap/enh/precip.mon.mean.nc" + "\n\nReferences:\n\nHuffman, G. J. and " + "co-authors, 1997: The Global Precipitation Climatology Project (GPCP) " + "combined data set. Bull. Amer. Meteor. Soc., 78, 5-20.\n\nReynolds, R." + " W., 1988: A real-time global sea surface temperature analysis. J. " + "Climate, 1, 75-86.\n\nSpencer, R. W., 1993: Global oceanic " + "precipitation from the MSU during 1979-91 and comparisons to other " + "climatologies. J. Climate, 6, 1301-1326.\n\nXie P., and P. A. Arkin, " + "1996: Global precipitation: a 17-year monthly analysis based on gauge " + "observations, satellite estimates, and numerical model outputs. Bull. " + "Amer. Meteor. Soc., 78, 2539-2558." + ) def download(self, url, tmp_dir=GS_TMP_DIR, filename=None): if not filename: @@ -122,6 +119,7 @@ def run(self): cdf_file = self.convert(os.path.join(self.tmp_dir, ncfile)) bands = get_band_count(cdf_file) img_list = self.get_mosaic_filenames(self.layer_name) + dst_files = [] for band in range(1, bands + 1): band_date = re.sub('[\-\.]+', '', self.get_date(band).isoformat()) img_name = '{}_{}T000000000Z.tif'.format(self.layer_name, band_date) @@ -136,7 +134,10 @@ def run(self): os.makedirs(dst_dir) if dst_file.endswith('.tif'): shutil.move(os.path.join(self.tmp_dir, band_tif), dst_file) - self.post_geoserver(dst_file, self.layer_name) + dst_files.append(dst_file) + sleep(RSYNC_WAIT_TIME * 2) + for dst_file in dst_files: + self.post_geoserver(dst_file, self.layer_name, sleeptime=0) if not style_exists(self.layer_name): with open(os.path.join(script_dir, diff --git a/dataqs/cmap/resources/cmap.sld b/dataqs/cmap/resources/cmap.sld index bf07030..9509e3f 100644 --- a/dataqs/cmap/resources/cmap.sld +++ b/dataqs/cmap/resources/cmap.sld @@ -9,11 +9,6 @@ 1.0 - - - {latest_band} - - diff --git a/dataqs/forecastio/forecastio_air.py b/dataqs/forecastio/forecastio_air.py index ee4d258..6da54b8 100755 --- a/dataqs/forecastio/forecastio_air.py +++ b/dataqs/forecastio/forecastio_air.py @@ -41,13 +41,16 @@ class ForecastIOAirTempProcessor(GeoDataMosaicProcessor): prefix = "forecast_io_airtemp" base_url = "http://maps.forecast.io/temperature/" layer_name = "forecast_io_airtemp" - description = """Project Quicksilver is an experimental new data product -that attempts to create the world's highest resolution real-time map of global -(near-surface) air temperature.\n\n -It is generated using the same source data models that power Forecast.io, -combined with a sophisticated microclimate model that adjusts the temperatures -based on the effects of elevation, terrain, proximity to water, foliage cover, -and other factors.\n\nSource: http://blog.forecast.io/project-quicksilver/""" + description = ( + "Project Quicksilver is an experimental new data product that attempts " + "to create the world's highest resolution real-time map of global " + "(near-surface) air temperature.\n\nIt is generated using the same " + "source data models that power Forecast.io, combined with a " + "sophisticated microclimate model that adjusts the temperatures based " + "on the effects of elevation, terrain, proximity to water, foliage " + "cover, and other factors.\n\nSource: " + "http://blog.forecast.io/project-quicksilver/" + ) def parse_name(self, img_date): imgstrtime = img_date.strftime("%Y-%m-%d %H:00") diff --git a/dataqs/gdacs/gdacs.py b/dataqs/gdacs/gdacs.py index c302c86..7422180 100755 --- a/dataqs/gdacs/gdacs.py +++ b/dataqs/gdacs/gdacs.py @@ -39,19 +39,23 @@ class GDACSProcessor(GeoDataProcessor): prefix = "gdacs_alerts" layer_title = 'Flood, Quake, Cyclone Alerts - GDACS' params = {} - base_url = \ - "http://www.gdacs.org/rss.aspx?profile=ARCHIVE&fromarchive=true&" + \ - "from={}&to={}&alertlevel=&country=&eventtype=EQ,TC,FL&map=true" - description = """GDACS (Global Disaster and Alert Coordination System) is a -collaboration platform for organisations providing information on humanitarian -disasters. From a technical point of view, GDACS links information of all -participating organisations using a variety of systems to have a harmonized list - of data sources.In 2011, the GDACS platform was completely revised to collect, -store and distribute resources explicitly by events. The system matches -information from all organisations (by translating unique identifiers), and make - these resources available for GDACS users and developers in the form of GDACS -Platform Services. The GDACS RSS feed automatically include a list of available - resources.\n\nSource: http://www.gdacs.org/resources.aspx""" + base_url = ( + "http://www.gdacs.org/rss.aspx?profile=ARCHIVE&fromarchive=true&" + "from={}&to={}&alertlevel=&country=&eventtype=EQ,TC,FL&map=true") + description = ( + "GDACS (Global Disaster and Alert Coordination System) is a " + "collaboration platform for organisations providing information on " + "humanitarian disasters. From a technical point of view, GDACS links " + "information of all participating organisations using a variety of " + "systems to have a harmonized list of data sources.In 2011, the GDACS " + "platform was completely revised to collect, store and distribute " + "resources explicitly by events. The system matches information from " + "all organisations (by translating unique identifiers), and make these " + "resources available for GDACS users and developers in the form of " + "GDACS Platform Services. The GDACS RSS feed automatically include a " + "list of available resources.\n\nSource: " + "http://www.gdacs.org/resources.aspx" + ) def __init__(self, *args, **kwargs): for key in kwargs.keys(): diff --git a/dataqs/gdacs/tests.py b/dataqs/gdacs/tests.py index fb818cb..3f8c300 100644 --- a/dataqs/gdacs/tests.py +++ b/dataqs/gdacs/tests.py @@ -55,7 +55,7 @@ def test_download(self): body=response) rssfile = self.processor.download(self.processor.base_url.format( self.processor.params['sdate'], self.processor.params['edate']), - self.processor.prefix + ".rss") + filename=self.processor.prefix + ".rss") rsspath = os.path.join( self.processor.tmp_dir, rssfile) self.assertTrue(os.path.exists(rsspath)) @@ -74,7 +74,7 @@ def test_cleanup(self): body=response) rssfile = self.processor.download(self.processor.base_url.format( self.processor.params['sdate'], self.processor.params['edate']), - self.processor.prefix + ".rss") + filename=self.processor.prefix + ".rss") rsspath = os.path.join( self.processor.tmp_dir, rssfile) self.assertTrue(os.path.exists(rsspath)) diff --git a/dataqs/gfms/gfms.py b/dataqs/gfms/gfms.py index 23532e2..1b8a4c9 100755 --- a/dataqs/gfms/gfms.py +++ b/dataqs/gfms/gfms.py @@ -52,18 +52,21 @@ class GFMSProcessor(GeoDataProcessor): layer_future = "gfms_latest" layer_current = "gfms_current" prefix = 'Flood_byStor_' - description = u"""The GFMS (Global Flood Management System) is a NASA-funded -experimental system using real-time TRMM Multi-satellite Precipitation Analysis -(TMPA) precipitation information as input to a quasi-global (50°N - 50°S) -hydrological runoff and routing model running on a 1/8th degree latitude / -longitude grid. Flood detection/intensity estimates are based on 13 years of -retrospective model runs with TMPA input, with flood thresholds derived for -each grid location using surface water storage statistics (95th percentile plus -parameters related to basin hydrologic characteristics). Streamflow,surface -water storage,inundation variables are also calculated at 1km resolution. -In addition, the latest maps of instantaneous precipitation and totals from the -last day, three days and seven days are displayed. -\n\nSource: http://eagle1.umd.edu/flood/""" + description = ( + u"The GFMS (Global Flood Management System) is a NASA-funded" + u"experimental system using real-time TRMM Multi-satellite " + u"Precipitation Analysis (TMPA) precipitation information as input to a" + u"quasi-global (50°N - 50°S) hydrological runoff and routing model " + u"running on a 1/8th degree latitude /longitude grid. Flood detection/" + u"intensity estimates are based on 13 years of retrospective model runs" + u"with TMPA input, with flood thresholds derived for each grid location" + u" using surface water storage statistics (95th percentile plus " + u"parameters related to basin hydrologic characteristics). Streamflow," + u" surface water storage,inundation variables are also calculated at 1" + u"km resolution. In addition, the latest maps of instantaneous " + u"precipitation and totals from the last day, three days and seven days" + u" are displayed.\n\nSource: http://eagle1.umd.edu/flood/" + ) def get_latest_future(self): """ diff --git a/dataqs/gistemp/gistemp.py b/dataqs/gistemp/gistemp.py index c361b30..cbcb2ec 100755 --- a/dataqs/gistemp/gistemp.py +++ b/dataqs/gistemp/gistemp.py @@ -23,8 +23,10 @@ import re import shutil from datetime import date +import time from dateutil.relativedelta import relativedelta -from dataqs.processor_base import GeoDataMosaicProcessor, GS_DATA_DIR +from dataqs.processor_base import GeoDataMosaicProcessor, GS_DATA_DIR, \ + RSYNC_WAIT_TIME from dataqs.helpers import get_band_count, gdal_translate, \ nc_convert, style_exists, cdo_fixlng, gunzip @@ -43,23 +45,24 @@ class GISTEMPProcessor(GeoDataMosaicProcessor): base_url = "http://data.giss.nasa.gov/pub/gistemp/gistemp1200_ERSSTv4.nc.gz" layer_name = 'gistemp1200_ERSSTv4' title = 'Global Monthly Air Temperature Anomalies, 1880/01/01 - {}' - abstract = """The GISTEMP analysis recalculates consistent temperature -anomaly series from 1880 to the present for a regularly spaced array of virtual -stations covering the whole globe. Those data are used to investigate regional -and global patterns and trends. Graphs and tables are updated around the -middle of every month using current data files from NOAA GHCN v3 (meteorological - stations), ERSST v4 (ocean areas), and SCAR (Antarctic stations). - - The displayed image is based on the most current month. - -Citations: - - GISTEMP Team, 2016: GISS Surface Temperature Analysis (GISTEMP). - NASA Goddard Institute for Space Studies. Dataset accessed monthly - since 8/2016 at http://data.giss.nasa.gov/gistemp/. - - Hansen, J., R. Ruedy, M. Sato, and K. Lo, 2010: Global surface - temperature change, Rev. Geophys., 48, RG4004, doi:10.1029/2010RG000345. - - """ + abstract = ( + "The GISTEMP analysis recalculates consistent temperature " + "anomaly series from 1880 to the present for a regularly spaced array " + "of virtual stations covering the whole globe. Those data are used to " + "investigate regional and global patterns and trends. Graphs and " + "tables are updated around the middle of every month using current data" + " files from NOAA GHCN v3 (meteorological stations), ERSST v4 (ocean " + "areas), and SCAR (Antarctic stations).\n\nThe displayed image is based" + " on the most current month." + "\n\nSource:http://data.giss.nasa.gov/gistemp/\n\nRaw data file: " + "http://data.giss.nasa.gov/pub/gistemp/gistemp1200_ERSSTv4.nc.gz" + "\n\nCitations:\n\n- GISTEMP Team, 2016: " + "GISS Surface Temperature Analysis (GISTEMP).\nNASA Goddard Institute " + "for Space Studies. Dataset accessed monthly since 8/2016 at " + "http://data.giss.nasa.gov/gistemp/.\n- Hansen, J., R. Ruedy, M. Sato," + " and K. Lo, 2010: Global surface temperature change, Rev. Geophys., " + "48, RG4004, doi:10.1029/2010RG000345." + ) def convert(self, nc_file): nc_transform = nc_convert(nc_file) @@ -91,6 +94,7 @@ def run(self): cdf_file = self.convert(ncfile) bands = get_band_count(cdf_file) img_list = self.get_mosaic_filenames(self.layer_name) + dst_files = [] for band in range(1, bands+1): band_date = re.sub('[\-\.]+', '', self.get_date(band).isoformat()) img_name = '{}_{}T000000000Z.tif'.format(self.layer_name, band_date) @@ -105,7 +109,10 @@ def run(self): os.makedirs(dst_dir) if dst_file.endswith('.tif'): shutil.move(os.path.join(self.tmp_dir, band_tif), dst_file) - self.post_geoserver(dst_file, self.layer_name) + dst_files.append(dst_file) + time.sleep(RSYNC_WAIT_TIME * 2) + for dst_file in dst_files: + self.post_geoserver(dst_file, self.layer_name, sleeptime=0) if not style_exists(self.layer_name): with open(os.path.join(script_dir, diff --git a/dataqs/gistemp/resources/gistemp.sld b/dataqs/gistemp/resources/gistemp.sld index 1307183..37b0d85 100644 --- a/dataqs/gistemp/resources/gistemp.sld +++ b/dataqs/gistemp/resources/gistemp.sld @@ -9,11 +9,6 @@ 1.0 - - - {latest_band} - - diff --git a/dataqs/helpers.py b/dataqs/helpers.py index e8d4f05..02c682d 100755 --- a/dataqs/helpers.py +++ b/dataqs/helpers.py @@ -29,12 +29,10 @@ import psycopg2 import re import sys +import unicodedata from StringIO import StringIO -import rasterio from osgeo import gdal, ogr -from rasterio.warp import RESAMPLING -from rasterio.warp import calculate_default_transform, reproject -import unicodedata +from osgeo.osr import SpatialReference from geonode.geoserver.helpers import ogc_server_settings import ogr2ogr @@ -88,9 +86,6 @@ def gdal_translate(src_filename, dst_filename, dst_format="GTiff", bands=None, Convert a raster image with the specified arguments (as if running from commandline) """ - from osgeo import gdal - from osr import SpatialReference - if not options: options = [] @@ -312,42 +307,38 @@ def gdal_band_subset(infile, bands, dst_filename, dst_format="GTiff"): out_ds = None -def warp_image(infile, outfile, dst_crs="EPSG:3857", dst_driver='GTiff'): +def warp_image(infile, outfile, dst_crs=3857, dst_driver='GTiff', + threshold=0.125, resampling=gdal.GRA_NearestNeighbour): """ - Use rasterio to warp an image from one projection to another + Use GDAL to warp an image from one projection to another :param infile: Origina raster image :param outfile: Warped raster image :param dst_crs: Output projection :param dst_driver: Output filetype driver + :param threshold: error threshold + :param resampling: Resampling method :return: None """ - with rasterio.drivers(CPL_DEBUG=False): - with rasterio.open(infile) as src: - res = None - dst_transform, dst_width, dst_height = calculate_default_transform( - src.crs, dst_crs, src.width, src.height, - *src.bounds, resolution=res) - out_kwargs = src.meta.copy() - out_kwargs.update({ - 'crs': dst_crs, - 'transform': dst_transform, - 'affine': dst_transform, - 'width': dst_width, - 'height': dst_height, - 'driver': dst_driver - }) - - with rasterio.open(outfile, 'w', **out_kwargs) as dst: - for i in range(1, src.count + 1): - reproject( - source=rasterio.band(src, i), - destination=rasterio.band(dst, i), - src_transform=src.affine, - src_crs=src.crs, - dst_transform=out_kwargs['transform'], - dst_crs=out_kwargs['crs'], - resampling=RESAMPLING.nearest, - num_threads=1) + # Open source dataset + src_ds = gdal.Open(infile) + + # Define target SRS + dst_srs = SpatialReference() + dst_srs.ImportFromEPSG(dst_crs) + dst_wkt = dst_srs.ExportToWkt() + + tmp_ds = gdal.AutoCreateWarpedVRT(src_ds, + None, + dst_wkt, + resampling, + threshold) + + # Create the final warped raster + try: + dst_ds = gdal.GetDriverByName(dst_driver).CreateCopy(outfile, tmp_ds) + dst_ds.FlushCache() + finally: + dst_ds = None def get_html(url=None): diff --git a/dataqs/mmwr/mmwr.py b/dataqs/mmwr/mmwr.py index 9f5202c..606fd79 100755 --- a/dataqs/mmwr/mmwr.py +++ b/dataqs/mmwr/mmwr.py @@ -61,18 +61,20 @@ class MortalityProcessor(GeoDataProcessor): '&mmwr_location=Click+here+for+all+Locations&mmwr_table=4A' \ '&mmwr_year={year}&mmwr_week={week:02d}' params = {} - description = """Mortality data voluntarily reported from 122 cities in the -United States, most of which have populations of 100,000 or more. A death is -reported by the place of its occurrence and by the week that the death -certificate was filed. Fetal deaths are not included. - -The Morbidity and Mortality Weekly Report (MMWR ) series is -prepared by the Centers for Disease Control and Prevention (CDC). Often called -“the voice of CDC,” the MMWR series is the agency’s primary vehicle for -scientific publication of timely, reliable, authoritative, accurate, objective, -and useful public health information and recommendations. The data in the weekly - MMWR are provisional, based on weekly reports to CDC by state health -departments.\n\nSource: http://wonder.cdc.gov/mmwr/mmwr_2015.asp""" + description = ( + u"Mortality data voluntarily reported from 122 cities in the United " + u"States, most with populations of 100,000 or more. A death is reported" + u" by the place of its occurrence and by the week that the death" + u" certificate was filed. Fetal deaths are not included.\n\nThe " + u"Morbidity and Mortality Weekly Report (MMWR) series is prepared by " + u"the Centers for Disease Control & Prevention (CDC). Often called the" + u" voice of CDC,” the MMWR series is the agency’s primary vehicle for " + u"scientific publication of timely, reliable, authoritative, accurate, " + u"objective, and useful public health information and recommendations. " + u"The data in the weekly MMWR are provisional, based on weekly reports" + u" to CDC by state health departments.\n\nSource: http://wonder.cdc.gov" + u"/mmwr/mmwr_2015.asp" + ) def __init__(self, *args, **kwargs): for key in kwargs.keys(): diff --git a/dataqs/nasa_gpm/nasa_gpm.py b/dataqs/nasa_gpm/nasa_gpm.py index 03f4053..67dd0e7 100755 --- a/dataqs/nasa_gpm/nasa_gpm.py +++ b/dataqs/nasa_gpm/nasa_gpm.py @@ -51,17 +51,19 @@ class GPMProcessor(GeoDataMosaicProcessor): prefix = '3B-HHR-E.MS.MRG.3IMERG.' layer_name = 'nasa_gpm_24hr' archive_hours = ("T12:00:00.000Z", "T12:30:00.000Z") - description = """NASA IMERG: Rainfall estimates combining data from all -passive-microwave instruments in the GPM Constellation. - -This algorithm is intended to intercalibrate, merge, and interpolate "all" -satellite microwave precipitation estimates, together with microwave-calibrated -infrared (IR) satellite estimates, precipitation gauge analyses, and -potentially other precipitation estimators at fine time and space scales for the - TRMM and GPM eras over the entire globe. The system is run several times for -each observation time, first giving a quick estimate (this image) and -successively providing better estimates as more data arrive. -\n\nSource: http://pmm.nasa.gov/data-access/downloads/gpm""" + description = ( + "NASA IMERG: Rainfall estimates combining data from all passive-" + "microwave instruments in the GPM Constellation.\n\nThis algorithm is " + "intended to intercalibrate, merge, and interpolate \"all\" satellite " + "microwave precipitation estimates, together with microwave-calibrated " + "infrared (IR) satellite estimates, precipitation gauge analyses, and " + "potentially other precipitation estimators at fine time and space " + "scales for the TRMM and GPM eras over the entire globe. The system is " + "run several times for each observation time, first giving a quick " + "estimate (this image) and successively providing better estimates as " + "more data arrive.\n\nSource: http://pmm.nasa.gov/data-access/downloads" + "/gpm" + ) def download(self, auth_account=GPM_ACCOUNT, tmp_dir=GS_TMP_DIR, days=1): ftp = FTP(self.base_url) diff --git a/dataqs/processor_base.py b/dataqs/processor_base.py index 667d5dd..48ea8d0 100755 --- a/dataqs/processor_base.py +++ b/dataqs/processor_base.py @@ -374,13 +374,13 @@ def del_mosaic_image(self, url): r.raise_for_status() return r.status_code, r.content - def post_geoserver(self, filepath, layer_name): + def post_geoserver(self, filepath, layer_name, sleeptime=RSYNC_WAIT_TIME): """ Add another image to a mosaic datastore :param filepath: Full path&name of GeoTIFF to import :param layer_name: Name of the layer & store (assumed to be same) """ - sleep(RSYNC_WAIT_TIME) + sleep(sleeptime) gs_url = self.gs_url.format(ogc_server_settings.hostname, self.workspace, layer_name) data = "file://{}".format(filepath) diff --git a/dataqs/spei/spei.py b/dataqs/spei/spei.py index e78a1f0..af40b55 100755 --- a/dataqs/spei/spei.py +++ b/dataqs/spei/spei.py @@ -38,27 +38,31 @@ class SPEIProcessor(GeoDataProcessor): 'spei01': 'SPEI Global Drought Monitor (past month)', 'spei03': 'SPEI Global Drought Monitor (past 3 months)'} base_url = "http://notos.eead.csic.es/spei/nc/" - description = """The SPEI Global Drought Monitor (http://sac.csic.es/spei/) -offers near real-time information about drought conditions at the global scale, -with a 0.5 degrees spatial resolution and a monthly time resolution. SPEI -time-scales between 1 and 48 months are provided. The calibration period for the - SPEI is January 1950 to December 2010. -\n\nThe dataset is updated during the first days of the following month based on - the most reliable and updated sources of climatic data. Mean temperature data -are obtained from the NOAA NCEP CPC GHCN_CAMS gridded dataset. Monthly -precipitation sums data are obtained from the Global Precipitation Climatology -Centre (GPCC). Data from the 'first guess' GPCC product, with an original -resolution of 1º, are interpolated to the resolution of 0.5º.\n\nCurrently, -the SPEI Global Drought Monitor is based on the Thortnthwaite -equation for estimating potential evapotranspiration, PET. This is due to the -lack of real-time data sources for computing more robust PET estimations which -have larger data requirements. The main advantage of the SPEI Global Drought -Monitor is thus its near real-time character, a characteristic best suited for -drought monitoring and early warning purposes. For long-term analysis, however, -other datasets are to be preferred that rely on more robust methods of PET -estimation. Use of the SPEIbase dataset, which is based on the FAO-56 -Penman-Monteith model, is thus recommended for climatological studies of -drought.\n\nSource: http://notos.eead.csic.es/spei/nc/""" + description = ( + u"The SPEI Global Drought Monitor (http://sac.csic.es/spei) offers near" + u" real-time information about drought conditions at the global scale, " + u"with a 0.5 degrees spatial resolution and a monthly time resolution. " + u"SPEI time-scales between 1 & 48 months are provided. The calibration" + u" period for the SPEI is January 1950 to December 2010.\n\nThe dataset" + u" is updated during the first days of the following month based on the" + u" most reliable and updated sources of climatic data. Mean temperature" + u" data are obtained from the NOAA NCEP CPC GHCN_CAMS gridded dataset. " + u"Monthly precipitation sums data are obtained from the Global " + u"Precipitation Climatology Centre (GPCC). Data from the 'first guess' " + u"GPCC product, with an original resolution of 1º, are interpolated to " + u"the resolution of 0.5º.\n\nCurrently, the SPEI Global Drought Monitor" + u" is based on the Thortnthwaite equation for estimating potential " + u"evapotranspiration, PET. This is due to the lack of real-time data " + u"sources for computing more robust PET estimations which have larger " + u"data requirements. The main advantage of the SPEI Global Drought " + u"Monitor is thus its near real-time character, a characteristic best " + u"suited for drought monitoring and early warning purposes. For long-" + u"term analysis, however, other datasets are to be preferred that rely " + u"on more robust methods of PET estimation. Use of the SPEIbase dataset" + u", which is based on the FAO-56 Penman-Monteith model, is thus " + u"recommended for climatological studies of drought.\n\nSource: http://" + u"notos.eead.csic.es/spei/nc/" + ) def convert(self, nc_file): tif_file = "{}.tif".format(nc_file) @@ -75,8 +79,8 @@ def run(self): object's spei_files property. """ for layer_name in self.spei_files.keys(): - self.download("{}{}.nc".format(self.base_url, layer_name)) - tif_file = self.convert(layer_name) + dlfile = self.download("{}{}.nc".format(self.base_url, layer_name)) + tif_file = self.convert(dlfile) self.post_geoserver(tif_file, layer_name) if not style_exists(layer_name): with open(os.path.join(script_dir, diff --git a/dataqs/usgs_quakes/usgs_quakes.py b/dataqs/usgs_quakes/usgs_quakes.py index b1ce9ca..8039635 100755 --- a/dataqs/usgs_quakes/usgs_quakes.py +++ b/dataqs/usgs_quakes/usgs_quakes.py @@ -46,9 +46,10 @@ class USGSQuakeProcessor(GeoDataProcessor): base_url = "http://earthquake.usgs.gov/fdsnws/event/1/query?" \ "format=geojson&starttime={}&endtime={}" params = {} - description = """Earthquake data from the US Geological Survey. -\n\nSource: http://earthquake.usgs.gov/fdsnws/event/1/ -""" + description = ( + "Earthquake data from the US Geological Survey.\n\nSource: http://" + "earthquake.usgs.gov/fdsnws/event/1/" + ) def __init__(self, *args, **kwargs): for key in kwargs.keys(): diff --git a/dataqs/whisp/whisp.py b/dataqs/whisp/whisp.py index 3178678..93e298d 100755 --- a/dataqs/whisp/whisp.py +++ b/dataqs/whisp/whisp.py @@ -80,27 +80,30 @@ class WhispProcessor(GeoDataProcessor): prefix = "whisp" base_url = "https://www.nwhc.usgs.gov/whispers/searchForm/recent100" title = "Wildlife Health Information Sharing Partnership - Events" - description = """WHISPers stands for Wildlife Health Information Sharing -Partnership - event reporting system. It is a partner-driven, web-based -repository for sharing basic information about historic and ongoing wildlife -mortality (death) and morbidity (illness) events involving five or more wild -animals. The primary goal of the system is to provide natural resource -management partners and the public with timely, accurate information on where -wildlife disease events are occurring or have occurred. The system is also a -searchable archive of historic wildlife mortality and morbidity event data. -The information is opportunistically collected and does not reflect all the -mortality events that occur in North America.\n\nEach record includes location -of the event (to county level), start and end dates, species involved, number of - sick and dead, cause of wildlife illness or death, the laboratory where cause -of disease/death was confirmed, and contact information for the agency that -submitted carcasses/samples to the laboratory.\n\nThis information is from a -variety of sources including federal, tribal, and state agencies who contacted -the USGS National Wildlife Health Center for diagnostic and epidemiological -assistance during mortality events; diagnostic case reports from other wildlife -diagnostic laboratories; and published reports in journals, newspapers, or other - reporting outlets such as ProMEDmail.org. -\n\nData source: http://www.nwhc.usgs.gov/whispers/ -""" + description = ( + "WHISPers stands for Wildlife Health Information Sharing Partnership -" + " event reporting system. It is a partner-driven, web-based repository " + "for sharing basic information about historic and ongoing wildlife " + "mortality (death) and morbidity (illness) events involving five or " + "more wild animals. The primary goal of the system is to provide " + "natural resource management partners and the public with timely, " + "accurate information on where wildlife disease events are occurring or" + " have occurred. The system is also a searchable archive of historic " + "wildlife mortality and morbidity event data. The information is " + "opportunistically collected and does not reflect all the mortality " + "events that occur in North America.\n\nEach record includes location " + "of the event (to county level), start and end dates, species involved," + " number of sick and dead, cause of wildlife illness or death, the " + "laboratory where cause of disease/death was confirmed, and contact " + "information for the agency that submitted carcasses/samples to the " + "laboratory.\n\nThis information is from a variety of sources including" + " federal, tribal, and state agencies who contacted the USGS National " + "Wildlife Health Center for diagnostic and epidemiological assistance " + "during mortality events; diagnostic case reports from other wildlife " + "diagnostic laboratories; and published reports in journals, newspapers" + ", or other reporting outlets such as ProMEDmail.org.\n\nData source: " + "http://www.nwhc.usgs.gov/whispers/" + ) def download(self, url=None, filename=None): if not url: diff --git a/dataqs/worldclim/worldclim.py b/dataqs/worldclim/worldclim.py index 740671e..977e3cf 100755 --- a/dataqs/worldclim/worldclim.py +++ b/dataqs/worldclim/worldclim.py @@ -86,14 +86,16 @@ class WorldClimProcessor(GeoDataProcessor): resolutions = ['10m', '5m', '2-5m'] - base_description = """WorldClim data layers were generated through -interpolation of average monthly climate data from weather stations on a 30 -arc-second resolution grid (often referred to as "1 km2" resolution). -\n\nMethods: http://worldclim.org/methods1 -\n\nSource: http://worldclim.org/version1 -\n\nCitation: Hijmans, R.J., S.E. Cameron, J.L. Parra, P.G. Jones and A. Jarvis, - 2005. Very high resolution interpolated climate surfaces for global land areas. - International Journal of Climatology 25: 1965-1978.""" + base_description = ( + "WorldClim data layers were generated through interpolation of average " + "monthly climate data from weather stations on a 30 arc-second " + "resolution grid (often referred to as '1 km2' resolution).\n\nMethods:" + " http://worldclim.org/methods1\n\nSource: http://worldclim.org/" + "version1\n\nCitation: Hijmans, R.J., S.E. Cameron, J.L. Parra, P.G. " + "Jones and A. Jarvis, 2005. Very high resolution interpolated climate " + "surfaces for global land areas. International Journal of Climatology " + "25: 1965-1978." + ) def custom_title(self, var_tuple, index): """ diff --git a/dataqs/wqp/wqp.py b/dataqs/wqp/wqp.py index d2f2994..4899d94 100755 --- a/dataqs/wqp/wqp.py +++ b/dataqs/wqp/wqp.py @@ -60,14 +60,17 @@ class WaterQualityPortalProcessor(GeoDataProcessor): station_table = "wqp_api_stations" suffix = "_map" base_url = "http://www.waterqualitydata.us/{}/search?countrycode=US" - description = """Water quality measurements obtained from the Water Quality -Data Portal (WQP), which includes data from:\n\nUSGS NWIS - Water-resources data - collected from approximately 1.5 million sites in all 50 states, the District -of Columbia, Puerto Rico, the Virgin Islands, Guam, American Samoa, and the -Commonwealth of the Northern Mariana Islands.\n\nUSEPA STORET - A data warehouse - for water quality, biological, and physical data used by state environmental -agencies, EPA, other federal agencies, universities, private citizens, and -others.\n\nSource: http://www.waterqualitydata.us""" + description = ( + "Water quality measurements obtained from the Water Quality Data Portal" + " (WQP), which includes data from:\n\nUSGS NWIS - Water-resources data " + "collected from approximately 1.5 million sites in all 50 states, the " + "District of Columbia, Puerto Rico, the Virgin Islands, Guam, American " + "Samoa, and the Commonwealth of the Northern Mariana Islands.\n\nUSEPA " + "STORET - A data warehouse for water quality, biological, and physical " + "data used by state environmental agencies, EPA, other federal agencies" + ", universities, private citizens, and others.\n\nSource: " + "http://www.waterqualitydata.us" + ) skip_errors = True def __init__(self, *args, **kwargs): diff --git a/local_settings.py b/local_settings.py.template similarity index 78% rename from local_settings.py rename to local_settings.py.template index a777f70..e6dc512 100644 --- a/local_settings.py +++ b/local_settings.py.template @@ -4,13 +4,18 @@ 'dataqs', 'dataqs.aqicn', 'dataqs.airnow', + 'dataqs.cmap', 'dataqs.forecastio', 'dataqs.gfms', + 'dataqs.gistemp', 'dataqs.gdacs', + 'dataqs.gfms', 'dataqs.hifld', + 'dataqs.mmwr', 'dataqs.nasa_gpm', 'dataqs.spei', 'dataqs.usgs_quakes', - 'dataqs.gistemp', + 'dataqs.whisp', 'dataqs.worldclim', + 'dataqs.wqp', ) diff --git a/setup.py b/setup.py index 1eb6a61..7488f7b 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,6 @@ 'shapely', 'pymongo', 'numpy', - 'rasterio==0.31.0', 'gdal==1.10' ] )