[WIP] Major fixes to datasets.py

.gitignore

@@ -99,3 +99,7 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# Dask-ml related
+src
+dask-ml

.travis.yml

@@ -24,8 +24,12 @@ install:
     - conda create $CHANNELS --name $ENV_NAME python=$PYTHON numpy=$NUMPY
     - source activate $ENV_NAME
 
+before_script:
+    - conda install -y requests jinja2
+    - ./fetch_dask-ml.py # Fetch dask-ml recipe from conda-forge
+
 script:
-    - conda build --python $PYTHON --numpy $NUMPY conda.recipe
+    - conda build -c conda-forge --python $PYTHON --numpy $NUMPY conda.recipe
 
 notifications:
   on_success: change

conda.recipe/meta.yaml

@@ -20,6 +20,9 @@ requirements:
     - dask
     - scikit-learn
     - six
+    - dask_ml
+    - attrs
+    - rasterio
 
 about:
   home: http://github.com/ContinuumIO/xarray_filters

environment_py27.yml

@@ -1,5 +1,6 @@
 name: xarray_filters_py27
 channels:
+ - defaults
  - conda-forge # essential for rasterio on osx
 dependencies:
  - python=2.7
@@ -16,3 +17,8 @@ dependencies:
  - attrs
  - conda-forge::rasterio
  - conda-forge::gdal=2.1.*
+ - cython # for dask-ml
+ - requests # for fetching dask-ml recipe
+ - jinja2 # for fetching dask-ml recipe
+ - pip:
+    - "--editable=git+https://github.com/dask/dask-ml.git@master#egg=dask-ml_master"

environment_py35.yml

@@ -1,9 +1,11 @@
 name: xarray_filters_py35
 channels:
+ - defaults
  - conda-forge # essential for rasterio on osx
 dependencies:
  - python=3.5
  - dask
+ - cython
  - numba
  - numpy
  - pandas
@@ -15,4 +17,8 @@ dependencies:
  - attrs
  - conda-forge::rasterio
  - conda-forge::gdal=2.1.*
-
+ - cython # for dask-ml
+ - requests # for fetching dask-ml recipe
+ - jinja2 # for fetching dask-ml recipe
+ - pip:
+    - "--editable=git+https://github.com/dask/dask-ml.git@master#egg=dask-ml_master"

environment_py36.yml

@@ -1,5 +1,6 @@
 name: xarray_filters_py36
 channels:
+ - defaults
  - conda-forge # essential for rasterio on osx
 dependencies:
  - python=3.6
@@ -15,4 +16,8 @@ dependencies:
  - attrs
  - conda-forge::rasterio
  - conda-forge::gdal=2.1.*
-
+ - cython # for dask-ml
+ - requests # for fetching dask-ml recipe
+ - jinja2 # for fetching dask-ml recipe
+ - pip:
+   - "--editable=git+https://github.com/dask/dask-ml.git@master#egg=dask-ml_master"

fetch_dask-ml.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import os
+import sys
+import requests
+import jinja2
+
+recipe_dpath = './dask_ml'
+meta_yml_fpath = os.path.join(recipe_dpath, 'meta.yaml')
+if os.path.isfile(meta_yml_fpath):
+    print('Recipe exists: skipping download.')
+    sys.exit(0)
+
+meta_yml_url = 'https://raw.githubusercontent.com/TomAugspurger/staged-recipes/dask-ml/recipes/dask-ml/meta.yaml'
+resp = requests.get(meta_yml_url)
+assert resp.status_code == 200, 'Error fetching meta.yaml file'
+template = jinja2.Template(resp.text)
+os.makedirs(recipe_dpath)
+print('Writing '+meta_yml_fpath)
+with open(meta_yml_fpath, 'w') as fp:
+    fp.write(template.render())