Merge pull request #514 from s22s/feature/ipy-display

DataFrame.display injection

Author: metasim

.circleci/config.yml

@@ -42,7 +42,8 @@ orbs:
         steps:
           - run:
               name: Install requirements
-              command: python -m pip install --progress-bar=off --user -r pyrasterframes/src/main/python/requirements.txt
+              command: /opt/conda/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt
+
 
   rasterframes:
     commands:
@@ -117,6 +118,7 @@ jobs:
       - checkout
       - sbt/setup
       - python/setup
+      - python/requirements
       - rasterframes/setup
       - rasterframes/restore-cache
       - sbt/compile

.gitignore

@@ -27,3 +27,4 @@ tour/*.tiff
 scoverage-report*
 
 zz-*
+rf-notebook/src/main/notebooks/.ipython

README.md

@@ -62,6 +62,6 @@ Additional, Python sepcific build instruction may be found at [pyrasterframes/sr
 
 ## Copyright and License
 
-RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2019.
+RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2020.
 
 

core/src/main/resources/reference.conf

@@ -1,6 +1,6 @@
 rasterframes {
   nominal-tile-size = 256
-  prefer-gdal = true
+  prefer-gdal = false
   showable-tiles = false
   showable-max-cells = 20
   max-truncate-row-element-length = 40

core/src/main/scala/org/locationtech/rasterframes/util/debug/package.scala

@@ -21,10 +21,46 @@
 
 package org.locationtech.rasterframes.util
 
+import java.lang.reflect.{AccessibleObject, Modifier}
+
+import org.apache.spark.Partition
+import org.apache.spark.rdd.RDD
+
+import scala.util.Try
+
 /**
  * Additional debugging routines. No guarantees these are or will remain stable.
  *
  * @since 4/6/18
  */
 package object debug {
+
+  implicit class DescribeablePartition(val p: Partition) extends AnyVal {
+    def describe: String = Try {
+      def acc[A <: AccessibleObject](a: A): A = {
+        a.setAccessible(true); a
+      }
+
+      val getters = p.getClass.getDeclaredMethods
+        .filter(_.getParameterCount == 0)
+        .filter(m ⇒ (m.getModifiers & Modifier.PUBLIC) > 0)
+        .filterNot(_.getName == "hashCode")
+        .map(acc)
+        .map(m ⇒ m.getName + "=" + String.valueOf(m.invoke(p)))
+
+      val fields = p.getClass.getDeclaredFields
+        .filter(f ⇒ (f.getModifiers & Modifier.PUBLIC) > 0)
+        .map(acc)
+        .map(m ⇒ m.getName + "=" + String.valueOf(m.get(p)))
+
+      p.getClass.getSimpleName + "(" + (fields ++ getters).mkString(", ") + ")"
+
+    }.getOrElse(p.toString)
+  }
+
+  implicit class RDDWithPartitionDescribe(val r: RDD[_]) extends AnyVal {
+    def describePartitions: String = r.partitions.map(p ⇒ ("Partition " + p.index) -> p.describe).mkString("\n")
+  }
+
 }
+

datasource/src/it/scala/org/locationtech/rasterframes/datasource/raster/RaterSourceDataSourceIT.scala

@@ -31,7 +31,7 @@ class RaterSourceDataSourceIT extends TestEnvironment with TestData {
       // A regression test.
       val rf = spark.read.raster
         .withSpatialIndex()
-        .load("https://s22s-test-geotiffs.s3.amazonaws.com/water_class/seasonality_90W_50N.tif")
+        .load("https://rasterframes.s3.amazonaws.com/samples/water_class/seasonality_90W_50N.tif")
 
       val target_rf =
         rf.select(rf_extent($"proj_raster").alias("extent"), rf_crs($"proj_raster").alias("crs"), rf_tile($"proj_raster").alias("target"))

datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala

@@ -153,6 +153,6 @@ case class RasterSourceRelation(
         .repartitionByRange(numParts,$"spatial_index")
       indexed.rdd
     }
-    else df.rdd
+    else df.repartition(numParts).rdd
   }
 }

datasource/src/test/resources/log4j.properties

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+# Set the default spark-shell log level to WARN. When running the spark-shell, the
+# log level for this class is used to overwrite the root logger's log level, so that
+# the user can have different defaults for the shell and regular Spark apps.
+log4j.logger.org.apache.spark.repl.Main=WARN
+
+
+log4j.logger.org.apache=ERROR
+log4j.logger.com.amazonaws=WARN
+log4j.logger.geotrellis=WARN
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.spark_project.jetty=WARN
+log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
+log4j.logger.org.locationtech.rasterframes=DEBUG
+log4j.logger.org.locationtech.rasterframes.ref=DEBUG
+log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
+
+# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
+log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
+
+log4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator=ERROR
+log4j.logger.org.apache.spark.sql.execution.WholeStageCodegenExec=ERROR
+log4j.logger.geotrellis.raster.gdal=ERROR

docs/src/main/paradox/index.md

@@ -29,18 +29,20 @@ The source code can be found on GitHub at [locationtech/rasterframes](https://gi
 
 ## Detailed Contents
 
-@@ toc { depth=4 }
+@@ toc { depth=3 }
 
 @@@ index
-* [Overview](description.md)
-* [Getting Started](getting-started.md)
-* [Concepts](concepts.md)
-* [Raster Data I/O](raster-io.md)
-* [Vector Data](vector-data.md)
-* [Raster Processing](raster-processing.md)
-* [Numpy and Pandas](numpy-pandas.md) 
-* [Scala and SQL](languages.md)
-* [Function Reference](reference.md)
-* [Release Notes](release-notes.md)
+* @ref:[Overview](description.md)
+* @ref:[Getting Started](getting-started.md)
+* @ref:[Concepts](concepts.md)
+* @ref:[Raster Data I/O](raster-io.md)
+* @ref:[Vector Data](vector-data.md)
+* @ref:[Raster Processing](raster-processing.md)
+* @ref:[Machine Learning](machine-learning.md)
+* @ref:[Numpy and Pandas](numpy-pandas.md) 
+* @ref:[IPython Extensions](ipython.md) 
+* @ref:[Scala and SQL](languages.md)
+* @ref:[Function Reference](reference.md)
+* @ref:[Release Notes](release-notes.md)
 @@@
 

docs/src/main/paradox/raster-processing.md

@@ -9,7 +9,6 @@
 * @ref:[Aggregation](aggregation.md)
 * @ref:[Time Series](time-series.md)
 * @ref:[Raster Join](raster-join.md)
-* @ref:[Machine Learning](machine-learning.md)
 
 @@@
 

docs/src/main/paradox/release-notes.md

@@ -5,10 +5,14 @@
 ### 0.9.1
 
 * Upgraded to Spark 2.4.7
+* Added `pyspark.sql.DataFrame.display(num_rows:int, truncate:bool)` extension method when `rf_ipython` is imported.
+* Added users' manual section on IPython display enhancements.
 * Added `method_name` parameter to the `rf_resample` method.
    * __BREAKING__: In SQL, the function `rf_resample` now takes 3 arguments. You can use `rf_resample_nearest` with two arguments or refactor to `rf_resample(t, v, "nearest")`.
 * Added resample method parameter to SQL and Python APIs. @ref:[See updated docs](raster-join.md).
-
+* Upgraded many of the pyrasterframes dependencies, including: 
+  `descartes`, `fiona`, `folium`, `geopandas`, `matplotlib`, `numpy`, `pandas`, `rasterio`, `shapely`
+* Changed `rasterframes.prefer-gdal` configuration parameter to default to `False`, as JVM GeoTIFF performs just as well for COGs as the GDAL one.
 
 ### 0.9.0
 

pyrasterframes/build.sbt

@@ -9,7 +9,7 @@ Python / doc := (Python / doc / target).toTask.dependsOn(
   Def.sequential(
     assembly,
     Test / compile,
-    pySetup.toTask(" pweave")
+    pySetup.toTask(" pweave --quick True")
   )
 ).value
 

pyrasterframes/src/main/python/README.md

@@ -38,6 +38,15 @@ Issue tracking is through [github](https://github.com/locationtech/rasterframes/
 
 Community contributions are always welcome. To get started, please review our [contribution guidelines](https://github.com/locationtech/rasterframes/blob/develop/CONTRIBUTING.md), [code of conduct](https://github.com/locationtech/rasterframes/blob/develop/CODE_OF_CONDUCT.md), and [developer's guide](../../../README.md).  Reach out to us on [gitter][gitter] so the community can help you get started!
 
+## Development environment setup
 
+For best results, we suggest using `conda` and the `conda-forge` channel to install the compiled dependencies before installing the packages in `setup.py`. Assuming you're in the same directory as this file:
+
+    conda create -n rasterframes python==3.7
+    conda install --file ./requirements-condaforge.txt
+    
+Then you can install the source dependencies:
+
+    pip install -e . 
 
 [gitter]: https://gitter.im/locationtech/rasterframes

pyrasterframes/src/main/python/docs/aggregation.pymd

@@ -71,7 +71,7 @@ rf.agg(rf_agg_local_mean('tile')) \
 We can also count the total number of data and NoData cells over all the _tiles_ in a DataFrame using @ref:[`rf_agg_data_cells`](reference.md#rf-agg-data-cells) and @ref:[`rf_agg_no_data_cells`](reference.md#rf-agg-no-data-cells). There are ~3.8 million data cells and ~1.9 million NoData cells in this DataFrame. See the section on @ref:["NoData" handling](nodata-handling.md) for additional discussion on handling missing data.
 
 ```python, cell_counts
-rf = spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/MCD43A4.006/11/05/2018233/MCD43A4.A2018233.h11v05.006.2018242035530_B02.TIF')
+rf = spark.read.raster('https://rasterframes.s3.amazonaws.com/samples/MCD43A4.006/11/05/2018233/MCD43A4.A2018233.h11v05.006.2018242035530_B02.TIF')
 stats = rf.agg(rf_agg_data_cells('proj_raster'), rf_agg_no_data_cells('proj_raster'))
 stats
 ```
@@ -83,7 +83,7 @@ The statistical summary functions return a summary of cell values: number of dat
 The @ref:[`rf_tile_stats`](reference.md#rf-tile-stats) function computes summary statistics separately for each row in a _tile_ column as shown below.
 
 ```python, tile_stats
-rf = spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B02.tif')
+rf = spark.read.raster('https://rasterframes.s3.amazonaws.com/samples/luray_snp/B02.tif')
 stats = rf.select(rf_tile_stats('proj_raster').alias('stats'))
 
 stats.printSchema()
@@ -125,7 +125,7 @@ The @ref:[`rf_tile_histogram`](reference.md#rf-tile-histogram) function computes
 ```python, tile_histogram
 import matplotlib.pyplot as plt
 
-rf = spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/MCD43A4.006/11/05/2018233/MCD43A4.A2018233.h11v05.006.2018242035530_B02.TIF')
+rf = spark.read.raster('https://rasterframes.s3.amazonaws.com/samples/MCD43A4.006/11/05/2018233/MCD43A4.A2018233.h11v05.006.2018242035530_B02.TIF')
 
 hist_df = rf.select(rf_tile_histogram('proj_raster')['bins'].alias('bins'))
 hist_df.printSchema()

pyrasterframes/src/main/python/docs/ipython.pymd

@@ -0,0 +1,94 @@
+# IPython/Jupyter Extensions
+
+The `pyrasterframes.rf_ipython` module injects a number of visualization extensions into the IPython environment, enhancing visualization of `DataFrame`s and `Tile`s.
+
+By default, the last expression's result in a IPython cell is passed to the `IPython.display.display` function. This function in turn looks for a [`DisplayFormatter`](https://ipython.readthedocs.io/en/stable/api/generated/IPython.core.formatters.html#IPython.core.formatters.DisplayFormatter) associated with the type, which in turn converts the instance to a display-appropriate representation, based on MIME type. For example, each `DisplayFormatter` may `plain/text` version for the IPython shell, and a `text/html` version for a Jupyter Notebook.
+
+This will be our setup for the following examples:
+
+```python setup
+from pyrasterframes import *
+from pyrasterframes.rasterfunctions import *
+from pyrasterframes.utils import create_rf_spark_session
+import pyrasterframes.rf_ipython
+from IPython.display import display
+import os.path
+spark = create_rf_spark_session()
+def scene(band):
+    b = str(band).zfill(2) # converts int 2 to '02'
+    return 'https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/' \
+             'MCD43A4.A2019059.h11v08.006.2019072203257_B{}.TIF'.format(b)
+rf = spark.read.raster(scene(2), tile_dimensions=(256, 256))
+```
+
+## Tile Samples
+
+We have some convenience methods to quickly visualize tiles (see discussion of the RasterFrame @ref:[schema](raster-read.md#single-raster) for orientation to the concept) when inspecting a subset of the data in a Notebook.
+
+In an IPython or Jupyter interpreter, a `Tile` object will be displayed as an image with limited metadata.
+
+```python, sample_tile
+sample_tile = rf.select(rf_tile('proj_raster').alias('tile')).first()['tile']
+sample_tile # or `display(sample_tile)`
+```
+
+## DataFrame Samples
+
+Within an IPython or Jupyter interpreter, a Spark and Pandas DataFrames containing a column of _tiles_ will be rendered as the samples discussed above. Simply import the `rf_ipython` submodule to enable enhanced HTML rendering of these DataFrame types.
+
+```python display_samples
+rf # or `display(rf)`, or `rf.display()`
+```
+
+### Changing Number of Rows
+
+By default the RasterFrame sample display renders 5 rows. Because the `IPython.display.display` function doesn't pass parameters to the underlying rendering functions, we have to provide a different means of passing parameters to the rendering code. Pandas approach to this is to use  global settings via `set_option`/`get_option`. We take a more functional approach and have the user invoke an explicit `display` method:
+
+```python custom_display, evaluate=False 
+rf.display(num_rows=1, truncate=True)
+```  
+
+```python custom_display_mime, echo=False 
+rf.display(num_rows=1, truncate=True, mimetype='text/markdown')
+```  
+
+### Pandas
+
+There is similar rendering support injected into the Pandas by the `rf_ipython` module, for Pandas Dataframes having Tiles in them: 
+
+```python pandas_dataframe
+# Limit copy of data from Spark to a few tiles.
+pandas_df = rf.select(rf_tile('proj_raster'), rf_extent('proj_raster')).limit(4).toPandas()
+pandas_df # or `display(pandas_df)`
+```
+
+## Sample Colorization
+
+RasterFrames uses the "Viridis" color ramp as the default color profile for tile column. There are other options for reasoning about how color should be applied in the results.
+
+### Color Composite 
+
+As shown in @ref:[Writing Raster Data section](raster-write.md) section, composites can be constructed for visualization:
+
+```python, png_color_composite
+from IPython.display import Image # For telling IPython how to interpret the PNG byte array
+# Select red, green, and blue, respectively
+three_band_rf = spark.read.raster(source=[[scene(1), scene(4), scene(3)]])
+composite_rf = three_band_rf.withColumn('png',
+                    rf_render_png('proj_raster_0', 'proj_raster_1', 'proj_raster_2'))
+png_bytes = composite_rf.select('png').first()['png'] 
+Image(png_bytes)
+```
+
+```python, png_render, echo=False
+from IPython.display import display_markdown
+display_markdown(pyrasterframes.rf_ipython.binary_to_html(png_bytes), raw=True)
+```
+
+### Custom Color Ramp
+
+You can also apply a different color ramp to a single-channel Tile using the @ref[`rf_render_color_ramp_png`](reference.md#rf-render-color-ramp-png) function. See the function documentation for information about the available color maps.
+
+```python, color_map
+rf.select(rf_render_color_ramp_png('proj_raster', 'Magma'))
+```

pyrasterframes/src/main/python/docs/local-algebra.pymd

@@ -35,7 +35,7 @@ This form of `(x - y) / (x + y)` is common in remote sensing and is called a nor
 
 ```python, read_rasters
 from pyspark.sql import Row
-uri_pattern = 'https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B0{}.tif'
+uri_pattern = 'https://rasterframes.s3.amazonaws.com/samples/luray_snp/B0{}.tif'
 catalog_df = spark.createDataFrame([
     Row(red=uri_pattern.format(4), nir=uri_pattern.format(8))
 ])

pyrasterframes/src/main/python/docs/masking.pymd

@@ -30,9 +30,9 @@ The first step is to create a catalog with our band of interest and the SCL band
 ```python, blue_scl_cat
 from pyspark.sql import Row
 
-blue_uri = 'https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B02.tif'
-green_uri = 'https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B03.tif'
-scl_uri = 'https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/SCL.tif'
+blue_uri = 'https://rasterframes.s3.amazonaws.com/samples/luray_snp/B02.tif'
+green_uri = 'https://rasterframes.s3.amazonaws.com/samples/luray_snp/B03.tif'
+scl_uri = 'https://rasterframes.s3.amazonaws.com/samples/luray_snp/SCL.tif'
 cat = spark.createDataFrame([Row(blue=blue_uri, green=green_uri, scl=scl_uri),])
 unmasked = spark.read.raster(cat, catalog_col_names=['blue', 'green', 'scl'])
 unmasked.printSchema()

pyrasterframes/src/main/python/docs/nodata-handling.pymd

@@ -40,7 +40,7 @@ CellType.float64()
 We can also inspect the cell type of a given _tile_ or `proj_raster` column.
 
 ```python, ct_from_sen
-cell_types = spark.read.raster('https://s22s-test-geotiffs.s3.amazonaws.com/luray_snp/B02.tif') \
+cell_types = spark.read.raster('https://rasterframes.s3.amazonaws.com/samples/luray_snp/B02.tif') \
     .select(rf_cell_type('proj_raster')).distinct()
 cell_types    
 ```