Merge pull request #83 from astrolabsoftware/0.8.2

Bump version 0.8.1 -> 0.8.2
astrolabsoftware · May 27, 2019 · 5d8a1e2 · 5d8a1e2
2 parents 4020f7d + c0d4228
commit 5d8a1e2
Show file tree

Hide file tree

Showing 16 changed files with 30 additions and 25 deletions.
diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@
 - [10/2018] **Release**: version 0.7.0, 0.7.1
 - [12/2018] **Release**: version 0.7.2
 - [03/2019] **Release**: version 0.7.3
-- [05/2019] **Release**: version 0.8.0, 0.8.1
+- [05/2019] **Release**: version 0.8.0, 0.8.1, 0.8.2
 
 ## spark-fits
 
@@ -32,20 +32,20 @@ data sources (CSV, JSON, Avro, Parquet, etc). Note that spark-fits follows Apach
 
 ```bash
 # Scala 2.11
-spark-submit --packages "com.github.astrolabsoftware:spark-fits_2.11:0.8.1" <...>
+spark-submit --packages "com.github.astrolabsoftware:spark-fits_2.11:0.8.2" <...>
 
 # Scala 2.12
-spark-submit --packages "com.github.astrolabsoftware:spark-fits_2.12:0.8.1" <...>
+spark-submit --packages "com.github.astrolabsoftware:spark-fits_2.12:0.8.2" <...>
 ```
 
 or you can link against this library in your program at the following coordinates in your build.sbt
 
 ```scala
 // Scala 2.11
-libraryDependencies += "com.github.astrolabsoftware" % "spark-fits_2.11" % "0.8.1"
+libraryDependencies += "com.github.astrolabsoftware" % "spark-fits_2.11" % "0.8.2"
 
 // Scala 2.12
-libraryDependencies += "com.github.astrolabsoftware" % "spark-fits_2.12" % "0.8.1"
+libraryDependencies += "com.github.astrolabsoftware" % "spark-fits_2.12" % "0.8.2"
 ```
 
 Currently available:

diff --git a/build.sbt b/build.sbt
@@ -19,7 +19,7 @@ import xerial.sbt.Sonatype._
 lazy val root = (project in file(".")).
  settings(
    inThisBuild(List(
-     version      := "0.8.1",
+     version      := "0.8.2",
      mainClass in Compile := Some("com.astrolabsoftware.sparkfits.ReadFits")
    )),
    // Name of the application

diff --git a/docs/01_installation.md b/docs/01_installation.md
@@ -21,10 +21,10 @@ You can link spark-fits to your project (either `spark-shell` or `spark-submit`)
 
 ```bash
 # Scala 2.11
-toto:~$ spark-submit --packages "com.github.astrolabsoftware:spark-fits_2.11:0.8.1" <...>
+toto:~$ spark-submit --packages "com.github.astrolabsoftware:spark-fits_2.11:0.8.2" <...>
 
 # Scala 2.12
-toto:~$ spark-submit --packages "com.github.astrolabsoftware:spark-fits_2.12:0.8.1" <...>
+toto:~$ spark-submit --packages "com.github.astrolabsoftware:spark-fits_2.12:0.8.2" <...>
 ```
 
 It might not contain the latest features though (see *Building from source*).

diff --git a/docs/02_api.md b/docs/02_api.md
@@ -20,10 +20,10 @@ coordinates in your `build.sbt`:
 
 ```scala
 // %% will automatically set the Scala version needed for spark-fits
-libraryDependencies += "com.github.astrolabsoftware" %% "spark-fits" % "0.8.1"
+libraryDependencies += "com.github.astrolabsoftware" %% "spark-fits" % "0.8.2"
 
 // Alternatively you can also specify directly the Scala version, e.g.
-libraryDependencies += "com.github.astrolabsoftware" % "spark-fits_2.11" % "0.8.1"
+libraryDependencies += "com.github.astrolabsoftware" % "spark-fits_2.11" % "0.8.2"
 ```
 
 #### Scala 2.10.6 and 2.11.X
@@ -47,6 +47,7 @@ object ReadFits extends App {
     .option("hdu", <Int>)                 // [mandatory] Which HDU you want to read.
     .option("columns", <String>)          // [optional]  Comma-separated column names to load. Default loads all columns.
     .option("recordlength", <Int>)        // [optional]  If you want to define yourself the length of a record.
+    .option("mode", <String>)             // [optional]  Discard empty files silently or fail fast.
     .option("verbose", <Boolean>)         // [optional]  If you want to print debugging messages on screen.
     .schema(<StructType>)                 // [optional]  If you want to bypass the header.
     .load(<String>)                       // [mandatory] Path to file or directory. Load data as DataFrame.
@@ -78,6 +79,8 @@ collector time. The maximum size allowed for a single record to be
 processed is 2\*\*31 - 1 (Int max value). But I doubt you ever need to
 go as high...
 
+The `mode` parameter controls the behaviour when reading many files. By default, it is set to `PERMISSIVE`, that is if there are empty files they will be silently discarded and the connector will not fail. Note that the empty files found will be printed on screen (WARN level of log). You can also set `mode` to `FAILFAST` to force the connector to crash if it encounters empty files. 
+
 Note that the schema is directly inferred from the HEADER of the HDU. In
 case the HEADER is not present or corrupted, you can also manually
 specify it:
@@ -134,6 +137,7 @@ if __name__ == "__main__":
     .option("hdu", int)\
     .option("columns", str)\
     .option("recordlength", int)\
+    .option("mode", str)\
     .option("verbose", bool)\
     .schema(StructType)\
     .load(str)
@@ -153,6 +157,7 @@ DataFrame df = spark.read()
   .option("hdu", <Int>)
   .option("columns", <String>)
   .option("recordlength", <Int>)
+  .option("mode", <String>)
   .option("verbose", <Boolean>)
   .schema(<StructType>)
   .load(<String>);

diff --git a/docs/03_interactive.md b/docs/03_interactive.md
@@ -14,21 +14,21 @@ option. For example, to include it when starting the spark shell:
 
 ```bash
 # Scala 2.11
-$SPARK_HOME/bin/spark-shell --packages com.github.astrolabsoftware:spark-fits_2.11:0.8.1
+$SPARK_HOME/bin/spark-shell --packages com.github.astrolabsoftware:spark-fits_2.11:0.8.2
 
 # Scala 2.12
-$SPARK_HOME/bin/spark-shell --packages com.github.astrolabsoftware:spark-fits_2.12:0.8.1
+$SPARK_HOME/bin/spark-shell --packages com.github.astrolabsoftware:spark-fits_2.12:0.8.2
 ```
 
 Using `--packages` ensures that this library and its dependencies will
 be added to the classpath (make sure you use the latest version). In Python, you would do the same
 
 ```bash
 # Scala 2.11
-$SPARK_HOME/bin/pyspark --packages com.github.astrolabsoftware:spark-fits_2.11:0.8.1
+$SPARK_HOME/bin/pyspark --packages com.github.astrolabsoftware:spark-fits_2.11:0.8.2
 
 # Scala 2.12
-$SPARK_HOME/bin/pyspark --packages com.github.astrolabsoftware:spark-fits_2.12:0.8.1
+$SPARK_HOME/bin/pyspark --packages com.github.astrolabsoftware:spark-fits_2.12:0.8.2
 ```
 
 Alternatively to have the latest development you can download this repo

diff --git a/docs/_pages/home.md b/docs/_pages/home.md
@@ -8,7 +8,7 @@ header:
   cta_url: "/docs/installation/"
   caption:
 intro:
-  - excerpt: '<p><font size="6">Distribute FITS data with Apache Spark: Binary tables, images and more!</font></p><br /><a href="https://github.com/astrolabsoftware/spark-fits/releases/tag/0.8.1">Latest release: 0.8.1</a>'
+  - excerpt: '<p><font size="6">Distribute FITS data with Apache Spark: Binary tables, images and more!</font></p><br /><a href="https://github.com/astrolabsoftware/spark-fits/releases/tag/0.8.2">Latest release: 0.8.2</a>'
 excerpt: '{::nomarkdown}<iframe style="display: inline-block;" src="https://ghbtns.com/github-btn.html?user=astrolabsoftware&repo=spark-fits&type=star&count=true&size=large" frameborder="0" scrolling="0" width="160px" height="30px"></iframe> <iframe style="display: inline-block;" src="https://ghbtns.com/github-btn.html?user=astrolabsoftware&repo=spark-fits&type=fork&count=true&size=large" frameborder="0" scrolling="0" width="158px" height="30px"></iframe>{:/nomarkdown}'
 feature_row:
   - image_path:

diff --git a/run_image_python.sh b/run_image_python.sh
@@ -18,7 +18,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_java.sh b/run_java.sh
@@ -18,7 +18,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_python.sh b/run_python.sh
@@ -18,7 +18,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_python_cluster.sh b/run_python_cluster.sh
@@ -18,7 +18,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_python_cori.sh b/run_python_cori.sh
@@ -14,7 +14,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_scala-2.11.sh b/run_scala-2.11.sh
@@ -18,7 +18,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_scala-2.12.sh b/run_scala-2.12.sh
@@ -18,7 +18,7 @@ SCALA_VERSION=2.12.8
 SCALA_VERSION_SPARK=2.12
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_scala_cluster.sh b/run_scala_cluster.sh
@@ -18,7 +18,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_scala_cluster_image.sh b/run_scala_cluster_image.sh
@@ -18,7 +18,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package

diff --git a/run_scala_cori.sh b/run_scala_cori.sh
@@ -14,7 +14,7 @@ SCALA_VERSION=2.11.8
 SCALA_VERSION_SPARK=2.11
 
 ## Package version
-VERSION=0.8.1
+VERSION=0.8.2
 
 # Package it
 sbt ++${SCALA_VERSION} package