Skip to content

Commit

Permalink
add case for when no records are present in the file
Browse files Browse the repository at this point in the history
* updated PAR1_END to \0PAR1, not all parquet files are written
  with \0\0PAR1
  • Loading branch information
Nick Rushton committed Nov 12, 2018
1 parent be0b013 commit 36e991b
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion src/main/scala/ParquetDumper.scala
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class StdinUnpacker(printerActor: ActorRef) extends Actor with akka.actor.ActorL
var numParOnes = 1
var sawParquetMrVersion = false
val PAR1_BEG = scala.collection.mutable.Queue("PAR1".toList.map(_.toByte): _*)
val PAR1_END = scala.collection.mutable.Queue[Byte](0.toByte, 0.toByte) ++ PAR1_BEG
val PAR1_END = scala.collection.mutable.Queue[Byte](0.toByte) ++ PAR1_BEG
var PARQUET_MR_VERSION = "parquet-mr version".toList.map(_.toByte).toVector
val q = scala.collection.mutable.Queue[Byte]()

Expand Down Expand Up @@ -168,6 +168,10 @@ class ParquetReaderActor(printerActor: ActorRef) extends Actor with akka.actor.A
linesSent += 1
record = reader.read()
}
// special case where the parquet file is empty
if (linesSent == 0) {
context.parent.tell(ParquetReaderFinished, self)
}
new java.io.File(input).delete
}
}
Expand Down

0 comments on commit 36e991b

Please sign in to comment.