Merge pull request #7 from jpbnetley/feature/logging

Feature/logging

Author: jpbnetley

build.sbt

@@ -3,9 +3,11 @@ name := "Mongo-csv-importer"
 version := "0.1"
 
 libraryDependencies ++= Seq(
-  "org.mongodb.scala" %% "mongo-scala-driver" % "2.8.0",
-  "io.monix"          %% "monix"              % "3.1.0",
-  "org.typelevel"     %% "cats-core"          % "2.1.0"
+  "org.mongodb.scala"           %% "mongo-scala-driver" % "2.8.0",
+  "io.monix"                    %% "monix"              % "3.1.0",
+  "org.typelevel"               %% "cats-core"          % "2.1.0",
+  "ch.qos.logback"              % "logback-classic"     % "1.2.3",
+  "com.typesafe.scala-logging"  %% "scala-logging"      % "3.9.2"
 )
 
 scalaVersion := "2.13.1"

src/main/resources/logback.xml

@@ -0,0 +1,11 @@
+<configuration>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <layout class="ch.qos.logback.classic.PatternLayout">
+            <pattern>%d %green([%thread]) %highlight(%level) %logger{50} - %msg%n</pattern>
+        </layout>
+    </appender>
+    <logger name="com.base22" level="TRACE"/>
+    <root level="error">
+        <appender-ref ref="STDOUT"/>
+    </root>
+</configuration>

src/main/scala/Util/DataBuilder/DataBuilder.scala

@@ -2,6 +2,7 @@ package Util.DataBuilder
 import org.bson.Document
 import cats.implicits._
 import monix.eval.Task
+import Util.Logging.log
 
 
 object DataBuilder {
@@ -13,17 +14,19 @@ object DataBuilder {
     * @return bsonDocument as String
     */
   def buildMongoDocuments(headers: Option[List[String]], lineItems: List[String]): Task[Either[Exception, List[Document]]] = Task {
+    log.debug("finding headers")
     val header              = Either.fromOption(headers, new Exception("No Headers found for csv"))
     header.map(headers => buildJsonObject(headers, lineItems))
   }
 
   /** Checks of the line items are the same length as the headers, otherwise adds NULL for the missing headers
     *
     * @param lineRow the single line item that is comma separated (csv)
-    * @param maxIndex
+    * @param maxIndex the max index for the line items
     * @return Csv with null for empty items
     */
   def handleShortColumnHeader(lineRow: String, maxIndex: Int): List[String] = {
+    @scala.annotation.tailrec
     def loopOverCsvString(acc: List[String], currentIndex: Int, maxIndex: Int): List[String] = {
       if(currentIndex == maxIndex)
         acc
@@ -53,7 +56,7 @@ object DataBuilder {
 
   /** Parses json with no Nulls
     *
-    * @param currentIndex
+    * @param currentIndex position of the parser
     * @param headerText for each object
     * @param item       value for the header text
     * @param maxIndex   max length of the csv file
@@ -79,7 +82,7 @@ object DataBuilder {
 
   /** Parses json with that contains a Null
     *
-    * @param currentIndex
+    * @param currentIndex value for the parser
     * @param headerText for each object
     * @param item       value for the header text
     * @param maxIndex   max length of the csv file
@@ -105,6 +108,7 @@ object DataBuilder {
     * @return List[Document]
     */
   def buildJsonObject(headerValue: List[String], lineItems: List[String]): List[Document] = {
+    log.debug("Parsing json")
     val maxHeaderIndex    = headerValue.length - 1
     val jsonObjects       = lineItems.map { line =>
       val nonEmptyCsvLine = handleShortColumnHeader(line, maxHeaderIndex)

src/main/scala/Util/DataBuilder/Processing.scala

@@ -11,6 +11,7 @@ import org.bson.Document
 import scala.concurrent.Await
 import scala.concurrent.duration.Duration
 import scala.reflect.io.File
+import Util.Logging.log
 
 object Processing {
 
@@ -30,7 +31,7 @@ object Processing {
         lineItems       =  fileLines.drop(1)
         collectionName  =  file.name.replace(".csv", "").toLowerCase
         documentResult  <- EitherT(buildMongoDocuments(headers, lineItems))
-        db              <- EitherT.right[Exception](database.getDatabase())
+        db              <- EitherT.right[Exception](database.getDatabase)
         dbInsert        <- EitherT.rightT[Task, Exception](db.getCollection[Document](collectionName).insertMany(documentResult))
       } yield {
         println(s"Inserting into db: $dbInsert")
@@ -39,6 +40,7 @@ object Processing {
       }).value
     }.map { result =>
       val (errors, _) = result.separate
+      log.error(errors.headOption.fold("No error found")(_.getMessage))
       errors.headOption.toLeft(())
     }
   }

src/main/scala/Util/Database/Database.scala

@@ -62,9 +62,7 @@ case object Database {
     *
     * @return
     */
-  def getDatabase: Task[MongoDatabase] = Task {
-    database
-  }
+  def getDatabase: Task[MongoDatabase] = Task(database)
 
   /** Closes the mongo connection
     *

src/main/scala/Util/File/FileHelper.scala

@@ -6,6 +6,7 @@ import monix.eval.Task
 
 import scala.io.Source
 import scala.reflect.io.{Directory, File, Path}
+import Util.Logging.log
 
 object FileHelper {
   /** Gets csv files
@@ -15,11 +16,13 @@ object FileHelper {
     * @return
     */
   def getCsvFiles(dir: Directory, skipFiles: List[File] = List.empty[File]): Task[Either[Exception, List[File]]] = {
+    log.debug("Skipping files that are not csv items")
     Task {
       val okFileExtensions = List("csv")
       Right(dir.files.filter(file => okFileExtensions.contains(file.extension)).toList)
     }.onErrorHandle { e =>
         val message = s"Could not read files from path: ${e.getMessage}"
+        log.error(message, e)
         Left(new Exception(message))
       }
   }
@@ -30,11 +33,16 @@ object FileHelper {
     * @return line items
     */
   def extractCsvFileLines(file: File): Task[Either[Exception, List[String]]] = Task{
+
     try {
+      log.debug("Reading in file")
       val reader = Source.fromFile(file.toURI, StandardCharsets.ISO_8859_1.name())
       Right(reader.getLines().toList)
     } catch {
-      case e: Exception => Left(new Exception("Failed to extract csv files: "+ e.getMessage, e))
+      case e: Exception =>
+        val message = "Failed to extract csv files: "+ e.getMessage
+        log.error(message)
+        Left(new Exception(message, e))
     }
   }
 
@@ -45,7 +53,12 @@ object FileHelper {
     * @return optional found file
     */
   def findFile(dir: Directory, fileName: String): Task[Either[Exception, File]] = Task {
-    Either.fromOption(dir.files.toList.find(_.name == fileName), new Exception("File not found: "+ fileName))
+    log.debug("Finding file by name")
+    Either.fromOption(dir.files.toList.find(_.name == fileName), {
+      val error = new Exception("File not found: " + fileName)
+      log.error(error.getMessage)
+      error
+    })
   }
 
   /** Casts a string to a url
@@ -55,10 +68,13 @@ object FileHelper {
     */
   def toDirectory(dir: String): Either[Exception, Directory] = {
     try{
+      log.debug("Build directory from path string")
       val directory = File(Path(dir))
       Right(Directory(directory))
     } catch {
-      case e: Exception => Left(new Exception(s"Could not convert to Directory $dir", e))
+      case e: Exception =>
+        val error = new Exception(s"Could not convert to Directory $dir", e)
+        Left(error)
     }
   }
 

src/main/scala/Util/Logging.scala

@@ -0,0 +1,8 @@
+package Util
+
+import com.typesafe.scalalogging.Logger
+import org.slf4j.LoggerFactory
+
+object Logging {
+val log: Logger = Logger(LoggerFactory.getLogger("Mongo csv importer"))
+}

src/main/scala/Util/UserPrompt.scala

@@ -5,6 +5,7 @@ import Util.Models.UserInput
 import cats.data.EitherT
 import cats.implicits._
 import monix.eval.Task
+import Util.Logging.log
 
 object UserPrompt {
 
@@ -14,6 +15,7 @@ object UserPrompt {
     */
   def promptUser(): Task[Either[Exception, UserInput]] = {
     try {
+      log.debug("Prompting user for input...")
       println("Please enter the path to the csv files: ")
       (for {
         directory <- EitherT.fromEither[Task](FileHelper.toDirectory(scala.io.StdIn.readLine()))
@@ -28,24 +30,29 @@ object UserPrompt {
     } catch {
       case e: Exception =>
         val message = s"Could not read user input: ${e.getMessage}"
+        log.error(message)
         Task.now(Left(new Exception(message)))
     }
   }
 
   /** Prompts the user to add files to skip
     *
-    * @param inputItems
+    * @param inputItems the items that will skipped
     * @return skipped items list
     */
   def addSkipItems(inputItems: List[String]): Either[Exception, List[String]] = {
     try {
+      log.debug("Prompting user to enter files to skip")
       println("Please enter any files to skip (enter :q to exit) eg. filename.extension: ")
       val input = scala.io.StdIn.readLine()
       if (input.equals(":q"))
         Right(inputItems)
       else addSkipItems(inputItems :+ input)
     } catch {
-      case e: Exception => Left(new Exception("Could not read user input: "+e.getMessage, e))
+      case e: Exception =>
+        val message = "Could not read user input: "+e.getMessage
+        log.error(message)
+        Left(new Exception(message, e))
     }
   }