Fix Amharic extraction common bugs

core/src/main/scala/org/dbpedia/extraction/config/dataparser/EthiopianDateParserConfig.scala

@@ -38,16 +38,39 @@ object EthiopianDateParserConfig {
     "መስከረም" -> 1,
     "ጥቅምት" -> 2,
     "ኅዳር" -> 3,
+    "ህዳር" -> 3,
+    "ኀዳር" -> 3,
+    "ታኅሣሥ" -> 4,
     "ታኅሳስ" -> 4,
+    "ታህሳስ" -> 4,
+    "ታኅሣስ" -> 4,
     "ጥር" -> 5,
     "የካቲት" -> 6,
     "መጋቢት" -> 7,
     "ሚያዝያ" -> 8,
+    "ሚያዚያ" -> 8,
     "ግንቦት" -> 9,
     "ሰኔ" -> 10,
     "ሐምሌ" -> 11,
+    "ሓምሌ" -> 11,
+    "ሃምሌ" -> 11,
+    "ኃምሌ" -> 11,
     "ነሐሴ" -> 12,
+    "ነሓሴ" -> 12,
+    "ነሃሴ" -> 12,
+    "ጳጉሜን" -> 13,
     "ጳጉሜ" -> 13
   )
 
+  val dayMarkers = Set("ቀን")
+
+  val eraMarkers = Set(
+    "ዓ.ም.",
+    "ዓ.ም",
+    "ዓ/ም",
+    "ዓመተ ምሕረት",
+    "ዓመተ ምኅረት",
+    "አ.ም."
+  )
+
 }

core/src/main/scala/org/dbpedia/extraction/config/mappings/InfoboxExtractorConfig.scala

@@ -12,13 +12,16 @@ object InfoboxExtractorConfig
 
     val ignoreProperties = Map (
         "en"-> Set("image", "image_photo", "map"),
-        "am"-> Set("ምስል", "ፎቶ", "ስዕል", "ካርታ", "አርማ"),
+        "am"-> Set("ምስል", "ፎቶ", "ስዕል", "ሥዕል", "ካርታ", "አርማ", "ባንዲራ", "ማኅተም", "ባንዲራ_ሥዕል", "ማኅተም_ሥዕል", "ካርታ_ሥዕል"),
         "ar"-> Set("صورة"),
         "id"-> Set("foto", "gambar"),
         "el"-> Set("εικόνα", "εικονα", "Εικόνα", "Εικονα", "χάρτης", "Χάρτης"),
         "eo"-> Set("dosiero"),
         "pt" -> Set("imagem", "foto","mapa")
     )
+
+    def ignoredProperties(wikiCode: String): Set[String] =
+        ignoreProperties("en") ++ ignoreProperties.getOrElse(wikiCode, Set.empty[String])
 
     // TODO: i18n
     val RankRegex = """(?i)([0-9]+)\s?(?:st|nd|rd|th)""".r

core/src/main/scala/org/dbpedia/extraction/dataparser/EthiopianDateParser.scala

@@ -1,5 +1,6 @@
 package org.dbpedia.extraction.dataparser
 import java.util.logging.{Logger, Level}
+import java.util.regex.Pattern
 import scala.util.matching.Regex
 import org.dbpedia.extraction.config.dataparser.{
   EthiopianDateParserConfig,
@@ -15,34 +16,50 @@ class EthiopianDateParser(datatype: Datatype, val strict: Boolean = false) {
 
   val geezNumberParser = new GeezNumberUtils()
   private val monthsMap = EthiopianDateParserConfig.monthsMap
-  private val monthsName = monthsMap.keys.mkString("|")
+  private val monthsName = monthsMap.keys.toSeq
+    .sortBy(name => -name.length)
+    .map(Pattern.quote)
+    .mkString("|")
   private val geezNumberDate =
-    EthiopianDateParserConfig.geezNumberDateMap.values.mkString("|")
+    EthiopianDateParserConfig.geezNumberDateMap.values.toSeq
+      .sortBy(number => -number.length)
+      .map(Pattern.quote)
+      .mkString("|")
+  private val dayMarkers = EthiopianDateParserConfig.dayMarkers.toSeq
+    .sortBy(marker => -marker.length)
+    .map(Pattern.quote)
+    .mkString("|")
+  private val eraMarkers = EthiopianDateParserConfig.eraMarkers.toSeq
+    .sortBy(marker => -marker.length)
+    .map(Pattern.quote)
+    .mkString("|")
+  private val optionalDayMarker = s"""(?:\\s+(?:$dayMarkers))?"""
+  private val optionalEraMarker = s"""(?:\\s+(?:$eraMarkers))?"""
 
   private val gregorianDateIndicator = s""".*(እ.ኤ.አ).*""".r
   private val prefix = if (strict) """\s*""" else """.*?"""
   private val postfix = if (strict) """\s*""" else ".*"
 
   // catches dd-mm-yyyy including a 13th month 21 13 2013, 21-13-2013, 21/13/2013, 21-13-2013, 21/13/2013
   private val dateRegex1: Regex =
-    s"""$prefix\\b(0?[1-9]|[12][0-9]|3[01])\\b[-/\\s]\\b(0?[1-9]|1[0-2]|13)\\b[-/\\s](\\d{4}|[\\u1369-\\u137C]+)$postfix""".r
+    s"""$prefix\\b(0?[1-9]|[12][0-9]|3[01])\\b[-/\\s]\\b(0?[1-9]|1[0-2]|13)\\b[-/\\s](\\d{4}|[\\u1369-\\u137C]+)$optionalEraMarker$postfix""".r
 
   // Regex for dates containing geez characters
   // catches dates like ጥቅምት-21-2013 or ጥቅምት/21/2013 or ጥቅምት 21 2013
   private val dateRegex2: Regex =
-    s"""$prefix($monthsName)[\\s/-](\\b(0?[1-9]|[12][0-9]|3[01])\\b)[\\s/-](\\d{4}|[\\u1369-\\u137C]+)$postfix""".r
+    s"""$prefix($monthsName)[\\s/-](\\b(?:0?[1-9]|[12][0-9]|3[01])\\b)$optionalDayMarker[\\s/-](\\d{4}|[\\u1369-\\u137C]+)$optionalEraMarker$postfix""".r
 
   // catches dates dd-month-yyyy like 21-ጥቅምት-2013 or 21/ጥቅምት/2013 or 21 ጥቅምት 2013
   private val dateRegex3: Regex =
-    s"""$prefix(\\b(0?[1-9]|[12][0-9]|3[01])\\b)[\\s/-]($monthsName)[\\s/-](\\d{4}|[\\u1369-\\u137C]+)$postfix""".r
+    s"""$prefix(\\b(?:0?[1-9]|[12][0-9]|3[01])\\b)[\\s/-]($monthsName)[\\s/-](\\d{4}|[\\u1369-\\u137C]+)$optionalEraMarker$postfix""".r
 
   // catches dates month-dd-yyyy ጥቅምት ፳፩ ፳፻፲፫ or ጥቅምት/፳፩/፳፻፲፫ or ጥቅምት ፳፩ ፳፻፲፫ mmmm-dd-yyyy
   private val dateRegex4: Regex =
-    s"""$prefix(\\b$monthsName)[\\s/-]($geezNumberDate|0?[1-9]|[12][0-9]|3[01])[\\s/-](\\d{4}|[\\u1369-\\u137C]+)$postfix""".r
+    s"""$prefix(\\b(?:$monthsName))[\\s/-]($geezNumberDate|0?[1-9]|[12][0-9]|3[01])$optionalDayMarker[\\s/-](\\d{4}|[\\u1369-\\u137C]+)$optionalEraMarker$postfix""".r
 
   // catches dates like ፳፩ ጥቅምት ፳፻፲፫ or ፳፩/ጥቅምት/፳፻፲፫ or 21/ጥቅምት/2013 dd-mmmm-yyyy
   private val dateRegex5: Regex =
-    s"""$prefix(\\b$geezNumberDate|0?[1-9]|[12][0-9]|3[01])[\\s/-]($monthsName)[\\s/-](\\d{4}|[\\u1369-\\u137C]+)$postfix""".r
+    s"""$prefix(\\b(?:$geezNumberDate|0?[1-9]|[12][0-9]|3[01]))[\\s/-]($monthsName)[\\s/-](\\d{4}|[\\u1369-\\u137C]+)$optionalEraMarker$postfix""".r
 
   def catchGeezDate(dateString: String): Option[(String, String, String)] = {
 
@@ -149,7 +166,8 @@ class EthiopianDateParser(datatype: Datatype, val strict: Boolean = false) {
     val F: Long = (30.6001 * E).toLong
     val gregorianDay: Int = (B - D - F + (Q - Z)).toInt
     val gregorianMonth: Long = if (E - 1 <= 12) E - 1 else E - 13
-    val gregorianYear: Long = if (month <= 2) C - 4715 else C - 4716
+    val gregorianYear: Long =
+      if (gregorianMonth <= 2) C - 4715 else C - 4716
 
     Some(
       new Date(

core/src/main/scala/org/dbpedia/extraction/mappings/InfoboxExtractor.scala

@@ -55,7 +55,7 @@ extends PageNodeExtractor
 
     private val ignoreTemplatesRegex = InfoboxExtractorConfig.ignoreTemplatesRegex
 
-    private val ignoreProperties = InfoboxExtractorConfig.ignoreProperties
+    private val ignoreProperties = InfoboxExtractorConfig.ignoredProperties(wikiCode)
 
     private val labelProperty = ontology.properties("rdfs:label")
     private val typeProperty = ontology.properties("rdf:type")
@@ -118,7 +118,7 @@ extends PageNodeExtractor
           if !ignoreTemplatesRegex.exists(regex => regex.unapplySeq(resolvedTitle).isDefined) 
         }
         {
-            val propertyList = template.children.filterNot(property => ignoreProperties.get(wikiCode).getOrElse(ignoreProperties("en")).contains(property.key.toLowerCase))
+            val propertyList = template.children.filterNot(property => ignoreProperties.contains(property.key.toLowerCase))
 
             var propertiesFound = false
 
@@ -136,7 +136,7 @@ extends PageNodeExtractor
                     //for(splitNode <- splitPropertyNodes; pr <- extractValue(splitNode); if pr.unit.nonEmpty)
                     //sh: removed pr.unit.nonEmpty as it kicked out all objectproperty triples from wikilinks,
                     // didn't test for further side-effects seems to work
-                    for(splitNode <- splitPropertyNodes; pr <- extractValue(splitNode))
+                    for(splitNode <- splitPropertyNodes; pr <- extractValue(splitNode); if pr.value.trim.nonEmpty)
                     {
                         val propertyUri = getPropertyUri(property.key)
                         try

core/src/main/scala/org/dbpedia/extraction/mappings/InfoboxReferencesExtractor.scala

@@ -44,7 +44,7 @@ extends PageNodeExtractor
 
     private val ignoreTemplatesRegex = InfoboxExtractorConfig.ignoreTemplatesRegex
 
-    private val ignoreProperties = InfoboxExtractorConfig.ignoreProperties
+    private val ignoreProperties = InfoboxExtractorConfig.ignoredProperties(wikiCode)
 
     private val rdfLangStrDt = ontology.datatypes("rdf:langString")
 
@@ -172,7 +172,7 @@ extends PageNodeExtractor
 
             var propertiesFound = false
 
-            val propertyList = template.children.filterNot(property => ignoreProperties.get(wikiCode).getOrElse(ignoreProperties("en")).contains(property.key.toLowerCase))
+            val propertyList = template.children.filterNot(property => ignoreProperties.contains(property.key.toLowerCase))
 
 
 

core/src/main/scala/org/dbpedia/extraction/mappings/MappingExtractor.scala

@@ -29,7 +29,9 @@ extends PageNodeExtractor
   {
     if(page.title.namespace != Namespace.Main && !ExtractorUtils.titleContainsCommonsMetadata(page.title)) return Seq.empty
 
-    val graph = extractNode(page, subjectUri)
+    val graph = extractNode(page, subjectUri).filter(quad =>
+      quad.datatype == null || quad.value.trim.nonEmpty
+    )
 
     if (graph.isEmpty) Seq.empty
     else splitInferredFromDirectTypes(graph, page, subjectUri)

core/src/test/scala/org/dbpedia/extraction/dataparser/EthiopianDateParserTest.scala

@@ -0,0 +1,89 @@
+package org.dbpedia.extraction.dataparser
+
+import org.dbpedia.extraction.ontology.datatypes.Datatype
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{FlatSpec, Matchers}
+
+@RunWith(classOf[JUnitRunner])
+class EthiopianDateParserTest extends FlatSpec with Matchers {
+
+  private val datatype = new Datatype("xsd:date")
+  private val parser = new EthiopianDateParser(datatype, strict = true)
+
+  private def parse(input: String): Option[String] =
+    parser.findGeezDate(input).map(_.toString)
+
+  "EthiopianDateParser" should "convert every Ethiopian month to the correct Gregorian year" in {
+    val conversions = Seq(
+      (2013, 1, 1, "2020-09-11"),
+      (2013, 2, 1, "2020-10-11"),
+      (2013, 3, 1, "2020-11-10"),
+      (2013, 4, 1, "2020-12-10"),
+      (2013, 5, 1, "2021-01-09"),
+      (2013, 6, 1, "2021-02-08"),
+      (2013, 7, 1, "2021-03-10"),
+      (2013, 8, 1, "2021-04-09"),
+      (2013, 9, 1, "2021-05-09"),
+      (2013, 10, 1, "2021-06-08"),
+      (2013, 11, 1, "2021-07-08"),
+      (2013, 12, 1, "2021-08-07"),
+      (2013, 13, 1, "2021-09-06")
+    )
+
+    conversions.foreach { case (year, month, day, expected) =>
+      parser.geezToGregorianDateConverter(year, month, day, datatype)
+        .map(_.toString) should equal(Some(expected))
+    }
+  }
+
+  it should "parse the standard Amharic day-marker format as written" in {
+    val articleDates = Seq(
+      "መስከረም ፳ ቀን ፲፱፻፶፱ ዓ.ም." -> "1966-09-30",
+      "ጥቅምት ፪ ቀን ፲፱፻፷፩ ዓ.ም." -> "1968-10-12",
+      "ጥር ፲፫ ቀን ፲፱፻፲፩ ዓ.ም." -> "1919-01-21",
+      "የካቲት ፲፩ ቀን ፲፱፻፶፯ ዓ.ም." -> "1965-02-18"
+    )
+
+    articleDates.foreach { case (input, expected) =>
+      parse(input) should equal(Some(expected))
+    }
+  }
+
+  it should "recognize supported Ethiopian era markers" in {
+    val eraMarkers = Seq("ዓ.ም.", "ዓ.ም", "ዓ/ም", "ዓመተ ምሕረት", "ዓመተ ምኅረት", "አ.ም.")
+
+    eraMarkers.foreach { marker =>
+      parse(s"መስከረም 1 ቀን 2013 $marker") should equal(Some("2020-09-11"))
+    }
+  }
+
+  it should "recognize observed Ethiopian month spelling variants" in {
+    val monthVariants = Seq(
+      "ኅዳር" -> "2020-11-10",
+      "ህዳር" -> "2020-11-10",
+      "ኀዳር" -> "2020-11-10",
+      "ታኅሣሥ" -> "2020-12-10",
+      "ታኅሳስ" -> "2020-12-10",
+      "ታህሳስ" -> "2020-12-10",
+      "ታኅሣስ" -> "2020-12-10",
+      "ሚያዚያ" -> "2021-04-09",
+      "ሓምሌ" -> "2021-07-08",
+      "ሃምሌ" -> "2021-07-08",
+      "ኃምሌ" -> "2021-07-08",
+      "ነሓሴ" -> "2021-08-07",
+      "ነሃሴ" -> "2021-08-07",
+      "ጳጉሜን" -> "2021-09-06"
+    )
+
+    monthVariants.foreach { case (month, expected) =>
+      parse(s"$month 1 2013") should equal(Some(expected))
+    }
+  }
+
+  it should "parse Arabic and single-digit Geez days in both supported orders" in {
+    parse("ጥቅምት 21 2013") should equal(Some("2020-10-31"))
+    parse("21 ጥቅምት 2013") should equal(Some("2020-10-31"))
+    parse("ታኅሣሥ ፪ ቀን ፲፱፻፶፮ ዓ.ም.") should equal(Some("1963-12-12"))
+  }
+}

core/src/test/scala/org/dbpedia/extraction/mappings/InfoboxExtractorConfigTest.scala

@@ -0,0 +1,22 @@
+package org.dbpedia.extraction.mappings
+
+import org.dbpedia.extraction.config.mappings.InfoboxExtractorConfig
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{FlatSpec, Matchers}
+
+@RunWith(classOf[JUnitRunner])
+class InfoboxExtractorConfigTest extends FlatSpec with Matchers {
+
+  "InfoboxExtractorConfig" should "combine the baseline and language-specific ignored properties" in {
+    InfoboxExtractorConfig.ignoreProperties.foreach { case (wikiCode, languageProperties) =>
+      val ignored = InfoboxExtractorConfig.ignoredProperties(wikiCode)
+      InfoboxExtractorConfig.ignoreProperties("en").subsetOf(ignored) should equal(true)
+      languageProperties.subsetOf(ignored) should equal(true)
+    }
+
+    val amharicIgnored = InfoboxExtractorConfig.ignoredProperties("am")
+    amharicIgnored should contain ("image")
+    amharicIgnored should contain ("ሥዕል")
+  }
+}

core/src/test/scala/org/dbpedia/extraction/mappings/MappingExtractorEmptyLiteralTest.scala

@@ -0,0 +1,42 @@
+package org.dbpedia.extraction.mappings
+
+import org.dbpedia.extraction.config.provenance.DBpediaDatasets
+import org.dbpedia.extraction.transform.Quad
+import org.dbpedia.extraction.util.Language
+import org.dbpedia.extraction.wikiparser.{TemplateNode, WikiPage, WikiParser, WikiTitle}
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{FlatSpec, Matchers}
+
+@RunWith(classOf[JUnitRunner])
+class MappingExtractorEmptyLiteralTest extends FlatSpec with Matchers {
+
+  "MappingExtractor" should "omit empty literals while preserving non-empty mapping output" in {
+    val mapping = new Extractor[TemplateNode] {
+      override val datasets = Set(DBpediaDatasets.OntologyPropertiesLiterals)
+
+      override def extract(input: TemplateNode, subjectUri: String): Seq[Quad] = Seq(
+        new Quad("am", DBpediaDatasets.OntologyPropertiesLiterals.encoded, subjectUri,
+          "http://dbpedia.org/ontology/name", "", input.sourceIri,
+          "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString"),
+        new Quad("am", DBpediaDatasets.OntologyPropertiesLiterals.encoded, subjectUri,
+          "http://dbpedia.org/ontology/name", "የሙከራ ስም", input.sourceIri,
+          "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString")
+      )
+    }
+    val context = new {
+      def mappings = new Mappings(Map("Test mapping" -> mapping), Nil)
+      def redirects = new Redirects(Map.empty)
+    }
+    val language = Language("am")
+    val page = new WikiPage(
+      WikiTitle.parse("የሙከራ ገጽ", language),
+      "{{Test mapping}}"
+    )
+    val pageNode = WikiParser.getInstance()(page).get
+
+    val quads = new MappingExtractor(context).extract(pageNode, page.title.resourceIri)
+
+    quads.map(_.value) should equal(Seq("የሙከራ ስም"))
+  }
+}