predict.cc

/*

  Copyright 2022 Flammie A Pirinen

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

*/

/*
  This is a toy commandline utility for testing spellers on standard io.
 */

#if HAVE_CONFIG_H
#include <config.h>
#else
#define PACKAGE_NAME
#define PACKAGE_BUGREPORT
#define PACKAGE_STRING
#endif
#if HAVE_GETOPT_H
#include <getopt.h>
#endif

#ifdef WINDOWS
#include <windows.h>
#endif

#include <cstdarg>
#include <errno.h>
#include <stdio.h>

#include "ZHfstOspeller.h"
#include "ol-exceptions.h"
#include "ospell.h"

using hfst_ospell::Transducer;
using hfst_ospell::ZHfstOspeller;

static bool quiet = false;
static bool verbose = false;
static bool analyse = false;
static unsigned long suggs = 0;
static hfst_ospell::Weight max_weight = -1.0;
static hfst_ospell::Weight beam = -1.0;
static float time_cutoff = 0.0;
static std::string error_model_filename = "";
static std::string lexicon_filename = "";
static std::string continuation_marker = "";
#ifdef WINDOWS
static bool output_to_console = false;
#endif

#ifdef WINDOWS
static std::string
wide_string_to_string(const std::wstring &wstr)
{
    int size_needed = WideCharToMultiByte(
        CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL);
    std::string str(size_needed, 0);
    WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &str[0],
                        size_needed, NULL, NULL);
    return str;
}
#endif

// C++20, https://stackoverflow.com/a/2072890/4109773
inline bool
ends_with(std::string const &value, std::string const &ending)
{
    if (ending.size() > value.size())
        return false;
    return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
}

static int
hfst_fprintf(FILE *stream, const char *format, ...)
{
    va_list args;
    va_start(args, format);
#ifdef WINDOWS
    if (output_to_console && (stream == stdout || stream == stderr))
    {
        char buffer[1024];
        int r = vsprintf(buffer, format, args);
        va_end(args);
        if (r < 0)
            return r;
        HANDLE stdHandle = GetStdHandle(STD_OUTPUT_HANDLE);
        if (stream == stderr)
            stdHandle = GetStdHandle(STD_ERROR_HANDLE);

        std::string pstr(buffer);
        DWORD numWritten = 0;
        int wchars_num
            = MultiByteToWideChar(CP_UTF8, 0, pstr.c_str(), -1, NULL, 0);
        wchar_t *wstr = new wchar_t[wchars_num];
        MultiByteToWideChar(CP_UTF8, 0, pstr.c_str(), -1, wstr, wchars_num);
        int retval = WriteConsoleW(stdHandle, wstr, wchars_num - 1,
                                   &numWritten, NULL);
        delete[] wstr;

        return retval;
    }
    else
    {
        int retval = vfprintf(stream, format, args);
        va_end(args);
        return retval;
    }
#else
    errno = 0;
    int retval = vfprintf(stream, format, args);
    if (retval < 0)
    {
        perror("hfst_fprintf");
    }
    va_end(args);
    return retval;
#endif
}

bool
print_usage(void)
{
    std::cout
        << "\n"
        << "Usage: " PACKAGE_NAME " [OPTIONS] [ZHFST-ARCHIVE]\n"
        << "Use automata in ZHFST-ARCHIVE or from OPTIONS to check and "
           "predict\n"
           "\n"
        << "  -h, --help                Print this help message\n"
        << "  -V, --version             Print version information\n"
        << "  -v, --verbose             Be verbose\n"
        << "  -q, --quiet               Don't be verbose (default)\n"
        << "  -s, --silent              Same as quiet\n"
        << "  -a, --analyse             Analyse strings and corrections\n"
        << "  -n, --limit=N             Show at most N predictions\n"
        << "  -w, --max-weight=W        Suppress corrections with weights "
           "above W\n"
        << "  -b, --beam=W              Suppress corrections worse than best "
           "candidate by more than W\n"
        << "  -t, --time-cutoff=T       Stop trying to find better "
           "corrections after T seconds (T is a float)\n"
        << "  -C, --continuation=C      Word-continuation character is C\n"
        << "  -m, --error-model         Use this error model (must also give "
           "lexicon as option)\n"
        << "  -l, --lexicon             Use this lexicon (must also give erro "
           "model as option)\n"
        <<
#ifdef WINDOWS
        "  -k, --output-to-console   Print output to console "
        "(Windows-specific)"
        <<
#endif
        "\n"
        << "\n"
        << "Report bugs to " PACKAGE_BUGREPORT "\n"
        << "\n";
    return true;
}

bool
print_version(void)
{
    std::cout << "\n" PACKAGE_STRING << std::endl
              << "copyright (C) 2009 - 2022 University of Helsinki\n";
    return true;
}

bool
print_short_help(void)
{
    print_usage();
    return true;
}

void
do_predict(ZHfstOspeller &speller, const std::string &str)
{
    if (verbose)
    {
        hfst_fprintf(stdout, "Suggesting for %s:\n", str.c_str());
    }
    hfst_ospell::CorrectionQueue corrections = speller.suggest(str);
    if (corrections.size() > 0)
    {
        hfst_fprintf(stdout, "Corrections for \"%s\":\n", str.c_str());
        while (corrections.size() > 0)
        {
            const std::string &corr = corrections.top().first;
            if (analyse)
            {
                hfst_ospell::AnalysisQueue anals = speller.analyse(corr, true);
                bool all_discarded = true;
                while (anals.size() > 0)
                {
                    if (anals.top().first.find("Use/SpellNoSugg")
                        != std::string::npos)
                    {
                        hfst_fprintf(stdout,
                                     "%s    %f    %s    "
                                     "[DISCARDED BY ANALYSES]\n",
                                     corr.c_str(), corrections.top().second,
                                     anals.top().first.c_str());
                    }
                    else
                    {
                        all_discarded = false;
                        hfst_fprintf(stdout, "%s    %f    %s\n", corr.c_str(),
                                     corrections.top().second,
                                     anals.top().first.c_str());
                    }
                    anals.pop();
                }
                if (all_discarded)
                {
                    hfst_fprintf(stdout, "All corrections were "
                                         "invalidated by analysis! "
                                         "No score!\n");
                }
            }
            else
            {
                if ((!continuation_marker.empty())
                    && (ends_with(corr, continuation_marker)))
                {
                    std::string chomped = corr.substr(
                        0, corr.size() - continuation_marker.size());
                    hfst_fprintf(stdout, "%s...    %f    (continuation %s)\n",
                                 chomped.c_str(), corrections.top().second,
                                 continuation_marker.c_str());
                }
                else
                {
                    hfst_fprintf(stdout, "%s    %f\n", corr.c_str(),
                                 corrections.top().second);
                }
            }
            corrections.pop();
        }
        hfst_fprintf(stdout, "\n");
    }
    else
    {
        hfst_fprintf(stdout, "Unable to correct \"%s\"!\n\n", str.c_str());
    }
}

void
do_spell(ZHfstOspeller &speller, const std::string &str)
{
    if (speller.spell(str))
    {
        hfst_fprintf(stdout, "\"%s\" is in the lexicon...\n", str.c_str());
        if (analyse)
        {
            hfst_fprintf(stdout, "analysing:\n");
            hfst_ospell::AnalysisQueue anals = speller.analyse(str, false);
            bool all_no_spell = true;
            while (anals.size() > 0)
            {
                if (anals.top().first.find("Use/-Spell") != std::string::npos)
                {
                    hfst_fprintf(stdout, "%s   %f [DISCARDED AS -Spell]\n",
                                 anals.top().first.c_str(),
                                 anals.top().second);
                }
                else
                {
                    all_no_spell = false;
                    hfst_fprintf(stdout, "%s   %f\n",
                                 anals.top().first.c_str(),
                                 anals.top().second);
                }
                anals.pop();
            }
            if (all_no_spell)
            {
                hfst_fprintf(stdout,
                             "All spellings were invalidated by analysis! "
                             ".:. Not in lexicon!\n");
            }
        }
        hfst_fprintf(stdout, "(but correcting anyways)\n", str.c_str());
        do_predict(speller, str);
    }
    else
    {
        hfst_fprintf(stdout, "\"%s\" is NOT in the lexicon:\n", str.c_str());
        do_predict(speller, str);
    }
}

int
zhfst_spell(char *zhfst_filename)
{
    ZHfstOspeller speller;
    try
    {
        speller.read_zhfst(zhfst_filename);
    }
    catch (hfst_ospell::ZHfstMetaDataParsingError &zhmdpe)
    {
        hfst_fprintf(stderr, "cannot finish reading zhfst archive %s:\n%s.\n",
                     zhfst_filename, zhmdpe.what());
        return EXIT_FAILURE;
    }
    catch (hfst_ospell::ZHfstZipReadingError &zhzre)
    {
        hfst_fprintf(stderr,
                     "cannot read zhfst archive %s:\n"
                     "%s.\n",
                     zhfst_filename, zhzre.what());
        return EXIT_FAILURE;
    }
    catch (hfst_ospell::ZHfstXmlParsingError &zhxpe)
    {
        hfst_fprintf(stderr,
                     "Cannot finish reading index.xml from %s:\n"
                     "%s.\n",
                     zhfst_filename, zhxpe.what());
        return EXIT_FAILURE;
    }
    if (verbose)
    {
        hfst_fprintf(stdout,
                     "Following metadata was read from ZHFST archive:\n"
                     "%s\n",
                     speller.metadata_dump().c_str());
    }
    speller.set_queue_limit(suggs);
    if (suggs != 0 && verbose)
    {
        hfst_fprintf(stdout, "Printing only %lu top predictions per line\n",
                     suggs);
    }
    speller.set_weight_limit(max_weight);
    if (max_weight >= 0.0 && verbose)
    {
        hfst_fprintf(stdout, "Not printing predictions worse than %f\n",
                     max_weight);
    }
    speller.set_beam(beam);
    if (beam >= 0.0 && verbose)
    {
        hfst_fprintf(stdout,
                     "Not printing predictions worse than best by margin %f\n",
                     beam);
    }
    speller.set_time_cutoff(time_cutoff);
    if (time_cutoff > 0.0 && verbose)
    {
        hfst_fprintf(
            stdout, "Not trying to find better predictions after %f seconds\n",
            time_cutoff);
    }
    if ((!continuation_marker.empty()) && verbose)
    {
        hfst_fprintf(stdout, "%s marks incomplete words\n",
                     continuation_marker.c_str());
    }
    char *str = (char *)malloc(2000);

#ifdef WINDOWS
    SetConsoleCP(65001);
    const HANDLE stdIn = GetStdHandle(STD_INPUT_HANDLE);
    WCHAR buffer[0x1000];
    DWORD numRead = 0;
    while (ReadConsoleW(stdIn, buffer, sizeof buffer, &numRead, NULL))
    {
        std::wstring wstr(buffer, numRead - 1); // skip the newline
        std::string linestr = wide_string_to_string(wstr);
        free(str);
        str = strdup(linestr.c_str());
#else
    while (!std::cin.eof())
    {
        std::cin.getline(str, 2000);
#endif
        if (str[0] == '\0')
        {
            continue;
        }
        if (str[strlen(str) - 1] == '\r')
        {
#ifdef WINDOWS
            str[strlen(str) - 1] = '\0';
#else
            hfst_fprintf(stderr, "There is a WINDOWS linebreak in this file\n"
                                 "Please convert with dos2unix or fromdos\n");
            exit(1);
#endif
        }
        do_spell(speller, str);
    }
    free(str);
    return EXIT_SUCCESS;
}

int
legacy_spell(hfst_ospell::Speller *s)
{
    ZHfstOspeller speller;
    speller.inject_speller(s);
    speller.set_queue_limit(suggs);
    if (suggs != 0 && verbose)
    {
        hfst_fprintf(stdout, "Printing only %lu top predictions per line\n",
                     suggs);
    }
    speller.set_weight_limit(max_weight);
    if (max_weight >= 0.0 && verbose)
    {
        hfst_fprintf(stdout, "Not printing predictions worse than %f\n",
                     suggs);
    }
    speller.set_beam(beam);
    if (beam >= 0.0 && verbose)
    {
        hfst_fprintf(stdout,
                     "Not printing predictions worse than best by margin %f\n",
                     suggs);
    }
    char *str = (char *)malloc(2000);

#ifdef WINDOWS
    SetConsoleCP(65001);
    const HANDLE stdIn = GetStdHandle(STD_INPUT_HANDLE);
    WCHAR buffer[0x1000];
    DWORD numRead = 0;
    while (ReadConsoleW(stdIn, buffer, sizeof buffer, &numRead, NULL))
    {
        std::wstring wstr(buffer, numRead - 1); // skip the newline
        std::string linestr = wide_string_to_string(wstr);
        free(str);
        str = strdup(linestr.c_str());
#else
    while (!std::cin.eof())
    {
        std::cin.getline(str, 2000);
#endif
        if (str[0] == '\0')
        {
            continue;
        }
        if (str[strlen(str) - 1] == '\r')
        {
#ifdef WINDOWS
            str[strlen(str) - 1] = '\0';
#else
            hfst_fprintf(stderr, "There is a WINDOWS linebreak in this file\n"
                                 "Please convert with dos2unix or fromdos\n");
            exit(1);
#endif
        }
        do_spell(speller, str);
    }
    free(str);
    return EXIT_SUCCESS;
}

int
main(int argc, char **argv)
{

#if HAVE_GETOPT_H
    int c;
    // std::locale::global(std::locale(""));
    while (true)
    {
        static struct option long_options[]
            = { // first the hfst-mandated options
                { "help", no_argument, 0, 'h' },
                { "version", no_argument, 0, 'V' },
                { "verbose", no_argument, 0, 'v' },
                { "quiet", no_argument, 0, 'q' },
                { "silent", no_argument, 0, 's' },
                { "analyse", no_argument, 0, 'a' },
                { "limit", required_argument, 0, 'n' },
                { "max-weight", required_argument, 0, 'w' },
                { "beam", required_argument, 0, 'b' },
                { "time-cutoff", required_argument, 0, 't' },
                { "real-word", no_argument, 0, 'X' },
                { "error-model", required_argument, 0, 'm' },
                { "lexicon", required_argument, 0, 'l' },
                { "continuation", required_argument, 0, 'C' },
#ifdef WINDOWS
                { "output-to-console", no_argument, 0, 'k' },
#endif
                { 0, 0, 0, 0 }
              };

        int option_index = 0;
        c = getopt_long(argc, argv, "hVvqsan:w:b:t:SXm:l:kC:", long_options,
                        &option_index);
        char *endptr = 0;

        if (c == -1) // no more options to look at
            break;

        switch (c)
        {
        case 'h':
            print_usage();
            return EXIT_SUCCESS;
            break;

        case 'V':
            print_version();
            return EXIT_SUCCESS;
            break;

        case 'v':
            verbose = true;
            quiet = false;
            break;

        case 'q': // fallthrough
        case 's':
            quiet = true;
            verbose = false;
            break;
        case 'a':
            analyse = true;
            break;
        case 'n':
            suggs = strtoul(optarg, &endptr, 10);
            if (endptr == optarg)
            {
                fprintf(stderr, "%s not a strtoul number\n", optarg);
                exit(1);
            }
            else if (*endptr != '\0')
            {
                fprintf(stderr, "%s truncated from limit parameter\n", endptr);
            }
            break;
        case 'w':
            max_weight = strtof(optarg, &endptr);
            if (endptr == optarg)
            {
                fprintf(stderr, "%s is not a float\n", optarg);
                exit(1);
            }
            else if (*endptr != '\0')
            {
                fprintf(stderr, "%s truncated from limit parameter\n", endptr);
            }

            break;
        case 'b':
            beam = strtof(optarg, &endptr);
            if (endptr == optarg)
            {
                fprintf(stderr, "%s is not a float\n", optarg);
                exit(1);
            }
            else if (*endptr != '\0')
            {
                fprintf(stderr, "%s truncated from limit parameter\n", endptr);
            }

            break;
        case 't':
            time_cutoff = strtof(optarg, &endptr);
            if (endptr == optarg)
            {
                fprintf(stderr, "%s is not a float\n", optarg);
                exit(1);
            }
            else if (*endptr != '\0')
            {
                fprintf(stderr, "%s truncated from limit parameter\n", endptr);
            }

            break;
#ifdef WINDOWS
        case 'k':
            output_to_console = true;
            break;
#endif
        case 'm':
            error_model_filename = optarg;
            break;
        case 'l':
            lexicon_filename = optarg;
            break;
        case 'C':
            continuation_marker = optarg;
            break;
        default:
            std::cerr << "Invalid option\n\n";
            print_short_help();
            return EXIT_FAILURE;
            break;
        }
    }
#else
    int optind = 1;
#endif
    // no more options, we should now be at the input filenames
    if (optind == (argc - 1))
    {
        if (error_model_filename != "" || lexicon_filename != "")
        {
            std::cerr << "Give *either* a zhfst speller or --error-model and "
                         "--lexicon"
                      << std::endl;
            print_short_help();
            return EXIT_FAILURE;
        }
        return zhfst_spell(argv[optind]);
    }
    else if (optind < (argc - 1))
    {
        std::cerr << "Too many file parameters" << std::endl;
        print_short_help();
        return EXIT_FAILURE;
    }
    else if (optind >= argc)
    {
        if (error_model_filename == "" || lexicon_filename == "")
        {
            std::cerr << "Give *either* a zhfst speller or --error-model and "
                         "--lexicon"
                      << std::endl;
            print_short_help();
            return EXIT_FAILURE;
        }
        FILE *err_file = fopen(error_model_filename.c_str(), "r");
        FILE *lex_file = fopen(lexicon_filename.c_str(), "r");
        hfst_ospell::Transducer err(err_file);
        hfst_ospell::Transducer lex(lex_file);
        hfst_ospell::Speller *s = new hfst_ospell::Speller(&err, &lex);
        return legacy_spell(s);
    }
    return EXIT_SUCCESS;
}