HfstTokenizer.h

Go to the documentation of this file.

// Copyright (c) 2016 University of Helsinki
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
// See the file COPYING included with this distribution for more
// information.

#ifndef _HFST_TOKENIZER_H_
#define _HFST_TOKENIZER_H_
#include "HfstSymbolDefs.h"
#include "HfstExceptionDefs.h"
#include <iostream>
#include <climits>
#include <string>

#include "hfstdll.h"

namespace hfst
{
  using hfst::String;
  using hfst::StringSet;
  using hfst::StringPair;
  using hfst::StringPairVector;

  // Copied from HfstDataTypes.h because including the file
  // causes problems with header file #ifndef _HEADER_FILE_H_ guards
  typedef std::vector<std::string> StringVector;
  typedef std::pair<float,StringVector> HfstOneLevelPath;

  class MultiCharSymbolTrie;
  typedef std::vector<MultiCharSymbolTrie*> MultiCharSymbolTrieVector;
  typedef std::vector<bool> SymbolEndVector;

  class MultiCharSymbolTrie
  {
  private:
    MultiCharSymbolTrieVector symbol_rests;
    SymbolEndVector is_leaf;
    bool is_end_of_string(const char * p) const ;
    void set_symbol_end(const char * p);
    void init_symbol_rests(const char * p);
    void add_symbol_rest(const char * p);
    bool is_symbol_end(const char * p) const;
    MultiCharSymbolTrie * get_symbol_rest_trie(const char * p) const;

  public:
    HFSTDLL MultiCharSymbolTrie(void);
    HFSTDLL ~MultiCharSymbolTrie(void);
    HFSTDLL void add(const char * p);
    HFSTDLL const char * find(const char * p) const;  
  };
  
  class HfstTokenizer
  {  
  private:
    MultiCharSymbolTrie multi_char_symbols;
    StringSet skip_symbol_set;
    int get_next_symbol_size(const char * symbol) const;
    bool is_skip_symbol(String &s) const;
  
  public:

    HFSTDLL HfstTokenizer();

    HFSTDLL void add_skip_symbol(const std::string &symbol);

    HFSTDLL void add_multichar_symbol(const std::string& symbol);

    HFSTDLL StringPairVector tokenize(const std::string &input_string) const;

    HFSTDLL StringVector tokenize_one_level(const std::string &input_string) const;

    HFSTDLL static StringPairVector tokenize_space_separated(const std::string & str);

    HFSTDLL StringPairVector tokenize(const std::string &input_string,
                              const std::string &output_string) const;

    HFSTDLL StringPairVector tokenize(const std::string &input_string,
                              const std::string &output_string,
                              void (*warn_about_pair)(const std::pair<std::string, std::string> &symbol_pair)) const;

    HFSTDLL StringPairVector tokenize_and_align_flag_diacritics
      (const std::string &input_string,
       const std::string &output_string,
       void (*warn_about_pair)(const std::pair<std::string, std::string> &symbol_pair)) const;

    HFSTDLL static void check_utf8_correctness(const std::string &input_string);
  };
}
#endif